/ Check-in [e8540377]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:The sqlite3_normalized_sql() interface should not be transforming quoted identifier names into wildcards. Fix this, and at the same time simplify the code substantially.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: e8540377ec66fa5f9ae3c93bedb5c094057698199c37fc211f7ea95429e815e4
User & Date: drh 2018-12-05 23:45:46
Context
2018-12-06
20:18
Experimental changes to (optionally) allow double-quoted strings to be checked against known identifiers. Leaf check-in: 73a6b8c1 user: mistachkin tags: normalize_v4
2018-12-05
23:56
Get rid of the hash table used to track IN operators in the sqlite3_normalized_sql() implementation. Use simple integer variables instead. check-in: 272dc74f user: drh tags: trunk
23:45
The sqlite3_normalized_sql() interface should not be transforming quoted identifier names into wildcards. Fix this, and at the same time simplify the code substantially. check-in: e8540377 user: drh tags: trunk
21:55
Fix an imbalanced lock problem in sqlite3Normalize() introduced by the simplification in [94ea6379178e3ff6a]. check-in: f6962437 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/expr.c.

2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
*/
int sqlite3IsRowid(const char *z){
  if( sqlite3StrICmp(z, "_ROWID_")==0 ) return 1;
  if( sqlite3StrICmp(z, "ROWID")==0 ) return 1;
  if( sqlite3StrICmp(z, "OID")==0 ) return 1;
  return 0;
}
#ifdef SQLITE_ENABLE_NORMALIZE
int sqlite3IsRowidN(const char *z, int n){
  if( sqlite3StrNICmp(z, "_ROWID_", n)==0 ) return 1;
  if( sqlite3StrNICmp(z, "ROWID", n)==0 ) return 1;
  if( sqlite3StrNICmp(z, "OID", n)==0 ) return 1;
  return 0;
}
#endif

/*
** pX is the RHS of an IN operator.  If pX is a SELECT statement 
** that can be simplified to a direct table access, then return
** a pointer to the SELECT statement.  If pX is not a SELECT statement,
** or if the SELECT statement needs to be manifested into a transient
** table, then return NULL.







<
<
<
<
<
<
<
<







2145
2146
2147
2148
2149
2150
2151








2152
2153
2154
2155
2156
2157
2158
*/
int sqlite3IsRowid(const char *z){
  if( sqlite3StrICmp(z, "_ROWID_")==0 ) return 1;
  if( sqlite3StrICmp(z, "ROWID")==0 ) return 1;
  if( sqlite3StrICmp(z, "OID")==0 ) return 1;
  return 0;
}









/*
** pX is the RHS of an IN operator.  If pX is a SELECT statement 
** that can be simplified to a direct table access, then return
** a pointer to the SELECT statement.  If pX is not a SELECT statement,
** or if the SELECT statement needs to be manifested into a transient
** table, then return NULL.

Changes to src/prepare.c.

706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
...
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
  rc = sqlite3ApiExit(db, rc);
  assert( (rc&db->errMask)==rc );
  sqlite3_mutex_leave(db->mutex);
  return rc;
}

#ifdef SQLITE_ENABLE_NORMALIZE
/*
** Checks if the specified token is a table, column, or function name,
** based on the databases associated with the statement being prepared.
** If the function fails, zero is returned and pRc is filled with the
** error code.
*/
static int shouldTreatAsIdentifier(
  sqlite3 *db,        /* Database handle. */
  const char *zToken, /* Pointer to start of token to be checked */
  int nToken,         /* Length of token to be checked */
  int *pRc            /* Pointer to error code upon failure */
){
  int bFound = 0;     /* Non-zero if token is an identifier name. */
  int i, j;           /* Database and column loop indexes. */
  Schema *pSchema;    /* Schema for current database. */
  Hash *pHash;        /* Hash table of tables for current database. */
  HashElem *e;        /* Hash element for hash table iteration. */
  Table *pTab;        /* Database table for columns being checked. */
  char *zId;          /* Zero terminated name of the identifier */
  char zSpace[50];    /* Static space for the zero-terminated name */

  if( nToken<sizeof(zSpace) ){
    memcpy(zSpace, zToken, nToken);
    zSpace[nToken] = 0;
    zId = zSpace;
  }else{
    zId = sqlite3DbStrNDup(db, zToken, nToken);
    if( zId==0 ) return 0;
  }
  if( sqlite3IsRowid(zId) ){
    bFound = 1;
    goto done1;
  }
  if( nToken>0 ){
    int hash = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zToken[0]], nToken);
    if( sqlite3FunctionSearch(hash, zId) ){
      bFound = 1;
      goto done1;
    }
  }
  assert( db!=0 );
  sqlite3_mutex_enter(db->mutex);
  sqlite3BtreeEnterAll(db);
  for(i=0; i<db->nDb; i++){
    pHash = &db->aFunc;
    if( sqlite3HashFind(pHash, zId) ){
      bFound = 1;
      break;
    }
    pSchema = db->aDb[i].pSchema;
    if( pSchema==0 ) continue;
    pHash = &pSchema->tblHash;
    if( sqlite3HashFind(pHash, zId) ){
      bFound = 1;
      break;
    }
    for(e=sqliteHashFirst(pHash); e; e=sqliteHashNext(e)){
      pTab = sqliteHashData(e);
      if( pTab==0 ) continue;
      pHash = pTab->pColHash;
      if( pHash==0 ){
        pTab->pColHash = pHash = sqlite3_malloc(sizeof(Hash));
        if( pHash ){
          sqlite3HashInit(pHash);
          for(j=0; j<pTab->nCol; j++){
            Column *pCol = &pTab->aCol[j];
            sqlite3HashInsert(pHash, pCol->zName, pCol);
          }
        }else{
          *pRc = SQLITE_NOMEM_BKPT;
          bFound = 0;
          goto done2;
        }
      }
      if( pHash && sqlite3HashFind(pHash, zId) ){
        bFound = 1;
        goto done2;
      }
    }
  }
done2:
  sqlite3BtreeLeaveAll(db);
  sqlite3_mutex_leave(db->mutex);
done1:
  if( zId!=zSpace ) sqlite3DbFree(db, zId);
  return bFound;
}

/*
** Attempt to estimate the final output buffer size needed for the fully
** normalized version of the specified SQL string.  This should take into
** account any potential expansion that could occur (e.g. via IN clauses
** being expanded, etc).  This size returned is the total number of bytes
** including the NUL terminator.
................................................................................
            break;
          }
        }
        if( j>0 && sqlite3IsIdChar(z[j-1]) && sqlite3IsIdChar(zSql[i]) ){
          z[j++] = ' ';
        }
        if( tokenType==TK_ID ){
          int i2 = i, n2 = n, rc = SQLITE_OK;
          if( nParen>0 ){
            assert( nParen<nSql );
            sqlite3HashInsert(&inHash, zSql+nParen, 0);
          }
          if( flags&SQLITE_TOKEN_QUOTED ){ i2++; n2-=2; }
          if( shouldTreatAsIdentifier(db, zSql+i2, n2, &rc)==0 ){
            if( rc!=SQLITE_OK ) goto normalizeError;
            if( sqlite3_keyword_check(zSql+i2, n2)==0 ){
              z[j++] = '?';
              break;
            }
          }
        }
        copyNormalizedToken(zSql, i, n, flags, z, &j);
        break;
      }
    }
  }
  assert( j<nZ && "one" );







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







|





<
<
<
<
<
<
<







706
707
708
709
710
711
712























































































713
714
715
716
717
718
719
...
881
882
883
884
885
886
887
888
889
890
891
892
893







894
895
896
897
898
899
900
  rc = sqlite3ApiExit(db, rc);
  assert( (rc&db->errMask)==rc );
  sqlite3_mutex_leave(db->mutex);
  return rc;
}

#ifdef SQLITE_ENABLE_NORMALIZE
























































































/*
** Attempt to estimate the final output buffer size needed for the fully
** normalized version of the specified SQL string.  This should take into
** account any potential expansion that could occur (e.g. via IN clauses
** being expanded, etc).  This size returned is the total number of bytes
** including the NUL terminator.
................................................................................
            break;
          }
        }
        if( j>0 && sqlite3IsIdChar(z[j-1]) && sqlite3IsIdChar(zSql[i]) ){
          z[j++] = ' ';
        }
        if( tokenType==TK_ID ){
          int i2 = i, n2 = n;
          if( nParen>0 ){
            assert( nParen<nSql );
            sqlite3HashInsert(&inHash, zSql+nParen, 0);
          }
          if( flags&SQLITE_TOKEN_QUOTED ){ i2++; n2-=2; }







        }
        copyNormalizedToken(zSql, i, n, flags, z, &j);
        break;
      }
    }
  }
  assert( j<nZ && "one" );

Changes to src/sqliteInt.h.

4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
#ifdef SQLITE_ENABLE_CURSOR_HINTS
int sqlite3ExprContainsSubquery(Expr*);
#endif
int sqlite3ExprIsInteger(Expr*, int*);
int sqlite3ExprCanBeNull(const Expr*);
int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
int sqlite3IsRowid(const char*);
#ifdef SQLITE_ENABLE_NORMALIZE
int sqlite3IsRowidN(const char*, int);
#endif
void sqlite3GenerateRowDelete(
    Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int);
void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int);
int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int);
void sqlite3ResolvePartIdxLabel(Parse*,int);
int sqlite3ExprReferencesUpdatedColumn(Expr*,int*,int);
void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int,







<
<
<







4023
4024
4025
4026
4027
4028
4029



4030
4031
4032
4033
4034
4035
4036
#ifdef SQLITE_ENABLE_CURSOR_HINTS
int sqlite3ExprContainsSubquery(Expr*);
#endif
int sqlite3ExprIsInteger(Expr*, int*);
int sqlite3ExprCanBeNull(const Expr*);
int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
int sqlite3IsRowid(const char*);



void sqlite3GenerateRowDelete(
    Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int);
void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int);
int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int);
void sqlite3ResolvePartIdxLabel(Parse*,int);
int sqlite3ExprReferencesUpdatedColumn(Expr*,int*,int);
void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int,

Changes to test/normalize.test.

203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
  {SELECT a FROM t1 WHERE x IN (1,2,3) AND hex8('abc');}
  0x2
  {0 {SELECT a FROM t1 WHERE x IN(?,?,?)AND hex8(?);}}

  430
  {SELECT "a" FROM t1 WHERE "x" IN ("1","2",'3');}
  0x2
  {0 {SELECT"a"FROM t1 WHERE"x"IN(?,?,?);}}

  440
  {SELECT 'a' FROM t1 WHERE 'x';}
  0x2
  {0 {SELECT?FROM t1 WHERE?;}}

  450







|







203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
  {SELECT a FROM t1 WHERE x IN (1,2,3) AND hex8('abc');}
  0x2
  {0 {SELECT a FROM t1 WHERE x IN(?,?,?)AND hex8(?);}}

  430
  {SELECT "a" FROM t1 WHERE "x" IN ("1","2",'3');}
  0x2
  {0 {SELECT"a"FROM t1 WHERE"x"IN("1","2",?);}}

  440
  {SELECT 'a' FROM t1 WHERE 'x';}
  0x2
  {0 {SELECT?FROM t1 WHERE?;}}

  450