/ Check-in [8ff2b8f5]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the "notindexed" option to fts4.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts4-notindexed
Files: files | file ages | folders
SHA1: 8ff2b8f5948ccddce70102e6d68d464c66e4f7ca
User & Date: dan 2013-06-21 17:30:47
Context
2013-06-21
18:18
Add a few more tests for the fts4 notindexed option. Closed-Leaf check-in: b53c0c41 user: dan tags: fts4-notindexed
17:30
Add the "notindexed" option to fts4. check-in: 8ff2b8f5 user: dan tags: fts4-notindexed
2013-06-20
14:07
Disable posix_fallocate() for all systems, all the time, unless the HAVE_POSIX_FALLOCATE compile-time macro is supplied. check-in: b6744622 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to ext/fts3/fts3.c.

1077
1078
1079
1080
1081
1082
1083


1084
1085
1086
1087
1088
1089
1090
1091
1092

1093








1094
1095


1096
1097
1098
1099
1100
1101
1102
....
1127
1128
1129
1130
1131
1132
1133
1134

1135
1136
1137
1138
1139
1140
1141
....
1193
1194
1195
1196
1197
1198
1199





1200
1201
1202
1203
1204
1205
1206
....
1264
1265
1266
1267
1268
1269
1270

1271
1272
1273
1274
1275
1276
1277
....
1297
1298
1299
1300
1301
1302
1303

1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
....
1320
1321
1322
1323
1324
1325
1326



















1327
1328
1329
1330
1331
1332
1333
1334
....
1361
1362
1363
1364
1365
1366
1367

1368

1369
1370
1371
1372
1373
1374
1375
  int bNoDocsize = 0;             /* True to omit %_docsize table */
  int bDescIdx = 0;               /* True to store descending indexes */
  char *zPrefix = 0;              /* Prefix parameter value (or NULL) */
  char *zCompress = 0;            /* compress=? parameter (or NULL) */
  char *zUncompress = 0;          /* uncompress=? parameter (or NULL) */
  char *zContent = 0;             /* content=? parameter (or NULL) */
  char *zLanguageid = 0;          /* languageid=? parameter (or NULL) */



  assert( strlen(argv[0])==4 );
  assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
       || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
  );

  nDb = (int)strlen(argv[1]) + 1;
  nName = (int)strlen(argv[2]) + 1;


  aCol = (const char **)sqlite3_malloc(sizeof(const char *) * (argc-2) );








  if( !aCol ) return SQLITE_NOMEM;
  memset((void *)aCol, 0, sizeof(const char *) * (argc-2));



  /* Loop through all of the arguments passed by the user to the FTS3/4
  ** module (i.e. all the column names and special arguments). This loop
  ** does the following:
  **
  **   + Figures out the number of columns the FTSX table will have, and
  **     the number of bytes of space that must be allocated to store copies
................................................................................
      } aFts4Opt[] = {
        { "matchinfo",   9 },     /* 0 -> MATCHINFO */
        { "prefix",      6 },     /* 1 -> PREFIX */
        { "compress",    8 },     /* 2 -> COMPRESS */
        { "uncompress", 10 },     /* 3 -> UNCOMPRESS */
        { "order",       5 },     /* 4 -> ORDER */
        { "content",     7 },     /* 5 -> CONTENT */
        { "languageid", 10 }      /* 6 -> LANGUAGEID */

      };

      int iOpt;
      if( !zVal ){
        rc = SQLITE_NOMEM;
      }else{
        for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){
................................................................................

            case 6:              /* LANGUAGEID */
              assert( iOpt==6 );
              sqlite3_free(zLanguageid);
              zLanguageid = zVal;
              zVal = 0;
              break;





          }
        }
        sqlite3_free(zVal);
      }
    }

    /* Otherwise, the argument is a column name. */
................................................................................
  }
  if( rc!=SQLITE_OK ) goto fts3_init_out;

  /* Allocate and populate the Fts3Table structure. */
  nByte = sizeof(Fts3Table) +                  /* Fts3Table */
          nCol * sizeof(char *) +              /* azColumn */
          nIndex * sizeof(struct Fts3Index) +  /* aIndex */

          nName +                              /* zName */
          nDb +                                /* zDb */
          nString;                             /* Space for azColumn strings */
  p = (Fts3Table*)sqlite3_malloc(nByte);
  if( p==0 ){
    rc = SQLITE_NOMEM;
    goto fts3_init_out;
................................................................................

  p->aIndex = (struct Fts3Index *)&p->azColumn[nCol];
  memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex);
  p->nIndex = nIndex;
  for(i=0; i<nIndex; i++){
    fts3HashInit(&p->aIndex[i].hPending, FTS3_HASH_STRING, 1);
  }


  /* Fill in the zName and zDb fields of the vtab structure. */
  zCsr = (char *)&p->aIndex[nIndex];
  p->zName = zCsr;
  memcpy(zCsr, argv[2], nName);
  zCsr += nName;
  p->zDb = zCsr;
  memcpy(zCsr, argv[1], nDb);
  zCsr += nDb;

................................................................................
    zCsr[n] = '\0';
    sqlite3Fts3Dequote(zCsr);
    p->azColumn[iCol] = zCsr;
    zCsr += n+1;
    assert( zCsr <= &((char *)p)[nByte] );
  }




















  if( (zCompress==0)!=(zUncompress==0) ){
    char const *zMiss = (zCompress==0 ? "compress" : "uncompress");
    rc = SQLITE_ERROR;
    *pzErr = sqlite3_mprintf("missing %s parameter in fts4 constructor", zMiss);
  }
  p->zReadExprlist = fts3ReadExprList(p, zUncompress, &rc);
  p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc);
  if( rc!=SQLITE_OK ) goto fts3_init_out;
................................................................................
fts3_init_out:
  sqlite3_free(zPrefix);
  sqlite3_free(aIndex);
  sqlite3_free(zCompress);
  sqlite3_free(zUncompress);
  sqlite3_free(zContent);
  sqlite3_free(zLanguageid);

  sqlite3_free((void *)aCol);

  if( rc!=SQLITE_OK ){
    if( p ){
      fts3DisconnectMethod((sqlite3_vtab *)p);
    }else if( pTokenizer ){
      pTokenizer->pModule->xDestroy(pTokenizer);
    }
  }else{







>
>









>
|
>
>
>
>
>
>
>
>
|
<
>
>







 







|
>







 







>
>
>
>
>







 







>







 







>


|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|







 







>

>







1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105

1106
1107
1108
1109
1110
1111
1112
1113
1114
....
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
....
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
....
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
....
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
....
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
....
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
  int bNoDocsize = 0;             /* True to omit %_docsize table */
  int bDescIdx = 0;               /* True to store descending indexes */
  char *zPrefix = 0;              /* Prefix parameter value (or NULL) */
  char *zCompress = 0;            /* compress=? parameter (or NULL) */
  char *zUncompress = 0;          /* uncompress=? parameter (or NULL) */
  char *zContent = 0;             /* content=? parameter (or NULL) */
  char *zLanguageid = 0;          /* languageid=? parameter (or NULL) */
  char **azNotindexed = 0;        /* The set of notindexed= columns */
  int nNotindexed = 0;            /* Size of azNotindexed[] array */

  assert( strlen(argv[0])==4 );
  assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
       || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
  );

  nDb = (int)strlen(argv[1]) + 1;
  nName = (int)strlen(argv[2]) + 1;

  nByte = sizeof(const char *) * (argc-2);
  aCol = (const char **)sqlite3_malloc(nByte);
  if( aCol ){
    memset(aCol, 0, nByte);
    azNotindexed = (char **)sqlite3_malloc(nByte);
  }
  if( azNotindexed ){
    memset(azNotindexed, 0, nByte);
  }
  if( !aCol || !azNotindexed ){
    rc = SQLITE_NOMEM;

    goto fts3_init_out;
  }

  /* Loop through all of the arguments passed by the user to the FTS3/4
  ** module (i.e. all the column names and special arguments). This loop
  ** does the following:
  **
  **   + Figures out the number of columns the FTSX table will have, and
  **     the number of bytes of space that must be allocated to store copies
................................................................................
      } aFts4Opt[] = {
        { "matchinfo",   9 },     /* 0 -> MATCHINFO */
        { "prefix",      6 },     /* 1 -> PREFIX */
        { "compress",    8 },     /* 2 -> COMPRESS */
        { "uncompress", 10 },     /* 3 -> UNCOMPRESS */
        { "order",       5 },     /* 4 -> ORDER */
        { "content",     7 },     /* 5 -> CONTENT */
        { "languageid", 10 },     /* 6 -> LANGUAGEID */
        { "notindexed", 10 }      /* 7 -> NOTINDEXED */
      };

      int iOpt;
      if( !zVal ){
        rc = SQLITE_NOMEM;
      }else{
        for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){
................................................................................

            case 6:              /* LANGUAGEID */
              assert( iOpt==6 );
              sqlite3_free(zLanguageid);
              zLanguageid = zVal;
              zVal = 0;
              break;

            case 7:              /* NOTINDEXED */
              azNotindexed[nNotindexed++] = zVal;
              zVal = 0;
              break;
          }
        }
        sqlite3_free(zVal);
      }
    }

    /* Otherwise, the argument is a column name. */
................................................................................
  }
  if( rc!=SQLITE_OK ) goto fts3_init_out;

  /* Allocate and populate the Fts3Table structure. */
  nByte = sizeof(Fts3Table) +                  /* Fts3Table */
          nCol * sizeof(char *) +              /* azColumn */
          nIndex * sizeof(struct Fts3Index) +  /* aIndex */
          nCol * sizeof(u8) +                  /* abNotindexed */
          nName +                              /* zName */
          nDb +                                /* zDb */
          nString;                             /* Space for azColumn strings */
  p = (Fts3Table*)sqlite3_malloc(nByte);
  if( p==0 ){
    rc = SQLITE_NOMEM;
    goto fts3_init_out;
................................................................................

  p->aIndex = (struct Fts3Index *)&p->azColumn[nCol];
  memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex);
  p->nIndex = nIndex;
  for(i=0; i<nIndex; i++){
    fts3HashInit(&p->aIndex[i].hPending, FTS3_HASH_STRING, 1);
  }
  p->abNotindexed = (u8 *)&p->aIndex[nIndex];

  /* Fill in the zName and zDb fields of the vtab structure. */
  zCsr = (char *)&p->abNotindexed[nCol];
  p->zName = zCsr;
  memcpy(zCsr, argv[2], nName);
  zCsr += nName;
  p->zDb = zCsr;
  memcpy(zCsr, argv[1], nDb);
  zCsr += nDb;

................................................................................
    zCsr[n] = '\0';
    sqlite3Fts3Dequote(zCsr);
    p->azColumn[iCol] = zCsr;
    zCsr += n+1;
    assert( zCsr <= &((char *)p)[nByte] );
  }

  /* Fill in the abNotindexed array */
  for(iCol=0; iCol<nCol; iCol++){
    int n = strlen(p->azColumn[iCol]);
    for(i=0; i<nNotindexed; i++){
      char *zNot = azNotindexed[i];
      if( zNot && 0==sqlite3_strnicmp(p->azColumn[iCol], zNot, n) ){
        p->abNotindexed[iCol] = 1;
        sqlite3_free(zNot);
        azNotindexed[i] = 0;
      }
    }
  }
  for(i=0; i<nNotindexed; i++){
    if( azNotindexed[i] ){
      *pzErr = sqlite3_mprintf("no such column: %s", azNotindexed[i]);
      rc = SQLITE_ERROR;
    }
  }

  if( rc==SQLITE_OK && (zCompress==0)!=(zUncompress==0) ){
    char const *zMiss = (zCompress==0 ? "compress" : "uncompress");
    rc = SQLITE_ERROR;
    *pzErr = sqlite3_mprintf("missing %s parameter in fts4 constructor", zMiss);
  }
  p->zReadExprlist = fts3ReadExprList(p, zUncompress, &rc);
  p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc);
  if( rc!=SQLITE_OK ) goto fts3_init_out;
................................................................................
fts3_init_out:
  sqlite3_free(zPrefix);
  sqlite3_free(aIndex);
  sqlite3_free(zCompress);
  sqlite3_free(zUncompress);
  sqlite3_free(zContent);
  sqlite3_free(zLanguageid);
  for(i=0; i<nNotindexed; i++) sqlite3_free(azNotindexed[i]);
  sqlite3_free((void *)aCol);
  sqlite3_free((void *)azNotindexed);
  if( rc!=SQLITE_OK ){
    if( p ){
      fts3DisconnectMethod((sqlite3_vtab *)p);
    }else if( pTokenizer ){
      pTokenizer->pModule->xDestroy(pTokenizer);
    }
  }else{

Changes to ext/fts3/fts3Int.h.

202
203
204
205
206
207
208

209
210
211
212
213
214
215
struct Fts3Table {
  sqlite3_vtab base;              /* Base class used by SQLite core */
  sqlite3 *db;                    /* The database connection */
  const char *zDb;                /* logical database name */
  const char *zName;              /* virtual table name */
  int nColumn;                    /* number of named columns in virtual table */
  char **azColumn;                /* column names.  malloced */

  sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
  char *zContentTbl;              /* content=xxx option, or NULL */
  char *zLanguageid;              /* languageid=xxx option, or NULL */
  u8 bAutoincrmerge;              /* True if automerge=1 */
  u32 nLeafAdd;                   /* Number of leaf blocks added this trans */

  /* Precompiled statements used by the implementation. Each of these 







>







202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
struct Fts3Table {
  sqlite3_vtab base;              /* Base class used by SQLite core */
  sqlite3 *db;                    /* The database connection */
  const char *zDb;                /* logical database name */
  const char *zName;              /* virtual table name */
  int nColumn;                    /* number of named columns in virtual table */
  char **azColumn;                /* column names.  malloced */
  u8 *abNotindexed;               /* True for 'notindexed' columns */
  sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
  char *zContentTbl;              /* content=xxx option, or NULL */
  char *zLanguageid;              /* languageid=xxx option, or NULL */
  u8 bAutoincrmerge;              /* True if automerge=1 */
  u32 nLeafAdd;                   /* Number of leaf blocks added this trans */

  /* Precompiled statements used by the implementation. Each of these 

Changes to ext/fts3/fts3_write.c.

896
897
898
899
900
901
902


903
904
905
906
907
908
909

910
911
912
913
914
915
916
....
1048
1049
1050
1051
1052
1053
1054


1055
1056
1057
1058

1059
1060
1061
1062
1063
1064
1065
....
3292
3293
3294
3295
3296
3297
3298

3299
3300
3301
3302

3303
3304
3305
3306
3307
3308
3309
....
5097
5098
5099
5100
5101
5102
5103

5104
5105
5106
5107
5108
5109
5110
....
5129
5130
5131
5132
5133
5134
5135

5136
5137
5138
5139
5140
5141
5142
  Fts3Table *p, 
  int iLangid, 
  sqlite3_value **apVal, 
  u32 *aSz
){
  int i;                          /* Iterator variable */
  for(i=2; i<p->nColumn+2; i++){


    const char *zText = (const char *)sqlite3_value_text(apVal[i]);
    int rc = fts3PendingTermsAdd(p, iLangid, zText, i-2, &aSz[i-2]);
    if( rc!=SQLITE_OK ){
      return rc;
    }
    aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
  }

  return SQLITE_OK;
}

/*
** This function is called by the xUpdate() method for an INSERT operation.
** The apVal parameter is passed a copy of the apVal argument passed by
** SQLite to the xUpdate() method. i.e:
................................................................................
  rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid);
  if( rc==SQLITE_OK ){
    if( SQLITE_ROW==sqlite3_step(pSelect) ){
      int i;
      int iLangid = langidFromSelect(p, pSelect);
      rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0));
      for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){


        const char *zText = (const char *)sqlite3_column_text(pSelect, i);
        rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[i-1]);
        aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i);
      }

      if( rc!=SQLITE_OK ){
        sqlite3_reset(pSelect);
        *pRC = rc;
        return;
      }
      *pbFound = 1;
    }
................................................................................

    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
      int iCol;
      int iLangid = langidFromSelect(p, pStmt);
      rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0));
      memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1));
      for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){

        const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1);
        rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]);
        aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1);
      }

      if( p->bHasDocsize ){
        fts3InsertDocsize(&rc, p, aSz);
      }
      if( rc!=SQLITE_OK ){
        sqlite3_finalize(pStmt);
        pStmt = 0;
      }else{
................................................................................
    sqlite3_tokenizer *pT = p->pTokenizer;
    sqlite3_tokenizer_module const *pModule = pT->pModule;
   
    assert( pCsr->isRequireSeek==0 );
    iDocid = sqlite3_column_int64(pCsr->pStmt, 0);
  
    for(i=0; i<p->nColumn && rc==SQLITE_OK; i++){

      const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1);
      sqlite3_tokenizer_cursor *pTC = 0;
  
      rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC);
      while( rc==SQLITE_OK ){
        char const *zToken;       /* Buffer containing token */
        int nToken = 0;           /* Number of bytes in token */
................................................................................
  
    for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
      if( pDef->pList ){
        rc = fts3PendingListAppendVarint(&pDef->pList, 0);
      }
    }
  }


  return rc;
}

int sqlite3Fts3DeferredTokenList(
  Fts3DeferredToken *p, 
  char **ppData, 







>
>
|
|





>







 







>
>
|
|
|
|
>







 







>




>







 







>







 







>







896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
....
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
....
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
....
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
....
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
  Fts3Table *p, 
  int iLangid, 
  sqlite3_value **apVal, 
  u32 *aSz
){
  int i;                          /* Iterator variable */
  for(i=2; i<p->nColumn+2; i++){
    int iCol = i-2;
    if( p->abNotindexed[iCol]==0 ){
      const char *zText = (const char *)sqlite3_value_text(apVal[i]);
      int rc = fts3PendingTermsAdd(p, iLangid, zText, iCol, &aSz[iCol]);
      if( rc!=SQLITE_OK ){
        return rc;
      }
      aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
    }
  }
  return SQLITE_OK;
}

/*
** This function is called by the xUpdate() method for an INSERT operation.
** The apVal parameter is passed a copy of the apVal argument passed by
** SQLite to the xUpdate() method. i.e:
................................................................................
  rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid);
  if( rc==SQLITE_OK ){
    if( SQLITE_ROW==sqlite3_step(pSelect) ){
      int i;
      int iLangid = langidFromSelect(p, pSelect);
      rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0));
      for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){
        int iCol = i-1;
        if( p->abNotindexed[iCol]==0 ){
          const char *zText = (const char *)sqlite3_column_text(pSelect, i);
          rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]);
          aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i);
        }
      }
      if( rc!=SQLITE_OK ){
        sqlite3_reset(pSelect);
        *pRC = rc;
        return;
      }
      *pbFound = 1;
    }
................................................................................

    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
      int iCol;
      int iLangid = langidFromSelect(p, pStmt);
      rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0));
      memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1));
      for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
        if( p->abNotindexed[iCol]==0 ){
          const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1);
          rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]);
          aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1);
        }
      }
      if( p->bHasDocsize ){
        fts3InsertDocsize(&rc, p, aSz);
      }
      if( rc!=SQLITE_OK ){
        sqlite3_finalize(pStmt);
        pStmt = 0;
      }else{
................................................................................
    sqlite3_tokenizer *pT = p->pTokenizer;
    sqlite3_tokenizer_module const *pModule = pT->pModule;
   
    assert( pCsr->isRequireSeek==0 );
    iDocid = sqlite3_column_int64(pCsr->pStmt, 0);
  
    for(i=0; i<p->nColumn && rc==SQLITE_OK; i++){
      if( p->abNotindexed[i]==0 ){
        const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1);
        sqlite3_tokenizer_cursor *pTC = 0;

        rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC);
        while( rc==SQLITE_OK ){
          char const *zToken;       /* Buffer containing token */
          int nToken = 0;           /* Number of bytes in token */
................................................................................

      for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
        if( pDef->pList ){
          rc = fts3PendingListAppendVarint(&pDef->pList, 0);
        }
      }
    }
  }

  return rc;
}

int sqlite3Fts3DeferredTokenList(
  Fts3DeferredToken *p, 
  char **ppData, 

Added test/fts4noti.test.































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
# 2013 June 21
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the notindexed=xxx FTS4 option.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set ::testprefix fts4noti

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}


#-------------------------------------------------------------------------
# Test that typos in "notindexed=" column names are detected.
#
do_execsql_test 1.0 {
  CREATE TABLE cc(a, b, c);
}
foreach {tn arg res} {
  1 "(b, c, notindexed=a)"                  {1 {no such column: a}}
  2 "(a, b, notindexed=a)"                                   {0 {}}
  3 "(a, b, notindexed=a, notindexed=a)"                     {0 {}}
  4 "(notindexed=a, a, b)"                                   {0 {}}
  5 "(notindexed=a, notindexed=b, notindexed=c, a, b, c, d)" {0 {}}
  6 "(notindexed=a, notindexed=B, notindexed=c, a, b, c, d)" {0 {}}
  7 "(notindexed=a, notindexed=b, notindexed=c, a, B, c, d)" {0 {}}
  8 "(notindexed=d, content=cc)" {1 {no such column: d}}
  9 "(notindexed=a, content=cc)" {0 {}}
} {
  do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts4 $arg" $res
  if {[lindex $res 0]==0} { execsql "DROP TABLE t1" }
}

do_execsql_test 1.x { SELECT name FROM sqlite_master } {cc}


#-------------------------------------------------------------------------
# Test that notindexed columns are not indexed.
#
do_execsql_test 2.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(a, b, c, notindexed=b);
  INSERT INTO t1 VALUES('one two', 'three four', 'five six');
  INSERT INTO t1 VALUES('three four', 'five six', 'one two');
}
do_execsql_test 2.2 { SELECT docid FROM t1 WHERE t1 MATCH 'one' } {1 2}
do_execsql_test 2.3 { SELECT docid FROM t1 WHERE t1 MATCH 'three' } {2}
do_execsql_test 2.4 { SELECT docid FROM t1 WHERE t1 MATCH 'five' } {1}

do_execsql_test 2.5 { INSERT INTO t1(t1) VALUES('optimize') }

do_execsql_test 2.6 { SELECT docid FROM t1 WHERE t1 MATCH 'one' } {1 2}
do_execsql_test 2.7 { SELECT docid FROM t1 WHERE t1 MATCH 'three' } {2}
do_execsql_test 2.8 { SELECT docid FROM t1 WHERE t1 MATCH 'five' } {1}

do_execsql_test 2.9 { INSERT INTO t1(t1) VALUES('rebuild') }

do_execsql_test 2.10 { SELECT docid FROM t1 WHERE t1 MATCH 'one' } {1 2}
do_execsql_test 2.11 { SELECT docid FROM t1 WHERE t1 MATCH 'three' } {2}
do_execsql_test 2.12 { SELECT docid FROM t1 WHERE t1 MATCH 'five' } {1}

do_execsql_test 2.13 { 
  SELECT * FROM t1 WHERE docid=1
} {{one two} {three four} {five six}}
do_execsql_test 2.14 { 
  SELECT * FROM t1 WHERE docid=2
} {{three four} {five six} {one two}}

do_execsql_test 2.x { DROP TABLE t1 }

#-------------------------------------------------------------------------
# Test that notindexed columns are not scanned for deferred tokens.
#

do_execsql_test 3.1 {
  CREATE VIRTUAL TABLE t2 USING fts4(x, y, notindexed=x);
}
do_test 3.2 {
  set v [string repeat " 1" 50000]
  set v1 "x $v"
  set v2 "y $v"
  execsql { 
    INSERT INTO t2 VALUES(1, 'x y z');
    INSERT INTO t2 VALUES(2, $v1);
    INSERT INTO t2 VALUES(3, $v2);
    INSERT INTO t2 VALUES(4, $v2);
    INSERT INTO t2 VALUES(5, $v2);
    INSERT INTO t2 VALUES(6, $v2);
  }
} {}

do_execsql_test 3.3 { SELECT x FROM t2 WHERE t2 MATCH '2' } {}
do_execsql_test 3.4 { SELECT x FROM t2 WHERE t2 MATCH '1' } {2 3 4 5 6}
do_execsql_test 3.5 { SELECT x FROM t2 WHERE t2 MATCH 'x' } {1 2}
do_execsql_test 3.6 { SELECT x FROM t2 WHERE t2 MATCH 'x 1' } {2}
  


finish_test

Changes to test/permutations.test.

190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
  fts3near.test fts3query.test fts3shared.test fts3snippet.test 
  fts3sort.test
  fts3fault.test fts3malloc.test fts3matchinfo.test
  fts3aux1.test fts3comp1.test fts3auto.test
  fts4aa.test fts4content.test
  fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
  fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test
  fts4check.test fts4unicode.test
}

test_suite "nofaultsim" -prefix "" -description {
  "Very" quick test suite. Runs in less than 5 minutes on a workstation. 
  This test suite is the same as the "quick" tests, except that some files
  that test malloc and IO errors are omitted.
} -files [







|







190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
  fts3near.test fts3query.test fts3shared.test fts3snippet.test 
  fts3sort.test
  fts3fault.test fts3malloc.test fts3matchinfo.test
  fts3aux1.test fts3comp1.test fts3auto.test
  fts4aa.test fts4content.test
  fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
  fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test
  fts4check.test fts4unicode.test fts4noti.test
}

test_suite "nofaultsim" -prefix "" -description {
  "Very" quick test suite. Runs in less than 5 minutes on a workstation. 
  This test suite is the same as the "quick" tests, except that some files
  that test malloc and IO errors are omitted.
} -files [