/ Check-in [1d27ea74]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add experimental 'content' option to FTS4.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts4-content
Files: files | file ages | folders
SHA1: 1d27ea741f61c624e18bdc6a3b1c2d8574a64ddc
User & Date: dan 2011-10-04 11:22:59
Context
2011-10-04
16:37
Add tests to check that modifying the schema of an FTS content table does not cause a crash in the FTS module. Also disable the deferred token optimization for content=xxx FTS tables. check-in: be86c706 user: dan tags: fts4-content
11:22
Add experimental 'content' option to FTS4. check-in: 1d27ea74 user: dan tags: fts4-content
2011-10-02
05:23
Update MSVC makefile to allow targets to be built with support for ICU. check-in: eb5da5e1 user: mistachkin tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

464
465
466
467
468
469
470

471
472
473
474
475
476
477
...
503
504
505
506
507
508
509
510
511


512
513
514

515

516
517
518
519
520
521
522
523
524
525
526
...
574
575
576
577
578
579
580
581
582
583



584
585
586
587
588
589
590
591
592
593
594
595
596
597


598
599
600
601
602
603
604
...
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
...
769
770
771
772
773
774
775

776
777
778
779
780
781
782
783
784
785











786
787
788
789
790
791
792
...
901
902
903
904
905
906
907





















































































908
909
910
911
912
913
914
...
946
947
948
949
950
951
952

953
954
955
956
957
958
959
...
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
....
1048
1049
1050
1051
1052
1053
1054






1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066





















1067
1068
1069
1070
1071
1072
1073
....
1104
1105
1106
1107
1108
1109
1110


1111
1112
1113
1114
1115
1116
1117
....
1165
1166
1167
1168
1169
1170
1171

1172
1173
1174
1175
1176
1177
1178
....
1330
1331
1332
1333
1334
1335
1336


1337
1338
1339
1340
1341



1342
1343
1344
1345
1346
1347
1348
....
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718


2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
....
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
....
3075
3076
3077
3078
3079
3080
3081

3082
3083
3084
3085


3086
3087
3088
3089
3090
3091
3092
  /* Free any prepared statements held */
  for(i=0; i<SizeofArray(p->aStmt); i++){
    sqlite3_finalize(p->aStmt[i]);
  }
  sqlite3_free(p->zSegmentsTbl);
  sqlite3_free(p->zReadExprlist);
  sqlite3_free(p->zWriteExprlist);


  /* Invoke the tokenizer destructor to free the tokenizer. */
  p->pTokenizer->pModule->xDestroy(p->pTokenizer);

  sqlite3_free(p);
  return SQLITE_OK;
}
................................................................................
  }
}

/*
** The xDestroy() virtual table method.
*/
static int fts3DestroyMethod(sqlite3_vtab *pVtab){
  int rc = SQLITE_OK;              /* Return code */
  Fts3Table *p = (Fts3Table *)pVtab;


  sqlite3 *db = p->db;

  /* Drop the shadow tables */

  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", p->zDb, p->zName);

  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", p->zDb,p->zName);
  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", p->zDb, p->zName);
  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", p->zDb, p->zName);
  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", p->zDb, p->zName);

  /* If everything has worked, invoke fts3DisconnectMethod() to free the
  ** memory associated with the Fts3Table structure and return SQLITE_OK.
  ** Otherwise, return an SQLite error code.
  */
  return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc);
}
................................................................................
** If the p->bHasDocsize boolean is true (indicating that this is an
** FTS4 table, not an FTS3 table) then also create the %_docsize and
** %_stat tables required by FTS4.
*/
static int fts3CreateTables(Fts3Table *p){
  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* Iterator variable */
  char *zContentCols;             /* Columns of %_content table */
  sqlite3 *db = p->db;            /* The database connection */




  /* Create a list of user columns for the content table */
  zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
  for(i=0; zContentCols && i<p->nColumn; i++){
    char *z = p->azColumn[i];
    zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
  }
  if( zContentCols==0 ) rc = SQLITE_NOMEM;

  /* Create the content table */
  fts3DbExec(&rc, db, 
     "CREATE TABLE %Q.'%q_content'(%s)",
     p->zDb, p->zName, zContentCols
  );
  sqlite3_free(zContentCols);


  /* Create other tables */
  fts3DbExec(&rc, db, 
      "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);",
      p->zDb, p->zName
  );
  fts3DbExec(&rc, db, 
      "CREATE TABLE %Q.'%q_segdir'("
................................................................................
    *(z++) = '"';
    *(z++) = '\0';
  }
  return zRet;
}

/*
** Return a list of comma separated SQL expressions that could be used
** in a SELECT statement such as the following:
**
**     SELECT <list of expressions> FROM %_content AS x ...
**
** to return the docid, followed by each column of text data in order
** from left to write. If parameter zFunc is not NULL, then instead of
** being returned directly each column of text data is passed to an SQL
** function named zFunc first. For example, if zFunc is "unzip" and the
** table has the three user-defined columns "a", "b", and "c", the following
** string is returned:
**
**     "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c')"
**
** The pointer returned points to a buffer allocated by sqlite3_malloc(). It
** is the responsibility of the caller to eventually free it.
**
** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and
** a NULL pointer is returned). Otherwise, if an OOM error is encountered
** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If
................................................................................
*/
static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){
  char *zRet = 0;
  char *zFree = 0;
  char *zFunction;
  int i;


  if( !zFunc ){
    zFunction = "";
  }else{
    zFree = zFunction = fts3QuoteId(zFunc);
  }
  fts3Appendf(pRc, &zRet, "docid");
  for(i=0; i<p->nColumn; i++){
    fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]);
  }
  sqlite3_free(zFree);











  return zRet;
}

/*
** Return a list of N comma separated question marks, where N is the number
** of columns in the %_content table (one for the docid plus one for each
** user-defined text column).
................................................................................
      aIndex[i].nPrefix = nPrefix;
      p++;
    }
  }

  return SQLITE_OK;
}






















































































/*
** This function is the implementation of both the xConnect and xCreate
** methods of the FTS3 virtual table.
**
** The argv[] array contains the following:
**
................................................................................

  /* The results of parsing supported FTS4 key=value options: */
  int bNoDocsize = 0;             /* True to omit %_docsize table */
  int bDescIdx = 0;               /* True to store descending indexes */
  char *zPrefix = 0;              /* Prefix parameter value (or NULL) */
  char *zCompress = 0;            /* compress=? parameter (or NULL) */
  char *zUncompress = 0;          /* uncompress=? parameter (or NULL) */


  assert( strlen(argv[0])==4 );
  assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
       || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
  );

  nDb = (int)strlen(argv[1]) + 1;
................................................................................
    }

    /* Check if it is an FTS4 special argument. */
    else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){
      struct Fts4Option {
        const char *zOpt;
        int nOpt;
        char **pzVar;
      } aFts4Opt[] = {
        { "matchinfo",   9, 0 },            /* 0 -> MATCHINFO */
        { "prefix",      6, 0 },            /* 1 -> PREFIX */
        { "compress",    8, 0 },            /* 2 -> COMPRESS */
        { "uncompress", 10, 0 },            /* 3 -> UNCOMPRESS */
        { "order",       5, 0 }             /* 4 -> ORDER */
      };

      int iOpt;
      if( !zVal ){
        rc = SQLITE_NOMEM;
      }else{
        for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){
................................................................................
               && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3)) 
              ){
                *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal);
                rc = SQLITE_ERROR;
              }
              bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
              break;






          }
        }
        sqlite3_free(zVal);
      }
    }

    /* Otherwise, the argument is a column name. */
    else {
      nString += (int)(strlen(z) + 1);
      aCol[nCol++] = z;
    }
  }





















  if( rc!=SQLITE_OK ) goto fts3_init_out;

  if( nCol==0 ){
    assert( nString==0 );
    aCol[0] = "content";
    nString = 8;
    nCol = 1;
................................................................................
  p->nPendingData = 0;
  p->azColumn = (char **)&p[1];
  p->pTokenizer = pTokenizer;
  p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
  p->bHasDocsize = (isFts4 && bNoDocsize==0);
  p->bHasStat = isFts4;
  p->bDescIdx = bDescIdx;


  TESTONLY( p->inTransaction = -1 );
  TESTONLY( p->mxSavepoint = -1 );

  p->aIndex = (struct Fts3Index *)&p->azColumn[nCol];
  memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex);
  p->nIndex = nIndex;
  for(i=0; i<nIndex; i++){
................................................................................
  fts3DeclareVtab(&rc, p);

fts3_init_out:
  sqlite3_free(zPrefix);
  sqlite3_free(aIndex);
  sqlite3_free(zCompress);
  sqlite3_free(zUncompress);

  sqlite3_free((void *)aCol);
  if( rc!=SQLITE_OK ){
    if( p ){
      fts3DisconnectMethod((sqlite3_vtab *)p);
    }else if( pTokenizer ){
      pTokenizer->pModule->xDestroy(pTokenizer);
    }
................................................................................
    sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
    pCsr->isRequireSeek = 0;
    if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
      return SQLITE_OK;
    }else{
      int rc = sqlite3_reset(pCsr->pStmt);
      if( rc==SQLITE_OK ){


        /* If no row was found and no error has occured, then the %_content
        ** table is missing a row that is present in the full-text index.
        ** The data structures are corrupt.
        */
        rc = SQLITE_CORRUPT_VTAB;



      }
      pCsr->isEof = 1;
      if( pContext ){
        sqlite3_result_error_code(pContext, rc);
      }
      return rc;
    }
................................................................................

  /* Compile a SELECT statement for this cursor. For a full-table-scan, the
  ** statement loops through all rows of the %_content table. For a
  ** full-text query or docid lookup, the statement retrieves a single
  ** row by docid.
  */
  if( idxNum==FTS3_FULLSCAN_SEARCH ){
    const char *zSort = (pCsr->bDesc ? "DESC" : "ASC");
    const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s";
    zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort);


  }else{
    const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?";
    zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName);
  }
  if( !zSql ) return SQLITE_NOMEM;
  rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
  sqlite3_free(zSql);
  if( rc!=SQLITE_OK ) return rc;

  if( idxNum==FTS3_DOCID_SEARCH ){
................................................................................
  }else if( iCol==p->nColumn ){
    /* The extra column whose name is the same as the table.
    ** Return a blob which is a pointer to the cursor.
    */
    sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
  }else{
    rc = fts3CursorSeek(0, pCsr);
    if( rc==SQLITE_OK ){
      sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
    }
  }

  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  return rc;
}
................................................................................
  int rc;                         /* Return Code */

  rc = sqlite3Fts3PendingTermsFlush(p);
  if( rc!=SQLITE_OK ){
    return rc;
  }


  fts3DbExec(&rc, db,
    "ALTER TABLE %Q.'%q_content'  RENAME TO '%q_content';",
    p->zDb, p->zName, zName
  );


  if( p->bHasDocsize ){
    fts3DbExec(&rc, db,
      "ALTER TABLE %Q.'%q_docsize'  RENAME TO '%q_docsize';",
      p->zDb, p->zName, zName
    );
  }
  if( p->bHasStat ){







>







 







<

>
>
|


>
|
>
|
|
|
|







 







<


>
>
>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>
>







 







|
|










|







 







>
|
|
|
|
|
|
|
|
|
|
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>







 







|
|
|
|
|
|
|







 







>
>
>
>
>
>












>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>







 







>







 







>
>
|
|
|
|
|
>
>
>







 







<
|
|
>
>

|
|







 







|







 







>
|
|
|
|
>
>







464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
...
504
505
506
507
508
509
510

511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
...
578
579
580
581
582
583
584

585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
...
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
...
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
...
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
....
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
....
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
....
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
....
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
....
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
....
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
....
2850
2851
2852
2853
2854
2855
2856

2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
....
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
....
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
  /* Free any prepared statements held */
  for(i=0; i<SizeofArray(p->aStmt); i++){
    sqlite3_finalize(p->aStmt[i]);
  }
  sqlite3_free(p->zSegmentsTbl);
  sqlite3_free(p->zReadExprlist);
  sqlite3_free(p->zWriteExprlist);
  sqlite3_free(p->zContentTbl);

  /* Invoke the tokenizer destructor to free the tokenizer. */
  p->pTokenizer->pModule->xDestroy(p->pTokenizer);

  sqlite3_free(p);
  return SQLITE_OK;
}
................................................................................
  }
}

/*
** The xDestroy() virtual table method.
*/
static int fts3DestroyMethod(sqlite3_vtab *pVtab){

  Fts3Table *p = (Fts3Table *)pVtab;
  int rc = SQLITE_OK;              /* Return code */
  const char *zDb = p->zDb;        /* Name of database (e.g. "main", "temp") */
  sqlite3 *db = p->db;             /* Database handle */

  /* Drop the shadow tables */
  if( p->zContentTbl==0 ){
    fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName);
  }
  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName);
  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName);
  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName);
  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName);

  /* If everything has worked, invoke fts3DisconnectMethod() to free the
  ** memory associated with the Fts3Table structure and return SQLITE_OK.
  ** Otherwise, return an SQLite error code.
  */
  return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc);
}
................................................................................
** If the p->bHasDocsize boolean is true (indicating that this is an
** FTS4 table, not an FTS3 table) then also create the %_docsize and
** %_stat tables required by FTS4.
*/
static int fts3CreateTables(Fts3Table *p){
  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* Iterator variable */

  sqlite3 *db = p->db;            /* The database connection */

  if( p->zContentTbl==0 ){
    char *zContentCols;           /* Columns of %_content table */

    /* Create a list of user columns for the content table */
    zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
    for(i=0; zContentCols && i<p->nColumn; i++){
      char *z = p->azColumn[i];
      zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
    }
    if( zContentCols==0 ) rc = SQLITE_NOMEM;
  
    /* Create the content table */
    fts3DbExec(&rc, db, 
       "CREATE TABLE %Q.'%q_content'(%s)",
       p->zDb, p->zName, zContentCols
    );
    sqlite3_free(zContentCols);
  }

  /* Create other tables */
  fts3DbExec(&rc, db, 
      "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);",
      p->zDb, p->zName
  );
  fts3DbExec(&rc, db, 
      "CREATE TABLE %Q.'%q_segdir'("
................................................................................
    *(z++) = '"';
    *(z++) = '\0';
  }
  return zRet;
}

/*
** Return a list of comma separated SQL expressions and a FROM clause that 
** could be used in a SELECT statement such as the following:
**
**     SELECT <list of expressions> FROM %_content AS x ...
**
** to return the docid, followed by each column of text data in order
** from left to write. If parameter zFunc is not NULL, then instead of
** being returned directly each column of text data is passed to an SQL
** function named zFunc first. For example, if zFunc is "unzip" and the
** table has the three user-defined columns "a", "b", and "c", the following
** string is returned:
**
**     "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x"
**
** The pointer returned points to a buffer allocated by sqlite3_malloc(). It
** is the responsibility of the caller to eventually free it.
**
** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and
** a NULL pointer is returned). Otherwise, if an OOM error is encountered
** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If
................................................................................
*/
static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){
  char *zRet = 0;
  char *zFree = 0;
  char *zFunction;
  int i;

  if( p->zContentTbl==0 ){
    if( !zFunc ){
      zFunction = "";
    }else{
      zFree = zFunction = fts3QuoteId(zFunc);
    }
    fts3Appendf(pRc, &zRet, "docid");
    for(i=0; i<p->nColumn; i++){
      fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]);
    }
    sqlite3_free(zFree);
  }else{
    fts3Appendf(pRc, &zRet, "rowid");
    for(i=0; i<p->nColumn; i++){
      fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]);
    }
  }
  fts3Appendf(pRc, &zRet, "FROM '%q'.'%q%s' AS x", 
      p->zDb,
      (p->zContentTbl ? p->zContentTbl : p->zName),
      (p->zContentTbl ? "" : "_content")
  );
  return zRet;
}

/*
** Return a list of N comma separated question marks, where N is the number
** of columns in the %_content table (one for the docid plus one for each
** user-defined text column).
................................................................................
      aIndex[i].nPrefix = nPrefix;
      p++;
    }
  }

  return SQLITE_OK;
}

/*
** This function is called when initializing an FTS4 table that uses the
** content=xxx option. It determines the number of and names of the columns
** of the new FTS4 table.
**
** The third argument passed to this function is the value passed to the
** config=xxx option (i.e. "xxx"). This function queries the database for
** a table of that name. If found, the output variables are populated
** as follows:
**
**   *pnCol:   Set to the number of columns table xxx has,
**
**   *pnStr:   Set to the total amount of space required to store a copy
**             of each columns name, including the nul-terminator.
**
**   *pazCol:  Set to point to an array of *pnCol strings. Each string is
**             the name of the corresponding column in table xxx. The array
**             and its contents are allocated using a single allocation. It
**             is the responsibility of the caller to free this allocation
**             by eventually passing the *pazCol value to sqlite3_free().
**
** If the table cannot be found, an error code is returned and the output
** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is
** returned (and the output variables are undefined).
*/
static int fts3ContentColumns(
  sqlite3 *db,                    /* Database handle */
  const char *zDb,                /* Name of db (i.e. "main", "temp" etc.) */
  const char *zTbl,               /* Name of content table */
  const char ***pazCol,           /* OUT: Malloc'd array of column names */
  int *pnCol,                     /* OUT: Size of array *pazCol */
  int *pnStr                      /* OUT: Bytes of string content */
){
  int rc = SQLITE_OK;             /* Return code */
  char *zSql;                     /* "SELECT *" statement on zTbl */  
  sqlite3_stmt *pStmt = 0;        /* Compiled version of zSql */

  zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl);
  if( !zSql ){
    rc = SQLITE_NOMEM;
  }else{
    rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
  }
  sqlite3_free(zSql);

  if( rc==SQLITE_OK ){
    const char **azCol;           /* Output array */
    int nStr = 0;                 /* Size of all column names (incl. 0x00) */
    int nCol;                     /* Number of table columns */
    int i;                        /* Used to iterate through columns */

    /* Loop through the returned columns. Set nStr to the number of bytes of
    ** space required to store a copy of each column name, including the
    ** nul-terminator byte.  */
    nCol = sqlite3_column_count(pStmt);
    for(i=0; i<nCol; i++){
      const char *zCol = sqlite3_column_name(pStmt, i);
      nStr += strlen(zCol) + 1;
    }

    /* Allocate and populate the array to return. */
    azCol = (const char **)sqlite3_malloc(sizeof(char *) * nCol + nStr);
    if( azCol==0 ){
      rc = SQLITE_NOMEM;
    }else{
      char *p = (char *)&azCol[nCol];
      for(i=0; i<nCol; i++){
        const char *zCol = sqlite3_column_name(pStmt, i);
        int n = strlen(zCol)+1;
        memcpy(p, zCol, n);
        azCol[i] = p;
        p += n;
      }
    }
    sqlite3_finalize(pStmt);

    /* Set the output variables. */
    *pnCol = nCol;
    *pnStr = nStr;
    *pazCol = azCol;
  }

  return rc;
}

/*
** This function is the implementation of both the xConnect and xCreate
** methods of the FTS3 virtual table.
**
** The argv[] array contains the following:
**
................................................................................

  /* The results of parsing supported FTS4 key=value options: */
  int bNoDocsize = 0;             /* True to omit %_docsize table */
  int bDescIdx = 0;               /* True to store descending indexes */
  char *zPrefix = 0;              /* Prefix parameter value (or NULL) */
  char *zCompress = 0;            /* compress=? parameter (or NULL) */
  char *zUncompress = 0;          /* uncompress=? parameter (or NULL) */
  char *zContent = 0;             /* content=? parameter (or NULL) */

  assert( strlen(argv[0])==4 );
  assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
       || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
  );

  nDb = (int)strlen(argv[1]) + 1;
................................................................................
    }

    /* Check if it is an FTS4 special argument. */
    else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){
      struct Fts4Option {
        const char *zOpt;
        int nOpt;
      } aFts4Opt[] = {
        { "matchinfo",   9 },     /* 0 -> MATCHINFO */
        { "prefix",      6 },     /* 1 -> PREFIX */
        { "compress",    8 },     /* 2 -> COMPRESS */
        { "uncompress", 10 },     /* 3 -> UNCOMPRESS */
        { "order",       5 },     /* 4 -> ORDER */
        { "content",     7 }      /* 5 -> CONTENT */
      };

      int iOpt;
      if( !zVal ){
        rc = SQLITE_NOMEM;
      }else{
        for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){
................................................................................
               && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3)) 
              ){
                *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal);
                rc = SQLITE_ERROR;
              }
              bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
              break;

            case 5:               /* CONTENT */
              sqlite3_free(zUncompress);
              zContent = zVal;
              zVal = 0;
              break;
          }
        }
        sqlite3_free(zVal);
      }
    }

    /* Otherwise, the argument is a column name. */
    else {
      nString += (int)(strlen(z) + 1);
      aCol[nCol++] = z;
    }
  }

  /* If a content=xxx option was specified, the following:
  **
  **   1. Ignore any compress= and uncompress= options.
  **
  **   2. Ignore any column names that were specified as part of the 
  **      the CREATE VIRTUAL TABLE statement.
  **
  **   3. Determine the actual column names to use for the FTS table 
  **      based on the columns of the content= table.
  */
  if( rc==SQLITE_OK && zContent ){
    sqlite3_free(aCol); 
    sqlite3_free(zCompress); 
    sqlite3_free(zUncompress); 
    zCompress = 0;
    zUncompress = 0;
    aCol = 0;
    rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString);
    assert( rc!=SQLITE_OK || nCol>0 );
  }
  if( rc!=SQLITE_OK ) goto fts3_init_out;

  if( nCol==0 ){
    assert( nString==0 );
    aCol[0] = "content";
    nString = 8;
    nCol = 1;
................................................................................
  p->nPendingData = 0;
  p->azColumn = (char **)&p[1];
  p->pTokenizer = pTokenizer;
  p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
  p->bHasDocsize = (isFts4 && bNoDocsize==0);
  p->bHasStat = isFts4;
  p->bDescIdx = bDescIdx;
  p->zContentTbl = zContent;
  zContent = 0;
  TESTONLY( p->inTransaction = -1 );
  TESTONLY( p->mxSavepoint = -1 );

  p->aIndex = (struct Fts3Index *)&p->azColumn[nCol];
  memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex);
  p->nIndex = nIndex;
  for(i=0; i<nIndex; i++){
................................................................................
  fts3DeclareVtab(&rc, p);

fts3_init_out:
  sqlite3_free(zPrefix);
  sqlite3_free(aIndex);
  sqlite3_free(zCompress);
  sqlite3_free(zUncompress);
  sqlite3_free(zContent);
  sqlite3_free((void *)aCol);
  if( rc!=SQLITE_OK ){
    if( p ){
      fts3DisconnectMethod((sqlite3_vtab *)p);
    }else if( pTokenizer ){
      pTokenizer->pModule->xDestroy(pTokenizer);
    }
................................................................................
    sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
    pCsr->isRequireSeek = 0;
    if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
      return SQLITE_OK;
    }else{
      int rc = sqlite3_reset(pCsr->pStmt);
      if( rc==SQLITE_OK ){
        Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
        if( p->zContentTbl==0 ){
          /* If no row was found and no error has occured, then the %_content
          ** table is missing a row that is present in the full-text index.
          ** The data structures are corrupt.
          */
          rc = SQLITE_CORRUPT_VTAB;
        }else{
          return SQLITE_OK;
        }
      }
      pCsr->isEof = 1;
      if( pContext ){
        sqlite3_result_error_code(pContext, rc);
      }
      return rc;
    }
................................................................................

  /* Compile a SELECT statement for this cursor. For a full-table-scan, the
  ** statement loops through all rows of the %_content table. For a
  ** full-text query or docid lookup, the statement retrieves a single
  ** row by docid.
  */
  if( idxNum==FTS3_FULLSCAN_SEARCH ){

    const char *zTmpl = "SELECT %s ORDER BY rowid %s";
    zSql = sqlite3_mprintf(zTmpl, 
        p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC")
    );
  }else{
    const char *zTmpl = "SELECT %s WHERE rowid = ?";
    zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist);
  }
  if( !zSql ) return SQLITE_NOMEM;
  rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
  sqlite3_free(zSql);
  if( rc!=SQLITE_OK ) return rc;

  if( idxNum==FTS3_DOCID_SEARCH ){
................................................................................
  }else if( iCol==p->nColumn ){
    /* The extra column whose name is the same as the table.
    ** Return a blob which is a pointer to the cursor.
    */
    sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
  }else{
    rc = fts3CursorSeek(0, pCsr);
    if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){
      sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
    }
  }

  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  return rc;
}
................................................................................
  int rc;                         /* Return Code */

  rc = sqlite3Fts3PendingTermsFlush(p);
  if( rc!=SQLITE_OK ){
    return rc;
  }

  if( p->zContentTbl==0 ){
    fts3DbExec(&rc, db,
      "ALTER TABLE %Q.'%q_content'  RENAME TO '%q_content';",
      p->zDb, p->zName, zName
    );
  }

  if( p->bHasDocsize ){
    fts3DbExec(&rc, db,
      "ALTER TABLE %Q.'%q_docsize'  RENAME TO '%q_docsize';",
      p->zDb, p->zName, zName
    );
  }
  if( p->bHasStat ){

Changes to ext/fts3/fts3Int.h.

180
181
182
183
184
185
186

187
188
189
190
191
192
193
...
241
242
243
244
245
246
247

248
249
250
251
252
253
254
  sqlite3_vtab base;              /* Base class used by SQLite core */
  sqlite3 *db;                    /* The database connection */
  const char *zDb;                /* logical database name */
  const char *zName;              /* virtual table name */
  int nColumn;                    /* number of named columns in virtual table */
  char **azColumn;                /* column names.  malloced */
  sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */


  /* Precompiled statements used by the implementation. Each of these 
  ** statements is run and reset within a single virtual table API call. 
  */
  sqlite3_stmt *aStmt[27];

  char *zReadExprlist;
................................................................................
** the xOpen method. Cursors are destroyed using the xClose method.
*/
struct Fts3Cursor {
  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
  i16 eSearch;                    /* Search strategy (see below) */
  u8 isEof;                       /* True if at End Of Results */
  u8 isRequireSeek;               /* True if must seek pStmt to %_content row */

  sqlite3_stmt *pStmt;            /* Prepared statement in use by the cursor */
  Fts3Expr *pExpr;                /* Parsed MATCH query string */
  int nPhrase;                    /* Number of matchable phrases in query */
  Fts3DeferredToken *pDeferred;   /* Deferred search tokens, if any */
  sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
  char *pNextId;                  /* Pointer into the body of aDoclist */
  char *aDoclist;                 /* List of docids for full-text queries */







>







 







>







180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
...
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
  sqlite3_vtab base;              /* Base class used by SQLite core */
  sqlite3 *db;                    /* The database connection */
  const char *zDb;                /* logical database name */
  const char *zName;              /* virtual table name */
  int nColumn;                    /* number of named columns in virtual table */
  char **azColumn;                /* column names.  malloced */
  sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
  char *zContentTbl;              /* content=xxx option, or NULL */

  /* Precompiled statements used by the implementation. Each of these 
  ** statements is run and reset within a single virtual table API call. 
  */
  sqlite3_stmt *aStmt[27];

  char *zReadExprlist;
................................................................................
** the xOpen method. Cursors are destroyed using the xClose method.
*/
struct Fts3Cursor {
  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
  i16 eSearch;                    /* Search strategy (see below) */
  u8 isEof;                       /* True if at End Of Results */
  u8 isRequireSeek;               /* True if must seek pStmt to %_content row */
  u8 isNullRow;                   /* True for a row of NULLs */
  sqlite3_stmt *pStmt;            /* Prepared statement in use by the cursor */
  Fts3Expr *pExpr;                /* Parsed MATCH query string */
  int nPhrase;                    /* Number of matchable phrases in query */
  Fts3DeferredToken *pDeferred;   /* Deferred search tokens, if any */
  sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
  char *pNextId;                  /* Pointer into the body of aDoclist */
  char *aDoclist;                 /* List of docids for full-text queries */

Changes to ext/fts3/fts3_snippet.c.

1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
....
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
          iMinPos = pT->iPos-pT->iOff;
          pTerm = pT;
        }
      }

      if( !pTerm ){
        /* All offsets for this column have been gathered. */
        break;
      }else{
        assert( iCurrent<=iMinPos );
        if( 0==(0xFE&*pTerm->pList) ){
          pTerm->pList = 0;
        }else{
          fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos);
        }
................................................................................
        }
        if( rc==SQLITE_OK ){
          char aBuffer[64];
          sqlite3_snprintf(sizeof(aBuffer), aBuffer, 
              "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart
          );
          rc = fts3StringAppend(&res, aBuffer, -1);
        }else if( rc==SQLITE_DONE ){
          rc = SQLITE_CORRUPT_VTAB;
        }
      }
    }
    if( rc==SQLITE_DONE ){
      rc = SQLITE_OK;
    }







|







 







|







1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
....
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
          iMinPos = pT->iPos-pT->iOff;
          pTerm = pT;
        }
      }

      if( !pTerm ){
        /* All offsets for this column have been gathered. */
        rc = SQLITE_DONE;
      }else{
        assert( iCurrent<=iMinPos );
        if( 0==(0xFE&*pTerm->pList) ){
          pTerm->pList = 0;
        }else{
          fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos);
        }
................................................................................
        }
        if( rc==SQLITE_OK ){
          char aBuffer[64];
          sqlite3_snprintf(sizeof(aBuffer), aBuffer, 
              "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart
          );
          rc = fts3StringAppend(&res, aBuffer, -1);
        }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){
          rc = SQLITE_CORRUPT_VTAB;
        }
      }
    }
    if( rc==SQLITE_DONE ){
      rc = SQLITE_OK;
    }

Changes to ext/fts3/fts3_write.c.

252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
...
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
...
775
776
777
778
779
780
781












782
783
784
785
786
787
788
...
826
827
828
829
830
831
832
833
834
835
836
837
838
839


840
841
842
843
844
845
846
847
....
2121
2122
2123
2124
2125
2126
2127





2128
2129
2130
2131
2132
2133

2134
2135
2136
2137
2138
2139
2140
....
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
....
2906
2907
2908
2909
2910
2911
2912














































































2913
2914
2915
2916
2917
2918
2919
....
2924
2925
2926
2927
2928
2929
2930


2931
2932
2933
2934
2935
2936
2937
....
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107

3108
3109



3110
3111
3112
3113
3114
3115
3116
....
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
....
3215
3216
3217
3218
3219
3220
3221

3222

3223
3224
3225
3226
3227
3228
3229
/* 0  */  "DELETE FROM %Q.'%q_content' WHERE rowid = ?",
/* 1  */  "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)",
/* 2  */  "DELETE FROM %Q.'%q_content'",
/* 3  */  "DELETE FROM %Q.'%q_segments'",
/* 4  */  "DELETE FROM %Q.'%q_segdir'",
/* 5  */  "DELETE FROM %Q.'%q_docsize'",
/* 6  */  "DELETE FROM %Q.'%q_stat'",
/* 7  */  "SELECT %s FROM %Q.'%q_content' AS x WHERE rowid=?",
/* 8  */  "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1",
/* 9  */  "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
/* 10 */  "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)",
/* 11 */  "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",

          /* Return segments in order from oldest to newest.*/ 
/* 12 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
................................................................................
  
  pStmt = p->aStmt[eStmt];
  if( !pStmt ){
    char *zSql;
    if( eStmt==SQL_CONTENT_INSERT ){
      zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist);
    }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){
      zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist, p->zDb, p->zName);
    }else{
      zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
    }
    if( !zSql ){
      rc = SQLITE_NOMEM;
    }else{
      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
................................................................................
static int fts3InsertData(
  Fts3Table *p,                   /* Full-text table */
  sqlite3_value **apVal,          /* Array of values to insert */
  sqlite3_int64 *piDocid          /* OUT: Docid for row just inserted */
){
  int rc;                         /* Return code */
  sqlite3_stmt *pContentInsert;   /* INSERT INTO %_content VALUES(...) */













  /* Locate the statement handle used to insert data into the %_content
  ** table. The SQL for this statement is:
  **
  **   INSERT INTO %_content VALUES(?, ?, ?, ...)
  **
  ** The statement features N '?' variables, where N is the number of user
................................................................................



/*
** Remove all data from the FTS3 table. Clear the hash table containing
** pending terms.
*/
static int fts3DeleteAll(Fts3Table *p){
  int rc = SQLITE_OK;             /* Return code */

  /* Discard the contents of the pending-terms hash table. */
  sqlite3Fts3PendingTermsClear(p);

  /* Delete everything from the %_content, %_segments and %_segdir tables. */


  fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0);
  fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0);
  fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
  if( p->bHasDocsize ){
    fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0);
  }
  if( p->bHasStat ){
    fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0);
................................................................................
** If successful, *pisEmpty is set to true if the table is empty except for
** document pRowid, or false otherwise, and SQLITE_OK is returned. If an
** error occurs, an SQLite error code is returned.
*/
static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){
  sqlite3_stmt *pStmt;
  int rc;





  rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid);
  if( rc==SQLITE_OK ){
    if( SQLITE_ROW==sqlite3_step(pStmt) ){
      *pisEmpty = sqlite3_column_int(pStmt, 0);
    }
    rc = sqlite3_reset(pStmt);

  }
  return rc;
}

/*
** Set *pnMax to the largest segment level in the database for the index
** iIndex.
................................................................................

/*
** Insert the sizes (in tokens) for each column of the document
** with docid equal to p->iPrevDocid.  The sizes are encoded as
** a blob of varints.
*/
static void fts3InsertDocsize(
  int *pRC,         /* Result code */
  Fts3Table *p,     /* Table into which to insert */
  u32 *aSz          /* Sizes of each column */
){
  char *pBlob;             /* The BLOB encoding of the document size */
  int nBlob;               /* Number of bytes in the BLOB */
  sqlite3_stmt *pStmt;     /* Statement used to insert the encoding */
  int rc;                  /* Result code from subfunctions */

  if( *pRC ) return;
................................................................................
    }
  }
  sqlite3Fts3SegmentsClose(p);
  sqlite3Fts3PendingTermsClear(p);

  return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc;
}















































































/*
** Handle a 'special' INSERT of the form:
**
**   "INSERT INTO tbl(tbl) VALUES(<expr>)"
**
** Argument pVal contains the result of <expr>. Currently the only 
................................................................................
  const char *zVal = (const char *)sqlite3_value_text(pVal);
  int nVal = sqlite3_value_bytes(pVal);

  if( !zVal ){
    return SQLITE_NOMEM;
  }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
    rc = fts3DoOptimize(p, 0);


#ifdef SQLITE_TEST
  }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
    p->nNodeSize = atoi(&zVal[9]);
    rc = SQLITE_OK;
  }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
    p->nMaxPendingData = atoi(&zVal[11]);
    rc = SQLITE_OK;
................................................................................
  int isEmpty = 0;
  int rc = fts3IsEmpty(p, pRowid, &isEmpty);
  if( rc==SQLITE_OK ){
    if( isEmpty ){
      /* Deleting this row means the whole table is empty. In this case
      ** delete the contents of all three tables and throw away any
      ** data in the pendingTerms hash table.  */
      rc = fts3DeleteAll(p);
      *pnDoc = *pnDoc - 1;
    }else{
      sqlite3_int64 iRemove = sqlite3_value_int64(pRowid);
      rc = fts3PendingTermsDocid(p, iRemove);
      fts3DeleteTerms(&rc, p, pRowid, aSzDel);

      fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid);
      if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1;



      if( p->bHasDocsize ){
        fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid);
      }
    }
  }

  return rc;
................................................................................
  **
  ** If the on-conflict mode is REPLACE, this means that the existing row
  ** should be deleted from the database before inserting the new row. Or,
  ** if the on-conflict mode is other than REPLACE, then this method must
  ** detect the conflict and return SQLITE_CONSTRAINT before beginning to
  ** modify the database file.
  */
  if( nArg>1 ){
    /* Find the value object that holds the new rowid value. */
    sqlite3_value *pNewRowid = apVal[3+p->nColumn];
    if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){
      pNewRowid = apVal[1];
    }

    if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( 
................................................................................
    iRemove = sqlite3_value_int64(apVal[0]);
  }
  
  /* If this is an INSERT or UPDATE operation, insert the new record. */
  if( nArg>1 && rc==SQLITE_OK ){
    if( bInsertDone==0 ){
      rc = fts3InsertData(p, apVal, pRowid);

      if( rc==SQLITE_CONSTRAINT ) rc = SQLITE_CORRUPT_VTAB;

    }
    if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){
      rc = fts3PendingTermsDocid(p, *pRowid);
    }
    if( rc==SQLITE_OK ){
      rc = fts3InsertTerms(p, apVal, aSzIns);
    }







|







 







|







 







>
>
>
>
>
>
>
>
>
>
>
>







 







|





|
>
>
|







 







>
>
>
>
>
|
|
|
|
|
|
>







 







|
|
|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>







 







|





>
|
|
>
>
>







 







|







 







>
|
>







252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
...
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
...
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
...
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
....
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
....
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
....
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
....
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
....
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
....
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
....
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
/* 0  */  "DELETE FROM %Q.'%q_content' WHERE rowid = ?",
/* 1  */  "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)",
/* 2  */  "DELETE FROM %Q.'%q_content'",
/* 3  */  "DELETE FROM %Q.'%q_segments'",
/* 4  */  "DELETE FROM %Q.'%q_segdir'",
/* 5  */  "DELETE FROM %Q.'%q_docsize'",
/* 6  */  "DELETE FROM %Q.'%q_stat'",
/* 7  */  "SELECT %s WHERE rowid=?",
/* 8  */  "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1",
/* 9  */  "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
/* 10 */  "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)",
/* 11 */  "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",

          /* Return segments in order from oldest to newest.*/ 
/* 12 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
................................................................................
  
  pStmt = p->aStmt[eStmt];
  if( !pStmt ){
    char *zSql;
    if( eStmt==SQL_CONTENT_INSERT ){
      zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist);
    }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){
      zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist);
    }else{
      zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
    }
    if( !zSql ){
      rc = SQLITE_NOMEM;
    }else{
      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
................................................................................
static int fts3InsertData(
  Fts3Table *p,                   /* Full-text table */
  sqlite3_value **apVal,          /* Array of values to insert */
  sqlite3_int64 *piDocid          /* OUT: Docid for row just inserted */
){
  int rc;                         /* Return code */
  sqlite3_stmt *pContentInsert;   /* INSERT INTO %_content VALUES(...) */

  if( p->zContentTbl ){
    sqlite3_value *pRowid = apVal[p->nColumn+3];
    if( sqlite3_value_type(pRowid)==SQLITE_NULL ){
      pRowid = apVal[1];
    }
    if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){
      return SQLITE_CONSTRAINT;
    }
    *piDocid = sqlite3_value_int64(pRowid);
    return SQLITE_OK;
  }

  /* Locate the statement handle used to insert data into the %_content
  ** table. The SQL for this statement is:
  **
  **   INSERT INTO %_content VALUES(?, ?, ?, ...)
  **
  ** The statement features N '?' variables, where N is the number of user
................................................................................



/*
** Remove all data from the FTS3 table. Clear the hash table containing
** pending terms.
*/
static int fts3DeleteAll(Fts3Table *p, int bContent){
  int rc = SQLITE_OK;             /* Return code */

  /* Discard the contents of the pending-terms hash table. */
  sqlite3Fts3PendingTermsClear(p);

  /* Delete everything from the shadow tables. Except, leave %_content as
  ** is if bContent is false.  */
  assert( p->zContentTbl==0 || bContent==0 );
  if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0);
  fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0);
  fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
  if( p->bHasDocsize ){
    fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0);
  }
  if( p->bHasStat ){
    fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0);
................................................................................
** If successful, *pisEmpty is set to true if the table is empty except for
** document pRowid, or false otherwise, and SQLITE_OK is returned. If an
** error occurs, an SQLite error code is returned.
*/
static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){
  sqlite3_stmt *pStmt;
  int rc;
  if( p->zContentTbl ){
    /* If using the content=xxx option, assume the table is never empty */
    *pisEmpty = 0;
    rc = SQLITE_OK;
  }else{
    rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid);
    if( rc==SQLITE_OK ){
      if( SQLITE_ROW==sqlite3_step(pStmt) ){
        *pisEmpty = sqlite3_column_int(pStmt, 0);
      }
      rc = sqlite3_reset(pStmt);
    }
  }
  return rc;
}

/*
** Set *pnMax to the largest segment level in the database for the index
** iIndex.
................................................................................

/*
** Insert the sizes (in tokens) for each column of the document
** with docid equal to p->iPrevDocid.  The sizes are encoded as
** a blob of varints.
*/
static void fts3InsertDocsize(
  int *pRC,                       /* Result code */
  Fts3Table *p,                   /* Table into which to insert */
  u32 *aSz                        /* Sizes of each column, in tokens */
){
  char *pBlob;             /* The BLOB encoding of the document size */
  int nBlob;               /* Number of bytes in the BLOB */
  sqlite3_stmt *pStmt;     /* Statement used to insert the encoding */
  int rc;                  /* Result code from subfunctions */

  if( *pRC ) return;
................................................................................
    }
  }
  sqlite3Fts3SegmentsClose(p);
  sqlite3Fts3PendingTermsClear(p);

  return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc;
}

/*
** This function is called when the user executes the following statement:
**
**     INSERT INTO <tbl>(<tbl>) VALUES('rebuild');
**
** The entire FTS index is discarded and rebuilt. If the table is one 
** created using the content=xxx option, then the new index is based on
** the current contents of the xxx table. Otherwise, it is rebuilt based
** on the contents of the %_content table.
*/
static int fts3DoRebuild(Fts3Table *p){
  int rc;                         /* Return Code */

  rc = fts3DeleteAll(p, 0);
  if( rc==SQLITE_OK ){
    u32 *aSz = 0;
    u32 *aSzIns;
    u32 *aSzDel;
    sqlite3_stmt *pStmt = 0;
    int nEntry = 0;

    /* Compose and prepare an SQL statement to loop through the content table */
    char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist);
    if( !zSql ){
      rc = SQLITE_NOMEM;
    }else{
      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
      sqlite3_free(zSql);
    }

    if( rc==SQLITE_OK ){
      int nByte = sizeof(u32) * (p->nColumn+1)*3;
      aSz = (u32 *)sqlite3_malloc(nByte);
      if( aSz==0 ){
        rc = SQLITE_NOMEM;
      }else{
        memset(aSz, 0, nByte);
        aSzIns = &aSz[p->nColumn+1];
        aSzDel = &aSzIns[p->nColumn+1];
      }
    }

    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
      int iCol;
      rc = fts3PendingTermsDocid(p, sqlite3_column_int64(pStmt, 0));
      aSz[p->nColumn] = 0;
      for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
        const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1);
        rc = fts3PendingTermsAdd(p, z, iCol, &aSz[iCol]);
        aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1);
      }
      if( p->bHasDocsize ){
        fts3InsertDocsize(&rc, p, aSz);
      }
      if( rc!=SQLITE_OK ){
        sqlite3_finalize(pStmt);
        pStmt = 0;
      }else{
        nEntry++;
        for(iCol=0; iCol<=p->nColumn; iCol++){
          aSzIns[iCol] += aSz[iCol];
        }
      }
    }
    if( p->bHasStat ){
      fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry);
    }
    sqlite3_free(aSz);

    if( pStmt ){
      assert( rc==SQLITE_OK );
      rc = sqlite3_finalize(pStmt);
    }
  }

  return rc;
}

/*
** Handle a 'special' INSERT of the form:
**
**   "INSERT INTO tbl(tbl) VALUES(<expr>)"
**
** Argument pVal contains the result of <expr>. Currently the only 
................................................................................
  const char *zVal = (const char *)sqlite3_value_text(pVal);
  int nVal = sqlite3_value_bytes(pVal);

  if( !zVal ){
    return SQLITE_NOMEM;
  }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
    rc = fts3DoOptimize(p, 0);
  }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){
    rc = fts3DoRebuild(p);
#ifdef SQLITE_TEST
  }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
    p->nNodeSize = atoi(&zVal[9]);
    rc = SQLITE_OK;
  }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
    p->nMaxPendingData = atoi(&zVal[11]);
    rc = SQLITE_OK;
................................................................................
  int isEmpty = 0;
  int rc = fts3IsEmpty(p, pRowid, &isEmpty);
  if( rc==SQLITE_OK ){
    if( isEmpty ){
      /* Deleting this row means the whole table is empty. In this case
      ** delete the contents of all three tables and throw away any
      ** data in the pendingTerms hash table.  */
      rc = fts3DeleteAll(p, 1);
      *pnDoc = *pnDoc - 1;
    }else{
      sqlite3_int64 iRemove = sqlite3_value_int64(pRowid);
      rc = fts3PendingTermsDocid(p, iRemove);
      fts3DeleteTerms(&rc, p, pRowid, aSzDel);
      if( p->zContentTbl==0 ){
        fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid);
        if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1;
      }else{
        *pnDoc = *pnDoc - 1;
      }
      if( p->bHasDocsize ){
        fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid);
      }
    }
  }

  return rc;
................................................................................
  **
  ** If the on-conflict mode is REPLACE, this means that the existing row
  ** should be deleted from the database before inserting the new row. Or,
  ** if the on-conflict mode is other than REPLACE, then this method must
  ** detect the conflict and return SQLITE_CONSTRAINT before beginning to
  ** modify the database file.
  */
  if( nArg>1 && p->zContentTbl==0 ){
    /* Find the value object that holds the new rowid value. */
    sqlite3_value *pNewRowid = apVal[3+p->nColumn];
    if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){
      pNewRowid = apVal[1];
    }

    if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( 
................................................................................
    iRemove = sqlite3_value_int64(apVal[0]);
  }
  
  /* If this is an INSERT or UPDATE operation, insert the new record. */
  if( nArg>1 && rc==SQLITE_OK ){
    if( bInsertDone==0 ){
      rc = fts3InsertData(p, apVal, pRowid);
      if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){
        rc = SQLITE_CORRUPT_VTAB;
      }
    }
    if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){
      rc = fts3PendingTermsDocid(p, *pRowid);
    }
    if( rc==SQLITE_OK ){
      rc = fts3InsertTerms(p, apVal, aSzIns);
    }

Added test/fts4content.test.





























































































































































































































































































































































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
# 2011 October 03
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the content=xxx FTS4 option.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set ::testprefix fts4content

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}

do_execsql_test 1.1 {
  CREATE TABLE t1(a, b, c);
  INSERT INTO t1 VALUES('w x', 'x y', 'y z');
  CREATE VIRTUAL TABLE ft1 USING fts4(content=t1);
}

do_execsql_test 1.2 {
  PRAGMA table_info(ft1);
} {
  0 a {} 0 {} 0 
  1 b {} 0 {} 0 
  2 c {} 0 {} 0
}

do_execsql_test 1.3 { SELECT *, rowid FROM ft1 } {{w x} {x y} {y z} 1}
do_execsql_test 1.4 { SELECT a, c FROM ft1 WHERE rowid=1 } {{w x} {y z}}

do_execsql_test 1.5 { INSERT INTO ft1(ft1) VALUES('rebuild') } {}
do_execsql_test 1.6 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'x' } {1}
do_execsql_test 1.7 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'a' } {}

#-------------------------------------------------------------------------
# The following block of tests - 2.* - test that a content=xxx FTS table
# can be queried. Also tested are cases where rows identified in the FTS
# are missing from the content table, and cases where the index is 
# inconsistent with the content table.
# 
do_execsql_test 2.0 {
  CREATE TABLE t2(x);
  INSERT INTO t2 VALUES('O S W W F U C R Q I C N P Z Y Y E Y Y E');  -- 1
  INSERT INTO t2 VALUES('Y X U V L B E H Y J C Y A I A P V F V K');  -- 2
  INSERT INTO t2 VALUES('P W I N J H I I N I F B K D U Q B Z S F');  -- 3
  INSERT INTO t2 VALUES('N R O R H J R H G M D I U U B O M P A U');  -- 4
  INSERT INTO t2 VALUES('Y O V O G T P N G T N F I V B U M J M G');  -- 5
  INSERT INTO t2 VALUES('J O B N K N E C H Z R K J O U G M K L S');  -- 6
  INSERT INTO t2 VALUES('S Z S R I Q U A P W R X H K C Z U L S P');  -- 7
  INSERT INTO t2 VALUES('J C H N R C K R V N M O F Z M Z A I H W');  -- 8
  INSERT INTO t2 VALUES('O Y G I S J U U W O D Z F J K N R P R L');  -- 9
  INSERT INTO t2 VALUES('B G L K U R U P V X Z I H V R W C Q A S');  -- 10
  INSERT INTO t2 VALUES('T F T J F F Y V F W N X K Q A Y L X W G');  -- 11
  INSERT INTO t2 VALUES('C J U H B Q X L C M M Y E G V F W V Z C');  -- 12
  INSERT INTO t2 VALUES('B W L T F S G X D P H N G M R I O A X I');  -- 13
  INSERT INTO t2 VALUES('N G Y O K Q K Z N M H U J E D H U W R K');  -- 14
  INSERT INTO t2 VALUES('U D T R U Y F J D S J X E H Q G V A S Z');  -- 15
  INSERT INTO t2 VALUES('M I W P J S H R J D Q I C G P C T P H R');  -- 16
  INSERT INTO t2 VALUES('J M N I S L X Q C A B F C B Y D H V R J');  -- 17
  INSERT INTO t2 VALUES('F V Z W J Q L P X Y E W B U Q N H X K T');  -- 18
  INSERT INTO t2 VALUES('R F S R Y O F Q E I E G H C B H R X Y N');  -- 19
  INSERT INTO t2 VALUES('U Q Q Q T E P D M F X P J G H X C Q D L');  -- 20
}

do_execsql_test 2.1 {
  CREATE VIRTUAL TABLE ft2 USING fts4(content=t2);
  INSERT INTO ft2(ft2) VALUES('rebuild');

  -- Modify the backing table a bit: Row 17 is missing and the contents 
  -- of row 20 do not match the FTS index contents. 
  DELETE FROM t2 WHERE rowid = 17;
  UPDATE t2 SET x = 'a b c d e f g h i j' WHERE rowid = 20;
}

foreach {tn match rowidlist} {
  1   {S}        {1 3 6 7 9 10 13 15 16 17 19}
  2   {"S R"}    {7 19}
  3   {"N K N"}  {6}
  4   {"Q Q"}    {20}
  5   {"B Y D"}  {17}
} {
  do_execsql_test 2.2.1.$tn {
    SELECT rowid FROM ft2 WHERE ft2 MATCH $match
  } $rowidlist

  do_execsql_test 2.2.2.$tn {
    SELECT docid FROM ft2 WHERE ft2 MATCH $match
  } $rowidlist
}

foreach {tn match result} {
  1   {"N K N"}  {{J O B N K N E C H Z R K J O U G M K L S}}
  2   {"Q Q"}    {{a b c d e f g h i j}}
  3   {"B Y D"}  {{}}
} {
  do_execsql_test 2.3.$tn {
    SELECT * FROM ft2 WHERE ft2 MATCH $match
  } $result
}

foreach {tn match result} {
  1   {"N K N"}  {{..O B [N] [K] [N] E..}}
  2   {"B Y D"}  {{}}
  3   {"Q Q"}    {{a [b] [c] [d] e f..}}
} {
  do_execsql_test 2.4.$tn {
    SELECT snippet(ft2, '[', ']', '..', -1, 6) FROM ft2 WHERE ft2 MATCH $match
  } $result
}

foreach {tn match result} {
  1   {"N K N"}  {{0 0 6 1 0 1 8 1 0 2 10 1}}
  2   {"B Y D"}  {{}}
  3   {"Q Q"}    {{0 0 2 1 0 0 4 1 0 1 4 1 0 1 6 1}}
  4   {"Q D L"}  {{}}
} {
  do_execsql_test 2.5.$tn {
    SELECT offsets(ft2) FROM ft2 WHERE ft2 MATCH $match
  } $result
}

#-------------------------------------------------------------------------
# The following block of tests - 3.* - test that the FTS index can be
# modified by writing to the table. But that this has no effect on the 
# content table.
# 

do_execsql_test 3.1 {
  CREATE TABLE t3(x, y);
  CREATE VIRTUAL TABLE ft3 USING fts4(content=t3);
}

do_catchsql_test 3.1.1 {
  INSERT INTO ft3 VALUES('a b c', 'd e f');
} {1 {constraint failed}}
do_execsql_test 3.1.2 {
  INSERT INTO ft3(docid, x, y) VALUES(21, 'a b c', 'd e f');
  SELECT rowid FROM ft3 WHERE ft3 MATCH '"a b c"';
} {21}
do_execsql_test 3.1.3 { SELECT * FROM t3 } {}

# This DELETE does not work, since there is no row in [t3] to base the
# DELETE on. So the SELECT on [ft3] still returns rowid 21.
do_execsql_test 3.1.4 { 
  DELETE FROM ft3;
  SELECT rowid FROM ft3 WHERE ft3 MATCH '"a b c"';
} {21}

# If the row is added to [t3] before the DELETE on [ft3], it works.
do_execsql_test 3.1.5 {
  INSERT INTO t3(rowid, x, y) VALUES(21, 'a b c', 'd e f');
  DELETE FROM ft3;
  SELECT rowid FROM ft3 WHERE ft3 MATCH '"a b c"';
} {}
do_execsql_test 3.1.6 { SELECT rowid FROM t3 } {21}

do_execsql_test 3.2.1 {
  INSERT INTO ft3(rowid, x, y) VALUES(0, 'R T M S M', 'A F O K H');
  INSERT INTO ft3(rowid, x, y) VALUES(1, 'C Z J O X', 'U S Q D K');
  INSERT INTO ft3(rowid, x, y) VALUES(2, 'N G H P O', 'N O P O C');
  INSERT INTO ft3(rowid, x, y) VALUES(3, 'V H S D R', 'K N G E C');
  INSERT INTO ft3(rowid, x, y) VALUES(4, 'J T R V U', 'U X S L C');
  INSERT INTO ft3(rowid, x, y) VALUES(5, 'N A Y N G', 'X D G P Y');
  INSERT INTO ft3(rowid, x, y) VALUES(6, 'I Q I S P', 'D R O Q B');
  INSERT INTO ft3(rowid, x, y) VALUES(7, 'T K T Z J', 'B W D G O');
  INSERT INTO ft3(rowid, x, y) VALUES(8, 'Y K F X T', 'D F G V G');
  INSERT INTO ft3(rowid, x, y) VALUES(9, 'E L E T L', 'P W N F Z');
  INSERT INTO ft3(rowid, x, y) VALUES(10, 'O G J G X', 'G J F E P');
  INSERT INTO ft3(rowid, x, y) VALUES(11, 'O L N N Z', 'K E Z F D');
  INSERT INTO ft3(rowid, x, y) VALUES(12, 'R Z M R J', 'X G I M Z');
  INSERT INTO ft3(rowid, x, y) VALUES(13, 'L X N N X', 'R R N S T');
  INSERT INTO ft3(rowid, x, y) VALUES(14, 'F L B J H', 'K W F L C');
  INSERT INTO ft3(rowid, x, y) VALUES(15, 'P E B M V', 'E A A B U');
  INSERT INTO ft3(rowid, x, y) VALUES(16, 'V E C F P', 'L U T V K');
  INSERT INTO ft3(rowid, x, y) VALUES(17, 'T N O Z N', 'T P Q X N');
  INSERT INTO ft3(rowid, x, y) VALUES(18, 'V W U W R', 'H O A A V');
  INSERT INTO ft3(rowid, x, y) VALUES(19, 'A H N L F', 'I G H B O');
}

foreach {tn match rowidlist} {
  1   "N A"    {5 19}
  2   "x:O"    {1 2 10 11 17}
  3   "y:O"    {0 2 6 7 18 19}
} {
  set res [list]
  foreach rowid $rowidlist { lappend res $rowid {} {} }

  do_execsql_test 3.2.2.$tn {
    SELECT rowid, * FROM ft3 WHERE ft3 MATCH $match
  } $res
  do_execsql_test 3.2.3.$tn {
    SELECT docid, * FROM ft3 WHERE ft3 MATCH $match
  } $res
}

do_execsql_test 3.3.1 {
  INSERT INTO t3(rowid, x, y) VALUES(0, 'R T M S M', 'A F O K H');
  INSERT INTO t3(rowid, x, y) VALUES(1, 'C Z J O X', 'U S Q D K');
  INSERT INTO t3(rowid, x, y) VALUES(2, 'N G H P O', 'N O P O C');
  INSERT INTO t3(rowid, x, y) VALUES(3, 'V H S D R', 'K N G E C');
  INSERT INTO t3(rowid, x, y) VALUES(4, 'J T R V U', 'U X S L C');
  INSERT INTO t3(rowid, x, y) VALUES(5, 'N A Y N G', 'X D G P Y');
  UPDATE ft3 SET x = y, y = x;
  DELETE FROM t3;
}

foreach {tn match rowidlist} {
  1   "N A"    {5 19}
  2   "x:O"    {0 2 10 11 17}
  3   "y:O"    {1 2 6 7 18 19}
} {
  set res [list]
  foreach rowid $rowidlist { lappend res $rowid {} {} }

  do_execsql_test 3.3.2.$tn {
    SELECT rowid, * FROM ft3 WHERE ft3 MATCH $match
  } $res
  do_execsql_test 3.3.3.$tn {
    SELECT docid, * FROM ft3 WHERE ft3 MATCH $match
  } $res
}

do_execsql_test 3.3.1 {
  INSERT INTO t3(rowid, x, y) VALUES(15, 'P E B M V', 'E A A B U');
  INSERT INTO t3(rowid, x, y) VALUES(16, 'V E C F P', 'L U T V K');
  INSERT INTO t3(rowid, x, y) VALUES(17, 'T N O Z N', 'T P Q X N');
  INSERT INTO t3(rowid, x, y) VALUES(18, 'V W U W R', 'H O A A V');
  INSERT INTO t3(rowid, x, y) VALUES(19, 'A H N L F', 'I G H B O');
  DELETE FROM ft3;
}

foreach {tn match rowidlist} {
  1   "N A"    {5}
  2   "x:O"    {0 2 10 11}
  3   "y:O"    {1 2 6 7}
} {
  set res [list]
  foreach rowid $rowidlist { lappend res $rowid {} {} }

  do_execsql_test 3.3.2.$tn {
    SELECT rowid, * FROM ft3 WHERE ft3 MATCH $match
  } $res
  do_execsql_test 3.3.3.$tn {
    SELECT docid, * FROM ft3 WHERE ft3 MATCH $match
  } $res
}


#-------------------------------------------------------------------------
# Test cases 4.* test the 'rebuild' command. On content=xxx and regular
# FTS tables.
# 
do_execsql_test 4.0 {
  CREATE TABLE t4(x);
  CREATE VIRTUAL TABLE ft4 USING fts4(content=t4);
  CREATE VIRTUAL TABLE ft4x USING fts4(x);
}

do_execsql_test 4.1.1 {
  INSERT INTO ft4x(ft4x) VALUES('rebuild');
  INSERT INTO ft4(ft4) VALUES('rebuild');
} {}
do_execsql_test 4.1.2 {
  SELECT id, quote(value) FROM ft4_stat
} {0 X'000000'}
do_execsql_test 4.1.3 {
  SELECT id, quote(value) FROM ft4x_stat
} {0 X'000000'}

do_execsql_test 4.2.1 {
  INSERT INTO ft4x VALUES('M G M F T');
  INSERT INTO ft4x VALUES('Z Q C A U');
  INSERT INTO ft4x VALUES('N L L V');
  INSERT INTO ft4x VALUES('T F D X D');
  INSERT INTO ft4x VALUES('Z H I S D');

  SELECT id, quote(value) FROM ft4x_stat
} {0 X'05182B'}

do_execsql_test 4.2.2 {
  INSERT INTO ft4(rowid, x) SELECT rowid, * FROM ft4x;
  SELECT id, quote(value) FROM ft4_stat
} {0 X'05182B'}

do_execsql_test 4.2.3 {
  SELECT docid, quote(size) FROM ft4_docsize
} {1 X'05' 2 X'05' 3 X'04' 4 X'05' 5 X'05'}

do_execsql_test 4.2.4 {
  INSERT INTO ft4x(ft4x) VALUES('rebuild');
  SELECT id, quote(value) FROM ft4x_stat;
  SELECT docid, quote(size) FROM ft4x_docsize
} {0 X'05182B' 1 X'05' 2 X'05' 3 X'04' 4 X'05' 5 X'05'}

do_execsql_test 4.2.5 {
  INSERT INTO ft4(ft4) VALUES('rebuild');
  SELECT id, quote(value) FROM ft4_stat;
  SELECT docid, quote(size) FROM ft4_docsize
} {0 X'000000'}

do_execsql_test 4.2.6 {
  INSERT INTO t4(rowid, x) SELECT rowid, x FROM ft4x;
  INSERT INTO ft4(ft4) VALUES('rebuild');
  SELECT id, quote(value) FROM ft4_stat;
  SELECT docid, quote(size) FROM ft4_docsize
} {0 X'05182B' 1 X'05' 2 X'05' 3 X'04' 4 X'05' 5 X'05'}


#-------------------------------------------------------------------------
# Test cases 5.* test that the following commands do not create/move or
# delete a %_content table when used with a content=xxx FTS table.
# 
do_execsql_test 5.1.1 {
  CREATE TABLE t5(a, b, c, d);
  CREATE VIRTUAL TABLE ft5 USING fts4(content=t5);
  SELECT name FROM sqlite_master WHERE name LIKE '%t5%';
} {
  t5 ft5 ft5_segments ft5_segdir 
  sqlite_autoindex_ft5_segdir_1 ft5_docsize ft5_stat
}
do_execsql_test 5.1.2 {
  ALTER TABLE ft5 RENAME TO ft6;
  SELECT name FROM sqlite_master WHERE name LIKE '%t5%';
} {
  t5
}
do_execsql_test 5.1.3 {
  SELECT name FROM sqlite_master WHERE name LIKE '%t6%';
} {
  ft6 ft6_segments ft6_segdir 
  sqlite_autoindex_ft6_segdir_1 ft6_docsize ft6_stat
}
do_execsql_test 5.1.4 {
  INSERT INTO t5 VALUES('a', 'b', 'c', 'd');
  INSERT INTO ft6(ft6) VALUES('rebuild');
  SELECT rowid FROM ft6 WHERE ft6 MATCH 'b';
} {1}
do_execsql_test 5.1.5 {
  DROP TABLE ft6;
  SELECT * FROM t5;
} {a b c d}
do_execsql_test 5.1.6 {
  SELECT name FROM sqlite_master WHERE name LIKE '%t6%';
} {
}
do_execsql_test 5.1.7 {
  CREATE VIRTUAL TABLE ft5 USING fts4(content=t5);
  CREATE TABLE t5_content(a, b);
  DROP TABLE ft5;
  SELECT name FROM sqlite_master WHERE name LIKE '%t5%';
} {
  t5 t5_content
}

finish_test

Changes to test/permutations.test.

176
177
178
179
180
181
182
183
184
185
186

187
188
189
190
191
192
193
  fts3aa.test fts3ab.test fts3ac.test fts3ad.test fts3ae.test
  fts3af.test fts3ag.test fts3ah.test fts3ai.test fts3aj.test
  fts3ak.test fts3al.test fts3am.test fts3an.test fts3ao.test
  fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test
  fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test 
  fts3near.test fts3query.test fts3shared.test fts3snippet.test 
  fts3sort.test

  fts3fault.test fts3malloc.test fts3matchinfo.test

  fts3aux1.test fts3comp1.test fts3auto.test

}


lappend ::testsuitelist xxx
#-------------------------------------------------------------------------
# Define the coverage related test suites:
#







<

<

>







176
177
178
179
180
181
182

183

184
185
186
187
188
189
190
191
192
  fts3aa.test fts3ab.test fts3ac.test fts3ad.test fts3ae.test
  fts3af.test fts3ag.test fts3ah.test fts3ai.test fts3aj.test
  fts3ak.test fts3al.test fts3am.test fts3an.test fts3ao.test
  fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test
  fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test 
  fts3near.test fts3query.test fts3shared.test fts3snippet.test 
  fts3sort.test

  fts3fault.test fts3malloc.test fts3matchinfo.test

  fts3aux1.test fts3comp1.test fts3auto.test
  fts4aa.test fts4content.test
}


lappend ::testsuitelist xxx
#-------------------------------------------------------------------------
# Define the coverage related test suites:
#