/ Check-in [be59bf49]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Change fts4 so that the prefix= parameter is passes a comma-separated list of integers. For each integer N, a separate index of all prefixes of length N bytes is created.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts3-prefix-search
Files: files | file ages | folders
SHA1: be59bf49402d2e2f4b95fb6668849f3745cb7bf2
User & Date: dan 2011-05-25 18:34:53
Context
2011-05-25
18:47
Merge trunk changes into experimental fts3-prefix-search branch. check-in: f0f0a03d user: dan tags: fts3-prefix-search
18:34
Change fts4 so that the prefix= parameter is passes a comma-separated list of integers. For each integer N, a separate index of all prefixes of length N bytes is created. check-in: be59bf49 user: dan tags: fts3-prefix-search
2011-05-24
18:49
If the fts4 option prefix=1 is specified, have the fts4 module maintain an index of prefixes as well as terms. check-in: b5bdc639 user: dan tags: fts3-prefix-search
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

   827    827     fts3Appendf(pRc, &zRet, "?");
   828    828     for(i=0; i<p->nColumn; i++){
   829    829       fts3Appendf(pRc, &zRet, ",%s(?)", zFunction);
   830    830     }
   831    831     sqlite3_free(zFree);
   832    832     return zRet;
   833    833   }
          834  +
          835  +static int fts3GobbleInt(const char **pp, int *pnOut){
          836  +  const char *p = *pp;
          837  +  int nInt = 0;
          838  +  for(p=*pp; p[0]>='0' && p[0]<='9'; p++){
          839  +    nInt = nInt * 10 + (p[0] - '0');
          840  +  }
          841  +  if( p==*pp ) return SQLITE_ERROR;
          842  +  *pnOut = nInt;
          843  +  *pp = p;
          844  +  return SQLITE_OK;
          845  +}
          846  +
          847  +
          848  +static int fts3PrefixParameter(
          849  +  const char *zParam,             /* ABC in prefix=ABC parameter to parse */
          850  +  int *pnIndex,                   /* OUT: size of *apIndex[] array */
          851  +  struct Fts3Index **apIndex,     /* OUT: Array of indexes for this table */
          852  +  struct Fts3Index **apFree       /* OUT: Free this with sqlite3_free() */
          853  +){
          854  +  struct Fts3Index *aIndex;
          855  +  int nIndex = 1;
          856  +
          857  +  if( zParam && zParam[0] ){
          858  +    const char *p;
          859  +    nIndex++;
          860  +    for(p=zParam; *p; p++){
          861  +      if( *p==',' ) nIndex++;
          862  +    }
          863  +  }
          864  +
          865  +  aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex);
          866  +  *apIndex = *apFree = aIndex;
          867  +  *pnIndex = nIndex;
          868  +  if( !aIndex ){
          869  +    return SQLITE_NOMEM;
          870  +  }
          871  +
          872  +  memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex);
          873  +  if( zParam ){
          874  +    const char *p = zParam;
          875  +    int i;
          876  +    for(i=1; i<nIndex; i++){
          877  +      int nPrefix;
          878  +      if( fts3GobbleInt(&p, &nPrefix) ) return SQLITE_ERROR;
          879  +      aIndex[i].nPrefix = nPrefix;
          880  +      p++;
          881  +    }
          882  +  }
          883  +
          884  +  return SQLITE_OK;
          885  +}
   834    886   
   835    887   /*
   836    888   ** This function is the implementation of both the xConnect and xCreate
   837    889   ** methods of the FTS3 virtual table.
   838    890   **
   839    891   ** The argv[] array contains the following:
   840    892   **
................................................................................
   861    913     int nString = 0;                /* Bytes required to hold all column names */
   862    914     int nCol = 0;                   /* Number of columns in the FTS table */
   863    915     char *zCsr;                     /* Space for holding column names */
   864    916     int nDb;                        /* Bytes required to hold database name */
   865    917     int nName;                      /* Bytes required to hold table name */
   866    918     int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
   867    919     int bNoDocsize = 0;             /* True to omit %_docsize table */
   868         -  int bPrefix = 0;                /* True to include a prefix-search index */
   869    920     const char **aCol;              /* Array of column names */
   870    921     sqlite3_tokenizer *pTokenizer = 0;        /* Tokenizer for this table */
          922  +
          923  +  char *zPrefix = 0;              /* Prefix parameter value (or NULL) */
          924  +  int nIndex;                     /* Size of aIndex[] array */
          925  +  struct Fts3Index *aIndex;       /* Array of indexes for this table */
          926  +  struct Fts3Index *aFree = 0;    /* Free this before returning */
   871    927   
   872    928     char *zCompress = 0;
   873    929     char *zUncompress = 0;
   874    930   
   875    931     assert( strlen(argv[0])==4 );
   876    932     assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
   877    933          || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
................................................................................
   925    981         }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){
   926    982           zCompress = zVal;
   927    983           zVal = 0;
   928    984         }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){
   929    985           zUncompress = zVal;
   930    986           zVal = 0;
   931    987         }else if( nKey==6 && 0==sqlite3_strnicmp(z, "prefix", 6) ){
   932         -        bPrefix = 1;
          988  +        sqlite3_free(zPrefix);
          989  +        zPrefix = zVal;
          990  +        zVal = 0;
   933    991         }else{
   934    992           *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
   935    993           rc = SQLITE_ERROR;
   936    994         }
   937    995         sqlite3_free(zVal);
   938    996       }
   939    997   
................................................................................
   954   1012   
   955   1013     if( pTokenizer==0 ){
   956   1014       rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
   957   1015       if( rc!=SQLITE_OK ) goto fts3_init_out;
   958   1016     }
   959   1017     assert( pTokenizer );
   960   1018   
         1019  +  rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree);
         1020  +  if( rc==SQLITE_ERROR ){
         1021  +    assert( zPrefix );
         1022  +    *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix);
         1023  +  }
         1024  +  if( rc ) goto fts3_init_out;
         1025  +
   961   1026   
   962   1027     /* Allocate and populate the Fts3Table structure. */
   963         -  nByte = sizeof(Fts3Table) +              /* Fts3Table */
         1028  +  nByte = sizeof(Fts3Table) +                  /* Fts3Table */
   964   1029             nCol * sizeof(char *) +              /* azColumn */
         1030  +          nIndex * sizeof(struct Fts3Index) +  /* aIndex */
   965   1031             nName +                              /* zName */
   966   1032             nDb +                                /* zDb */
   967   1033             nString;                             /* Space for azColumn strings */
   968   1034     p = (Fts3Table*)sqlite3_malloc(nByte);
   969   1035     if( p==0 ){
   970   1036       rc = SQLITE_NOMEM;
   971   1037       goto fts3_init_out;
................................................................................
   978   1044     p->pTokenizer = pTokenizer;
   979   1045     p->nNodeSize = 1000;
   980   1046     p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
   981   1047     p->bHasDocsize = (isFts4 && bNoDocsize==0);
   982   1048     p->bHasStat = isFts4;
   983   1049     TESTONLY( p->inTransaction = -1 );
   984   1050     TESTONLY( p->mxSavepoint = -1 );
   985         -  p->bPrefix = bPrefix;
   986         -  fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1);
   987         -  fts3HashInit(&p->pendingPrefixes, FTS3_HASH_STRING, 1);
         1051  +
         1052  +  p->aIndex = (struct Fts3Index *)&p->azColumn[nCol];
         1053  +  memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex);
         1054  +  p->nIndex = nIndex;
         1055  +  for(i=0; i<nIndex; i++){
         1056  +    fts3HashInit(&p->aIndex[i].hPending, FTS3_HASH_STRING, 1);
         1057  +  }
   988   1058   
   989   1059     /* Fill in the zName and zDb fields of the vtab structure. */
   990         -  zCsr = (char *)&p->azColumn[nCol];
         1060  +  zCsr = (char *)&p->aIndex[nIndex];
   991   1061     p->zName = zCsr;
   992   1062     memcpy(zCsr, argv[2], nName);
   993   1063     zCsr += nName;
   994   1064     p->zDb = zCsr;
   995   1065     memcpy(zCsr, argv[1], nDb);
   996   1066     zCsr += nDb;
   997   1067   
................................................................................
  1030   1100     */
  1031   1101     fts3DatabasePageSize(&rc, p);
  1032   1102   
  1033   1103     /* Declare the table schema to SQLite. */
  1034   1104     fts3DeclareVtab(&rc, p);
  1035   1105   
  1036   1106   fts3_init_out:
         1107  +  sqlite3_free(zPrefix);
         1108  +  sqlite3_free(aFree);
  1037   1109     sqlite3_free(zCompress);
  1038   1110     sqlite3_free(zUncompress);
  1039   1111     sqlite3_free((void *)aCol);
  1040   1112     if( rc!=SQLITE_OK ){
  1041   1113       if( p ){
  1042   1114         fts3DisconnectMethod((sqlite3_vtab *)p);
  1043   1115       }else if( pTokenizer ){
................................................................................
  2140   2212       if( !*ppOut ) return SQLITE_NOMEM;
  2141   2213       sqlite3Fts3PutVarint(*ppOut, docid);
  2142   2214     }
  2143   2215   
  2144   2216     return SQLITE_OK;
  2145   2217   }
  2146   2218   
         2219  +/*
         2220  +** Append SegReader object pNew to the end of the pCsr->apSegment[] array.
         2221  +*/
  2147   2222   static int fts3SegReaderCursorAppend(
  2148   2223     Fts3SegReaderCursor *pCsr, 
  2149   2224     Fts3SegReader *pNew
  2150   2225   ){
  2151   2226     if( (pCsr->nSegment%16)==0 ){
  2152   2227       Fts3SegReader **apNew;
  2153   2228       int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
................................................................................
  2159   2234       pCsr->apSegment = apNew;
  2160   2235     }
  2161   2236     pCsr->apSegment[pCsr->nSegment++] = pNew;
  2162   2237     return SQLITE_OK;
  2163   2238   }
  2164   2239   
  2165   2240   /*
  2166         -** Set up a cursor object for iterating through the full-text index or 
  2167         -** a single level therein.
         2241  +** Set up a cursor object for iterating through a full-text index or a 
         2242  +** single level therein.
  2168   2243   */
  2169   2244   int sqlite3Fts3SegReaderCursor(
  2170   2245     Fts3Table *p,                   /* FTS3 table handle */
         2246  +  int iIndex,                     /* Index to search (from 0 to p->nIndex-1) */
  2171   2247     int iLevel,                     /* Level of segments to scan */
  2172   2248     const char *zTerm,              /* Term to query for */
  2173   2249     int nTerm,                      /* Size of zTerm in bytes */
  2174   2250     int isPrefix,                   /* True for a prefix search */
  2175   2251     int isScan,                     /* True to scan from zTerm to EOF */
  2176   2252     Fts3SegReaderCursor *pCsr       /* Cursor object to populate */
  2177   2253   ){
  2178   2254     int rc = SQLITE_OK;
  2179   2255     int rc2;
  2180   2256     int iAge = 0;
  2181   2257     sqlite3_stmt *pStmt = 0;
  2182   2258   
  2183         -  assert( iLevel==FTS3_SEGCURSOR_ALL_TERM
         2259  +  assert( iIndex>=0 && iIndex<p->nIndex );
         2260  +  assert( iLevel==FTS3_SEGCURSOR_ALL
  2184   2261         ||  iLevel==FTS3_SEGCURSOR_PENDING 
  2185         -      ||  iLevel==FTS3_SEGCURSOR_PENDING_PREFIX
  2186         -      ||  iLevel==FTS3_SEGCURSOR_ALL_PREFIX
  2187   2262         ||  iLevel>=0
  2188   2263     );
  2189         -  assert( 0>FTS3_SEGCURSOR_ALL_TERM
  2190         -      &&  0>FTS3_SEGCURSOR_PENDING 
  2191         -      &&  0>FTS3_SEGCURSOR_PENDING_PREFIX
  2192         -      &&  0>FTS3_SEGCURSOR_ALL_PREFIX
  2193         -  );
  2194         -  assert( iLevel==FTS3_SEGCURSOR_ALL_TERM
  2195         -       || iLevel==FTS3_SEGCURSOR_ALL_PREFIX 
  2196         -       || (zTerm==0 && isPrefix==1) 
  2197         -  );
         2264  +  assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
         2265  +  assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 );
         2266  +  assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) );
  2198   2267     assert( isPrefix==0 || isScan==0 );
         2268  +
         2269  +  /* "isScan" is only set to true by the ft4aux module, an ordinary
         2270  +  ** full-text tables. */
         2271  +  assert( isScan==0 || p->aIndex==0 );
  2199   2272   
  2200   2273     memset(pCsr, 0, sizeof(Fts3SegReaderCursor));
  2201   2274   
  2202         -  /* "isScan" is only set to true by the ft4aux module, not an ordinary
  2203         -  ** full-text table. The pendingTerms and pendingPrefixes tables must be
  2204         -  ** empty in this case. */
  2205         -  assert( isScan==0 || fts3HashCount(&p->pendingTerms)==0 );
  2206         -  assert( isScan==0 || fts3HashCount(&p->pendingPrefixes)==0 );
  2207         -
  2208         -  /* If iLevel is less than 0, include a seg-reader for the pending-terms. */
  2209         -  if( iLevel<0 && isScan==0 ){
  2210         -    int bPrefix = (
  2211         -        iLevel==FTS3_SEGCURSOR_PENDING_PREFIX 
  2212         -     || iLevel==FTS3_SEGCURSOR_ALL_PREFIX
  2213         -    );
  2214         -    Fts3SegReader *pPending = 0;
  2215         -
  2216         -    rc = sqlite3Fts3SegReaderPending(p,zTerm,nTerm,isPrefix,bPrefix,&pPending);
  2217         -    if( rc==SQLITE_OK && pPending ){
  2218         -      rc = fts3SegReaderCursorAppend(pCsr, pPending);
         2275  +  /* If iLevel is less than 0 and this is not a scan, include a seg-reader 
         2276  +  ** for the pending-terms. If this is a scan, then this call must be being
         2277  +  ** made by an fts4aux module, not an FTS table. In this case calling
         2278  +  ** Fts3SegReaderPending might segfault, as the data structures used by 
         2279  +  ** fts4aux are not completely populated. So it's easiest to filter these
         2280  +  ** calls out here.  */
         2281  +  if( iLevel<0 && p->aIndex ){
         2282  +    Fts3SegReader *pSeg = 0;
         2283  +    rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg);
         2284  +    if( rc==SQLITE_OK && pSeg ){
         2285  +      rc = fts3SegReaderCursorAppend(pCsr, pSeg);
  2219   2286       }
  2220   2287     }
  2221   2288   
  2222         -  if( iLevel!=FTS3_SEGCURSOR_PENDING && iLevel!=FTS3_SEGCURSOR_PENDING_PREFIX ){
         2289  +  if( iLevel!=FTS3_SEGCURSOR_PENDING ){
  2223   2290       if( rc==SQLITE_OK ){
  2224         -      rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt);
         2291  +      rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt);
  2225   2292       }
         2293  +
  2226   2294       while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
  2227   2295         Fts3SegReader *pSeg = 0;
  2228   2296   
  2229   2297         /* Read the values returned by the SELECT into local variables. */
  2230   2298         sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1);
  2231   2299         sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2);
  2232   2300         sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3);
................................................................................
  2269   2337     Fts3SegReaderCursor *pSegcsr;   /* Object to allocate and return */
  2270   2338     int rc = SQLITE_NOMEM;          /* Return code */
  2271   2339   
  2272   2340     pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor));
  2273   2341     if( pSegcsr ){
  2274   2342       int i;
  2275   2343       int nCost = 0;
         2344  +    int bFound = 0;               /* True once an index has been found */
  2276   2345       Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
  2277   2346   
  2278         -    if( isPrefix && p->bPrefix && nTerm<=FTS3_MAX_PREFIX ){
  2279         -      rc = sqlite3Fts3SegReaderCursor(
  2280         -          p, FTS3_SEGCURSOR_ALL_PREFIX, zTerm, nTerm, 0, 0, pSegcsr);
         2347  +    if( isPrefix ){
         2348  +      for(i=1; i<p->nIndex; i++){
         2349  +        if( p->aIndex[i].nPrefix==nTerm ){
         2350  +          bFound = 1;
         2351  +          rc = sqlite3Fts3SegReaderCursor(
         2352  +              p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr);
         2353  +          break;
         2354  +        }
         2355  +      }
         2356  +    }
  2281   2357   
  2282         -    }else{
         2358  +    if( bFound==0 ){
  2283   2359         rc = sqlite3Fts3SegReaderCursor(
  2284         -          p, FTS3_SEGCURSOR_ALL_TERM, zTerm, nTerm, isPrefix, 0, pSegcsr);
         2360  +          p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr
         2361  +      );
  2285   2362       }
  2286         -  
  2287   2363       for(i=0; rc==SQLITE_OK && i<pSegcsr->nSegment; i++){
  2288   2364         rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost);
  2289   2365       }
  2290   2366       pSegcsr->nCost = nCost;
  2291   2367     }
  2292   2368   
  2293   2369     *ppSegcsr = pSegcsr;
................................................................................
  3336   3412   */
  3337   3413   static int fts3RollbackMethod(sqlite3_vtab *pVtab){
  3338   3414     Fts3Table *p = (Fts3Table*)pVtab;
  3339   3415     sqlite3Fts3PendingTermsClear(p);
  3340   3416     assert( p->inTransaction!=0 );
  3341   3417     TESTONLY( p->inTransaction = 0 );
  3342   3418     TESTONLY( p->mxSavepoint = -1; );
  3343         -  sqlite3Fts3PendingPrefixesClear((Fts3Table *)pVtab);
  3344   3419     return SQLITE_OK;
  3345   3420   }
  3346   3421   
  3347   3422   /*
  3348   3423   ** Load the doclist associated with expression pExpr to pExpr->aDoclist.
  3349   3424   ** The loaded doclist contains positions as well as the document ids.
  3350   3425   ** This is used by the matchinfo(), snippet() and offsets() auxillary
................................................................................
  3711   3786   }
  3712   3787   static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
  3713   3788     Fts3Table *p = (Fts3Table*)pVtab;
  3714   3789     assert( p->inTransaction );
  3715   3790     assert( p->mxSavepoint >= iSavepoint );
  3716   3791     TESTONLY( p->mxSavepoint = iSavepoint );
  3717   3792     sqlite3Fts3PendingTermsClear(p);
  3718         -  sqlite3Fts3PendingPrefixesClear((Fts3Table *)pVtab);
  3719   3793     return SQLITE_OK;
  3720   3794   }
  3721   3795   
  3722   3796   static const sqlite3_module fts3Module = {
  3723   3797     /* iVersion      */ 2,
  3724   3798     /* xCreate       */ fts3CreateMethod,
  3725   3799     /* xConnect      */ fts3ConnectMethod,

Changes to ext/fts3/fts3Int.h.

    19     19   # define NDEBUG 1
    20     20   #endif
    21     21   
    22     22   #include "sqlite3.h"
    23     23   #include "fts3_tokenizer.h"
    24     24   #include "fts3_hash.h"
    25     25   
    26         -#define FTS3_MAX_PREFIX 8
    27         -
    28     26   /*
    29     27   ** This constant controls how often segments are merged. Once there are
    30     28   ** FTS3_MERGE_COUNT segments of level N, they are merged into a single
    31     29   ** segment of level N+1.
    32     30   */
    33     31   #define FTS3_MERGE_COUNT 16
    34     32   
................................................................................
    52     50   /*
    53     51   ** Maximum length of a varint encoded integer. The varint format is different
    54     52   ** from that used by SQLite, so the maximum length is 10, not 9.
    55     53   */
    56     54   #define FTS3_VARINT_MAX 10
    57     55   
    58     56   /*
    59         -** FTS4 virtual tables may maintain two separate indexes. One that indexes
    60         -** all document terms (the same index FTS3 tables maintain) and another used
    61         -** for prefixes. B+-trees that are part of the prefix index have values for
    62         -** the %_segdir.level column that are equal to or greater than the following
    63         -** value.
           57  +** FTS4 virtual tables may maintain multiple indexes - one index of all terms
           58  +** in the document set and zero or more prefix indexes. All indexes are stored
           59  +** as one or more b+-trees in the %_segments and %_segdir tables. 
           60  +**
           61  +** It is possible to determine which index a b+-tree belongs to based on the
           62  +** value stored in the "%_segdir.level" column. Given this value L, the index
           63  +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with
           64  +** level values between 0 and 1023 (inclusive) belong to index 0, all levels
           65  +** between 1024 and 2047 to index 1, and so on.
    64     66   **
    65         -** It is considered impossible for the regular index to use levels this large.
    66         -** In theory it could, but that would require that at least 2^1024 separate
    67         -** write operations to be made within the lifetime of the database.
           67  +** It is considered impossible for an index to use more than 1024 levels. In 
           68  +** theory though this may happen, but only after at least 
           69  +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables.
    68     70   */
    69         -#define FTS3_SEGDIR_PREFIXLEVEL 1024
    70         -#define FTS3_SEGDIR_PREFIXLEVEL_STR "1024"
           71  +#define FTS3_SEGDIR_MAXLEVEL      1024
           72  +#define FTS3_SEGDIR_MAXLEVEL_STR "1024"
    71     73   
    72     74   /*
    73     75   ** The testcase() macro is only used by the amalgamation.  If undefined,
    74     76   ** make it a no-op.
    75     77   */
    76     78   #ifndef testcase
    77     79   # define testcase(X)
................................................................................
   168    170   
   169    171     char *zReadExprlist;
   170    172     char *zWriteExprlist;
   171    173   
   172    174     int nNodeSize;                  /* Soft limit for node size */
   173    175     u8 bHasStat;                    /* True if %_stat table exists */
   174    176     u8 bHasDocsize;                 /* True if %_docsize table exists */
   175         -  u8 bPrefix;                     /* True if there is a prefix index */
   176    177     int nPgsz;                      /* Page size for host database */
   177    178     char *zSegmentsTbl;             /* Name of %_segments table */
   178    179     sqlite3_blob *pSegments;        /* Blob handle open on %_segments table */
   179    180   
   180         -  /* The following hash table is used to buffer pending index updates during
          181  +  /* TODO: Fix the first paragraph of this comment.
          182  +  **
          183  +  ** The following hash table is used to buffer pending index updates during
   181    184     ** transactions. Variable nPendingData estimates the memory size of the 
   182    185     ** pending data, including hash table overhead, but not malloc overhead. 
   183    186     ** When nPendingData exceeds nMaxPendingData, the buffer is flushed 
   184    187     ** automatically. Variable iPrevDocid is the docid of the most recently
   185    188     ** inserted record.
          189  +  **
          190  +  ** A single FTS4 table may have multiple full-text indexes. For each index
          191  +  ** there is an entry in the aIndex[] array. Index 0 is an index of all the
          192  +  ** terms that appear in the document set. Each subsequent index in aIndex[]
          193  +  ** is an index of prefixes of a specific length.
   186    194     */
   187         -  int nMaxPendingData;
   188         -  int nPendingData;
   189         -  sqlite_int64 iPrevDocid;
   190         -  Fts3Hash pendingTerms;
   191         -  Fts3Hash pendingPrefixes;
          195  +  int nIndex;                     /* Size of aIndex[] */
          196  +  struct Fts3Index {
          197  +    int nPrefix;                  /* Prefix length (0 for main terms index) */
          198  +    Fts3Hash hPending;            /* Pending terms table for this index */
          199  +  } *aIndex;
          200  +  int nMaxPendingData;            /* Max pending data before flush to disk */
          201  +  int nPendingData;               /* Current bytes of pending data */
          202  +  sqlite_int64 iPrevDocid;        /* Docid of most recently inserted document */
   192    203   
   193    204   #if defined(SQLITE_DEBUG)
   194    205     /* State variables used for validating that the transaction control
   195    206     ** methods of the virtual table are called at appropriate times.  These
   196    207     ** values do not contribution to the FTS computation; they are used for
   197    208     ** verifying the SQLite core.
   198    209     */
................................................................................
   332    343   int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
   333    344   int sqlite3Fts3PendingTermsFlush(Fts3Table *);
   334    345   void sqlite3Fts3PendingTermsClear(Fts3Table *);
   335    346   int sqlite3Fts3Optimize(Fts3Table *);
   336    347   int sqlite3Fts3SegReaderNew(int, sqlite3_int64,
   337    348     sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
   338    349   int sqlite3Fts3SegReaderPending(
   339         -  Fts3Table*,const char*,int,int,int,Fts3SegReader**);
          350  +  Fts3Table*,int,const char*,int,int,Fts3SegReader**);
   340    351   void sqlite3Fts3SegReaderFree(Fts3SegReader *);
   341    352   int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
   342         -int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **);
          353  +int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **);
   343    354   int sqlite3Fts3ReadLock(Fts3Table *);
   344    355   int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);
   345    356   
   346    357   int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
   347    358   int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
   348    359   
   349    360   void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
................................................................................
   351    362   int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
   352    363   void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
   353    364   char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);
   354    365   void sqlite3Fts3SegmentsClose(Fts3Table *);
   355    366   
   356    367   /* Special values interpreted by sqlite3SegReaderCursor() */
   357    368   #define FTS3_SEGCURSOR_PENDING        -1
   358         -#define FTS3_SEGCURSOR_PENDING_PREFIX -2
   359         -#define FTS3_SEGCURSOR_ALL_PREFIX     -3
   360         -#define FTS3_SEGCURSOR_ALL_TERM       -4
          369  +#define FTS3_SEGCURSOR_ALL            -2
   361    370   
   362    371   int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*);
   363    372   int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *);
   364    373   void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *);
   365    374   int sqlite3Fts3SegReaderCursor(
   366         -    Fts3Table *, int, const char *, int, int, int, Fts3SegReaderCursor *);
          375  +    Fts3Table *, int, int, const char *, int, int, int, Fts3SegReaderCursor *);
   367    376   
   368    377   /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
   369    378   #define FTS3_SEGMENT_REQUIRE_POS   0x00000001
   370    379   #define FTS3_SEGMENT_IGNORE_EMPTY  0x00000002
   371    380   #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
   372    381   #define FTS3_SEGMENT_PREFIX        0x00000008
   373    382   #define FTS3_SEGMENT_SCAN          0x00000010

Changes to ext/fts3/fts3_aux.c.

    92     92     if( !p ) return SQLITE_NOMEM;
    93     93     memset(p, 0, nByte);
    94     94   
    95     95     p->pFts3Tab = (Fts3Table *)&p[1];
    96     96     p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
    97     97     p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
    98     98     p->pFts3Tab->db = db;
           99  +  p->pFts3Tab->nIndex = 1;
    99    100   
   100    101     memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
   101    102     memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
   102    103     sqlite3Fts3Dequote((char *)p->pFts3Tab->zName);
   103    104   
   104    105     *ppVtab = (sqlite3_vtab *)p;
   105    106     return SQLITE_OK;
................................................................................
   371    372     if( idxNum&FTS4AUX_LE_CONSTRAINT ){
   372    373       int iIdx = (idxNum&FTS4AUX_GE_CONSTRAINT) ? 1 : 0;
   373    374       pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx]));
   374    375       pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]);
   375    376       if( pCsr->zStop==0 ) return SQLITE_NOMEM;
   376    377     }
   377    378   
   378         -  rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL_TERM,
          379  +  rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL,
   379    380         pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr
   380    381     );
   381    382     if( rc==SQLITE_OK ){
   382    383       rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
   383    384     }
   384    385   
   385    386     if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor);

Changes to ext/fts3/fts3_term.c.

    23     23   #include <assert.h>
    24     24   
    25     25   typedef struct Fts3termTable Fts3termTable;
    26     26   typedef struct Fts3termCursor Fts3termCursor;
    27     27   
    28     28   struct Fts3termTable {
    29     29     sqlite3_vtab base;              /* Base class used by SQLite core */
    30         -  int bPrefix;                    /* True for an fts4prefix table */
           30  +  int iIndex;                     /* Index for Fts3Table.aIndex[] */
    31     31     Fts3Table *pFts3Tab;
    32     32   };
    33     33   
    34     34   struct Fts3termCursor {
    35     35     sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
    36     36     Fts3SegReaderCursor csr;        /* Must be right after "base" */
    37     37     Fts3SegFilter filter;
................................................................................
    66     66     char const *zDb;                /* Name of database (e.g. "main") */
    67     67     char const *zFts3;              /* Name of fts3 table */
    68     68     int nDb;                        /* Result of strlen(zDb) */
    69     69     int nFts3;                      /* Result of strlen(zFts3) */
    70     70     int nByte;                      /* Bytes of space to allocate here */
    71     71     int rc;                         /* value returned by declare_vtab() */
    72     72     Fts3termTable *p;                /* Virtual table object to return */
           73  +  int iIndex = 0;
           74  +
           75  +  if( argc==5 ){
           76  +    iIndex = atoi(argv[4]);
           77  +    argc--;
           78  +  }
    73     79   
    74     80     /* The user should specify a single argument - the name of an fts3 table. */
    75     81     if( argc!=4 ){
    76     82       *pzErr = sqlite3_mprintf(
    77     83           "wrong number of arguments to fts4term constructor"
    78     84       );
    79     85       return SQLITE_ERROR;
................................................................................
    92     98     if( !p ) return SQLITE_NOMEM;
    93     99     memset(p, 0, nByte);
    94    100   
    95    101     p->pFts3Tab = (Fts3Table *)&p[1];
    96    102     p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
    97    103     p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
    98    104     p->pFts3Tab->db = db;
    99         -  p->bPrefix = (int)pCtx;
          105  +  p->pFts3Tab->nIndex = iIndex+1;
          106  +  p->iIndex = iIndex;
   100    107   
   101    108     memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
   102    109     memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
   103    110     sqlite3Fts3Dequote((char *)p->pFts3Tab->zName);
   104    111   
   105    112     *ppVtab = (sqlite3_vtab *)p;
   106    113     return SQLITE_OK;
................................................................................
   259    266     testcase(pCsr->filter.zTerm);
   260    267     sqlite3Fts3SegReaderFinish(&pCsr->csr);
   261    268     memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr);
   262    269   
   263    270     pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
   264    271     pCsr->filter.flags |= FTS3_SEGMENT_SCAN;
   265    272   
   266         -  rc = sqlite3Fts3SegReaderCursor(pFts3, 
   267         -      p->bPrefix ? FTS3_SEGCURSOR_ALL_PREFIX : FTS3_SEGCURSOR_ALL_TERM,
          273  +  rc = sqlite3Fts3SegReaderCursor(pFts3, p->iIndex, FTS3_SEGCURSOR_ALL,
   268    274         pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr
   269    275     );
   270    276     if( rc==SQLITE_OK ){
   271    277       rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
   272    278     }
   273    279     if( rc==SQLITE_OK ){
   274    280       rc = fts3termNextMethod(pCursor);
................................................................................
   351    357        0,                           /* xRollback     */
   352    358        0,                           /* xFindFunction */
   353    359        0                            /* xRename       */
   354    360     };
   355    361     int rc;                         /* Return code */
   356    362   
   357    363     rc = sqlite3_create_module(db, "fts4term", &fts3term_module, 0);
   358         -  if( rc==SQLITE_OK ){
   359         -    rc = sqlite3_create_module(db, "fts4prefix", &fts3term_module, (void*)1);
   360         -  }
   361    364     return rc;
   362    365   }
   363    366   
   364    367   #endif
   365    368   #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */

Changes to ext/fts3/fts3_write.c.

   179    179   #define SQL_DELETE_ALL_STAT            6
   180    180   #define SQL_SELECT_CONTENT_BY_ROWID    7
   181    181   #define SQL_NEXT_SEGMENT_INDEX         8
   182    182   #define SQL_INSERT_SEGMENTS            9
   183    183   #define SQL_NEXT_SEGMENTS_ID          10
   184    184   #define SQL_INSERT_SEGDIR             11
   185    185   #define SQL_SELECT_LEVEL              12
   186         -#define SQL_SELECT_ALL_LEVEL          13
          186  +#define SQL_SELECT_LEVEL_RANGE        13
   187    187   #define SQL_SELECT_LEVEL_COUNT        14
   188    188   #define SQL_SELECT_SEGDIR_MAX_LEVEL   15
   189         -#define SQL_DELETE_SEGDIR_BY_LEVEL    16
          189  +#define SQL_DELETE_SEGDIR_LEVEL       16
   190    190   #define SQL_DELETE_SEGMENTS_RANGE     17
   191    191   #define SQL_CONTENT_INSERT            18
   192    192   #define SQL_DELETE_DOCSIZE            19
   193    193   #define SQL_REPLACE_DOCSIZE           20
   194    194   #define SQL_SELECT_DOCSIZE            21
   195    195   #define SQL_SELECT_DOCTOTAL           22
   196    196   #define SQL_REPLACE_DOCTOTAL          23
          197  +
   197    198   #define SQL_SELECT_ALL_PREFIX_LEVEL   24
   198         -
   199    199   #define SQL_DELETE_ALL_TERMS_SEGDIR   25
   200         -#define SQL_DELETE_ALL_PREFIX_SEGDIR  26
          200  +
          201  +#define SQL_DELETE_SEGDIR_RANGE       26
   201    202   
   202    203   /*
   203    204   ** This function is used to obtain an SQLite prepared statement handle
   204    205   ** for the statement identified by the second argument. If successful,
   205    206   ** *pp is set to the requested statement handle and SQLITE_OK returned.
   206    207   ** Otherwise, an SQLite error code is returned and *pp is set to 0.
   207    208   **
................................................................................
   230    231   /* 10 */  "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)",
   231    232   /* 11 */  "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",
   232    233   
   233    234             /* Return segments in order from oldest to newest.*/ 
   234    235   /* 12 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
   235    236               "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
   236    237   /* 13 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
   237         -            "FROM %Q.'%q_segdir' WHERE level < " FTS3_SEGDIR_PREFIXLEVEL_STR
   238         -            " ORDER BY level DESC, idx ASC",
          238  +            "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?"
          239  +            "ORDER BY level DESC, idx ASC",
   239    240   
   240    241   /* 14 */  "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
   241         -/* 15 */  "SELECT max(level) FROM %Q.'%q_segdir' WHERE level < (?+1)*"
   242         -            FTS3_SEGDIR_PREFIXLEVEL_STR,
          242  +/* 15 */  "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
   243    243   
   244    244   /* 16 */  "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
   245    245   /* 17 */  "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
   246    246   /* 18 */  "INSERT INTO %Q.'%q_content' VALUES(%s)",
   247    247   /* 19 */  "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
   248    248   /* 20 */  "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
   249    249   /* 21 */  "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
   250    250   /* 22 */  "SELECT value FROM %Q.'%q_stat' WHERE id=0",
   251    251   /* 23 */  "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
   252         -/* 24 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
   253         -            "FROM %Q.'%q_segdir' WHERE level >= " FTS3_SEGDIR_PREFIXLEVEL_STR
   254         -            " ORDER BY level DESC, idx ASC",
   255         -/* 25 */ "DELETE FROM %Q.'%q_segdir' WHERE level<" FTS3_SEGDIR_PREFIXLEVEL_STR,
   256         -/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level>=" FTS3_SEGDIR_PREFIXLEVEL_STR,
          252  +/* 24 */  "",
          253  +/* 25 */  "",
          254  +
          255  +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
          256  +
   257    257     };
   258    258     int rc = SQLITE_OK;
   259    259     sqlite3_stmt *pStmt;
   260    260   
   261    261     assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
   262    262     assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
   263    263     
................................................................................
   405    405   **
   406    406   **   0: idx
   407    407   **   1: start_block
   408    408   **   2: leaves_end_block
   409    409   **   3: end_block
   410    410   **   4: root
   411    411   */
   412         -int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){
          412  +int sqlite3Fts3AllSegdirs(
          413  +  Fts3Table *p,                   /* FTS3 table */
          414  +  int iIndex,                     /* Index for p->aIndex[] */
          415  +  int iLevel,                     /* Level to select */
          416  +  sqlite3_stmt **ppStmt           /* OUT: Compiled statement */
          417  +){
   413    418     int rc;
   414    419     sqlite3_stmt *pStmt = 0;
   415         -  if( iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){
   416         -    rc = fts3SqlStmt(p, SQL_SELECT_ALL_PREFIX_LEVEL, &pStmt, 0);
   417         -  }else if( iLevel==FTS3_SEGCURSOR_ALL_TERM ){
   418         -    rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0);
          420  +
          421  +  assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 );
          422  +  assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
          423  +  assert( iIndex>=0 && iIndex<p->nIndex );
          424  +
          425  +  if( iLevel==FTS3_SEGCURSOR_ALL ){
          426  +    /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */
          427  +    rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0);
          428  +    if( rc==SQLITE_OK ){ 
          429  +      sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
          430  +      sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1);
          431  +    }
   419    432     }else{
   420         -    assert( iLevel>=0 );
          433  +    /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */
   421    434       rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
   422         -    if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel);
          435  +    if( rc==SQLITE_OK ){ 
          436  +      sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL);
          437  +    }
   423    438     }
   424    439     *ppStmt = pStmt;
   425    440     return rc;
   426    441   }
   427    442   
   428    443   
   429    444   /*
................................................................................
   560    575       }
   561    576     }
   562    577     if( rc==SQLITE_OK ){
   563    578       p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
   564    579     }
   565    580     return rc;
   566    581   }
   567         -
   568         - 
   569         -
   570    582   
   571    583   /*
   572    584   ** Tokenize the nul-terminated string zText and add all tokens to the
   573    585   ** pending-terms hash-table. The docid used is that currently stored in
   574    586   ** p->iPrevDocid, and the column is specified by argument iCol.
   575    587   **
   576    588   ** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
................................................................................
   612    624     }
   613    625     pCsr->pTokenizer = pTokenizer;
   614    626   
   615    627     xNext = pModule->xNext;
   616    628     while( SQLITE_OK==rc
   617    629         && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos))
   618    630     ){
          631  +    int i;
   619    632       if( iPos>=nWord ) nWord = iPos+1;
   620    633   
   621    634       /* Positions cannot be negative; we use -1 as a terminator internally.
   622    635       ** Tokens must have a non-zero length.
   623    636       */
   624    637       if( iPos<0 || !zToken || nToken<=0 ){
   625    638         rc = SQLITE_ERROR;
   626    639         break;
   627    640       }
   628    641   
   629         -    rc = fts3PendingTermsAddOne(p,iCol,iPos,&p->pendingTerms,zToken,nToken);
   630         -    if( p->bPrefix ){
   631         -      int n = (nToken > FTS3_MAX_PREFIX ? FTS3_MAX_PREFIX : nToken);
   632         -      for(; n>0 && rc==SQLITE_OK; n--){
   633         -        rc = fts3PendingTermsAddOne(p,iCol,iPos,&p->pendingPrefixes,zToken,n);
   634         -      }
          642  +    /* Add the term to the terms index */
          643  +    rc = fts3PendingTermsAddOne(
          644  +        p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken
          645  +    );
          646  +    
          647  +    /* Add the term to each of the prefix indexes that it is not too 
          648  +    ** short for. */
          649  +    for(i=1; rc==SQLITE_OK && i<p->nIndex; i++){
          650  +      struct Fts3Index *pIndex = &p->aIndex[i];
          651  +      if( nToken<pIndex->nPrefix ) continue;
          652  +      rc = fts3PendingTermsAddOne(
          653  +          p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix
          654  +      );
   635    655       }
   636    656     }
   637    657   
   638    658     pModule->xClose(pCsr);
   639    659     *pnWord = nWord;
   640    660     return (rc==SQLITE_DONE ? SQLITE_OK : rc);
   641    661   }
................................................................................
   657    677       if( rc!=SQLITE_OK ) return rc;
   658    678     }
   659    679     p->iPrevDocid = iDocid;
   660    680     return SQLITE_OK;
   661    681   }
   662    682   
   663    683   /*
   664         -** Discard the contents of the pending-terms hash table. 
          684  +** Discard the contents of the pending-terms hash tables. 
   665    685   */
   666    686   void sqlite3Fts3PendingTermsClear(Fts3Table *p){
   667         -  Fts3HashElem *pElem;
   668         -  for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
   669         -    sqlite3_free(fts3HashData(pElem));
          687  +  int i;
          688  +  for(i=0; i<p->nIndex; i++){
          689  +    Fts3HashElem *pElem;
          690  +    Fts3Hash *pHash = &p->aIndex[i].hPending;
          691  +    for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){
          692  +      sqlite3_free(fts3HashData(pElem));
          693  +    }
          694  +    fts3HashClear(pHash);
   670    695     }
   671         -  fts3HashClear(&p->pendingTerms);
   672    696     p->nPendingData = 0;
   673    697   }
   674    698   
   675         -/*
   676         -** Discard the contents of the pending-prefixes hash table. 
   677         -*/
   678         -void sqlite3Fts3PendingPrefixesClear(Fts3Table *p){
   679         -  Fts3HashElem *pElem;
   680         -  for(pElem=fts3HashFirst(&p->pendingPrefixes); pElem; pElem=fts3HashNext(pElem)){
   681         -    sqlite3_free(fts3HashData(pElem));
   682         -  }
   683         -  fts3HashClear(&p->pendingPrefixes);
   684         -}
   685         -
   686    699   /*
   687    700   ** This function is called by the xUpdate() method as part of an INSERT
   688    701   ** operation. It adds entries for each term in the new record to the
   689    702   ** pendingTerms hash table.
   690    703   **
   691    704   ** Argument apVal is the same as the similarly named argument passed to
   692    705   ** fts3InsertData(). Parameter iDocid is the docid of the new row.
................................................................................
   776    789   ** pending terms.
   777    790   */
   778    791   static int fts3DeleteAll(Fts3Table *p){
   779    792     int rc = SQLITE_OK;             /* Return code */
   780    793   
   781    794     /* Discard the contents of the pending-terms hash table. */
   782    795     sqlite3Fts3PendingTermsClear(p);
   783         -  sqlite3Fts3PendingPrefixesClear(p);
   784    796   
   785    797     /* Delete everything from the %_content, %_segments and %_segdir tables. */
   786    798     fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0);
   787    799     fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0);
   788    800     fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
   789    801     if( p->bHasDocsize ){
   790    802       fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0);
................................................................................
   832    844     *pRC = rc;
   833    845   }
   834    846   
   835    847   /*
   836    848   ** Forward declaration to account for the circular dependency between
   837    849   ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx().
   838    850   */
   839         -static int fts3SegmentMerge(Fts3Table *, int);
          851  +static int fts3SegmentMerge(Fts3Table *, int, int);
   840    852   
   841    853   /* 
   842    854   ** This function allocates a new level iLevel index in the segdir table.
   843    855   ** Usually, indexes are allocated within a level sequentially starting
   844    856   ** with 0, so the allocated index is one greater than the value returned
   845    857   ** by:
   846    858   **
................................................................................
   849    861   ** However, if there are already FTS3_MERGE_COUNT indexes at the requested
   850    862   ** level, they are merged into a single level (iLevel+1) segment and the 
   851    863   ** allocated index is 0.
   852    864   **
   853    865   ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK
   854    866   ** returned. Otherwise, an SQLite error code is returned.
   855    867   */
   856         -static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){
          868  +static int fts3AllocateSegdirIdx(
          869  +  Fts3Table *p, 
          870  +  int iIndex,                     /* Index for p->aIndex */
          871  +  int iLevel, 
          872  +  int *piIdx
          873  +){
   857    874     int rc;                         /* Return Code */
   858    875     sqlite3_stmt *pNextIdx;         /* Query for next idx at level iLevel */
   859    876     int iNext = 0;                  /* Result of query pNextIdx */
   860    877   
   861    878     /* Set variable iNext to the next available segdir index at level iLevel. */
   862    879     rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0);
   863    880     if( rc==SQLITE_OK ){
   864         -    sqlite3_bind_int(pNextIdx, 1, iLevel);
          881  +    sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel);
   865    882       if( SQLITE_ROW==sqlite3_step(pNextIdx) ){
   866    883         iNext = sqlite3_column_int(pNextIdx, 0);
   867    884       }
   868    885       rc = sqlite3_reset(pNextIdx);
   869    886     }
   870    887   
   871    888     if( rc==SQLITE_OK ){
   872    889       /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already
   873    890       ** full, merge all segments in level iLevel into a single iLevel+1
   874    891       ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
   875    892       ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
   876    893       */
   877    894       if( iNext>=FTS3_MERGE_COUNT ){
   878         -      rc = fts3SegmentMerge(p, iLevel);
          895  +      rc = fts3SegmentMerge(p, iIndex, iLevel);
   879    896         *piIdx = 0;
   880    897       }else{
   881    898         *piIdx = iNext;
   882    899       }
   883    900     }
   884    901   
   885    902     return rc;
................................................................................
  1305   1322   **
  1306   1323   ** Whereas if isPrefixIter is zero, the terms visited are:
  1307   1324   **
  1308   1325   **   firebird mysql sqlite
  1309   1326   */
  1310   1327   int sqlite3Fts3SegReaderPending(
  1311   1328     Fts3Table *p,                   /* Virtual table handle */
         1329  +  int iIndex,                     /* Index for p->aIndex */
  1312   1330     const char *zTerm,              /* Term to search for */
  1313   1331     int nTerm,                      /* Size of buffer zTerm */
  1314         -  int isMultiTerm,                /* True to visit multiple terms */
  1315         -  int isPrefixIter,               /* 0->pendingTerms, 1->pendingPrefixes */
         1332  +  int bPrefix,                    /* True for a prefix iterator */
  1316   1333     Fts3SegReader **ppReader        /* OUT: SegReader for pending-terms */
  1317   1334   ){
  1318   1335     Fts3SegReader *pReader = 0;     /* Fts3SegReader object to return */
  1319   1336     Fts3HashElem **aElem = 0;       /* Array of term hash entries to scan */
  1320   1337     int nElem = 0;                  /* Size of array at aElem */
  1321   1338     int rc = SQLITE_OK;             /* Return Code */
         1339  +  Fts3Hash *pHash;
  1322   1340   
  1323         -  if( isMultiTerm ){
         1341  +  pHash = &p->aIndex[iIndex].hPending;
         1342  +  if( bPrefix ){
  1324   1343       int nAlloc = 0;               /* Size of allocated array at aElem */
  1325   1344       Fts3HashElem *pE = 0;         /* Iterator variable */
  1326         -    Fts3Hash *pHash;
  1327         -
  1328         -    pHash = (isPrefixIter ? &p->pendingPrefixes : &p->pendingTerms);
  1329   1345   
  1330   1346       for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){
  1331   1347         char *zKey = (char *)fts3HashKey(pE);
  1332   1348         int nKey = fts3HashKeysize(pE);
  1333   1349         if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){
  1334   1350           if( nElem==nAlloc ){
  1335   1351             Fts3HashElem **aElem2;
................................................................................
  1356   1372       if( nElem>1 ){
  1357   1373         qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm);
  1358   1374       }
  1359   1375   
  1360   1376     }else{
  1361   1377       /* The query is a simple term lookup that matches at most one term in
  1362   1378       ** the index. All that is required is a straight hash-lookup. */
  1363         -    Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm);
         1379  +    Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm);
  1364   1380       if( pE ){
  1365   1381         aElem = &pE;
  1366   1382         nElem = 1;
  1367   1383       }
  1368   1384     }
  1369   1385   
  1370   1386     if( nElem>0 ){
................................................................................
  1376   1392         memset(pReader, 0, nByte);
  1377   1393         pReader->iIdx = 0x7FFFFFFF;
  1378   1394         pReader->ppNextElem = (Fts3HashElem **)&pReader[1];
  1379   1395         memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *));
  1380   1396       }
  1381   1397     }
  1382   1398   
  1383         -  if( isMultiTerm ){
         1399  +  if( bPrefix ){
  1384   1400       sqlite3_free(aElem);
  1385   1401     }
  1386   1402     *ppReader = pReader;
  1387   1403     return rc;
  1388   1404   }
  1389   1405   
  1390   1406   /*
................................................................................
  1988   2004       }
  1989   2005       rc = sqlite3_reset(pStmt);
  1990   2006     }
  1991   2007     return rc;
  1992   2008   }
  1993   2009   
  1994   2010   /*
  1995         -** Set *pnMax to the largest segment level in the database for either the
  1996         -** terms index (if parameter bPrefixIndex is 0) or the prefixes index (if
  1997         -** parameter bPrefixIndex is 1).
         2011  +** Set *pnMax to the largest segment level in the database for the index
         2012  +** iIndex.
  1998   2013   **
  1999   2014   ** Segment levels are stored in the 'level' column of the %_segdir table.
  2000   2015   **
  2001   2016   ** Return SQLITE_OK if successful, or an SQLite error code if not.
  2002   2017   */
  2003         -static int fts3SegmentMaxLevel(Fts3Table *p, int bPrefixIndex, int *pnMax){
         2018  +static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){
  2004   2019     sqlite3_stmt *pStmt;
  2005   2020     int rc;
  2006         -  assert( bPrefixIndex==0 || bPrefixIndex==1 );
         2021  +  assert( iIndex>=0 && iIndex<p->nIndex );
  2007   2022   
  2008   2023     /* Set pStmt to the compiled version of:
  2009   2024     **
  2010         -  **   SELECT max(level) FROM %Q.'%q_segdir' WHERE level < (?+1) * 1024 
         2025  +  **   SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?
  2011   2026     **
  2012   2027     ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR).
  2013   2028     */
  2014   2029     rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0);
  2015   2030     if( rc!=SQLITE_OK ) return rc;
  2016         -  sqlite3_bind_int(pStmt, 1, bPrefixIndex);
         2031  +  sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
         2032  +  sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1);
  2017   2033     if( SQLITE_ROW==sqlite3_step(pStmt) ){
  2018   2034       *pnMax = sqlite3_column_int(pStmt, 0);
  2019   2035     }
  2020   2036     return sqlite3_reset(pStmt);
  2021   2037   }
  2022   2038   
  2023   2039   /*
................................................................................
  2032   2048   **   2) deletes all %_segdir entries with level iLevel, or all %_segdir
  2033   2049   **      entries regardless of level if (iLevel<0).
  2034   2050   **
  2035   2051   ** SQLITE_OK is returned if successful, otherwise an SQLite error code.
  2036   2052   */
  2037   2053   static int fts3DeleteSegdir(
  2038   2054     Fts3Table *p,                   /* Virtual table handle */
         2055  +  int iIndex,                     /* Index for p->aIndex */
  2039   2056     int iLevel,                     /* Level of %_segdir entries to delete */
  2040   2057     Fts3SegReader **apSegment,      /* Array of SegReader objects */
  2041   2058     int nReader                     /* Size of array apSegment */
  2042   2059   ){
  2043   2060     int rc;                         /* Return Code */
  2044   2061     int i;                          /* Iterator variable */
  2045   2062     sqlite3_stmt *pDelete;          /* SQL statement to delete rows */
................................................................................
  2054   2071         rc = sqlite3_reset(pDelete);
  2055   2072       }
  2056   2073     }
  2057   2074     if( rc!=SQLITE_OK ){
  2058   2075       return rc;
  2059   2076     }
  2060   2077   
  2061         -  assert( iLevel>=0 
  2062         -       || iLevel==FTS3_SEGCURSOR_ALL_TERM
  2063         -       || iLevel==FTS3_SEGCURSOR_ALL_PREFIX 
  2064         -       || iLevel==FTS3_SEGCURSOR_PENDING
  2065         -       || iLevel==FTS3_SEGCURSOR_PENDING_PREFIX 
  2066         -  );
  2067         -  if( iLevel==FTS3_SEGCURSOR_ALL_TERM ){
  2068         -    fts3SqlExec(&rc, p, SQL_DELETE_ALL_TERMS_SEGDIR, 0);
  2069         -  }else if( iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){
  2070         -    fts3SqlExec(&rc, p, SQL_DELETE_ALL_PREFIX_SEGDIR, 0);
  2071         -  }else if( iLevel==FTS3_SEGCURSOR_PENDING_PREFIX ){
  2072         -    sqlite3Fts3PendingPrefixesClear(p);
  2073         -  }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
  2074         -    sqlite3Fts3PendingTermsClear(p);
  2075         -  }else if( iLevel>=0 ){
  2076         -    rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0);
         2078  +  assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL );
         2079  +  if( iLevel==FTS3_SEGCURSOR_ALL ){
         2080  +    rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0);
         2081  +    if( rc==SQLITE_OK ){
         2082  +      sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
         2083  +      sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1);
         2084  +    }
         2085  +  }else{
         2086  +    rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0);
  2077   2087       if( rc==SQLITE_OK ){
  2078         -      sqlite3_bind_int(pDelete, 1, iLevel);
  2079         -      sqlite3_step(pDelete);
  2080         -      rc = sqlite3_reset(pDelete);
         2088  +      sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel);
  2081   2089       }
  2082   2090     }
         2091  +
         2092  +  if( rc==SQLITE_OK ){
         2093  +    sqlite3_step(pDelete);
         2094  +    rc = sqlite3_reset(pDelete);
         2095  +  }
  2083   2096   
  2084   2097     return rc;
  2085   2098   }
  2086   2099   
  2087   2100   /*
  2088   2101   ** When this function is called, buffer *ppList (size *pnList bytes) contains 
  2089   2102   ** a position list that may (or may not) feature multiple columns. This
................................................................................
  2319   2332   ** currently present in the database.
  2320   2333   **
  2321   2334   ** If this function is called with iLevel<0, but there is only one
  2322   2335   ** segment in the database, SQLITE_DONE is returned immediately. 
  2323   2336   ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, 
  2324   2337   ** an SQLite error code is returned.
  2325   2338   */
  2326         -static int fts3SegmentMerge(Fts3Table *p, int iLevel){
         2339  +static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){
  2327   2340     int rc;                         /* Return code */
  2328   2341     int iIdx = 0;                   /* Index of new segment */
  2329         -  int iNewLevel = 0;              /* Level to create new segment at */
         2342  +  int iNewLevel = 0;              /* Level/index to create new segment at */
  2330   2343     SegmentWriter *pWriter = 0;     /* Used to write the new, merged, segment */
  2331   2344     Fts3SegFilter filter;           /* Segment term filter condition */
  2332   2345     Fts3SegReaderCursor csr;        /* Cursor to iterate through level(s) */
  2333   2346     int bIgnoreEmpty = 0;           /* True to ignore empty segments */
  2334   2347   
  2335         -  rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, 0, &csr);
         2348  +  assert( iLevel==FTS3_SEGCURSOR_ALL
         2349  +       || iLevel==FTS3_SEGCURSOR_PENDING
         2350  +       || iLevel>=0
         2351  +  );
         2352  +  assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
         2353  +  assert( iIndex>=0 && iIndex<p->nIndex );
         2354  +
         2355  +  rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr);
  2336   2356     if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished;
  2337   2357   
  2338         -  if( iLevel==FTS3_SEGCURSOR_ALL_TERM || iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){
         2358  +  if( iLevel==FTS3_SEGCURSOR_ALL ){
  2339   2359       /* This call is to merge all segments in the database to a single
  2340   2360       ** segment. The level of the new segment is equal to the the numerically 
  2341         -    ** greatest segment level currently present in the database. The index
  2342         -    ** of the new segment is always 0.  */
         2361  +    ** greatest segment level currently present in the database for this
         2362  +    ** index. The idx of the new segment is always 0.  */
  2343   2363       if( csr.nSegment==1 ){
  2344   2364         rc = SQLITE_DONE;
  2345   2365         goto finished;
  2346   2366       }
  2347         -    rc = fts3SegmentMaxLevel(p, iLevel==FTS3_SEGCURSOR_ALL_PREFIX, &iNewLevel);
         2367  +    rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel);
  2348   2368       bIgnoreEmpty = 1;
         2369  +
         2370  +  }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
         2371  +    iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL; 
         2372  +    rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx);
  2349   2373     }else{
  2350   2374       /* This call is to merge all segments at level iLevel. find the next
  2351   2375       ** available segment index at level iLevel+1. The call to
  2352   2376       ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to 
  2353   2377       ** a single iLevel+2 segment if necessary.  */
  2354         -    if( iLevel==FTS3_SEGCURSOR_PENDING ){
  2355         -      iNewLevel = 0;
  2356         -    }else if( iLevel==FTS3_SEGCURSOR_PENDING_PREFIX ){
  2357         -      iNewLevel = FTS3_SEGDIR_PREFIXLEVEL;
  2358         -    }else{
  2359         -      iNewLevel = iLevel+1;
  2360         -    }
  2361         -    rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx);
         2378  +    rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx);
         2379  +    iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1;
  2362   2380     }
  2363   2381     if( rc!=SQLITE_OK ) goto finished;
  2364   2382     assert( csr.nSegment>0 );
  2365         -  assert( iNewLevel>=0 );
         2383  +  assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) );
         2384  +  assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) );
  2366   2385   
  2367   2386     memset(&filter, 0, sizeof(Fts3SegFilter));
  2368   2387     filter.flags = FTS3_SEGMENT_REQUIRE_POS;
  2369   2388     filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
  2370   2389   
  2371   2390     rc = sqlite3Fts3SegReaderStart(p, &csr, &filter);
  2372   2391     while( SQLITE_OK==rc ){
................................................................................
  2374   2393       if( rc!=SQLITE_ROW ) break;
  2375   2394       rc = fts3SegWriterAdd(p, &pWriter, 1, 
  2376   2395           csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
  2377   2396     }
  2378   2397     if( rc!=SQLITE_OK ) goto finished;
  2379   2398     assert( pWriter );
  2380   2399   
  2381         -  rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment);
  2382         -  if( rc!=SQLITE_OK ) goto finished;
         2400  +  if( iLevel!=FTS3_SEGCURSOR_PENDING ){
         2401  +    rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment);
         2402  +    if( rc!=SQLITE_OK ) goto finished;
         2403  +  }
  2383   2404     rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
  2384   2405   
  2385   2406    finished:
  2386   2407     fts3SegWriterFree(pWriter);
  2387   2408     sqlite3Fts3SegReaderFinish(&csr);
  2388   2409     return rc;
  2389   2410   }
  2390   2411   
  2391   2412   
  2392   2413   /* 
  2393         -** Flush the contents of pendingTerms to a level 0 segment.
         2414  +** Flush the contents of pendingTerms to level 0 segments.
  2394   2415   */
  2395   2416   int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
  2396   2417     int rc = SQLITE_OK;
  2397         -  if( p->bPrefix ){
  2398         -    rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING_PREFIX);
         2418  +  int i;
         2419  +  for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
         2420  +    rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING);
         2421  +    if( rc==SQLITE_DONE ) rc = SQLITE_OK;
  2399   2422     }
  2400         -  if( rc==SQLITE_OK || rc==SQLITE_DONE ){
  2401         -    rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING);
  2402         -  }
  2403         -  if( rc==SQLITE_DONE ){
  2404         -    rc = SQLITE_OK;
  2405         -  }
         2423  +  sqlite3Fts3PendingTermsClear(p);
  2406   2424     return rc;
  2407   2425   }
  2408   2426   
  2409   2427   /*
  2410   2428   ** Encode N integers as varints into a blob.
  2411   2429   */
  2412   2430   static void fts3EncodeIntArray(
................................................................................
  2549   2567       return;
  2550   2568     }
  2551   2569     sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC);
  2552   2570     sqlite3_step(pStmt);
  2553   2571     *pRC = sqlite3_reset(pStmt);
  2554   2572     sqlite3_free(a);
  2555   2573   }
         2574  +
         2575  +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
         2576  +  int i;
         2577  +  int bSeenDone = 0;
         2578  +  int rc = SQLITE_OK;
         2579  +  for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
         2580  +    rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL);
         2581  +    if( rc==SQLITE_DONE ){
         2582  +      bSeenDone = 1;
         2583  +      rc = SQLITE_OK;
         2584  +    }
         2585  +  }
         2586  +  sqlite3Fts3SegmentsClose(p);
         2587  +  sqlite3Fts3PendingTermsClear(p);
         2588  +
         2589  +  return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc;
         2590  +}
  2556   2591   
  2557   2592   /*
  2558   2593   ** Handle a 'special' INSERT of the form:
  2559   2594   **
  2560   2595   **   "INSERT INTO tbl(tbl) VALUES(<expr>)"
  2561   2596   **
  2562   2597   ** Argument pVal contains the result of <expr>. Currently the only 
................................................................................
  2566   2601     int rc;                         /* Return Code */
  2567   2602     const char *zVal = (const char *)sqlite3_value_text(pVal);
  2568   2603     int nVal = sqlite3_value_bytes(pVal);
  2569   2604   
  2570   2605     if( !zVal ){
  2571   2606       return SQLITE_NOMEM;
  2572   2607     }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
  2573         -    rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_PREFIX);
  2574         -    if( rc==SQLITE_OK ){
  2575         -      rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_TERM);
  2576         -    }
         2608  +    rc = fts3DoOptimize(p, 0);
  2577   2609   #ifdef SQLITE_TEST
  2578   2610     }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
  2579   2611       p->nNodeSize = atoi(&zVal[9]);
  2580   2612       rc = SQLITE_OK;
  2581   2613     }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
  2582   2614       p->nMaxPendingData = atoi(&zVal[11]);
  2583   2615       rc = SQLITE_OK;
  2584   2616   #endif
  2585   2617     }else{
  2586   2618       rc = SQLITE_ERROR;
  2587   2619     }
  2588   2620   
  2589         -  sqlite3Fts3SegmentsClose(p);
  2590         -  sqlite3Fts3PendingTermsClear(p);
  2591         -  sqlite3Fts3PendingPrefixesClear(p);
  2592   2621     return rc;
  2593   2622   }
  2594   2623   
  2595   2624   /*
  2596   2625   ** Return the deferred doclist associated with deferred token pDeferred.
  2597   2626   ** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already
  2598   2627   ** been called to allocate and populate the doclist.
................................................................................
  2901   2930   /* 
  2902   2931   ** Flush any data in the pending-terms hash table to disk. If successful,
  2903   2932   ** merge all segments in the database (including the new segment, if 
  2904   2933   ** there was any data to flush) into a single segment. 
  2905   2934   */
  2906   2935   int sqlite3Fts3Optimize(Fts3Table *p){
  2907   2936     int rc;
  2908         -  int bReturnDone = 0;
  2909   2937     rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0);
  2910   2938     if( rc==SQLITE_OK ){
  2911         -    rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_PREFIX);
  2912         -    if( rc==SQLITE_OK ){
  2913         -      rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_TERM);
  2914         -    }
  2915         -    if( rc==SQLITE_DONE ){
  2916         -      bReturnDone = 1;
  2917         -      rc = SQLITE_OK;
  2918         -    }
  2919         -    if( rc==SQLITE_OK ){
  2920         -      rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
  2921         -      if( rc==SQLITE_OK ){
  2922         -        sqlite3Fts3PendingTermsClear(p);
  2923         -        sqlite3Fts3PendingPrefixesClear(p);
  2924         -      }
         2939  +    rc = fts3DoOptimize(p, 1);
         2940  +    if( rc==SQLITE_OK || rc==SQLITE_DONE ){
         2941  +      int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
         2942  +      if( rc2!=SQLITE_OK ) rc = rc2;
  2925   2943       }else{
  2926   2944         sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0);
  2927   2945         sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
  2928   2946       }
  2929   2947     }
  2930   2948     sqlite3Fts3SegmentsClose(p);
  2931         -  return ((rc==SQLITE_OK && bReturnDone) ? SQLITE_DONE : rc);
         2949  +  return rc;
  2932   2950   }
  2933   2951   
  2934   2952   #endif

Changes to test/fts3prefix.test.

    11     11   # This file implements regression tests for SQLite library.  The
    12     12   # focus of this script is testing the FTS3 module.
    13     13   #
    14     14   
    15     15   set testdir [file dirname $argv0]
    16     16   source $testdir/tester.tcl
    17     17   set testprefix fts3prefix
           18  +
           19  +ifcapable !fts3 {
           20  +  finish_test
           21  +  return
           22  +}
    18     23   
    19     24   # This proc tests that the prefixes index appears to represent the same content
    20     25   # as the terms index.
    21     26   #
    22         -proc fts3_terms_and_prefixes {db tbl} {
    23         -  $db eval "CREATE VIRTUAL TABLE fts3check1 USING fts4term($tbl);"
    24         -  $db eval "CREATE VIRTUAL TABLE fts3check2 USING fts4prefix($tbl);"
    25         -
    26         -  $db eval {
    27         -    CREATE TEMP TABLE terms AS SELECT * FROM fts3check1;
    28         -    CREATE TEMP TABLE prefixes AS SELECT * FROM fts3check2;
    29         -    CREATE INDEX temp.idx ON prefixes(term);
    30         -    DROP TABLE fts3check1;
    31         -    DROP TABLE fts3check2;
    32         -  }
    33         -
    34         -  $db eval { SELECT term, docid, col, pos FROM temp.terms } a {
    35         -    set nMax [expr [string length $a(term)] - 1]
    36         -    if {$nMax>8} {set nMax 8}
    37         -    for {set n 0} {$n < $nMax} {incr n} {
    38         -      set t [string range $a(term) 0 $n]
           27  +proc fts3_terms_and_prefixes {db tbl prefixlengths} {
           28  +
           29  +  set iIndex 0
           30  +  foreach len $prefixlengths {
           31  +    incr iIndex
           32  +    $db eval {
           33  +      DROP TABLE IF EXISTS fts3check1;
           34  +      DROP TABLE IF EXISTS fts3check2;
           35  +    }
           36  +    $db eval "CREATE VIRTUAL TABLE fts3check1 USING fts4term($tbl, 0);"
           37  +    $db eval "CREATE VIRTUAL TABLE fts3check2 USING fts4term($tbl, $iIndex);"
           38  +
           39  +    $db eval {
           40  +      DROP TABLE IF EXISTS temp.terms;
           41  +      DROP TABLE IF EXISTS temp.prefixes;
           42  +      CREATE TEMP TABLE terms AS SELECT * FROM fts3check1;
           43  +      CREATE TEMP TABLE prefixes AS SELECT * FROM fts3check2;
           44  +      CREATE INDEX temp.idx ON prefixes(term);
           45  +      DROP TABLE fts3check1;
           46  +      DROP TABLE fts3check2;
           47  +    }
           48  +
           49  +    set nExpect 0
           50  +    $db eval { SELECT term, docid, col, pos FROM temp.terms } a {
           51  +      if {[string length $a(term)]<$len} continue
           52  +      incr nExpect
           53  +      set prefix [string range $a(term) 0 [expr $len-1]]
    39     54         set r [$db one { 
    40     55           SELECT count(*) FROM temp.prefixes WHERE 
    41         -        term = $t AND docid = $a(docid) AND col = $a(col) AND pos = $a(pos)
           56  +        term = $prefix AND docid = $a(docid) AND col = $a(col) AND pos = $a(pos)
    42     57         }]
    43     58         if {$r != 1} {
    44     59           error "$t, $a(docid), $a(col), $a(pos)"
    45     60         }
    46     61       }
    47         -  }
    48     62   
    49         -  execsql { DROP TABLE temp.prefixes }
    50         -  execsql { DROP TABLE temp.terms }
           63  +    set nCount [$db one {SELECT count(*) FROM temp.prefixes}]
           64  +    if {$nCount != $nExpect} {
           65  +      error "prefixes.count(*) is $nCount expected $nExpect"
           66  +    }
           67  +  
           68  +    execsql { DROP TABLE temp.prefixes }
           69  +    execsql { DROP TABLE temp.terms }
    51     70   
    52         -  set terms_layout [$db eval " 
    53         -    SELECT level, idx FROM ${tbl}_segdir WHERE level < 1024 ORDER by 1, 2
    54         -  "]
    55         -  set prefixes_layout [$db eval " 
    56         -    SELECT level-1024, idx FROM ${tbl}_segdir WHERE level >= 1024 ORDER by 1, 2
    57         -  "]
           71  +    set list [list]
           72  +    $db eval "
           73  +      SELECT sum( 1 << (16*(level%1024)) ) AS total, (level/1024) AS tree 
           74  +      FROM ${tbl}_segdir GROUP BY tree
           75  +    " {
           76  +      lappend list [list $total $tree]
           77  +    }
    58     78   
    59         -  if {$terms_layout != $prefixes_layout} {
    60         -    puts "TERMS LAYOUT:  $terms_layout"
    61         -    puts "PREFIX LAYOUT: $prefixes_layout"
    62         -    error "Terms and prefixes are comprised of different b-trees"
           79  +    if { [lsort -integer -index 0 $list] != [lsort -integer -index 1 $list] } {
           80  +      error "inconsistent tree structures: $list"
           81  +    }
    63     82     }
    64     83   
    65     84     return ""
    66     85   }
    67         -proc fts3_tap_test {tn db tbl} {
    68         -  uplevel [list do_test $tn [list fts3_terms_and_prefixes $db $tbl] ""]
           86  +proc fts3_tap_test {tn db tbl lens} {
           87  +  uplevel [list do_test $tn [list fts3_terms_and_prefixes $db $tbl $lens] ""]
    69     88   }
    70     89   
    71     90   #-------------------------------------------------------------------------
    72     91   # Test cases 1.* are a sanity check. They test that the prefixes index is
    73     92   # being constructed correctly for the simplest possible case.
    74     93   #
    75     94   do_execsql_test 1.1 {
    76         -  CREATE VIRTUAL TABLE t1 USING fts4(prefix=1);
    77         -  CREATE VIRTUAL TABLE prefixes USING fts4prefix(t1);
           95  +  CREATE VIRTUAL TABLE t1 USING fts4(prefix='1,3,6');
           96  +
           97  +  CREATE VIRTUAL TABLE p1 USING fts4term(t1, 1);
           98  +  CREATE VIRTUAL TABLE p2 USING fts4term(t1, 2);
           99  +  CREATE VIRTUAL TABLE p3 USING fts4term(t1, 3);
    78    100     CREATE VIRTUAL TABLE terms USING fts4term(t1);
    79    101   }
    80    102   do_execsql_test 1.2 {
    81    103     INSERT INTO t1 VALUES('sqlite mysql firebird');
    82    104   }
    83         -do_execsql_test 1.3 {
    84         -  SELECT term FROM prefixes;
    85         -} {f fi fir fire fireb firebi firebir firebird m my mys mysq mysql s sq sql sqli sqlit sqlite}
          105  +do_execsql_test 1.3.1 { SELECT term FROM p1 } {f m s}
          106  +do_execsql_test 1.3.2 { SELECT term FROM p2 } {fir mys sql}
          107  +do_execsql_test 1.3.3 { SELECT term FROM p3 } {firebi sqlite}
    86    108   do_execsql_test 1.4 {
    87    109     SELECT term FROM terms;
    88    110   } {firebird mysql sqlite}
    89    111   
    90         -fts3_tap_test 1.5 db t1
          112  +fts3_tap_test 1.5 db t1 {1 3 6}
    91    113   
    92    114   #-------------------------------------------------------------------------
    93    115   # A slightly more complicated dataset. This test also verifies that DELETE
    94    116   # operations do not corrupt the prefixes index.
    95    117   #
    96    118   do_execsql_test 2.1 {
    97    119     INSERT INTO t1 VALUES('FTS3 and FTS4 are an SQLite virtual table modules');
................................................................................
   111    133     INSERT INTO t1 VALUES('modules for SQLite. There are known issues with');
   112    134     INSERT INTO t1 VALUES('these older modules and their use should be');
   113    135     INSERT INTO t1 VALUES('avoided. Portions of the original FTS3 code were');
   114    136     INSERT INTO t1 VALUES('contributed to the SQLite project by Scott Hess of');
   115    137     INSERT INTO t1 VALUES('Google. It is now developed and maintained as part');
   116    138     INSERT INTO t1 VALUES('of SQLite. ');
   117    139   }
   118         -fts3_tap_test 2.2 db t1
          140  +fts3_tap_test 2.2 db t1 {1 3 6}
   119    141   do_execsql_test 2.3 { DELETE FROM t1 WHERE docid%2; }
   120         -fts3_tap_test 2.4 db t1
          142  +fts3_tap_test 2.4 db t1 {1 3 6}
   121    143   
   122    144   do_execsql_test 2.5 { INSERT INTO t1(t1) VALUES('optimize') }
   123         -fts3_tap_test 2.6 db t1
          145  +fts3_tap_test 2.6 db t1 {1 3 6}
   124    146   
   125    147   do_execsql_test 3.1 {
   126         -  CREATE VIRTUAL TABLE t2 USING fts4(prefix=1);
          148  +  CREATE VIRTUAL TABLE t2 USING fts4(prefix='1,2,3');
   127    149     INSERT INTO t2 VALUES('On 12 September the wind direction turned and');
   128    150     INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the');
   129    151     INSERT INTO t2 VALUES('fleet was forced to take shelter at');
   130    152     INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind');
   131    153     INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet');
   132    154     INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey');
   133    155     INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved');
................................................................................
   144    166     INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved');
   145    167     INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he');
   146    168     INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of');
   147    169     INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland');
   148    170     INSERT INTO t2 VALUES('and waited for Harold''s return from the north.');
   149    171   }
   150    172   
   151         -fts3_tap_test 3.2 db t2
          173  +fts3_tap_test 3.2 db t2 {1 2 3}
   152    174   do_execsql_test 3.3 { SELECT optimize(t2) FROM t2 LIMIT 1 } {{Index optimized}}
   153         -fts3_tap_test 3.4 db t2
          175  +fts3_tap_test 3.4 db t2 {1 2 3}
   154    176   
   155    177   
   156    178   #-------------------------------------------------------------------------
   157    179   # Simple tests for reading the prefix-index.
   158    180   #
   159    181   do_execsql_test 4.1 {
   160         -  CREATE VIRTUAL TABLE t3 USING fts4(prefix=1);
          182  +  CREATE VIRTUAL TABLE t3 USING fts4(prefix="1,4");
   161    183     INSERT INTO t3 VALUES('one two three');
   162    184     INSERT INTO t3 VALUES('four five six');
   163    185     INSERT INTO t3 VALUES('seven eight nine');
   164    186   }
   165    187   do_execsql_test 4.2 {
   166    188     SELECT * FROM t3 WHERE t3 MATCH 'f*'
   167    189   } {{four five six}}