/ Check-in [b1f9c1e0]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge fts3-prefix-search branch with trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: b1f9c1e0ac51cedfb05ac073a603343f6df865b5
User & Date: dan 2011-06-14 11:50:09
References
2013-03-24
20:45 New ticket [38b1ae01] Assertion fault on valid FTS4 query. artifact: cb548670 user: drh
Context
2011-06-14
14:18
Fix a memory leak that can follow an OOM error in a user-function that uses sqlite3_set_auxdata(). check-in: 0185c4b6 user: dan tags: trunk
11:50
Merge fts3-prefix-search branch with trunk. check-in: b1f9c1e0 user: dan tags: trunk
11:32
Add a couple of extra tests. Closed-Leaf check-in: aefd46df user: dan tags: fts3-prefix-search
2011-06-13
12:19
Use only unsigned values in the implementatin of LIKE and GLOB so that values won't overflow to negative when dealing with malformed UTF8. check-in: 77f01578 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   375    375     $(TOP)/src/test_syscall.c \
   376    376     $(TOP)/src/test_stat.c \
   377    377     $(TOP)/src/test_tclvar.c \
   378    378     $(TOP)/src/test_thread.c \
   379    379     $(TOP)/src/test_vfs.c \
   380    380     $(TOP)/src/test_wholenumber.c \
   381    381     $(TOP)/src/test_wsd.c       \
   382         -  $(TOP)/ext/fts3/fts3_term.c 
          382  +  $(TOP)/ext/fts3/fts3_term.c \
          383  +  $(TOP)/ext/fts3/fts3_test.c 
   383    384   
   384    385   # Source code to the library files needed by the test fixture
   385    386   #
   386    387   TESTSRC2 = \
   387    388     $(TOP)/src/attach.c \
   388    389     $(TOP)/src/backup.c \
   389    390     $(TOP)/src/bitvec.c \

Changes to ext/fts3/fts3.c.

   419    419     *pVal += iVal;
   420    420   }
   421    421   
   422    422   /*
   423    423   ** When this function is called, *pp points to the first byte following a
   424    424   ** varint that is part of a doclist (or position-list, or any other list
   425    425   ** of varints). This function moves *pp to point to the start of that varint,
   426         -** and decrements the value stored in *pVal by the varint value.
          426  +** and sets *pVal by the varint value.
   427    427   **
   428    428   ** Argument pStart points to the first byte of the doclist that the
   429    429   ** varint is part of.
   430    430   */
   431         -static void fts3GetReverseDeltaVarint(
          431  +static void fts3GetReverseVarint(
   432    432     char **pp, 
   433    433     char *pStart, 
   434    434     sqlite3_int64 *pVal
   435    435   ){
   436    436     sqlite3_int64 iVal;
   437    437     char *p = *pp;
   438    438   
................................................................................
   440    440     ** interested in. So, unless the doclist is corrupt, the 0x80 bit is
   441    441     ** clear on character p[-1]. */
   442    442     for(p = (*pp)-2; p>=pStart && *p&0x80; p--);
   443    443     p++;
   444    444     *pp = p;
   445    445   
   446    446     sqlite3Fts3GetVarint(p, &iVal);
   447         -  *pVal -= iVal;
   448         -}
   449         -
   450         -/*
   451         -** As long as *pp has not reached its end (pEnd), then do the same
   452         -** as fts3GetDeltaVarint(): read a single varint and add it to *pVal.
   453         -** But if we have reached the end of the varint, just set *pp=0 and
   454         -** leave *pVal unchanged.
   455         -*/
   456         -static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){
   457         -  if( *pp>=pEnd ){
   458         -    *pp = 0;
   459         -  }else{
   460         -    fts3GetDeltaVarint(pp, pVal);
   461         -  }
          447  +  *pVal = iVal;
   462    448   }
   463    449   
   464    450   /*
   465    451   ** The xDisconnect() virtual table method.
   466    452   */
   467    453   static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
   468    454     Fts3Table *p = (Fts3Table *)pVtab;
................................................................................
   827    813     fts3Appendf(pRc, &zRet, "?");
   828    814     for(i=0; i<p->nColumn; i++){
   829    815       fts3Appendf(pRc, &zRet, ",%s(?)", zFunction);
   830    816     }
   831    817     sqlite3_free(zFree);
   832    818     return zRet;
   833    819   }
          820  +
          821  +static int fts3GobbleInt(const char **pp, int *pnOut){
          822  +  const char *p = *pp;
          823  +  int nInt = 0;
          824  +  for(p=*pp; p[0]>='0' && p[0]<='9'; p++){
          825  +    nInt = nInt * 10 + (p[0] - '0');
          826  +  }
          827  +  if( p==*pp ) return SQLITE_ERROR;
          828  +  *pnOut = nInt;
          829  +  *pp = p;
          830  +  return SQLITE_OK;
          831  +}
          832  +
          833  +
          834  +static int fts3PrefixParameter(
          835  +  const char *zParam,             /* ABC in prefix=ABC parameter to parse */
          836  +  int *pnIndex,                   /* OUT: size of *apIndex[] array */
          837  +  struct Fts3Index **apIndex,     /* OUT: Array of indexes for this table */
          838  +  struct Fts3Index **apFree       /* OUT: Free this with sqlite3_free() */
          839  +){
          840  +  struct Fts3Index *aIndex;
          841  +  int nIndex = 1;
          842  +
          843  +  if( zParam && zParam[0] ){
          844  +    const char *p;
          845  +    nIndex++;
          846  +    for(p=zParam; *p; p++){
          847  +      if( *p==',' ) nIndex++;
          848  +    }
          849  +  }
          850  +
          851  +  aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex);
          852  +  *apIndex = *apFree = aIndex;
          853  +  *pnIndex = nIndex;
          854  +  if( !aIndex ){
          855  +    return SQLITE_NOMEM;
          856  +  }
          857  +
          858  +  memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex);
          859  +  if( zParam ){
          860  +    const char *p = zParam;
          861  +    int i;
          862  +    for(i=1; i<nIndex; i++){
          863  +      int nPrefix;
          864  +      if( fts3GobbleInt(&p, &nPrefix) ) return SQLITE_ERROR;
          865  +      aIndex[i].nPrefix = nPrefix;
          866  +      p++;
          867  +    }
          868  +  }
          869  +
          870  +  return SQLITE_OK;
          871  +}
   834    872   
   835    873   /*
   836    874   ** This function is the implementation of both the xConnect and xCreate
   837    875   ** methods of the FTS3 virtual table.
   838    876   **
   839    877   ** The argv[] array contains the following:
   840    878   **
................................................................................
   860    898     int iCol;                       /* Column index */
   861    899     int nString = 0;                /* Bytes required to hold all column names */
   862    900     int nCol = 0;                   /* Number of columns in the FTS table */
   863    901     char *zCsr;                     /* Space for holding column names */
   864    902     int nDb;                        /* Bytes required to hold database name */
   865    903     int nName;                      /* Bytes required to hold table name */
   866    904     int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
   867         -  int bNoDocsize = 0;             /* True to omit %_docsize table */
   868    905     const char **aCol;              /* Array of column names */
   869    906     sqlite3_tokenizer *pTokenizer = 0;        /* Tokenizer for this table */
   870    907   
   871         -  char *zCompress = 0;
   872         -  char *zUncompress = 0;
          908  +  int nIndex;                     /* Size of aIndex[] array */
          909  +  struct Fts3Index *aIndex;       /* Array of indexes for this table */
          910  +  struct Fts3Index *aFree = 0;    /* Free this before returning */
          911  +
          912  +  /* The results of parsing supported FTS4 key=value options: */
          913  +  int bNoDocsize = 0;             /* True to omit %_docsize table */
          914  +  int bDescIdx = 0;               /* True to store descending indexes */
          915  +  char *zPrefix = 0;              /* Prefix parameter value (or NULL) */
          916  +  char *zCompress = 0;            /* compress=? parameter (or NULL) */
          917  +  char *zUncompress = 0;          /* uncompress=? parameter (or NULL) */
   873    918   
   874    919     assert( strlen(argv[0])==4 );
   875    920     assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
   876    921          || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
   877    922     );
   878    923   
   879    924     nDb = (int)strlen(argv[1]) + 1;
................................................................................
   906    951        && 0==sqlite3Fts3IsIdChar(z[8])
   907    952       ){
   908    953         rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr);
   909    954       }
   910    955   
   911    956       /* Check if it is an FTS4 special argument. */
   912    957       else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){
          958  +      struct Fts4Option {
          959  +        const char *zOpt;
          960  +        int nOpt;
          961  +        char **pzVar;
          962  +      } aFts4Opt[] = {
          963  +        { "matchinfo",   9, 0 },            /* 0 -> MATCHINFO */
          964  +        { "prefix",      6, 0 },            /* 1 -> PREFIX */
          965  +        { "compress",    8, 0 },            /* 2 -> COMPRESS */
          966  +        { "uncompress", 10, 0 },            /* 3 -> UNCOMPRESS */
          967  +        { "order",       5, 0 }             /* 4 -> ORDER */
          968  +      };
          969  +
          970  +      int iOpt;
   913    971         if( !zVal ){
   914    972           rc = SQLITE_NOMEM;
   915         -        goto fts3_init_out;
   916         -      }
   917         -      if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){
   918         -        if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){
   919         -          bNoDocsize = 1;
   920         -        }else{
   921         -          *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal);
          973  +      }else{
          974  +        for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){
          975  +          struct Fts4Option *pOp = &aFts4Opt[iOpt];
          976  +          if( nKey==pOp->nOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){
          977  +            break;
          978  +          }
          979  +        }
          980  +        if( iOpt==SizeofArray(aFts4Opt) ){
          981  +          *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
   922    982             rc = SQLITE_ERROR;
   923         -        }
   924         -      }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){
   925         -        zCompress = zVal;
   926         -        zVal = 0;
   927         -      }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){
   928         -        zUncompress = zVal;
   929         -        zVal = 0;
   930         -      }else{
   931         -        *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
   932         -        rc = SQLITE_ERROR;
   933         -      }
   934         -      sqlite3_free(zVal);
          983  +        }else{
          984  +          switch( iOpt ){
          985  +            case 0:               /* MATCHINFO */
          986  +              if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
          987  +                *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal);
          988  +                rc = SQLITE_ERROR;
          989  +              }
          990  +              bNoDocsize = 1;
          991  +              break;
          992  +
          993  +            case 1:               /* PREFIX */
          994  +              sqlite3_free(zPrefix);
          995  +              zPrefix = zVal;
          996  +              zVal = 0;
          997  +              break;
          998  +
          999  +            case 2:               /* COMPRESS */
         1000  +              sqlite3_free(zCompress);
         1001  +              zCompress = zVal;
         1002  +              zVal = 0;
         1003  +              break;
         1004  +
         1005  +            case 3:               /* UNCOMPRESS */
         1006  +              sqlite3_free(zUncompress);
         1007  +              zUncompress = zVal;
         1008  +              zVal = 0;
         1009  +              break;
         1010  +
         1011  +            case 4:               /* ORDER */
         1012  +              if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) 
         1013  +               && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3)) 
         1014  +              ){
         1015  +                *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal);
         1016  +                rc = SQLITE_ERROR;
         1017  +              }
         1018  +              bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
         1019  +              break;
         1020  +          }
         1021  +        }
         1022  +        sqlite3_free(zVal);
         1023  +      }
   935   1024       }
   936   1025   
   937   1026       /* Otherwise, the argument is a column name. */
   938   1027       else {
   939   1028         nString += (int)(strlen(z) + 1);
   940   1029         aCol[nCol++] = z;
   941   1030       }
................................................................................
   951   1040   
   952   1041     if( pTokenizer==0 ){
   953   1042       rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
   954   1043       if( rc!=SQLITE_OK ) goto fts3_init_out;
   955   1044     }
   956   1045     assert( pTokenizer );
   957   1046   
         1047  +  rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree);
         1048  +  if( rc==SQLITE_ERROR ){
         1049  +    assert( zPrefix );
         1050  +    *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix);
         1051  +  }
         1052  +  if( rc!=SQLITE_OK ) goto fts3_init_out;
   958   1053   
   959   1054     /* Allocate and populate the Fts3Table structure. */
   960         -  nByte = sizeof(Fts3Table) +              /* Fts3Table */
         1055  +  nByte = sizeof(Fts3Table) +                  /* Fts3Table */
   961   1056             nCol * sizeof(char *) +              /* azColumn */
         1057  +          nIndex * sizeof(struct Fts3Index) +  /* aIndex */
   962   1058             nName +                              /* zName */
   963   1059             nDb +                                /* zDb */
   964   1060             nString;                             /* Space for azColumn strings */
   965   1061     p = (Fts3Table*)sqlite3_malloc(nByte);
   966   1062     if( p==0 ){
   967   1063       rc = SQLITE_NOMEM;
   968   1064       goto fts3_init_out;
................................................................................
   969   1065     }
   970   1066     memset(p, 0, nByte);
   971   1067     p->db = db;
   972   1068     p->nColumn = nCol;
   973   1069     p->nPendingData = 0;
   974   1070     p->azColumn = (char **)&p[1];
   975   1071     p->pTokenizer = pTokenizer;
   976         -  p->nNodeSize = 1000;
   977   1072     p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
   978   1073     p->bHasDocsize = (isFts4 && bNoDocsize==0);
   979   1074     p->bHasStat = isFts4;
         1075  +  p->bDescIdx = bDescIdx;
   980   1076     TESTONLY( p->inTransaction = -1 );
   981   1077     TESTONLY( p->mxSavepoint = -1 );
   982         -  fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1);
         1078  +
         1079  +  p->aIndex = (struct Fts3Index *)&p->azColumn[nCol];
         1080  +  memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex);
         1081  +  p->nIndex = nIndex;
         1082  +  for(i=0; i<nIndex; i++){
         1083  +    fts3HashInit(&p->aIndex[i].hPending, FTS3_HASH_STRING, 1);
         1084  +  }
   983   1085   
   984   1086     /* Fill in the zName and zDb fields of the vtab structure. */
   985         -  zCsr = (char *)&p->azColumn[nCol];
         1087  +  zCsr = (char *)&p->aIndex[nIndex];
   986   1088     p->zName = zCsr;
   987   1089     memcpy(zCsr, argv[2], nName);
   988   1090     zCsr += nName;
   989   1091     p->zDb = zCsr;
   990   1092     memcpy(zCsr, argv[1], nDb);
   991   1093     zCsr += nDb;
   992   1094   
................................................................................
  1016   1118     ** database. TODO: For xConnect(), it could verify that said tables exist.
  1017   1119     */
  1018   1120     if( isCreate ){
  1019   1121       rc = fts3CreateTables(p);
  1020   1122     }
  1021   1123   
  1022   1124     /* Figure out the page-size for the database. This is required in order to
  1023         -  ** estimate the cost of loading large doclists from the database (see 
  1024         -  ** function sqlite3Fts3SegReaderCost() for details).
  1025         -  */
         1125  +  ** estimate the cost of loading large doclists from the database.  */
  1026   1126     fts3DatabasePageSize(&rc, p);
         1127  +  p->nNodeSize = p->nPgsz-35;
  1027   1128   
  1028   1129     /* Declare the table schema to SQLite. */
  1029   1130     fts3DeclareVtab(&rc, p);
  1030   1131   
  1031   1132   fts3_init_out:
         1133  +  sqlite3_free(zPrefix);
         1134  +  sqlite3_free(aFree);
  1032   1135     sqlite3_free(zCompress);
  1033   1136     sqlite3_free(zUncompress);
  1034   1137     sqlite3_free((void *)aCol);
  1035   1138     if( rc!=SQLITE_OK ){
  1036   1139       if( p ){
  1037   1140         fts3DisconnectMethod((sqlite3_vtab *)p);
  1038   1141       }else if( pTokenizer ){
  1039   1142         pTokenizer->pModule->xDestroy(pTokenizer);
  1040   1143       }
  1041   1144     }else{
         1145  +    assert( p->pSegments==0 );
  1042   1146       *ppVTab = &p->base;
  1043   1147     }
  1044   1148     return rc;
  1045   1149   }
  1046   1150   
  1047   1151   /*
  1048   1152   ** The xConnect() and xCreate() methods for the virtual table. All the
................................................................................
  1132   1236       struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0];
  1133   1237       if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){
  1134   1238         if( pOrder->desc ){
  1135   1239           pInfo->idxStr = "DESC";
  1136   1240         }else{
  1137   1241           pInfo->idxStr = "ASC";
  1138   1242         }
         1243  +      pInfo->orderByConsumed = 1;
  1139   1244       }
  1140         -    pInfo->orderByConsumed = 1;
  1141   1245     }
  1142   1246   
         1247  +  assert( p->pSegments==0 );
  1143   1248     return SQLITE_OK;
  1144   1249   }
  1145   1250   
  1146   1251   /*
  1147   1252   ** Implementation of xOpen method.
  1148   1253   */
  1149   1254   static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
................................................................................
  1171   1276     Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  1172   1277     assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  1173   1278     sqlite3_finalize(pCsr->pStmt);
  1174   1279     sqlite3Fts3ExprFree(pCsr->pExpr);
  1175   1280     sqlite3Fts3FreeDeferredTokens(pCsr);
  1176   1281     sqlite3_free(pCsr->aDoclist);
  1177   1282     sqlite3_free(pCsr->aMatchinfo);
         1283  +  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  1178   1284     sqlite3_free(pCsr);
  1179   1285     return SQLITE_OK;
  1180   1286   }
  1181   1287   
  1182   1288   /*
  1183   1289   ** Position the pCsr->pStmt statement so that it is on the row
  1184   1290   ** of the %_content table that contains the last match.  Return
  1185   1291   ** SQLITE_OK on success.  
  1186   1292   */
  1187   1293   static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
  1188   1294     if( pCsr->isRequireSeek ){
  1189         -    pCsr->isRequireSeek = 0;
  1190   1295       sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
         1296  +    pCsr->isRequireSeek = 0;
  1191   1297       if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
  1192   1298         return SQLITE_OK;
  1193   1299       }else{
  1194   1300         int rc = sqlite3_reset(pCsr->pStmt);
  1195   1301         if( rc==SQLITE_OK ){
  1196   1302           /* If no row was found and no error has occured, then the %_content
  1197   1303           ** table is missing a row that is present in the full-text index.
................................................................................
  1364   1470     assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) );
  1365   1471   
  1366   1472     if( rc==SQLITE_OK && iHeight>1 ){
  1367   1473       char *zBlob = 0;              /* Blob read from %_segments table */
  1368   1474       int nBlob;                    /* Size of zBlob in bytes */
  1369   1475   
  1370   1476       if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){
  1371         -      rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob);
         1477  +      rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0);
  1372   1478         if( rc==SQLITE_OK ){
  1373   1479           rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0);
  1374   1480         }
  1375   1481         sqlite3_free(zBlob);
  1376   1482         piLeaf = 0;
  1377   1483         zBlob = 0;
  1378   1484       }
  1379   1485   
  1380   1486       if( rc==SQLITE_OK ){
  1381         -      rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob);
         1487  +      rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0);
  1382   1488       }
  1383   1489       if( rc==SQLITE_OK ){
  1384   1490         rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
  1385   1491       }
  1386   1492       sqlite3_free(zBlob);
  1387   1493     }
  1388   1494   
................................................................................
  1750   1856     }
  1751   1857     *p++ = 0x00;
  1752   1858     *pp = p;
  1753   1859     return 1;
  1754   1860   }
  1755   1861   
  1756   1862   /*
  1757         -** Merge two position-lists as required by the NEAR operator.
         1863  +** Merge two position-lists as required by the NEAR operator. The argument
         1864  +** position lists correspond to the left and right phrases of an expression 
         1865  +** like:
         1866  +**
         1867  +**     "phrase 1" NEAR "phrase number 2"
         1868  +**
         1869  +** Position list *pp1 corresponds to the left-hand side of the NEAR 
         1870  +** expression and *pp2 to the right. As usual, the indexes in the position 
         1871  +** lists are the offsets of the last token in each phrase (tokens "1" and "2" 
         1872  +** in the example above).
         1873  +**
         1874  +** The output position list - written to *pp - is a copy of *pp2 with those
         1875  +** entries that are not sufficiently NEAR entries in *pp1 removed.
  1758   1876   */
  1759   1877   static int fts3PoslistNearMerge(
  1760   1878     char **pp,                      /* Output buffer */
  1761   1879     char *aTmp,                     /* Temporary buffer space */
  1762   1880     int nRight,                     /* Maximum difference in token positions */
  1763   1881     int nLeft,                      /* Maximum difference in token positions */
  1764   1882     char **pp1,                     /* IN/OUT: Left input list */
  1765   1883     char **pp2                      /* IN/OUT: Right input list */
  1766   1884   ){
  1767   1885     char *p1 = *pp1;
  1768   1886     char *p2 = *pp2;
  1769   1887   
  1770         -  if( !pp ){
  1771         -    if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1;
  1772         -    *pp1 = p1;
  1773         -    *pp2 = p2;
  1774         -    return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1);
  1775         -  }else{
  1776         -    char *pTmp1 = aTmp;
  1777         -    char *pTmp2;
  1778         -    char *aTmp2;
  1779         -    int res = 1;
  1780         -
  1781         -    fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2);
  1782         -    aTmp2 = pTmp2 = pTmp1;
  1783         -    *pp1 = p1;
  1784         -    *pp2 = p2;
  1785         -    fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1);
  1786         -    if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
  1787         -      fts3PoslistMerge(pp, &aTmp, &aTmp2);
  1788         -    }else if( pTmp1!=aTmp ){
  1789         -      fts3PoslistCopy(pp, &aTmp);
  1790         -    }else if( pTmp2!=aTmp2 ){
  1791         -      fts3PoslistCopy(pp, &aTmp2);
  1792         -    }else{
  1793         -      res = 0;
  1794         -    }
  1795         -
  1796         -    return res;
  1797         -  }
  1798         -}
  1799         -
  1800         -/*
  1801         -** Values that may be used as the first parameter to fts3DoclistMerge().
  1802         -*/
  1803         -#define MERGE_NOT        2        /* D + D -> D */
  1804         -#define MERGE_AND        3        /* D + D -> D */
  1805         -#define MERGE_OR         4        /* D + D -> D */
  1806         -#define MERGE_POS_OR     5        /* P + P -> P */
  1807         -#define MERGE_PHRASE     6        /* P + P -> D */
  1808         -#define MERGE_POS_PHRASE 7        /* P + P -> P */
  1809         -#define MERGE_NEAR       8        /* P + P -> D */
  1810         -#define MERGE_POS_NEAR   9        /* P + P -> P */
  1811         -
  1812         -/*
  1813         -** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2
  1814         -** (size n2 bytes). The output is written to pre-allocated buffer aBuffer,
  1815         -** which is guaranteed to be large enough to hold the results. The number
  1816         -** of bytes written to aBuffer is stored in *pnBuffer before returning.
  1817         -**
  1818         -** If successful, SQLITE_OK is returned. Otherwise, if a malloc error
  1819         -** occurs while allocating a temporary buffer as part of the merge operation,
  1820         -** SQLITE_NOMEM is returned.
  1821         -*/
  1822         -static int fts3DoclistMerge(
  1823         -  int mergetype,                  /* One of the MERGE_XXX constants */
  1824         -  int nParam1,                    /* Used by MERGE_NEAR and MERGE_POS_NEAR */
  1825         -  int nParam2,                    /* Used by MERGE_NEAR and MERGE_POS_NEAR */
  1826         -  char *aBuffer,                  /* Pre-allocated output buffer */
  1827         -  int *pnBuffer,                  /* OUT: Bytes written to aBuffer */
  1828         -  char *a1,                       /* Buffer containing first doclist */
  1829         -  int n1,                         /* Size of buffer a1 */
  1830         -  char *a2,                       /* Buffer containing second doclist */
  1831         -  int n2,                         /* Size of buffer a2 */
  1832         -  int *pnDoc                      /* OUT: Number of docids in output */
  1833         -){
  1834         -  sqlite3_int64 i1 = 0;
  1835         -  sqlite3_int64 i2 = 0;
  1836         -  sqlite3_int64 iPrev = 0;
  1837         -
  1838         -  char *p = aBuffer;
  1839         -  char *p1 = a1;
  1840         -  char *p2 = a2;
  1841         -  char *pEnd1 = &a1[n1];
  1842         -  char *pEnd2 = &a2[n2];
  1843         -  int nDoc = 0;
  1844         -
  1845         -  assert( mergetype==MERGE_OR     || mergetype==MERGE_POS_OR 
  1846         -       || mergetype==MERGE_AND    || mergetype==MERGE_NOT
  1847         -       || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
  1848         -       || mergetype==MERGE_NEAR   || mergetype==MERGE_POS_NEAR
  1849         -  );
  1850         -
  1851         -  if( !aBuffer ){
  1852         -    *pnBuffer = 0;
  1853         -    return SQLITE_NOMEM;
  1854         -  }
  1855         -
  1856         -  /* Read the first docid from each doclist */
  1857         -  fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1858         -  fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1859         -
  1860         -  switch( mergetype ){
  1861         -    case MERGE_OR:
  1862         -    case MERGE_POS_OR:
  1863         -      while( p1 || p2 ){
  1864         -        if( p2 && p1 && i1==i2 ){
  1865         -          fts3PutDeltaVarint(&p, &iPrev, i1);
  1866         -          if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2);
  1867         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1868         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1869         -        }else if( !p2 || (p1 && i1<i2) ){
  1870         -          fts3PutDeltaVarint(&p, &iPrev, i1);
  1871         -          if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p1);
  1872         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1873         -        }else{
  1874         -          fts3PutDeltaVarint(&p, &iPrev, i2);
  1875         -          if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p2);
  1876         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1877         -        }
  1878         -      }
  1879         -      break;
  1880         -
  1881         -    case MERGE_AND:
  1882         -      while( p1 && p2 ){
  1883         -        if( i1==i2 ){
  1884         -          fts3PutDeltaVarint(&p, &iPrev, i1);
  1885         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1886         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1887         -          nDoc++;
  1888         -        }else if( i1<i2 ){
  1889         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1890         -        }else{
  1891         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1892         -        }
  1893         -      }
  1894         -      break;
  1895         -
  1896         -    case MERGE_NOT:
  1897         -      while( p1 ){
  1898         -        if( p2 && i1==i2 ){
  1899         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1900         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1901         -        }else if( !p2 || i1<i2 ){
  1902         -          fts3PutDeltaVarint(&p, &iPrev, i1);
  1903         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1904         -        }else{
  1905         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1906         -        }
  1907         -      }
  1908         -      break;
  1909         -
  1910         -    case MERGE_POS_PHRASE:
  1911         -    case MERGE_PHRASE: {
  1912         -      char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p);
  1913         -      while( p1 && p2 ){
  1914         -        if( i1==i2 ){
  1915         -          char *pSave = p;
  1916         -          sqlite3_int64 iPrevSave = iPrev;
  1917         -          fts3PutDeltaVarint(&p, &iPrev, i1);
  1918         -          if( 0==fts3PoslistPhraseMerge(ppPos, nParam1, 0, 1, &p1, &p2) ){
  1919         -            p = pSave;
  1920         -            iPrev = iPrevSave;
  1921         -          }else{
  1922         -            nDoc++;
  1923         -          }
  1924         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1925         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1926         -        }else if( i1<i2 ){
  1927         -          fts3PoslistCopy(0, &p1);
  1928         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1929         -        }else{
  1930         -          fts3PoslistCopy(0, &p2);
  1931         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1932         -        }
  1933         -      }
  1934         -      break;
  1935         -    }
  1936         -
  1937         -    default: assert( mergetype==MERGE_POS_NEAR || mergetype==MERGE_NEAR ); {
  1938         -      char *aTmp = 0;
  1939         -      char **ppPos = 0;
  1940         -
  1941         -      if( mergetype==MERGE_POS_NEAR ){
  1942         -        ppPos = &p;
  1943         -        aTmp = sqlite3_malloc(2*(n1+n2+1));
  1944         -        if( !aTmp ){
  1945         -          return SQLITE_NOMEM;
  1946         -        }
  1947         -      }
  1948         -
  1949         -      while( p1 && p2 ){
  1950         -        if( i1==i2 ){
  1951         -          char *pSave = p;
  1952         -          sqlite3_int64 iPrevSave = iPrev;
  1953         -          fts3PutDeltaVarint(&p, &iPrev, i1);
  1954         -
  1955         -          if( !fts3PoslistNearMerge(ppPos, aTmp, nParam1, nParam2, &p1, &p2) ){
  1956         -            iPrev = iPrevSave;
  1957         -            p = pSave;
  1958         -          }
  1959         -
  1960         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1961         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1962         -        }else if( i1<i2 ){
  1963         -          fts3PoslistCopy(0, &p1);
  1964         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1965         -        }else{
  1966         -          fts3PoslistCopy(0, &p2);
  1967         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1968         -        }
  1969         -      }
  1970         -      sqlite3_free(aTmp);
  1971         -      break;
  1972         -    }
  1973         -  }
  1974         -
  1975         -  if( pnDoc ) *pnDoc = nDoc;
  1976         -  *pnBuffer = (int)(p-aBuffer);
  1977         -  return SQLITE_OK;
         1888  +  char *pTmp1 = aTmp;
         1889  +  char *pTmp2;
         1890  +  char *aTmp2;
         1891  +  int res = 1;
         1892  +
         1893  +  fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2);
         1894  +  aTmp2 = pTmp2 = pTmp1;
         1895  +  *pp1 = p1;
         1896  +  *pp2 = p2;
         1897  +  fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1);
         1898  +  if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
         1899  +    fts3PoslistMerge(pp, &aTmp, &aTmp2);
         1900  +  }else if( pTmp1!=aTmp ){
         1901  +    fts3PoslistCopy(pp, &aTmp);
         1902  +  }else if( pTmp2!=aTmp2 ){
         1903  +    fts3PoslistCopy(pp, &aTmp2);
         1904  +  }else{
         1905  +    res = 0;
         1906  +  }
         1907  +
         1908  +  return res;
  1978   1909   }
  1979   1910   
  1980   1911   /* 
  1981   1912   ** A pointer to an instance of this structure is used as the context 
  1982   1913   ** argument to sqlite3Fts3SegReaderIterate()
  1983   1914   */
  1984   1915   typedef struct TermSelect TermSelect;
  1985   1916   struct TermSelect {
  1986   1917     int isReqPos;
  1987   1918     char *aaOutput[16];             /* Malloc'd output buffer */
  1988   1919     int anOutput[16];               /* Size of output in bytes */
  1989   1920   };
         1921  +
         1922  +
         1923  +static void fts3GetDeltaVarint3(
         1924  +  char **pp, 
         1925  +  char *pEnd, 
         1926  +  int bDescIdx,
         1927  +  sqlite3_int64 *pVal
         1928  +){
         1929  +  if( *pp>=pEnd ){
         1930  +    *pp = 0;
         1931  +  }else{
         1932  +    sqlite3_int64 iVal;
         1933  +    *pp += sqlite3Fts3GetVarint(*pp, &iVal);
         1934  +    if( bDescIdx ){
         1935  +      *pVal -= iVal;
         1936  +    }else{
         1937  +      *pVal += iVal;
         1938  +    }
         1939  +  }
         1940  +}
         1941  +
         1942  +static void fts3PutDeltaVarint3(
         1943  +  char **pp,                      /* IN/OUT: Output pointer */
         1944  +  int bDescIdx,                   /* True for descending docids */
         1945  +  sqlite3_int64 *piPrev,          /* IN/OUT: Previous value written to list */
         1946  +  int *pbFirst,                   /* IN/OUT: True after first int written */
         1947  +  sqlite3_int64 iVal              /* Write this value to the list */
         1948  +){
         1949  +  sqlite3_int64 iWrite;
         1950  +  if( bDescIdx==0 || *pbFirst==0 ){
         1951  +    iWrite = iVal - *piPrev;
         1952  +  }else{
         1953  +    iWrite = *piPrev - iVal;
         1954  +  }
         1955  +  assert( *pbFirst || *piPrev==0 );
         1956  +  assert( *pbFirst==0 || iWrite>0 );
         1957  +  *pp += sqlite3Fts3PutVarint(*pp, iWrite);
         1958  +  *piPrev = iVal;
         1959  +  *pbFirst = 1;
         1960  +}
         1961  +
         1962  +#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2))
         1963  +
         1964  +static int fts3DoclistOrMerge(
         1965  +  int bDescIdx,                   /* True if arguments are desc */
         1966  +  char *a1, int n1,               /* First doclist */
         1967  +  char *a2, int n2,               /* Second doclist */
         1968  +  char **paOut, int *pnOut        /* OUT: Malloc'd doclist */
         1969  +){
         1970  +  sqlite3_int64 i1 = 0;
         1971  +  sqlite3_int64 i2 = 0;
         1972  +  sqlite3_int64 iPrev = 0;
         1973  +  char *pEnd1 = &a1[n1];
         1974  +  char *pEnd2 = &a2[n2];
         1975  +  char *p1 = a1;
         1976  +  char *p2 = a2;
         1977  +  char *p;
         1978  +  char *aOut;
         1979  +  int bFirstOut = 0;
         1980  +
         1981  +  *paOut = 0;
         1982  +  *pnOut = 0;
         1983  +  aOut = sqlite3_malloc(n1+n2);
         1984  +  if( !aOut ) return SQLITE_NOMEM;
         1985  +
         1986  +  p = aOut;
         1987  +  fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
         1988  +  fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
         1989  +  while( p1 || p2 ){
         1990  +    sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
         1991  +
         1992  +    if( p2 && p1 && iDiff==0 ){
         1993  +      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
         1994  +      fts3PoslistMerge(&p, &p1, &p2);
         1995  +      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
         1996  +      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
         1997  +    }else if( !p2 || (p1 && iDiff<0) ){
         1998  +      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
         1999  +      fts3PoslistCopy(&p, &p1);
         2000  +      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
         2001  +    }else{
         2002  +      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2);
         2003  +      fts3PoslistCopy(&p, &p2);
         2004  +      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
         2005  +    }
         2006  +  }
         2007  +
         2008  +  *paOut = aOut;
         2009  +  *pnOut = (p-aOut);
         2010  +  return SQLITE_OK;
         2011  +}
         2012  +
         2013  +static void fts3DoclistPhraseMerge(
         2014  +  int bDescIdx,                   /* True if arguments are desc */
         2015  +  int nDist,                      /* Distance from left to right (1=adjacent) */
         2016  +  char *aLeft, int nLeft,         /* Left doclist */
         2017  +  char *aRight, int *pnRight      /* IN/OUT: Right/output doclist */
         2018  +){
         2019  +  sqlite3_int64 i1 = 0;
         2020  +  sqlite3_int64 i2 = 0;
         2021  +  sqlite3_int64 iPrev = 0;
         2022  +  char *pEnd1 = &aLeft[nLeft];
         2023  +  char *pEnd2 = &aRight[*pnRight];
         2024  +  char *p1 = aLeft;
         2025  +  char *p2 = aRight;
         2026  +  char *p;
         2027  +  int bFirstOut = 0;
         2028  +  char *aOut = aRight;
         2029  +
         2030  +  assert( nDist>0 );
         2031  +
         2032  +  p = aOut;
         2033  +  fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
         2034  +  fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
         2035  +
         2036  +  while( p1 && p2 ){
         2037  +    sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
         2038  +    if( iDiff==0 ){
         2039  +      char *pSave = p;
         2040  +      sqlite3_int64 iPrevSave = iPrev;
         2041  +      int bFirstOutSave = bFirstOut;
         2042  +
         2043  +      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
         2044  +      if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){
         2045  +        p = pSave;
         2046  +        iPrev = iPrevSave;
         2047  +        bFirstOut = bFirstOutSave;
         2048  +      }
         2049  +      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
         2050  +      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
         2051  +    }else if( iDiff<0 ){
         2052  +      fts3PoslistCopy(0, &p1);
         2053  +      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
         2054  +    }else{
         2055  +      fts3PoslistCopy(0, &p2);
         2056  +      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
         2057  +    }
         2058  +  }
         2059  +
         2060  +  *pnRight = p - aOut;
         2061  +}
         2062  +
  1990   2063   
  1991   2064   /*
  1992   2065   ** Merge all doclists in the TermSelect.aaOutput[] array into a single
  1993   2066   ** doclist stored in TermSelect.aaOutput[0]. If successful, delete all
  1994   2067   ** other doclists (except the aaOutput[0] one) and return SQLITE_OK.
  1995   2068   **
  1996   2069   ** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
  1997   2070   ** the responsibility of the caller to free any doclists left in the
  1998   2071   ** TermSelect.aaOutput[] array.
  1999   2072   */
  2000         -static int fts3TermSelectMerge(TermSelect *pTS){
  2001         -  int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
         2073  +static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){
  2002   2074     char *aOut = 0;
  2003   2075     int nOut = 0;
  2004   2076     int i;
  2005   2077   
  2006   2078     /* Loop through the doclists in the aaOutput[] array. Merge them all
  2007   2079     ** into a single doclist.
  2008   2080     */
................................................................................
  2009   2081     for(i=0; i<SizeofArray(pTS->aaOutput); i++){
  2010   2082       if( pTS->aaOutput[i] ){
  2011   2083         if( !aOut ){
  2012   2084           aOut = pTS->aaOutput[i];
  2013   2085           nOut = pTS->anOutput[i];
  2014   2086           pTS->aaOutput[i] = 0;
  2015   2087         }else{
  2016         -        int nNew = nOut + pTS->anOutput[i];
  2017         -        char *aNew = sqlite3_malloc(nNew);
  2018         -        if( !aNew ){
         2088  +        int nNew;
         2089  +        char *aNew;
         2090  +
         2091  +        int rc = fts3DoclistOrMerge(p->bDescIdx, 
         2092  +            pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew
         2093  +        );
         2094  +        if( rc!=SQLITE_OK ){
  2019   2095             sqlite3_free(aOut);
  2020         -          return SQLITE_NOMEM;
         2096  +          return rc;
  2021   2097           }
  2022         -        fts3DoclistMerge(mergetype, 0, 0,
  2023         -            aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0
  2024         -        );
         2098  +
  2025   2099           sqlite3_free(pTS->aaOutput[i]);
  2026   2100           sqlite3_free(aOut);
  2027   2101           pTS->aaOutput[i] = 0;
  2028   2102           aOut = aNew;
  2029   2103           nOut = nNew;
  2030   2104         }
  2031   2105       }
................................................................................
  2053   2127   
  2054   2128     UNUSED_PARAMETER(p);
  2055   2129     UNUSED_PARAMETER(zTerm);
  2056   2130     UNUSED_PARAMETER(nTerm);
  2057   2131   
  2058   2132     if( pTS->aaOutput[0]==0 ){
  2059   2133       /* If this is the first term selected, copy the doclist to the output
  2060         -    ** buffer using memcpy(). TODO: Add a way to transfer control of the
  2061         -    ** aDoclist buffer from the caller so as to avoid the memcpy().
  2062         -    */
         2134  +    ** buffer using memcpy(). */
  2063   2135       pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
  2064   2136       pTS->anOutput[0] = nDoclist;
  2065   2137       if( pTS->aaOutput[0] ){
  2066   2138         memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
  2067   2139       }else{
  2068   2140         return SQLITE_NOMEM;
  2069   2141       }
  2070   2142     }else{
  2071         -    int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
  2072   2143       char *aMerge = aDoclist;
  2073   2144       int nMerge = nDoclist;
  2074   2145       int iOut;
  2075   2146   
  2076   2147       for(iOut=0; iOut<SizeofArray(pTS->aaOutput); iOut++){
  2077         -      char *aNew;
  2078         -      int nNew;
  2079   2148         if( pTS->aaOutput[iOut]==0 ){
  2080   2149           assert( iOut>0 );
  2081   2150           pTS->aaOutput[iOut] = aMerge;
  2082   2151           pTS->anOutput[iOut] = nMerge;
  2083   2152           break;
  2084         -      }
  2085         -
  2086         -      nNew = nMerge + pTS->anOutput[iOut];
  2087         -      aNew = sqlite3_malloc(nNew);
  2088         -      if( !aNew ){
  2089         -        if( aMerge!=aDoclist ){
  2090         -          sqlite3_free(aMerge);
  2091         -        }
  2092         -        return SQLITE_NOMEM;
  2093         -      }
  2094         -      fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, 
  2095         -          pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0
  2096         -      );
  2097         -
  2098         -      if( iOut>0 ) sqlite3_free(aMerge);
  2099         -      sqlite3_free(pTS->aaOutput[iOut]);
  2100         -      pTS->aaOutput[iOut] = 0;
  2101         -
  2102         -      aMerge = aNew;
  2103         -      nMerge = nNew;
  2104         -      if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
  2105         -        pTS->aaOutput[iOut] = aMerge;
  2106         -        pTS->anOutput[iOut] = nMerge;
  2107         -      }
  2108         -    }
  2109         -  }
         2153  +      }else{
         2154  +        char *aNew;
         2155  +        int nNew;
         2156  +
         2157  +        int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, 
         2158  +            pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew
         2159  +        );
         2160  +        if( rc!=SQLITE_OK ){
         2161  +          if( aMerge!=aDoclist ) sqlite3_free(aMerge);
         2162  +          return rc;
         2163  +        }
         2164  +
         2165  +        if( aMerge!=aDoclist ) sqlite3_free(aMerge);
         2166  +        sqlite3_free(pTS->aaOutput[iOut]);
         2167  +        pTS->aaOutput[iOut] = 0;
         2168  +  
         2169  +        aMerge = aNew;
         2170  +        nMerge = nNew;
         2171  +        if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
         2172  +          pTS->aaOutput[iOut] = aMerge;
         2173  +          pTS->anOutput[iOut] = nMerge;
         2174  +        }
         2175  +      }
         2176  +    }
         2177  +  }
         2178  +  return SQLITE_OK;
         2179  +}
         2180  +
         2181  +/*
         2182  +** Append SegReader object pNew to the end of the pCsr->apSegment[] array.
         2183  +*/
         2184  +static int fts3SegReaderCursorAppend(
         2185  +  Fts3MultiSegReader *pCsr, 
         2186  +  Fts3SegReader *pNew
         2187  +){
         2188  +  if( (pCsr->nSegment%16)==0 ){
         2189  +    Fts3SegReader **apNew;
         2190  +    int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
         2191  +    apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte);
         2192  +    if( !apNew ){
         2193  +      sqlite3Fts3SegReaderFree(pNew);
         2194  +      return SQLITE_NOMEM;
         2195  +    }
         2196  +    pCsr->apSegment = apNew;
         2197  +  }
         2198  +  pCsr->apSegment[pCsr->nSegment++] = pNew;
  2110   2199     return SQLITE_OK;
  2111   2200   }
  2112   2201   
  2113         -static int fts3DeferredTermSelect(
  2114         -  Fts3DeferredToken *pToken,      /* Phrase token */
  2115         -  int isTermPos,                  /* True to include positions */
  2116         -  int *pnOut,                     /* OUT: Size of list */
  2117         -  char **ppOut                    /* OUT: Body of list */
  2118         -){
  2119         -  char *aSource;
  2120         -  int nSource;
  2121         -
  2122         -  aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource);
  2123         -  if( !aSource ){
  2124         -    *pnOut = 0;
  2125         -    *ppOut = 0;
  2126         -  }else if( isTermPos ){
  2127         -    *ppOut = sqlite3_malloc(nSource);
  2128         -    if( !*ppOut ) return SQLITE_NOMEM;
  2129         -    memcpy(*ppOut, aSource, nSource);
  2130         -    *pnOut = nSource;
  2131         -  }else{
  2132         -    sqlite3_int64 docid;
  2133         -    *pnOut = sqlite3Fts3GetVarint(aSource, &docid);
  2134         -    *ppOut = sqlite3_malloc(*pnOut);
  2135         -    if( !*ppOut ) return SQLITE_NOMEM;
  2136         -    sqlite3Fts3PutVarint(*ppOut, docid);
  2137         -  }
  2138         -
  2139         -  return SQLITE_OK;
  2140         -}
  2141         -
  2142         -int sqlite3Fts3SegReaderCursor(
         2202  +static int fts3SegReaderCursor(
  2143   2203     Fts3Table *p,                   /* FTS3 table handle */
         2204  +  int iIndex,                     /* Index to search (from 0 to p->nIndex-1) */
  2144   2205     int iLevel,                     /* Level of segments to scan */
  2145   2206     const char *zTerm,              /* Term to query for */
  2146   2207     int nTerm,                      /* Size of zTerm in bytes */
  2147   2208     int isPrefix,                   /* True for a prefix search */
  2148   2209     int isScan,                     /* True to scan from zTerm to EOF */
  2149         -  Fts3SegReaderCursor *pCsr       /* Cursor object to populate */
         2210  +  Fts3MultiSegReader *pCsr       /* Cursor object to populate */
  2150   2211   ){
  2151   2212     int rc = SQLITE_OK;
  2152   2213     int rc2;
  2153         -  int iAge = 0;
  2154   2214     sqlite3_stmt *pStmt = 0;
  2155         -  Fts3SegReader *pPending = 0;
  2156   2215   
  2157         -  assert( iLevel==FTS3_SEGCURSOR_ALL 
  2158         -      ||  iLevel==FTS3_SEGCURSOR_PENDING 
  2159         -      ||  iLevel>=0
  2160         -  );
  2161         -  assert( FTS3_SEGCURSOR_PENDING<0 );
  2162         -  assert( FTS3_SEGCURSOR_ALL<0 );
  2163         -  assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) );
  2164         -  assert( isPrefix==0 || isScan==0 );
  2165         -
  2166         -
  2167         -  memset(pCsr, 0, sizeof(Fts3SegReaderCursor));
  2168         -
  2169         -  /* If iLevel is less than 0, include a seg-reader for the pending-terms. */
  2170         -  assert( isScan==0 || fts3HashCount(&p->pendingTerms)==0 );
  2171         -  if( iLevel<0 && isScan==0 ){
  2172         -    rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pPending);
  2173         -    if( rc==SQLITE_OK && pPending ){
  2174         -      int nByte = (sizeof(Fts3SegReader *) * 16);
  2175         -      pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte);
  2176         -      if( pCsr->apSegment==0 ){
  2177         -        rc = SQLITE_NOMEM;
  2178         -      }else{
  2179         -        pCsr->apSegment[0] = pPending;
  2180         -        pCsr->nSegment = 1;
  2181         -        pPending = 0;
  2182         -      }
         2216  +  /* If iLevel is less than 0 and this is not a scan, include a seg-reader 
         2217  +  ** for the pending-terms. If this is a scan, then this call must be being
         2218  +  ** made by an fts4aux module, not an FTS table. In this case calling
         2219  +  ** Fts3SegReaderPending might segfault, as the data structures used by 
         2220  +  ** fts4aux are not completely populated. So it's easiest to filter these
         2221  +  ** calls out here.  */
         2222  +  if( iLevel<0 && p->aIndex ){
         2223  +    Fts3SegReader *pSeg = 0;
         2224  +    rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg);
         2225  +    if( rc==SQLITE_OK && pSeg ){
         2226  +      rc = fts3SegReaderCursorAppend(pCsr, pSeg);
  2183   2227       }
  2184   2228     }
  2185   2229   
  2186   2230     if( iLevel!=FTS3_SEGCURSOR_PENDING ){
  2187   2231       if( rc==SQLITE_OK ){
  2188         -      rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt);
         2232  +      rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt);
  2189   2233       }
         2234  +
  2190   2235       while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
         2236  +      Fts3SegReader *pSeg = 0;
  2191   2237   
  2192   2238         /* Read the values returned by the SELECT into local variables. */
  2193   2239         sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1);
  2194   2240         sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2);
  2195   2241         sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3);
  2196   2242         int nRoot = sqlite3_column_bytes(pStmt, 4);
  2197   2243         char const *zRoot = sqlite3_column_blob(pStmt, 4);
  2198   2244   
  2199         -      /* If nSegment is a multiple of 16 the array needs to be extended. */
  2200         -      if( (pCsr->nSegment%16)==0 ){
  2201         -        Fts3SegReader **apNew;
  2202         -        int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
  2203         -        apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte);
  2204         -        if( !apNew ){
  2205         -          rc = SQLITE_NOMEM;
  2206         -          goto finished;
  2207         -        }
  2208         -        pCsr->apSegment = apNew;
  2209         -      }
  2210         -
  2211   2245         /* If zTerm is not NULL, and this segment is not stored entirely on its
  2212   2246         ** root node, the range of leaves scanned can be reduced. Do this. */
  2213   2247         if( iStartBlock && zTerm ){
  2214   2248           sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0);
  2215   2249           rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi);
  2216   2250           if( rc!=SQLITE_OK ) goto finished;
  2217   2251           if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock;
  2218   2252         }
  2219   2253    
  2220         -      rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock,
  2221         -          iEndBlock, zRoot, nRoot, &pCsr->apSegment[pCsr->nSegment]
         2254  +      rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, 
         2255  +          iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg
  2222   2256         );
  2223   2257         if( rc!=SQLITE_OK ) goto finished;
  2224         -      pCsr->nSegment++;
  2225         -      iAge++;
         2258  +      rc = fts3SegReaderCursorAppend(pCsr, pSeg);
  2226   2259       }
  2227   2260     }
  2228   2261   
  2229   2262    finished:
  2230   2263     rc2 = sqlite3_reset(pStmt);
  2231   2264     if( rc==SQLITE_DONE ) rc = rc2;
  2232         -  sqlite3Fts3SegReaderFree(pPending);
  2233   2265   
  2234   2266     return rc;
  2235   2267   }
  2236   2268   
         2269  +/*
         2270  +** Set up a cursor object for iterating through a full-text index or a 
         2271  +** single level therein.
         2272  +*/
         2273  +int sqlite3Fts3SegReaderCursor(
         2274  +  Fts3Table *p,                   /* FTS3 table handle */
         2275  +  int iIndex,                     /* Index to search (from 0 to p->nIndex-1) */
         2276  +  int iLevel,                     /* Level of segments to scan */
         2277  +  const char *zTerm,              /* Term to query for */
         2278  +  int nTerm,                      /* Size of zTerm in bytes */
         2279  +  int isPrefix,                   /* True for a prefix search */
         2280  +  int isScan,                     /* True to scan from zTerm to EOF */
         2281  +  Fts3MultiSegReader *pCsr       /* Cursor object to populate */
         2282  +){
         2283  +  assert( iIndex>=0 && iIndex<p->nIndex );
         2284  +  assert( iLevel==FTS3_SEGCURSOR_ALL
         2285  +      ||  iLevel==FTS3_SEGCURSOR_PENDING 
         2286  +      ||  iLevel>=0
         2287  +  );
         2288  +  assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
         2289  +  assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 );
         2290  +  assert( isPrefix==0 || isScan==0 );
  2237   2291   
  2238         -static int fts3TermSegReaderCursor(
         2292  +  /* "isScan" is only set to true by the ft4aux module, an ordinary
         2293  +  ** full-text tables. */
         2294  +  assert( isScan==0 || p->aIndex==0 );
         2295  +
         2296  +  memset(pCsr, 0, sizeof(Fts3MultiSegReader));
         2297  +
         2298  +  return fts3SegReaderCursor(
         2299  +      p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
         2300  +  );
         2301  +}
         2302  +
         2303  +static int fts3SegReaderCursorAddZero(
         2304  +  Fts3Table *p,
         2305  +  const char *zTerm,
         2306  +  int nTerm,
         2307  +  Fts3MultiSegReader *pCsr
         2308  +){
         2309  +  return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr);
         2310  +}
         2311  +
         2312  +
         2313  +int sqlite3Fts3TermSegReaderCursor(
  2239   2314     Fts3Cursor *pCsr,               /* Virtual table cursor handle */
  2240   2315     const char *zTerm,              /* Term to query for */
  2241   2316     int nTerm,                      /* Size of zTerm in bytes */
  2242   2317     int isPrefix,                   /* True for a prefix search */
  2243         -  Fts3SegReaderCursor **ppSegcsr  /* OUT: Allocated seg-reader cursor */
         2318  +  Fts3MultiSegReader **ppSegcsr   /* OUT: Allocated seg-reader cursor */
  2244   2319   ){
  2245         -  Fts3SegReaderCursor *pSegcsr;   /* Object to allocate and return */
         2320  +  Fts3MultiSegReader *pSegcsr;   /* Object to allocate and return */
  2246   2321     int rc = SQLITE_NOMEM;          /* Return code */
  2247   2322   
  2248         -  pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor));
         2323  +  pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader));
  2249   2324     if( pSegcsr ){
  2250         -    Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
  2251   2325       int i;
  2252         -    int nCost = 0;
  2253         -    rc = sqlite3Fts3SegReaderCursor(
  2254         -        p, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr);
  2255         -  
  2256         -    for(i=0; rc==SQLITE_OK && i<pSegcsr->nSegment; i++){
  2257         -      rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost);
         2326  +    int bFound = 0;               /* True once an index has been found */
         2327  +    Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
         2328  +
         2329  +    if( isPrefix ){
         2330  +      for(i=1; bFound==0 && i<p->nIndex; i++){
         2331  +        if( p->aIndex[i].nPrefix==nTerm ){
         2332  +          bFound = 1;
         2333  +          rc = sqlite3Fts3SegReaderCursor(
         2334  +              p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr);
         2335  +          pSegcsr->bLookup = 1;
         2336  +        }
         2337  +      }
         2338  +
         2339  +      for(i=1; bFound==0 && i<p->nIndex; i++){
         2340  +        if( p->aIndex[i].nPrefix==nTerm+1 ){
         2341  +          bFound = 1;
         2342  +          rc = sqlite3Fts3SegReaderCursor(
         2343  +              p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr
         2344  +          );
         2345  +          if( rc==SQLITE_OK ){
         2346  +            rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr);
         2347  +          }
         2348  +        }
         2349  +      }
  2258   2350       }
  2259         -    pSegcsr->nCost = nCost;
         2351  +
         2352  +    if( bFound==0 ){
         2353  +      rc = sqlite3Fts3SegReaderCursor(
         2354  +          p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr
         2355  +      );
         2356  +      pSegcsr->bLookup = !isPrefix;
         2357  +    }
  2260   2358     }
  2261   2359   
  2262   2360     *ppSegcsr = pSegcsr;
  2263   2361     return rc;
  2264   2362   }
  2265   2363   
  2266         -static void fts3SegReaderCursorFree(Fts3SegReaderCursor *pSegcsr){
         2364  +static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){
  2267   2365     sqlite3Fts3SegReaderFinish(pSegcsr);
  2268   2366     sqlite3_free(pSegcsr);
  2269   2367   }
  2270   2368   
  2271   2369   /*
  2272   2370   ** This function retreives the doclist for the specified term (or term
  2273   2371   ** prefix) from the database. 
................................................................................
  2284   2382     Fts3PhraseToken *pTok,          /* Token to query for */
  2285   2383     int iColumn,                    /* Column to query (or -ve for all columns) */
  2286   2384     int isReqPos,                   /* True to include position lists in output */
  2287   2385     int *pnOut,                     /* OUT: Size of buffer at *ppOut */
  2288   2386     char **ppOut                    /* OUT: Malloced result buffer */
  2289   2387   ){
  2290   2388     int rc;                         /* Return code */
  2291         -  Fts3SegReaderCursor *pSegcsr;   /* Seg-reader cursor for this term */
         2389  +  Fts3MultiSegReader *pSegcsr;   /* Seg-reader cursor for this term */
  2292   2390     TermSelect tsc;                 /* Context object for fts3TermSelectCb() */
  2293   2391     Fts3SegFilter filter;           /* Segment term filter configuration */
  2294   2392   
  2295   2393     pSegcsr = pTok->pSegcsr;
  2296   2394     memset(&tsc, 0, sizeof(TermSelect));
  2297   2395     tsc.isReqPos = isReqPos;
  2298   2396   
................................................................................
  2310   2408     ){
  2311   2409       rc = fts3TermSelectCb(p, (void *)&tsc, 
  2312   2410           pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist
  2313   2411       );
  2314   2412     }
  2315   2413   
  2316   2414     if( rc==SQLITE_OK ){
  2317         -    rc = fts3TermSelectMerge(&tsc);
         2415  +    rc = fts3TermSelectMerge(p, &tsc);
  2318   2416     }
  2319   2417     if( rc==SQLITE_OK ){
  2320   2418       *ppOut = tsc.aaOutput[0];
  2321   2419       *pnOut = tsc.anOutput[0];
  2322   2420     }else{
  2323   2421       int i;
  2324   2422       for(i=0; i<SizeofArray(tsc.aaOutput); i++){
................................................................................
  2360   2458         }
  2361   2459       }
  2362   2460     }
  2363   2461   
  2364   2462     return nDoc;
  2365   2463   }
  2366   2464   
  2367         -/*
  2368         -** Call sqlite3Fts3DeferToken() for each token in the expression pExpr.
  2369         -*/
  2370         -static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){
  2371         -  int rc = SQLITE_OK;
  2372         -  if( pExpr ){
  2373         -    rc = fts3DeferExpression(pCsr, pExpr->pLeft);
  2374         -    if( rc==SQLITE_OK ){
  2375         -      rc = fts3DeferExpression(pCsr, pExpr->pRight);
  2376         -    }
  2377         -    if( pExpr->eType==FTSQUERY_PHRASE ){
  2378         -      int iCol = pExpr->pPhrase->iColumn;
  2379         -      int i;
  2380         -      for(i=0; rc==SQLITE_OK && i<pExpr->pPhrase->nToken; i++){
  2381         -        Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
  2382         -        if( pToken->pDeferred==0 ){
  2383         -          rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol);
  2384         -        }
  2385         -      }
  2386         -    }
  2387         -  }
  2388         -  return rc;
  2389         -}
  2390         -
  2391         -/*
  2392         -** This function removes the position information from a doclist. When
  2393         -** called, buffer aList (size *pnList bytes) contains a doclist that includes
  2394         -** position information. This function removes the position information so
  2395         -** that aList contains only docids, and adjusts *pnList to reflect the new
  2396         -** (possibly reduced) size of the doclist.
  2397         -*/
  2398         -static void fts3DoclistStripPositions(
  2399         -  char *aList,                    /* IN/OUT: Buffer containing doclist */
  2400         -  int *pnList                     /* IN/OUT: Size of doclist in bytes */
  2401         -){
  2402         -  if( aList ){
  2403         -    char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */
  2404         -    char *p = aList;              /* Input cursor */
  2405         -    char *pOut = aList;           /* Output cursor */
  2406         -  
  2407         -    while( p<aEnd ){
  2408         -      sqlite3_int64 delta;
  2409         -      p += sqlite3Fts3GetVarint(p, &delta);
  2410         -      fts3PoslistCopy(0, &p);
  2411         -      pOut += sqlite3Fts3PutVarint(pOut, delta);
  2412         -    }
  2413         -
  2414         -    *pnList = (int)(pOut - aList);
  2415         -  }
  2416         -}
  2417         -
  2418         -/* 
  2419         -** Return a DocList corresponding to the phrase *pPhrase.
  2420         -**
  2421         -** If this function returns SQLITE_OK, but *pnOut is set to a negative value,
  2422         -** then no tokens in the phrase were looked up in the full-text index. This
  2423         -** is only possible when this function is called from within xFilter(). The
  2424         -** caller should assume that all documents match the phrase. The actual
  2425         -** filtering will take place in xNext().
  2426         -*/
  2427         -static int fts3PhraseSelect(
  2428         -  Fts3Cursor *pCsr,               /* Virtual table cursor handle */
  2429         -  Fts3Phrase *pPhrase,            /* Phrase to return a doclist for */
  2430         -  int isReqPos,                   /* True if output should contain positions */
  2431         -  char **paOut,                   /* OUT: Pointer to malloc'd result buffer */
  2432         -  int *pnOut                      /* OUT: Size of buffer at *paOut */
  2433         -){
  2434         -  char *pOut = 0;
  2435         -  int nOut = 0;
  2436         -  int rc = SQLITE_OK;
  2437         -  int ii;
  2438         -  int iCol = pPhrase->iColumn;
  2439         -  int isTermPos = (pPhrase->nToken>1 || isReqPos);
  2440         -  Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
  2441         -  int isFirst = 1;
  2442         -
  2443         -  int iPrevTok = 0;
  2444         -  int nDoc = 0;
  2445         -
  2446         -  /* If this is an xFilter() evaluation, create a segment-reader for each
  2447         -  ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo
  2448         -  ** evaluation, only create segment-readers if there are no Fts3DeferredToken
  2449         -  ** objects attached to the phrase-tokens.
  2450         -  */
  2451         -  for(ii=0; ii<pPhrase->nToken; ii++){
  2452         -    Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
  2453         -    if( pTok->pSegcsr==0 ){
  2454         -      if( (pCsr->eEvalmode==FTS3_EVAL_FILTER)
  2455         -       || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) 
  2456         -       || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) 
  2457         -      ){
  2458         -        rc = fts3TermSegReaderCursor(
  2459         -            pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr
  2460         -        );
  2461         -        if( rc!=SQLITE_OK ) return rc;
  2462         -      }
  2463         -    }
  2464         -  }
  2465         -
  2466         -  for(ii=0; ii<pPhrase->nToken; ii++){
  2467         -    Fts3PhraseToken *pTok;        /* Token to find doclist for */
  2468         -    int iTok = 0;                 /* The token being queried this iteration */
  2469         -    char *pList = 0;              /* Pointer to token doclist */
  2470         -    int nList = 0;                /* Size of buffer at pList */
  2471         -
  2472         -    /* Select a token to process. If this is an xFilter() call, then tokens 
  2473         -    ** are processed in order from least to most costly. Otherwise, tokens 
  2474         -    ** are processed in the order in which they occur in the phrase.
  2475         -    */
  2476         -    if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){
  2477         -      assert( isReqPos );
  2478         -      iTok = ii;
  2479         -      pTok = &pPhrase->aToken[iTok];
  2480         -      if( pTok->bFulltext==0 ) continue;
  2481         -    }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){
  2482         -      iTok = ii;
  2483         -      pTok = &pPhrase->aToken[iTok];
  2484         -    }else{
  2485         -      int nMinCost = 0x7FFFFFFF;
  2486         -      int jj;
  2487         -
  2488         -      /* Find the remaining token with the lowest cost. */
  2489         -      for(jj=0; jj<pPhrase->nToken; jj++){
  2490         -        Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[jj].pSegcsr;
  2491         -        if( pSegcsr && pSegcsr->nCost<nMinCost ){
  2492         -          iTok = jj;
  2493         -          nMinCost = pSegcsr->nCost;
  2494         -        }
  2495         -      }
  2496         -      pTok = &pPhrase->aToken[iTok];
  2497         -
  2498         -      /* This branch is taken if it is determined that loading the doclist
  2499         -      ** for the next token would require more IO than loading all documents
  2500         -      ** currently identified by doclist pOut/nOut. No further doclists will
  2501         -      ** be loaded from the full-text index for this phrase.
  2502         -      */
  2503         -      if( nMinCost>nDoc && ii>0 ){
  2504         -        rc = fts3DeferExpression(pCsr, pCsr->pExpr);
  2505         -        break;
  2506         -      }
  2507         -    }
  2508         -
  2509         -    if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){
  2510         -      rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList);
  2511         -    }else{
  2512         -      if( pTok->pSegcsr ){
  2513         -        rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList);
  2514         -      }
  2515         -      pTok->bFulltext = 1;
  2516         -    }
  2517         -    assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pSegcsr==0 );
  2518         -    if( rc!=SQLITE_OK ) break;
  2519         -
  2520         -    if( isFirst ){
  2521         -      pOut = pList;
  2522         -      nOut = nList;
  2523         -      if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){
  2524         -        nDoc = fts3DoclistCountDocids(1, pOut, nOut);
  2525         -      }
  2526         -      isFirst = 0;
  2527         -      iPrevTok = iTok;
  2528         -    }else{
  2529         -      /* Merge the new term list and the current output. */
  2530         -      char *aLeft, *aRight;
  2531         -      int nLeft, nRight;
  2532         -      int nDist;
  2533         -      int mt;
  2534         -
  2535         -      /* If this is the final token of the phrase, and positions were not
  2536         -      ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE.
  2537         -      ** This drops the position information from the output list.
  2538         -      */
  2539         -      mt = MERGE_POS_PHRASE;
  2540         -      if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE;
  2541         -
  2542         -      assert( iPrevTok!=iTok );
  2543         -      if( iPrevTok<iTok ){
  2544         -        aLeft = pOut;
  2545         -        nLeft = nOut;
  2546         -        aRight = pList;
  2547         -        nRight = nList;
  2548         -        nDist = iTok-iPrevTok;
  2549         -        iPrevTok = iTok;
  2550         -      }else{
  2551         -        aRight = pOut;
  2552         -        nRight = nOut;
  2553         -        aLeft = pList;
  2554         -        nLeft = nList;
  2555         -        nDist = iPrevTok-iTok;
  2556         -      }
  2557         -      pOut = aRight;
  2558         -      fts3DoclistMerge(
  2559         -          mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight, &nDoc
  2560         -      );
  2561         -      sqlite3_free(aLeft);
  2562         -    }
  2563         -    assert( nOut==0 || pOut!=0 );
  2564         -  }
  2565         -
  2566         -  if( rc==SQLITE_OK ){
  2567         -    if( ii!=pPhrase->nToken ){
  2568         -      assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 );
  2569         -      fts3DoclistStripPositions(pOut, &nOut);
  2570         -    }
  2571         -    *paOut = pOut;
  2572         -    *pnOut = nOut;
  2573         -  }else{
  2574         -    sqlite3_free(pOut);
  2575         -  }
  2576         -  return rc;
  2577         -}
  2578         -
  2579         -/*
  2580         -** This function merges two doclists according to the requirements of a
  2581         -** NEAR operator.
  2582         -**
  2583         -** Both input doclists must include position information. The output doclist 
  2584         -** includes position information if the first argument to this function
  2585         -** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR.
  2586         -*/
  2587         -static int fts3NearMerge(
  2588         -  int mergetype,                  /* MERGE_POS_NEAR or MERGE_NEAR */
  2589         -  int nNear,                      /* Parameter to NEAR operator */
  2590         -  int nTokenLeft,                 /* Number of tokens in LHS phrase arg */
  2591         -  char *aLeft,                    /* Doclist for LHS (incl. positions) */
  2592         -  int nLeft,                      /* Size of LHS doclist in bytes */
  2593         -  int nTokenRight,                /* As nTokenLeft */
  2594         -  char *aRight,                   /* As aLeft */
  2595         -  int nRight,                     /* As nRight */
  2596         -  char **paOut,                   /* OUT: Results of merge (malloced) */
  2597         -  int *pnOut                      /* OUT: Sized of output buffer */
  2598         -){
  2599         -  char *aOut;                     /* Buffer to write output doclist to */
  2600         -  int rc;                         /* Return code */
  2601         -
  2602         -  assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR );
  2603         -
  2604         -  aOut = sqlite3_malloc(nLeft+nRight+1);
  2605         -  if( aOut==0 ){
  2606         -    rc = SQLITE_NOMEM;
  2607         -  }else{
  2608         -    rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, 
  2609         -      aOut, pnOut, aLeft, nLeft, aRight, nRight, 0
  2610         -    );
  2611         -    if( rc!=SQLITE_OK ){
  2612         -      sqlite3_free(aOut);
  2613         -      aOut = 0;
  2614         -    }
  2615         -  }
  2616         -
  2617         -  *paOut = aOut;
  2618         -  return rc;
  2619         -}
  2620         -
  2621         -/*
  2622         -** This function is used as part of the processing for the snippet() and
  2623         -** offsets() functions.
  2624         -**
  2625         -** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both
  2626         -** have their respective doclists (including position information) loaded
  2627         -** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from
  2628         -** each doclist that are not within nNear tokens of a corresponding entry
  2629         -** in the other doclist.
  2630         -*/
  2631         -int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){
  2632         -  int rc;                         /* Return code */
  2633         -
  2634         -  assert( pLeft->eType==FTSQUERY_PHRASE );
  2635         -  assert( pRight->eType==FTSQUERY_PHRASE );
  2636         -  assert( pLeft->isLoaded && pRight->isLoaded );
  2637         -
  2638         -  if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){
  2639         -    sqlite3_free(pLeft->aDoclist);
  2640         -    sqlite3_free(pRight->aDoclist);
  2641         -    pRight->aDoclist = 0;
  2642         -    pLeft->aDoclist = 0;
  2643         -    rc = SQLITE_OK;
  2644         -  }else{
  2645         -    char *aOut;                   /* Buffer in which to assemble new doclist */
  2646         -    int nOut;                     /* Size of buffer aOut in bytes */
  2647         -
  2648         -    rc = fts3NearMerge(MERGE_POS_NEAR, nNear, 
  2649         -        pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
  2650         -        pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist,
  2651         -        &aOut, &nOut
  2652         -    );
  2653         -    if( rc!=SQLITE_OK ) return rc;
  2654         -    sqlite3_free(pRight->aDoclist);
  2655         -    pRight->aDoclist = aOut;
  2656         -    pRight->nDoclist = nOut;
  2657         -
  2658         -    rc = fts3NearMerge(MERGE_POS_NEAR, nNear, 
  2659         -        pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist,
  2660         -        pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
  2661         -        &aOut, &nOut
  2662         -    );
  2663         -    sqlite3_free(pLeft->aDoclist);
  2664         -    pLeft->aDoclist = aOut;
  2665         -    pLeft->nDoclist = nOut;
  2666         -  }
  2667         -  return rc;
  2668         -}
  2669         -
  2670         -
  2671         -/*
  2672         -** Allocate an Fts3SegReaderArray for each token in the expression pExpr. 
  2673         -** The allocated objects are stored in the Fts3PhraseToken.pArray member
  2674         -** variables of each token structure.
  2675         -*/
  2676         -static int fts3ExprAllocateSegReaders(
  2677         -  Fts3Cursor *pCsr,               /* FTS3 table */
  2678         -  Fts3Expr *pExpr,                /* Expression to create seg-readers for */
  2679         -  int *pnExpr                     /* OUT: Number of AND'd expressions */
  2680         -){
  2681         -  int rc = SQLITE_OK;             /* Return code */
  2682         -
  2683         -  assert( pCsr->eEvalmode==FTS3_EVAL_FILTER );
  2684         -  if( pnExpr && pExpr->eType!=FTSQUERY_AND ){
  2685         -    (*pnExpr)++;
  2686         -    pnExpr = 0;
  2687         -  }
  2688         -
  2689         -  if( pExpr->eType==FTSQUERY_PHRASE ){
  2690         -    Fts3Phrase *pPhrase = pExpr->pPhrase;
  2691         -    int ii;
  2692         -
  2693         -    for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){
  2694         -      Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
  2695         -      if( pTok->pSegcsr==0 ){
  2696         -        rc = fts3TermSegReaderCursor(
  2697         -            pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr
  2698         -        );
  2699         -      }
  2700         -    }
  2701         -  }else{ 
  2702         -    rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr);
  2703         -    if( rc==SQLITE_OK ){
  2704         -      rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr);
  2705         -    }
  2706         -  }
  2707         -  return rc;
  2708         -}
  2709         -
  2710         -/*
  2711         -** Free the Fts3SegReaderArray objects associated with each token in the
  2712         -** expression pExpr. In other words, this function frees the resources
  2713         -** allocated by fts3ExprAllocateSegReaders().
  2714         -*/
  2715         -static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){
  2716         -  if( pExpr ){
  2717         -    Fts3Phrase *pPhrase = pExpr->pPhrase;
  2718         -    if( pPhrase ){
  2719         -      int kk;
  2720         -      for(kk=0; kk<pPhrase->nToken; kk++){
  2721         -        fts3SegReaderCursorFree(pPhrase->aToken[kk].pSegcsr);
  2722         -        pPhrase->aToken[kk].pSegcsr = 0;
  2723         -      }
  2724         -    }
  2725         -    fts3ExprFreeSegReaders(pExpr->pLeft);
  2726         -    fts3ExprFreeSegReaders(pExpr->pRight);
  2727         -  }
  2728         -}
  2729         -
  2730         -/*
  2731         -** Return the sum of the costs of all tokens in the expression pExpr. This
  2732         -** function must be called after Fts3SegReaderArrays have been allocated
  2733         -** for all tokens using fts3ExprAllocateSegReaders().
  2734         -*/
  2735         -static int fts3ExprCost(Fts3Expr *pExpr){
  2736         -  int nCost;                      /* Return value */
  2737         -  if( pExpr->eType==FTSQUERY_PHRASE ){
  2738         -    Fts3Phrase *pPhrase = pExpr->pPhrase;
  2739         -    int ii;
  2740         -    nCost = 0;
  2741         -    for(ii=0; ii<pPhrase->nToken; ii++){
  2742         -      Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[ii].pSegcsr;
  2743         -      if( pSegcsr ) nCost += pSegcsr->nCost;
  2744         -    }
  2745         -  }else{
  2746         -    nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight);
  2747         -  }
  2748         -  return nCost;
  2749         -}
  2750         -
  2751         -/*
  2752         -** The following is a helper function (and type) for fts3EvalExpr(). It
  2753         -** must be called after Fts3SegReaders have been allocated for every token
  2754         -** in the expression. See the context it is called from in fts3EvalExpr()
  2755         -** for further explanation.
  2756         -*/
  2757         -typedef struct ExprAndCost ExprAndCost;
  2758         -struct ExprAndCost {
  2759         -  Fts3Expr *pExpr;
  2760         -  int nCost;
  2761         -};
  2762         -static void fts3ExprAssignCosts(
  2763         -  Fts3Expr *pExpr,                /* Expression to create seg-readers for */
  2764         -  ExprAndCost **ppExprCost        /* OUT: Write to *ppExprCost */
  2765         -){
  2766         -  if( pExpr->eType==FTSQUERY_AND ){
  2767         -    fts3ExprAssignCosts(pExpr->pLeft, ppExprCost);
  2768         -    fts3ExprAssignCosts(pExpr->pRight, ppExprCost);
  2769         -  }else{
  2770         -    (*ppExprCost)->pExpr = pExpr;
  2771         -    (*ppExprCost)->nCost = fts3ExprCost(pExpr);
  2772         -    (*ppExprCost)++;
  2773         -  }
  2774         -}
  2775         -
  2776         -/*
  2777         -** Evaluate the full-text expression pExpr against FTS3 table pTab. Store
  2778         -** the resulting doclist in *paOut and *pnOut. This routine mallocs for
  2779         -** the space needed to store the output. The caller is responsible for
  2780         -** freeing the space when it has finished.
  2781         -**
  2782         -** This function is called in two distinct contexts:
  2783         -**
  2784         -**   * From within the virtual table xFilter() method. In this case, the
  2785         -**     output doclist contains entries for all rows in the table, based on
  2786         -**     data read from the full-text index.
  2787         -**
  2788         -**     In this case, if the query expression contains one or more tokens that 
  2789         -**     are very common, then the returned doclist may contain a superset of 
  2790         -**     the documents that actually match the expression.
  2791         -**
  2792         -**   * From within the virtual table xNext() method. This call is only made
  2793         -**     if the call from within xFilter() found that there were very common 
  2794         -**     tokens in the query expression and did return a superset of the 
  2795         -**     matching documents. In this case the returned doclist contains only
  2796         -**     entries that correspond to the current row of the table. Instead of
  2797         -**     reading the data for each token from the full-text index, the data is
  2798         -**     already available in-memory in the Fts3PhraseToken.pDeferred structures.
  2799         -**     See fts3EvalDeferred() for how it gets there.
  2800         -**
  2801         -** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is
  2802         -** required) Fts3Cursor.doDeferred==1.
  2803         -**
  2804         -** If the SQLite invokes the snippet(), offsets() or matchinfo() function
  2805         -** as part of a SELECT on an FTS3 table, this function is called on each
  2806         -** individual phrase expression in the query. If there were very common tokens
  2807         -** found in the xFilter() call, then this function is called once for phrase
  2808         -** for each row visited, and the returned doclist contains entries for the
  2809         -** current row only. Otherwise, if there were no very common tokens, then this
  2810         -** function is called once only for each phrase in the query and the returned
  2811         -** doclist contains entries for all rows of the table.
  2812         -**
  2813         -** Fts3Cursor.doDeferred==1 when this function is called on phrases as a
  2814         -** result of a snippet(), offsets() or matchinfo() invocation.
  2815         -*/
  2816         -static int fts3EvalExpr(
  2817         -  Fts3Cursor *p,                  /* Virtual table cursor handle */
  2818         -  Fts3Expr *pExpr,                /* Parsed fts3 expression */
  2819         -  char **paOut,                   /* OUT: Pointer to malloc'd result buffer */
  2820         -  int *pnOut,                     /* OUT: Size of buffer at *paOut */
  2821         -  int isReqPos                    /* Require positions in output buffer */
  2822         -){
  2823         -  int rc = SQLITE_OK;             /* Return code */
  2824         -
  2825         -  /* Zero the output parameters. */
  2826         -  *paOut = 0;
  2827         -  *pnOut = 0;
  2828         -
  2829         -  if( pExpr ){
  2830         -    assert( pExpr->eType==FTSQUERY_NEAR   || pExpr->eType==FTSQUERY_OR     
  2831         -         || pExpr->eType==FTSQUERY_AND    || pExpr->eType==FTSQUERY_NOT
  2832         -         || pExpr->eType==FTSQUERY_PHRASE
  2833         -    );
  2834         -    assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 );
  2835         -
  2836         -    if( pExpr->eType==FTSQUERY_PHRASE ){
  2837         -      rc = fts3PhraseSelect(p, pExpr->pPhrase,
  2838         -          isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR),
  2839         -          paOut, pnOut
  2840         -      );
  2841         -      fts3ExprFreeSegReaders(pExpr);
  2842         -    }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){
  2843         -      ExprAndCost *aExpr = 0;     /* Array of AND'd expressions and costs */
  2844         -      int nExpr = 0;              /* Size of aExpr[] */
  2845         -      char *aRet = 0;             /* Doclist to return to caller */
  2846         -      int nRet = 0;               /* Length of aRet[] in bytes */
  2847         -      int nDoc = 0x7FFFFFFF;
  2848         -
  2849         -      assert( !isReqPos );
  2850         -
  2851         -      rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr);
  2852         -      if( rc==SQLITE_OK ){
  2853         -        assert( nExpr>1 );
  2854         -        aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr);
  2855         -        if( !aExpr ) rc = SQLITE_NOMEM;
  2856         -      }
  2857         -      if( rc==SQLITE_OK ){
  2858         -        int ii;                   /* Used to iterate through expressions */
  2859         -
  2860         -        fts3ExprAssignCosts(pExpr, &aExpr);
  2861         -        aExpr -= nExpr;
  2862         -        for(ii=0; ii<nExpr; ii++){
  2863         -          char *aNew;
  2864         -          int nNew;
  2865         -          int jj;
  2866         -          ExprAndCost *pBest = 0;
  2867         -  
  2868         -          for(jj=0; jj<nExpr; jj++){
  2869         -            ExprAndCost *pCand = &aExpr[jj];
  2870         -            if( pCand->pExpr && (pBest==0 || pCand->nCost<pBest->nCost) ){
  2871         -              pBest = pCand;
  2872         -            }
  2873         -          }
  2874         -  
  2875         -          if( pBest->nCost>nDoc ){
  2876         -            rc = fts3DeferExpression(p, p->pExpr);
  2877         -            break;
  2878         -          }else{
  2879         -            rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0);
  2880         -            if( rc!=SQLITE_OK ) break;
  2881         -            pBest->pExpr = 0;
  2882         -            if( ii==0 ){
  2883         -              aRet = aNew;
  2884         -              nRet = nNew;
  2885         -              nDoc = fts3DoclistCountDocids(0, aRet, nRet);
  2886         -            }else{
  2887         -              fts3DoclistMerge(
  2888         -                  MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc
  2889         -              );
  2890         -              sqlite3_free(aNew);
  2891         -            }
  2892         -          }
  2893         -        }
  2894         -      }
  2895         -
  2896         -      if( rc==SQLITE_OK ){
  2897         -        *paOut = aRet;
  2898         -        *pnOut = nRet;
  2899         -      }else{
  2900         -        assert( *paOut==0 );
  2901         -        sqlite3_free(aRet);
  2902         -      }
  2903         -      sqlite3_free(aExpr);
  2904         -      fts3ExprFreeSegReaders(pExpr);
  2905         -
  2906         -    }else{
  2907         -      char *aLeft;
  2908         -      char *aRight;
  2909         -      int nLeft;
  2910         -      int nRight;
  2911         -
  2912         -      assert( pExpr->eType==FTSQUERY_NEAR 
  2913         -           || pExpr->eType==FTSQUERY_OR
  2914         -           || pExpr->eType==FTSQUERY_NOT
  2915         -           || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT)
  2916         -      );
  2917         -
  2918         -      if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos))
  2919         -       && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos))
  2920         -      ){
  2921         -        switch( pExpr->eType ){
  2922         -          case FTSQUERY_NEAR: {
  2923         -            Fts3Expr *pLeft;
  2924         -            Fts3Expr *pRight;
  2925         -            int mergetype = MERGE_NEAR;
  2926         -            if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
  2927         -              mergetype = MERGE_POS_NEAR;
  2928         -            }
  2929         -            pLeft = pExpr->pLeft;
  2930         -            while( pLeft->eType==FTSQUERY_NEAR ){ 
  2931         -              pLeft=pLeft->pRight;
  2932         -            }
  2933         -            pRight = pExpr->pRight;
  2934         -            assert( pRight->eType==FTSQUERY_PHRASE );
  2935         -            assert( pLeft->eType==FTSQUERY_PHRASE );
  2936         -
  2937         -            rc = fts3NearMerge(mergetype, pExpr->nNear, 
  2938         -                pLeft->pPhrase->nToken, aLeft, nLeft,
  2939         -                pRight->pPhrase->nToken, aRight, nRight,
  2940         -                paOut, pnOut
  2941         -            );
  2942         -            sqlite3_free(aLeft);
  2943         -            break;
  2944         -          }
  2945         -
  2946         -          case FTSQUERY_OR: {
  2947         -            /* Allocate a buffer for the output. The maximum size is the
  2948         -            ** sum of the sizes of the two input buffers. The +1 term is
  2949         -            ** so that a buffer of zero bytes is never allocated - this can
  2950         -            ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM.
  2951         -            */
  2952         -            char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
  2953         -            rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
  2954         -                aLeft, nLeft, aRight, nRight, 0
  2955         -            );
  2956         -            *paOut = aBuffer;
  2957         -            sqlite3_free(aLeft);
  2958         -            break;
  2959         -          }
  2960         -
  2961         -          default: {
  2962         -            assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
  2963         -            fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
  2964         -                aLeft, nLeft, aRight, nRight, 0
  2965         -            );
  2966         -            *paOut = aLeft;
  2967         -            break;
  2968         -          }
  2969         -        }
  2970         -      }
  2971         -      sqlite3_free(aRight);
  2972         -    }
  2973         -  }
  2974         -
  2975         -  assert( rc==SQLITE_OK || *paOut==0 );
  2976         -  return rc;
  2977         -}
  2978         -
  2979         -/*
  2980         -** This function is called from within xNext() for each row visited by
  2981         -** an FTS3 query. If evaluating the FTS3 query expression within xFilter()
  2982         -** was able to determine the exact set of matching rows, this function sets
  2983         -** *pbRes to true and returns SQLITE_IO immediately.
  2984         -**
  2985         -** Otherwise, if evaluating the query expression within xFilter() returned a
  2986         -** superset of the matching documents instead of an exact set (this happens
  2987         -** when the query includes very common tokens and it is deemed too expensive to
  2988         -** load their doclists from disk), this function tests if the current row
  2989         -** really does match the FTS3 query.
  2990         -**
  2991         -** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK
  2992         -** is returned and *pbRes is set to true if the current row matches the
  2993         -** FTS3 query (and should be included in the results returned to SQLite), or
  2994         -** false otherwise.
  2995         -*/
  2996         -static int fts3EvalDeferred(
  2997         -  Fts3Cursor *pCsr,               /* FTS3 cursor pointing at row to test */
  2998         -  int *pbRes                      /* OUT: Set to true if row is a match */
  2999         -){
  3000         -  int rc = SQLITE_OK;
  3001         -  if( pCsr->pDeferred==0 ){
  3002         -    *pbRes = 1;
  3003         -  }else{
  3004         -    rc = fts3CursorSeek(0, pCsr);
  3005         -    if( rc==SQLITE_OK ){
  3006         -      sqlite3Fts3FreeDeferredDoclists(pCsr);
  3007         -      rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
  3008         -    }
  3009         -    if( rc==SQLITE_OK ){
  3010         -      char *a = 0;
  3011         -      int n = 0;
  3012         -      rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0);
  3013         -      assert( n>=0 );
  3014         -      *pbRes = (n>0);
  3015         -      sqlite3_free(a);
  3016         -    }
  3017         -  }
  3018         -  return rc;
  3019         -}
  3020         -
  3021   2465   /*
  3022   2466   ** Advance the cursor to the next row in the %_content table that
  3023   2467   ** matches the search criteria.  For a MATCH search, this will be
  3024   2468   ** the next row that matches. For a full-table scan, this will be
  3025   2469   ** simply the next row in the %_content table.  For a docid lookup,
  3026   2470   ** this routine simply sets the EOF flag.
  3027   2471   **
  3028   2472   ** Return SQLITE_OK if nothing goes wrong.  SQLITE_OK is returned
  3029   2473   ** even if we reach end-of-file.  The fts3EofMethod() will be called
  3030   2474   ** subsequently to determine whether or not an EOF was hit.
  3031   2475   */
  3032   2476   static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
  3033         -  int res;
  3034         -  int rc = SQLITE_OK;             /* Return code */
         2477  +  int rc;
  3035   2478     Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  3036         -
  3037         -  pCsr->eEvalmode = FTS3_EVAL_NEXT;
  3038         -  do {
  3039         -    if( pCsr->aDoclist==0 ){
  3040         -      if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
  3041         -        pCsr->isEof = 1;
  3042         -        rc = sqlite3_reset(pCsr->pStmt);
  3043         -        break;
  3044         -      }
         2479  +  if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){
         2480  +    if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
         2481  +      pCsr->isEof = 1;
         2482  +      rc = sqlite3_reset(pCsr->pStmt);
         2483  +    }else{
  3045   2484         pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
  3046         -    }else{
  3047         -      if( pCsr->desc==0 ){
  3048         -        if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
  3049         -          pCsr->isEof = 1;
  3050         -          break;
  3051         -        }
  3052         -        fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
  3053         -      }else{
  3054         -        fts3GetReverseDeltaVarint(&pCsr->pNextId,pCsr->aDoclist,&pCsr->iPrevId);
  3055         -        if( pCsr->pNextId<=pCsr->aDoclist ){
  3056         -          pCsr->isEof = 1;
  3057         -          break;
  3058         -        }
  3059         -      }
  3060         -      sqlite3_reset(pCsr->pStmt);
  3061         -      pCsr->isRequireSeek = 1;
  3062         -      pCsr->isMatchinfoNeeded = 1;
         2485  +      rc = SQLITE_OK;
  3063   2486       }
  3064         -  }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 );
  3065         -
         2487  +  }else{
         2488  +    rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor);
         2489  +  }
         2490  +  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  3066   2491     return rc;
  3067   2492   }
  3068   2493   
  3069   2494   /*
  3070   2495   ** This is the xFilter interface for the virtual table.  See
  3071   2496   ** the virtual table xFilter method documentation for additional
  3072   2497   ** information.
................................................................................
  3085   2510   static int fts3FilterMethod(
  3086   2511     sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
  3087   2512     int idxNum,                     /* Strategy index */
  3088   2513     const char *idxStr,             /* Unused */
  3089   2514     int nVal,                       /* Number of elements in apVal */
  3090   2515     sqlite3_value **apVal           /* Arguments for the indexing scheme */
  3091   2516   ){
  3092         -  const char *azSql[] = {
  3093         -    "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?",   /* non-full-scan */
  3094         -    "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s", /* full-scan */
  3095         -  };
  3096         -  int rc;                         /* Return code */
         2517  +  int rc;
  3097   2518     char *zSql;                     /* SQL statement used to access %_content */
  3098   2519     Fts3Table *p = (Fts3Table *)pCursor->pVtab;
  3099   2520     Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  3100   2521   
  3101   2522     UNUSED_PARAMETER(idxStr);
  3102   2523     UNUSED_PARAMETER(nVal);
  3103   2524   
................................................................................
  3108   2529   
  3109   2530     /* In case the cursor has been used before, clear it now. */
  3110   2531     sqlite3_finalize(pCsr->pStmt);
  3111   2532     sqlite3_free(pCsr->aDoclist);
  3112   2533     sqlite3Fts3ExprFree(pCsr->pExpr);
  3113   2534     memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
  3114   2535   
         2536  +  if( idxStr ){
         2537  +    pCsr->bDesc = (idxStr[0]=='D');
         2538  +  }else{
         2539  +    pCsr->bDesc = p->bDescIdx;
         2540  +  }
         2541  +  pCsr->eSearch = (i16)idxNum;
         2542  +
  3115   2543     if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){
  3116   2544       int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
  3117   2545       const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);
  3118   2546   
  3119   2547       if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
  3120   2548         return SQLITE_NOMEM;
  3121   2549       }
  3122   2550   
  3123   2551       rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn, 
  3124   2552           iCol, zQuery, -1, &pCsr->pExpr
  3125   2553       );
  3126   2554       if( rc!=SQLITE_OK ){
  3127   2555         if( rc==SQLITE_ERROR ){
  3128         -        p->base.zErrMsg = sqlite3_mprintf("malformed MATCH expression: [%s]",
  3129         -                                          zQuery);
         2556  +        static const char *zErr = "malformed MATCH expression: [%s]";
         2557  +        p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery);
  3130   2558         }
  3131   2559         return rc;
  3132   2560       }
  3133   2561   
  3134   2562       rc = sqlite3Fts3ReadLock(p);
  3135   2563       if( rc!=SQLITE_OK ) return rc;
  3136   2564   
  3137         -    rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0);
         2565  +    rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1);
         2566  +
  3138   2567       sqlite3Fts3SegmentsClose(p);
  3139   2568       if( rc!=SQLITE_OK ) return rc;
  3140   2569       pCsr->pNextId = pCsr->aDoclist;
  3141   2570       pCsr->iPrevId = 0;
  3142   2571     }
  3143   2572   
  3144   2573     /* Compile a SELECT statement for this cursor. For a full-table-scan, the
  3145   2574     ** statement loops through all rows of the %_content table. For a
  3146   2575     ** full-text query or docid lookup, the statement retrieves a single
  3147   2576     ** row by docid.
  3148   2577     */
  3149         -  zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH];
  3150         -  zSql = sqlite3_mprintf(
  3151         -      zSql, p->zReadExprlist, p->zDb, p->zName, (idxStr ? idxStr : "ASC")
  3152         -  );
  3153         -  if( !zSql ){
  3154         -    rc = SQLITE_NOMEM;
         2578  +  if( idxNum==FTS3_FULLSCAN_SEARCH ){
         2579  +    const char *zSort = (pCsr->bDesc ? "DESC" : "ASC");
         2580  +    const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s";
         2581  +    zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort);
  3155   2582     }else{
  3156         -    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
  3157         -    sqlite3_free(zSql);
         2583  +    const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?";
         2584  +    zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName);
  3158   2585     }
  3159         -  if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){
  3160         -    rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
  3161         -  }
  3162         -  pCsr->eSearch = (i16)idxNum;
  3163         -
  3164         -  assert( pCsr->desc==0 );
         2586  +  if( !zSql ) return SQLITE_NOMEM;
         2587  +  rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
         2588  +  sqlite3_free(zSql);
  3165   2589     if( rc!=SQLITE_OK ) return rc;
  3166         -  if( rc==SQLITE_OK && pCsr->nDoclist>0 && idxStr && idxStr[0]=='D' ){
  3167         -    sqlite3_int64 iDocid = 0;
  3168         -    char *csr = pCsr->aDoclist;
  3169         -    while( csr<&pCsr->aDoclist[pCsr->nDoclist] ){
  3170         -      fts3GetDeltaVarint(&csr, &iDocid);
  3171         -    }
  3172         -    pCsr->pNextId = csr;
  3173         -    pCsr->iPrevId = iDocid;
  3174         -    pCsr->desc = 1;
  3175         -    pCsr->isRequireSeek = 1;
  3176         -    pCsr->isMatchinfoNeeded = 1;
  3177         -    pCsr->eEvalmode = FTS3_EVAL_NEXT;
  3178         -    return SQLITE_OK;
         2590  +
         2591  +  if( idxNum==FTS3_DOCID_SEARCH ){
         2592  +    rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
         2593  +    if( rc!=SQLITE_OK ) return rc;
  3179   2594     }
         2595  +
  3180   2596     return fts3NextMethod(pCursor);
  3181   2597   }
  3182   2598   
  3183   2599   /* 
  3184   2600   ** This is the xEof method of the virtual table. SQLite calls this 
  3185   2601   ** routine to find out if it has reached the end of a result set.
  3186   2602   */
................................................................................
  3192   2608   ** This is the xRowid method. The SQLite core calls this routine to
  3193   2609   ** retrieve the rowid for the current row of the result set. fts3
  3194   2610   ** exposes %_content.docid as the rowid for the virtual table. The
  3195   2611   ** rowid should be written to *pRowid.
  3196   2612   */
  3197   2613   static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
  3198   2614     Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
  3199         -  if( pCsr->aDoclist ){
  3200         -    *pRowid = pCsr->iPrevId;
  3201         -  }else{
  3202         -    /* This branch runs if the query is implemented using a full-table scan
  3203         -    ** (not using the full-text index). In this case grab the rowid from the
  3204         -    ** SELECT statement.
  3205         -    */
  3206         -    assert( pCsr->isRequireSeek==0 );
  3207         -    *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
  3208         -  }
         2615  +  *pRowid = pCsr->iPrevId;
  3209   2616     return SQLITE_OK;
  3210   2617   }
  3211   2618   
  3212   2619   /* 
  3213   2620   ** This is the xColumn method, called by SQLite to request a value from
  3214   2621   ** the row that the supplied cursor currently points to.
  3215   2622   */
  3216   2623   static int fts3ColumnMethod(
  3217   2624     sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
  3218   2625     sqlite3_context *pContext,      /* Context for sqlite3_result_xxx() calls */
  3219   2626     int iCol                        /* Index of column to read value from */
  3220   2627   ){
  3221         -  int rc;                         /* Return Code */
         2628  +  int rc = SQLITE_OK;             /* Return Code */
  3222   2629     Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
  3223   2630     Fts3Table *p = (Fts3Table *)pCursor->pVtab;
  3224   2631   
  3225   2632     /* The column value supplied by SQLite must be in range. */
  3226   2633     assert( iCol>=0 && iCol<=p->nColumn+1 );
  3227   2634   
  3228   2635     if( iCol==p->nColumn+1 ){
  3229   2636       /* This call is a request for the "docid" column. Since "docid" is an 
  3230   2637       ** alias for "rowid", use the xRowid() method to obtain the value.
  3231   2638       */
  3232         -    sqlite3_int64 iRowid;
  3233         -    rc = fts3RowidMethod(pCursor, &iRowid);
  3234         -    sqlite3_result_int64(pContext, iRowid);
         2639  +    sqlite3_result_int64(pContext, pCsr->iPrevId);
  3235   2640     }else if( iCol==p->nColumn ){
  3236   2641       /* The extra column whose name is the same as the table.
  3237   2642       ** Return a blob which is a pointer to the cursor.
  3238   2643       */
  3239   2644       sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
  3240         -    rc = SQLITE_OK;
  3241   2645     }else{
  3242   2646       rc = fts3CursorSeek(0, pCsr);
  3243   2647       if( rc==SQLITE_OK ){
  3244   2648         sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
  3245   2649       }
  3246   2650     }
         2651  +
         2652  +  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  3247   2653     return rc;
  3248   2654   }
  3249   2655   
  3250   2656   /* 
  3251   2657   ** This function is the implementation of the xUpdate callback used by 
  3252   2658   ** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
  3253   2659   ** inserted, updated or deleted.
................................................................................
  3273   2679   
  3274   2680   /*
  3275   2681   ** Implementation of xBegin() method. This is a no-op.
  3276   2682   */
  3277   2683   static int fts3BeginMethod(sqlite3_vtab *pVtab){
  3278   2684     UNUSED_PARAMETER(pVtab);
  3279   2685     TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
         2686  +  assert( p->pSegments==0 );
  3280   2687     assert( p->nPendingData==0 );
  3281   2688     assert( p->inTransaction!=1 );
  3282   2689     TESTONLY( p->inTransaction = 1 );
  3283   2690     TESTONLY( p->mxSavepoint = -1; );
  3284   2691     return SQLITE_OK;
  3285   2692   }
  3286   2693   
................................................................................
  3290   2697   ** by fts3SyncMethod().
  3291   2698   */
  3292   2699   static int fts3CommitMethod(sqlite3_vtab *pVtab){
  3293   2700     UNUSED_PARAMETER(pVtab);
  3294   2701     TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
  3295   2702     assert( p->nPendingData==0 );
  3296   2703     assert( p->inTransaction!=0 );
         2704  +  assert( p->pSegments==0 );
  3297   2705     TESTONLY( p->inTransaction = 0 );
  3298   2706     TESTONLY( p->mxSavepoint = -1; );
  3299   2707     return SQLITE_OK;
  3300   2708   }
  3301   2709   
  3302   2710   /*
  3303   2711   ** Implementation of xRollback(). Discard the contents of the pending-terms
................................................................................
  3307   2715     Fts3Table *p = (Fts3Table*)pVtab;
  3308   2716     sqlite3Fts3PendingTermsClear(p);
  3309   2717     assert( p->inTransaction!=0 );
  3310   2718     TESTONLY( p->inTransaction = 0 );
  3311   2719     TESTONLY( p->mxSavepoint = -1; );
  3312   2720     return SQLITE_OK;
  3313   2721   }
  3314         -
  3315         -/*
  3316         -** Load the doclist associated with expression pExpr to pExpr->aDoclist.
  3317         -** The loaded doclist contains positions as well as the document ids.
  3318         -** This is used by the matchinfo(), snippet() and offsets() auxillary
  3319         -** functions.
  3320         -*/
  3321         -int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){
  3322         -  int rc;
  3323         -  assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
  3324         -  assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
  3325         -  rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1);
  3326         -  return rc;
  3327         -}
  3328         -
  3329         -int sqlite3Fts3ExprLoadFtDoclist(
  3330         -  Fts3Cursor *pCsr, 
  3331         -  Fts3Expr *pExpr,
  3332         -  char **paDoclist,
  3333         -  int *pnDoclist
  3334         -){
  3335         -  int rc;
  3336         -  assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
  3337         -  assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
  3338         -  pCsr->eEvalmode = FTS3_EVAL_MATCHINFO;
  3339         -  rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1);
  3340         -  pCsr->eEvalmode = FTS3_EVAL_NEXT;
  3341         -  return rc;
  3342         -}
  3343         -
  3344   2722   
  3345   2723   /*
  3346   2724   ** When called, *ppPoslist must point to the byte immediately following the
  3347   2725   ** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function
  3348   2726   ** moves *ppPoslist so that it instead points to the first byte of the
  3349   2727   ** same position list.
  3350   2728   */
  3351   2729   static void fts3ReversePoslist(char *pStart, char **ppPoslist){
  3352         -  char *p = &(*ppPoslist)[-3];
  3353         -  char c = p[1];
         2730  +  char *p = &(*ppPoslist)[-2];
         2731  +  char c;
         2732  +
         2733  +  while( p>pStart && (c=*p--)==0 );
  3354   2734     while( p>pStart && (*p & 0x80) | c ){ 
  3355   2735       c = *p--; 
  3356   2736     }
  3357   2737     if( p>pStart ){ p = &p[2]; }
  3358   2738     while( *p++&0x80 );
  3359   2739     *ppPoslist = p;
  3360   2740   }
  3361   2741   
  3362         -
  3363         -/*
  3364         -** After ExprLoadDoclist() (see above) has been called, this function is
  3365         -** used to iterate/search through the position lists that make up the doclist
  3366         -** stored in pExpr->aDoclist.
  3367         -*/
  3368         -char *sqlite3Fts3FindPositions(
  3369         -  Fts3Cursor *pCursor,            /* Associate FTS3 cursor */
  3370         -  Fts3Expr *pExpr,                /* Access this expressions doclist */
  3371         -  sqlite3_int64 iDocid,           /* Docid associated with requested pos-list */
  3372         -  int iCol                        /* Column of requested pos-list */
  3373         -){
  3374         -  assert( pExpr->isLoaded );
  3375         -  if( pExpr->aDoclist ){
  3376         -    char *pEnd = &pExpr->aDoclist[pExpr->nDoclist];
  3377         -    char *pCsr;
  3378         -
  3379         -    if( pExpr->pCurrent==0 ){
  3380         -      if( pCursor->desc==0 ){
  3381         -        pExpr->pCurrent = pExpr->aDoclist;
  3382         -        pExpr->iCurrent = 0;
  3383         -        fts3GetDeltaVarint(&pExpr->pCurrent, &pExpr->iCurrent);
  3384         -      }else{
  3385         -        pCsr = pExpr->aDoclist;
  3386         -        while( pCsr<pEnd ){
  3387         -          fts3GetDeltaVarint(&pCsr, &pExpr->iCurrent);
  3388         -          fts3PoslistCopy(0, &pCsr);
  3389         -        }
  3390         -        fts3ReversePoslist(pExpr->aDoclist, &pCsr);
  3391         -        pExpr->pCurrent = pCsr;
  3392         -      }
  3393         -    }
  3394         -    pCsr = pExpr->pCurrent;
  3395         -    assert( pCsr );
  3396         -
  3397         -    while( (pCursor->desc==0 && pCsr<pEnd) 
  3398         -        || (pCursor->desc && pCsr>pExpr->aDoclist) 
  3399         -    ){
  3400         -      if( pCursor->desc==0 && pExpr->iCurrent<iDocid ){
  3401         -        fts3PoslistCopy(0, &pCsr);
  3402         -        if( pCsr<pEnd ){
  3403         -          fts3GetDeltaVarint(&pCsr, &pExpr->iCurrent);
  3404         -        }
  3405         -        pExpr->pCurrent = pCsr;
  3406         -      }else if( pCursor->desc && pExpr->iCurrent>iDocid ){
  3407         -        fts3GetReverseDeltaVarint(&pCsr, pExpr->aDoclist, &pExpr->iCurrent);
  3408         -        fts3ReversePoslist(pExpr->aDoclist, &pCsr);
  3409         -        pExpr->pCurrent = pCsr;
  3410         -      }else{
  3411         -        if( pExpr->iCurrent==iDocid ){
  3412         -          int iThis = 0;
  3413         -          if( iCol<0 ){
  3414         -            /* If iCol is negative, return a pointer to the start of the
  3415         -            ** position-list (instead of a pointer to the start of a list
  3416         -            ** of offsets associated with a specific column).
  3417         -            */
  3418         -            return pCsr;
  3419         -          }
  3420         -          while( iThis<iCol ){
  3421         -            fts3ColumnlistCopy(0, &pCsr);
  3422         -            if( *pCsr==0x00 ) return 0;
  3423         -            pCsr++;
  3424         -            pCsr += sqlite3Fts3GetVarint32(pCsr, &iThis);
  3425         -          }
  3426         -          if( iCol==iThis && (*pCsr&0xFE) ) return pCsr;
  3427         -        }
  3428         -        return 0;
  3429         -      }
  3430         -    }
  3431         -  }
  3432         -
  3433         -  return 0;
  3434         -}
  3435         -
  3436   2742   /*
  3437   2743   ** Helper function used by the implementation of the overloaded snippet(),
  3438   2744   ** offsets() and optimize() SQL functions.
  3439   2745   **
  3440   2746   ** If the value passed as the third argument is a blob of size
  3441   2747   ** sizeof(Fts3Cursor*), then the blob contents are copied to the 
  3442   2748   ** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error
................................................................................
  3660   2966       "ALTER TABLE %Q.'%q_segdir'   RENAME TO '%q_segdir';",
  3661   2967       p->zDb, p->zName, zName
  3662   2968     );
  3663   2969     return rc;
  3664   2970   }
  3665   2971   
  3666   2972   static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  3667         -  Fts3Table *p = (Fts3Table*)pVtab;
  3668   2973     UNUSED_PARAMETER(iSavepoint);
  3669         -  assert( p->inTransaction );
  3670         -  assert( p->mxSavepoint < iSavepoint );
  3671         -  TESTONLY( p->mxSavepoint = iSavepoint );
  3672         -  return sqlite3Fts3PendingTermsFlush(p);
         2974  +  assert( ((Fts3Table *)pVtab)->inTransaction );
         2975  +  assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
         2976  +  TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
         2977  +  return fts3SyncMethod(pVtab);
  3673   2978   }
  3674   2979   static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
  3675   2980     TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
  3676   2981     UNUSED_PARAMETER(iSavepoint);
  3677   2982     UNUSED_PARAMETER(pVtab);
  3678   2983     assert( p->inTransaction );
  3679   2984     assert( p->mxSavepoint >= iSavepoint );
................................................................................
  3836   3141     const sqlite3_api_routines *pApi
  3837   3142   ){
  3838   3143     SQLITE_EXTENSION_INIT2(pApi)
  3839   3144     return sqlite3Fts3Init(db);
  3840   3145   }
  3841   3146   #endif
  3842   3147   
         3148  +
         3149  +/*
         3150  +** Allocate an Fts3MultiSegReader for each token in the expression headed
         3151  +** by pExpr. 
         3152  +**
         3153  +** An Fts3SegReader object is a cursor that can seek or scan a range of
         3154  +** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple
         3155  +** Fts3SegReader objects internally to provide an interface to seek or scan
         3156  +** within the union of all segments of a b-tree. Hence the name.
         3157  +**
         3158  +** If the allocated Fts3MultiSegReader just seeks to a single entry in a
         3159  +** segment b-tree (if the term is not a prefix or it is a prefix for which
         3160  +** there exists prefix b-tree of the right length) then it may be traversed
         3161  +** and merged incrementally. Otherwise, it has to be merged into an in-memory 
         3162  +** doclist and then traversed.
         3163  +*/
         3164  +static void fts3EvalAllocateReaders(
         3165  +  Fts3Cursor *pCsr, 
         3166  +  Fts3Expr *pExpr, 
         3167  +  int *pnToken,                   /* OUT: Total number of tokens in phrase. */
         3168  +  int *pnOr,                      /* OUT: Total number of OR nodes in expr. */
         3169  +  int *pRc
         3170  +){
         3171  +  if( pExpr && SQLITE_OK==*pRc ){
         3172  +    if( pExpr->eType==FTSQUERY_PHRASE ){
         3173  +      int i;
         3174  +      int nToken = pExpr->pPhrase->nToken;
         3175  +      *pnToken += nToken;
         3176  +      for(i=0; i<nToken; i++){
         3177  +        Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
         3178  +        int rc = sqlite3Fts3TermSegReaderCursor(pCsr, 
         3179  +            pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr
         3180  +        );
         3181  +        if( rc!=SQLITE_OK ){
         3182  +          *pRc = rc;
         3183  +          return;
         3184  +        }
         3185  +      }
         3186  +    }else{
         3187  +      *pnOr += (pExpr->eType==FTSQUERY_OR);
         3188  +      fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc);
         3189  +      fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc);
         3190  +    }
         3191  +  }
         3192  +}
         3193  +
         3194  +static int fts3EvalPhraseLoad(
         3195  +  Fts3Cursor *pCsr, 
         3196  +  Fts3Phrase *p
         3197  +){
         3198  +  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
         3199  +  int iToken;
         3200  +  int rc = SQLITE_OK;
         3201  +
         3202  +  char *aDoclist = 0;
         3203  +  int nDoclist = 0;
         3204  +  int iPrev = -1;
         3205  +
         3206  +  for(iToken=0; rc==SQLITE_OK && iToken<p->nToken; iToken++){
         3207  +    Fts3PhraseToken *pToken = &p->aToken[iToken];
         3208  +    assert( pToken->pSegcsr || pToken->pDeferred );
         3209  +
         3210  +    if( pToken->pDeferred==0 ){
         3211  +      int nThis = 0;
         3212  +      char *pThis = 0;
         3213  +      rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis);
         3214  +      if( rc==SQLITE_OK ){
         3215  +        if( pThis==0 ){
         3216  +          sqlite3_free(aDoclist);
         3217  +          aDoclist = 0;
         3218  +          nDoclist = 0;
         3219  +          break;
         3220  +        }else if( aDoclist==0 ){
         3221  +          aDoclist = pThis;
         3222  +          nDoclist = nThis;
         3223  +        }else{
         3224  +          assert( iPrev>=0 );
         3225  +          fts3DoclistPhraseMerge(pTab->bDescIdx,
         3226  +              iToken-iPrev, aDoclist, nDoclist, pThis, &nThis
         3227  +          );
         3228  +          sqlite3_free(aDoclist);
         3229  +          aDoclist = pThis;
         3230  +          nDoclist = nThis;
         3231  +        }
         3232  +        iPrev = iToken;
         3233  +      }
         3234  +    }
         3235  +  }
         3236  +
         3237  +  if( rc==SQLITE_OK ){
         3238  +    p->doclist.aAll = aDoclist;
         3239  +    p->doclist.nAll = nDoclist;
         3240  +  }else{
         3241  +    sqlite3_free(aDoclist);
         3242  +  }
         3243  +  return rc;
         3244  +}
         3245  +
         3246  +static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
         3247  +  int iToken;
         3248  +  int rc = SQLITE_OK;
         3249  +
         3250  +  int nMaxUndeferred = -1;
         3251  +  char *aPoslist = 0;
         3252  +  int nPoslist = 0;
         3253  +  int iPrev = -1;
         3254  +
         3255  +  assert( pPhrase->doclist.bFreeList==0 );
         3256  +
         3257  +  for(iToken=0; rc==SQLITE_OK && iToken<pPhrase->nToken; iToken++){
         3258  +    Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
         3259  +    Fts3DeferredToken *pDeferred = pToken->pDeferred;
         3260  +
         3261  +    if( pDeferred ){
         3262  +      char *pList;
         3263  +      int nList;
         3264  +      rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList);
         3265  +      if( rc!=SQLITE_OK ) return rc;
         3266  +
         3267  +      if( pList==0 ){
         3268  +        sqlite3_free(aPoslist);
         3269  +        pPhrase->doclist.pList = 0;
         3270  +        pPhrase->doclist.nList = 0;
         3271  +        return SQLITE_OK;
         3272  +
         3273  +      }else if( aPoslist==0 ){
         3274  +        aPoslist = pList;
         3275  +        nPoslist = nList;
         3276  +
         3277  +      }else{
         3278  +        assert( iPrev>=0 );
         3279  +
         3280  +        char *aOut = pList;
         3281  +        char *p1 = aPoslist;
         3282  +        char *p2 = aOut;
         3283  +
         3284  +        fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2);
         3285  +        sqlite3_free(aPoslist);
         3286  +        aPoslist = pList;
         3287  +        nPoslist = aOut - aPoslist;
         3288  +        if( nPoslist==0 ){
         3289  +          sqlite3_free(aPoslist);
         3290  +          pPhrase->doclist.pList = 0;
         3291  +          pPhrase->doclist.nList = 0;
         3292  +          return SQLITE_OK;
         3293  +        }
         3294  +      }
         3295  +      iPrev = iToken;
         3296  +    }else{
         3297  +      nMaxUndeferred = iToken;
         3298  +    }
         3299  +  }
         3300  +
         3301  +  if( iPrev>=0 ){
         3302  +    if( nMaxUndeferred<0 ){
         3303  +      pPhrase->doclist.pList = aPoslist;
         3304  +      pPhrase->doclist.nList = nPoslist;
         3305  +      pPhrase->doclist.iDocid = pCsr->iPrevId;
         3306  +      pPhrase->doclist.bFreeList = 1;
         3307  +    }else{
         3308  +      int nDistance;
         3309  +      char *p1;
         3310  +      char *p2;
         3311  +      char *aOut;
         3312  +
         3313  +      if( nMaxUndeferred>iPrev ){
         3314  +        p1 = aPoslist;
         3315  +        p2 = pPhrase->doclist.pList;
         3316  +        nDistance = nMaxUndeferred - iPrev;
         3317  +      }else{
         3318  +        p1 = pPhrase->doclist.pList;
         3319  +        p2 = aPoslist;
         3320  +        nDistance = iPrev - nMaxUndeferred;
         3321  +      }
         3322  +
         3323  +      aOut = (char *)sqlite3_malloc(nPoslist+8);
         3324  +      if( !aOut ){
         3325  +        sqlite3_free(aPoslist);
         3326  +        return SQLITE_NOMEM;
         3327  +      }
         3328  +      
         3329  +      pPhrase->doclist.pList = aOut;
         3330  +      if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){
         3331  +        pPhrase->doclist.bFreeList = 1;
         3332  +        pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList);
         3333  +      }else{
         3334  +        sqlite3_free(aOut);
         3335  +        pPhrase->doclist.pList = 0;
         3336  +        pPhrase->doclist.nList = 0;
         3337  +      }
         3338  +      sqlite3_free(aPoslist);
         3339  +    }
         3340  +  }
         3341  +
         3342  +  return SQLITE_OK;
         3343  +}
         3344  +
         3345  +/*
         3346  +** This function is called for each Fts3Phrase in a full-text query 
         3347  +** expression to initialize the mechanism for returning rows. Once this
         3348  +** function has been called successfully on an Fts3Phrase, it may be
         3349  +** used with fts3EvalPhraseNext() to iterate through the matching docids.
         3350  +*/
         3351  +static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
         3352  +  int rc;
         3353  +  Fts3PhraseToken *pFirst = &p->aToken[0];
         3354  +  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
         3355  +
         3356  +  assert( p->doclist.aAll==0 );
         3357  +  if( pCsr->bDesc==pTab->bDescIdx && bOptOk==1 && p->nToken==1 
         3358  +   && pFirst->pSegcsr && pFirst->pSegcsr->bLookup 
         3359  +  ){
         3360  +    /* Use the incremental approach. */
         3361  +    int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn);
         3362  +    rc = sqlite3Fts3MsrIncrStart(
         3363  +        pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n);
         3364  +    p->bIncr = 1;
         3365  +
         3366  +  }else{
         3367  +    /* Load the full doclist for the phrase into memory. */
         3368  +    rc = fts3EvalPhraseLoad(pCsr, p);
         3369  +    p->bIncr = 0;
         3370  +  }
         3371  +
         3372  +  assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr );
         3373  +  return rc;
         3374  +}
         3375  +
         3376  +/*
         3377  +** This function is used to iterate backwards (from the end to start) 
         3378  +** through doclists.
         3379  +*/
         3380  +void sqlite3Fts3DoclistPrev(
         3381  +  int bDescIdx,                   /* True if the doclist is desc */
         3382  +  char *aDoclist,                 /* Pointer to entire doclist */
         3383  +  int nDoclist,                   /* Length of aDoclist in bytes */
         3384  +  char **ppIter,                  /* IN/OUT: Iterator pointer */
         3385  +  sqlite3_int64 *piDocid,         /* IN/OUT: Docid pointer */
         3386  +  int *pnList,                    /* IN/OUT: List length pointer */
         3387  +  u8 *pbEof                       /* OUT: End-of-file flag */
         3388  +){
         3389  +  char *p = *ppIter;
         3390  +
         3391  +  assert( nDoclist>0 );
         3392  +  assert( *pbEof==0 );
         3393  +  assert( p || *piDocid==0 );
         3394  +  assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) );
         3395  +
         3396  +  if( p==0 ){
         3397  +    sqlite3_int64 iDocid = 0;
         3398  +    char *pNext = 0;
         3399  +    char *pDocid = aDoclist;
         3400  +    char *pEnd = &aDoclist[nDoclist];
         3401  +    int iMul = 1;
         3402  +
         3403  +    while( pDocid<pEnd ){
         3404  +      sqlite3_int64 iDelta;
         3405  +      pDocid += sqlite3Fts3GetVarint(pDocid, &iDelta);
         3406  +      iDocid += (iMul * iDelta);
         3407  +      pNext = pDocid;
         3408  +      fts3PoslistCopy(0, &pDocid);
         3409  +      while( pDocid<pEnd && *pDocid==0 ) pDocid++;
         3410  +      iMul = (bDescIdx ? -1 : 1);
         3411  +    }
         3412  +
         3413  +    *pnList = pEnd - pNext;
         3414  +    *ppIter = pNext;
         3415  +    *piDocid = iDocid;
         3416  +  }else{
         3417  +    int iMul = (bDescIdx ? -1 : 1);
         3418  +    sqlite3_int64 iDelta;
         3419  +    fts3GetReverseVarint(&p, aDoclist, &iDelta);
         3420  +    *piDocid -= (iMul * iDelta);
         3421  +
         3422  +    if( p==aDoclist ){
         3423  +      *pbEof = 1;
         3424  +    }else{
         3425  +      char *pSave = p;
         3426  +      fts3ReversePoslist(aDoclist, &p);
         3427  +      *pnList = (pSave - p);
         3428  +    }
         3429  +    *ppIter = p;
         3430  +  }
         3431  +}
         3432  +
         3433  +/*
         3434  +** Attempt to move the phrase iterator to point to the next matching docid. 
         3435  +** If an error occurs, return an SQLite error code. Otherwise, return 
         3436  +** SQLITE_OK.
         3437  +**
         3438  +** If there is no "next" entry and no error occurs, then *pbEof is set to
         3439  +** 1 before returning. Otherwise, if no error occurs and the iterator is
         3440  +** successfully advanced, *pbEof is set to 0.
         3441  +*/
         3442  +static int fts3EvalPhraseNext(
         3443  +  Fts3Cursor *pCsr, 
         3444  +  Fts3Phrase *p, 
         3445  +  u8 *pbEof
         3446  +){
         3447  +  int rc = SQLITE_OK;
         3448  +  Fts3Doclist *pDL = &p->doclist;
         3449  +  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
         3450  +
         3451  +  if( p->bIncr ){
         3452  +    assert( p->nToken==1 );
         3453  +    assert( pDL->pNextDocid==0 );
         3454  +    rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, 
         3455  +        &pDL->iDocid, &pDL->pList, &pDL->nList
         3456  +    );
         3457  +    if( rc==SQLITE_OK && !pDL->pList ){
         3458  +      *pbEof = 1;
         3459  +    }
         3460  +  }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){
         3461  +    sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, 
         3462  +        &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof
         3463  +    );
         3464  +    pDL->pList = pDL->pNextDocid;
         3465  +  }else{
         3466  +    char *pIter;                            /* Used to iterate through aAll */
         3467  +    char *pEnd = &pDL->aAll[pDL->nAll];     /* 1 byte past end of aAll */
         3468  +    if( pDL->pNextDocid ){
         3469  +      pIter = pDL->pNextDocid;
         3470  +    }else{
         3471  +      pIter = pDL->aAll;
         3472  +    }
         3473  +
         3474  +    if( pIter>=pEnd ){
         3475  +      /* We have already reached the end of this doclist. EOF. */
         3476  +      *pbEof = 1;
         3477  +    }else{
         3478  +      sqlite3_int64 iDelta;
         3479  +      pIter += sqlite3Fts3GetVarint(pIter, &iDelta);
         3480  +      if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){
         3481  +        pDL->iDocid += iDelta;
         3482  +      }else{
         3483  +        pDL->iDocid -= iDelta;
         3484  +      }
         3485  +      pDL->pList = pIter;
         3486  +      fts3PoslistCopy(0, &pIter);
         3487  +      pDL->nList = (pIter - pDL->pList);
         3488  +
         3489  +      /* pIter now points just past the 0x00 that terminates the position-
         3490  +      ** list for document pDL->iDocid. However, if this position-list was
         3491  +      ** edited in place by fts3EvalNearTrim2(), then pIter may not actually
         3492  +      ** point to the start of the next docid value. The following line deals
         3493  +      ** with this case by advancing pIter past the zero-padding added by
         3494  +      ** fts3EvalNearTrim2().  */
         3495  +      while( pIter<pEnd && *pIter==0 ) pIter++;
         3496  +
         3497  +      pDL->pNextDocid = pIter;
         3498  +      assert( *pIter || pIter>=&pDL->aAll[pDL->nAll] );
         3499  +      *pbEof = 0;
         3500  +    }
         3501  +  }
         3502  +
         3503  +  return rc;
         3504  +}
         3505  +
         3506  +static void fts3EvalStartReaders(
         3507  +  Fts3Cursor *pCsr, 
         3508  +  Fts3Expr *pExpr, 
         3509  +  int bOptOk,
         3510  +  int *pRc
         3511  +){
         3512  +  if( pExpr && SQLITE_OK==*pRc ){
         3513  +    if( pExpr->eType==FTSQUERY_PHRASE ){
         3514  +      int i;
         3515  +      int nToken = pExpr->pPhrase->nToken;
         3516  +      for(i=0; i<nToken; i++){
         3517  +        if( pExpr->pPhrase->aToken[i].pDeferred==0 ) break;
         3518  +      }
         3519  +      pExpr->bDeferred = (i==nToken);
         3520  +      *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase);
         3521  +    }else{
         3522  +      fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc);
         3523  +      fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc);
         3524  +      pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred);
         3525  +    }
         3526  +  }
         3527  +}
         3528  +
         3529  +
         3530  +typedef struct Fts3TokenAndCost Fts3TokenAndCost;
         3531  +struct Fts3TokenAndCost {
         3532  +  Fts3PhraseToken *pToken;
         3533  +  Fts3Expr *pRoot;
         3534  +  int nOvfl;
         3535  +  int iCol;
         3536  +};
         3537  +
         3538  +static void fts3EvalTokenCosts(
         3539  +  Fts3Cursor *pCsr, 
         3540  +  Fts3Expr *pRoot, 
         3541  +  Fts3Expr *pExpr, 
         3542  +  Fts3TokenAndCost **ppTC,
         3543  +  Fts3Expr ***ppOr,
         3544  +  int *pRc
         3545  +){
         3546  +  if( *pRc==SQLITE_OK && pExpr ){
         3547  +    if( pExpr->eType==FTSQUERY_PHRASE ){
         3548  +      Fts3Phrase *pPhrase = pExpr->pPhrase;
         3549  +      int i;
         3550  +      for(i=0; *pRc==SQLITE_OK && i<pPhrase->nToken; i++){
         3551  +        Fts3TokenAndCost *pTC = (*ppTC)++;
         3552  +        pTC->pRoot = pRoot;
         3553  +        pTC->pToken = &pPhrase->aToken[i];
         3554  +        pTC->iCol = pPhrase->iColumn;
         3555  +        *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl);
         3556  +      }
         3557  +    }else if( pExpr->eType!=FTSQUERY_NOT ){
         3558  +      if( pExpr->eType==FTSQUERY_OR ){
         3559  +        pRoot = pExpr->pLeft;
         3560  +        **ppOr = pRoot;
         3561  +        (*ppOr)++;
         3562  +      }
         3563  +      fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc);
         3564  +      if( pExpr->eType==FTSQUERY_OR ){
         3565  +        pRoot = pExpr->pRight;
         3566  +        **ppOr = pRoot;
         3567  +        (*ppOr)++;
         3568  +      }
         3569  +      fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
         3570  +    }
         3571  +  }
         3572  +}
         3573  +
         3574  +static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
         3575  +  if( pCsr->nRowAvg==0 ){
         3576  +    /* The average document size, which is required to calculate the cost
         3577  +     ** of each doclist, has not yet been determined. Read the required 
         3578  +     ** data from the %_stat table to calculate it.
         3579  +     **
         3580  +     ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 
         3581  +     ** varints, where nCol is the number of columns in the FTS3 table.
         3582  +     ** The first varint is the number of documents currently stored in
         3583  +     ** the table. The following nCol varints contain the total amount of
         3584  +     ** data stored in all rows of each column of the table, from left
         3585  +     ** to right.
         3586  +     */
         3587  +    int rc;
         3588  +    Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
         3589  +    sqlite3_stmt *pStmt;
         3590  +    sqlite3_int64 nDoc = 0;
         3591  +    sqlite3_int64 nByte = 0;
         3592  +    const char *pEnd;
         3593  +    const char *a;
         3594  +
         3595  +    rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
         3596  +    if( rc!=SQLITE_OK ) return rc;
         3597  +    a = sqlite3_column_blob(pStmt, 0);
         3598  +    assert( a );
         3599  +
         3600  +    pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
         3601  +    a += sqlite3Fts3GetVarint(a, &nDoc);
         3602  +    while( a<pEnd ){
         3603  +      a += sqlite3Fts3GetVarint(a, &nByte);
         3604  +    }
         3605  +    if( nDoc==0 || nByte==0 ){
         3606  +      sqlite3_reset(pStmt);
         3607  +      return SQLITE_CORRUPT_VTAB;
         3608  +    }
         3609  +
         3610  +    pCsr->nDoc = nDoc;
         3611  +    pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz);
         3612  +    assert( pCsr->nRowAvg>0 ); 
         3613  +    rc = sqlite3_reset(pStmt);
         3614  +    if( rc!=SQLITE_OK ) return rc;
         3615  +  }
         3616  +
         3617  +  *pnPage = pCsr->nRowAvg;
         3618  +  return SQLITE_OK;
         3619  +}
         3620  +
         3621  +static int fts3EvalSelectDeferred(
         3622  +  Fts3Cursor *pCsr,
         3623  +  Fts3Expr *pRoot,
         3624  +  Fts3TokenAndCost *aTC,
         3625  +  int nTC
         3626  +){
         3627  +  int nDocSize = 0;
         3628  +  int nDocEst = 0;
         3629  +  int rc = SQLITE_OK;
         3630  +  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
         3631  +  int ii;
         3632  +
         3633  +  int nOvfl = 0;
         3634  +  int nTerm = 0;
         3635  +
         3636  +  for(ii=0; ii<nTC; ii++){
         3637  +    if( aTC[ii].pRoot==pRoot ){
         3638  +      nOvfl += aTC[ii].nOvfl;
         3639  +      nTerm++;
         3640  +    }
         3641  +  }
         3642  +  if( nOvfl==0 || nTerm<2 ) return SQLITE_OK;
         3643  +
         3644  +  rc = fts3EvalAverageDocsize(pCsr, &nDocSize);
         3645  +
         3646  +  for(ii=0; ii<nTerm && rc==SQLITE_OK; ii++){
         3647  +    int jj;
         3648  +    Fts3TokenAndCost *pTC = 0;
         3649  +
         3650  +    for(jj=0; jj<nTC; jj++){
         3651  +      if( aTC[jj].pToken && aTC[jj].pRoot==pRoot 
         3652  +       && (!pTC || aTC[jj].nOvfl<pTC->nOvfl) 
         3653  +      ){
         3654  +        pTC = &aTC[jj];
         3655  +      }
         3656  +    }
         3657  +    assert( pTC );
         3658  +
         3659  +    /* At this point pTC points to the cheapest remaining token. */
         3660  +    if( ii==0 ){
         3661  +      if( pTC->nOvfl ){
         3662  +        nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10;
         3663  +      }else{
         3664  +        /* TODO: Fix this so that the doclist need not be read twice. */
         3665  +        Fts3PhraseToken *pToken = pTC->pToken;
         3666  +        int nList = 0;
         3667  +        char *pList = 0;
         3668  +        rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList);
         3669  +        if( rc==SQLITE_OK ){
         3670  +          nDocEst = fts3DoclistCountDocids(1, pList, nList);
         3671  +        }
         3672  +        sqlite3_free(pList);
         3673  +        if( rc==SQLITE_OK ){
         3674  +          rc = sqlite3Fts3TermSegReaderCursor(pCsr, 
         3675  +              pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr
         3676  +          );
         3677  +        }
         3678  +      }
         3679  +    }else{
         3680  +      if( pTC->nOvfl>=(nDocEst*nDocSize) ){
         3681  +        Fts3PhraseToken *pToken = pTC->pToken;
         3682  +        rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol);
         3683  +        fts3SegReaderCursorFree(pToken->pSegcsr);
         3684  +        pToken->pSegcsr = 0;
         3685  +      }
         3686  +      nDocEst = 1 + (nDocEst/4);
         3687  +    }
         3688  +    pTC->pToken = 0;
         3689  +  }
         3690  +
         3691  +  return rc;
         3692  +}
         3693  +
         3694  +int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){
         3695  +  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
         3696  +  int rc = SQLITE_OK;
         3697  +  int nToken = 0;
         3698  +  int nOr = 0;
         3699  +
         3700  +  /* Allocate a MultiSegReader for each token in the expression. */
         3701  +  fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc);
         3702  +
         3703  +  /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO:
         3704  +  ** This call will eventually also be responsible for determining which
         3705  +  ** tokens are 'deferred' until the document text is loaded into memory.
         3706  +  **
         3707  +  ** Each token in each phrase is dealt with using one of the following
         3708  +  ** three strategies:
         3709  +  **
         3710  +  **   1. Entire doclist loaded into memory as part of the
         3711  +  **      fts3EvalStartReaders() call.
         3712  +  **
         3713  +  **   2. Doclist loaded into memory incrementally, as part of each
         3714  +  **      sqlite3Fts3EvalNext() call.
         3715  +  **
         3716  +  **   3. Token doclist is never loaded. Instead, documents are loaded into
         3717  +  **      memory and scanned for the token as part of the sqlite3Fts3EvalNext()
         3718  +  **      call. This is known as a "deferred" token.
         3719  +  */
         3720  +
         3721  +  /* If bOptOk is true, check if there are any tokens that should be deferred.
         3722  +  */
         3723  +  if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){
         3724  +    Fts3TokenAndCost *aTC;
         3725  +    Fts3Expr **apOr;
         3726  +    aTC = (Fts3TokenAndCost *)sqlite3_malloc(
         3727  +        sizeof(Fts3TokenAndCost) * nToken
         3728  +      + sizeof(Fts3Expr *) * nOr * 2
         3729  +    );
         3730  +    apOr = (Fts3Expr **)&aTC[nToken];
         3731  +
         3732  +    if( !aTC ){
         3733  +      rc = SQLITE_NOMEM;
         3734  +    }else{
         3735  +      int ii;
         3736  +      Fts3TokenAndCost *pTC = aTC;
         3737  +      Fts3Expr **ppOr = apOr;
         3738  +
         3739  +      fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc);
         3740  +      nToken = pTC-aTC;
         3741  +      nOr = ppOr-apOr;
         3742  +
         3743  +      if( rc==SQLITE_OK ){
         3744  +        rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken);
         3745  +        for(ii=0; rc==SQLITE_OK && ii<nOr; ii++){
         3746  +          rc = fts3EvalSelectDeferred(pCsr, apOr[ii], aTC, nToken);
         3747  +        }
         3748  +      }
         3749  +
         3750  +      sqlite3_free(aTC);
         3751  +    }
         3752  +  }
         3753  +
         3754  +  fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc);
         3755  +  return rc;
         3756  +}
         3757  +
         3758  +static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){
         3759  +  if( pPhrase->doclist.bFreeList ){
         3760  +    sqlite3_free(pPhrase->doclist.pList);
         3761  +  }
         3762  +  pPhrase->doclist.pList = 0;
         3763  +  pPhrase->doclist.nList = 0;
         3764  +  pPhrase->doclist.bFreeList = 0;
         3765  +}
         3766  +
         3767  +static int fts3EvalNearTrim2(
         3768  +  int nNear,
         3769  +  char *aTmp,                     /* Temporary space to use */
         3770  +  char **paPoslist,               /* IN/OUT: Position list */
         3771  +  int *pnToken,                   /* IN/OUT: Tokens in phrase of *paPoslist */
         3772  +  Fts3Phrase *pPhrase             /* The phrase object to trim the doclist of */
         3773  +){
         3774  +  int nParam1 = nNear + pPhrase->nToken;
         3775  +  int nParam2 = nNear + *pnToken;
         3776  +  int nNew;
         3777  +  char *p2; 
         3778  +  char *pOut; 
         3779  +  int res;
         3780  +
         3781  +  assert( pPhrase->doclist.pList );
         3782  +
         3783  +  p2 = pOut = pPhrase->doclist.pList;
         3784  +  res = fts3PoslistNearMerge(
         3785  +    &pOut, aTmp, nParam1, nParam2, paPoslist, &p2
         3786  +  );
         3787  +  if( res ){
         3788  +    nNew = (pOut - pPhrase->doclist.pList) - 1;
         3789  +    assert( pPhrase->doclist.pList[nNew]=='\0' );
         3790  +    assert( nNew<=pPhrase->doclist.nList && nNew>0 );
         3791  +    memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew);
         3792  +    pPhrase->doclist.nList = nNew;
         3793  +    *paPoslist = pPhrase->doclist.pList;
         3794  +    *pnToken = pPhrase->nToken;
         3795  +  }
         3796  +
         3797  +  return res;
         3798  +}
         3799  +
         3800  +static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
         3801  +  int res = 1;
         3802  +
         3803  +  /* The following block runs if pExpr is the root of a NEAR query.
         3804  +  ** For example, the query:
         3805  +  **
         3806  +  **         "w" NEAR "x" NEAR "y" NEAR "z"
         3807  +  **
         3808  +  ** which is represented in tree form as:
         3809  +  **
         3810  +  **                               |
         3811  +  **                          +--NEAR--+      <-- root of NEAR query
         3812  +  **                          |        |
         3813  +  **                     +--NEAR--+   "z"
         3814  +  **                     |        |
         3815  +  **                +--NEAR--+   "y"
         3816  +  **                |        |
         3817  +  **               "w"      "x"
         3818  +  **
         3819  +  ** The right-hand child of a NEAR node is always a phrase. The 
         3820  +  ** left-hand child may be either a phrase or a NEAR node. There are
         3821  +  ** no exceptions to this.
         3822  +  */
         3823  +  if( *pRc==SQLITE_OK 
         3824  +   && pExpr->eType==FTSQUERY_NEAR 
         3825  +   && pExpr->bEof==0
         3826  +   && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
         3827  +  ){
         3828  +    Fts3Expr *p; 
         3829  +    int nTmp = 0;                 /* Bytes of temp space */
         3830  +    char *aTmp;                   /* Temp space for PoslistNearMerge() */
         3831  +
         3832  +    /* Allocate temporary working space. */
         3833  +    for(p=pExpr; p->pLeft; p=p->pLeft){
         3834  +      nTmp += p->pRight->pPhrase->doclist.nList;
         3835  +    }
         3836  +    nTmp += p->pPhrase->doclist.nList;
         3837  +    aTmp = sqlite3_malloc(nTmp*2);
         3838  +    if( !aTmp ){
         3839  +      *pRc = SQLITE_NOMEM;
         3840  +      res = 0;
         3841  +    }else{
         3842  +      char *aPoslist = p->pPhrase->doclist.pList;
         3843  +      int nToken = p->pPhrase->nToken;
         3844  +
         3845  +      for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
         3846  +        Fts3Phrase *pPhrase = p->pRight->pPhrase;
         3847  +        int nNear = p->nNear;
         3848  +        res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
         3849  +      }
         3850  +  
         3851  +      aPoslist = pExpr->pRight->pPhrase->doclist.pList;
         3852  +      nToken = pExpr->pRight->pPhrase->nToken;
         3853  +      for(p=pExpr->pLeft; p && res; p=p->pLeft){
         3854  +        int nNear = p->pParent->nNear;
         3855  +        Fts3Phrase *pPhrase = (
         3856  +            p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
         3857  +        );
         3858  +        res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
         3859  +      }
         3860  +    }
         3861  +
         3862  +    sqlite3_free(aTmp);
         3863  +  }
         3864  +
         3865  +  return res;
         3866  +}
         3867  +
         3868  +/*
         3869  +** This macro is used by the fts3EvalNext() function. The two arguments are
         3870  +** 64-bit docid values. If the current query is "ORDER BY docid ASC", then
         3871  +** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then
         3872  +** it returns (i2 - i1). This allows the same code to be used for merging
         3873  +** doclists in ascending or descending order.
         3874  +*/
         3875  +#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2))
         3876  +
         3877  +static void fts3EvalNext(
         3878  +  Fts3Cursor *pCsr, 
         3879  +  Fts3Expr *pExpr, 
         3880  +  int *pRc
         3881  +){
         3882  +  if( *pRc==SQLITE_OK ){
         3883  +    assert( pExpr->bEof==0 );
         3884  +    pExpr->bStart = 1;
         3885  +
         3886  +    switch( pExpr->eType ){
         3887  +      case FTSQUERY_NEAR:
         3888  +      case FTSQUERY_AND: {
         3889  +        Fts3Expr *pLeft = pExpr->pLeft;
         3890  +        Fts3Expr *pRight = pExpr->pRight;
         3891  +        assert( !pLeft->bDeferred || !pRight->bDeferred );
         3892  +        if( pLeft->bDeferred ){
         3893  +          fts3EvalNext(pCsr, pRight, pRc);
         3894  +          pExpr->iDocid = pRight->iDocid;
         3895  +          pExpr->bEof = pRight->bEof;
         3896  +        }else if( pRight->bDeferred ){
         3897  +          fts3EvalNext(pCsr, pLeft, pRc);
         3898  +          pExpr->iDocid = pLeft->iDocid;
         3899  +          pExpr->bEof = pLeft->bEof;
         3900  +        }else{
         3901  +          fts3EvalNext(pCsr, pLeft, pRc);
         3902  +          fts3EvalNext(pCsr, pRight, pRc);
         3903  +
         3904  +          while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
         3905  +            sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
         3906  +            if( iDiff==0 ) break;
         3907  +            if( iDiff<0 ){
         3908  +              fts3EvalNext(pCsr, pLeft, pRc);
         3909  +            }else{
         3910  +              fts3EvalNext(pCsr, pRight, pRc);
         3911  +            }
         3912  +          }
         3913  +
         3914  +          pExpr->iDocid = pLeft->iDocid;
         3915  +          pExpr->bEof = (pLeft->bEof || pRight->bEof);
         3916  +        }
         3917  +        break;
         3918  +      }
         3919  +  
         3920  +      case FTSQUERY_OR: {
         3921  +        Fts3Expr *pLeft = pExpr->pLeft;
         3922  +        Fts3Expr *pRight = pExpr->pRight;
         3923  +        sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
         3924  +
         3925  +        assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
         3926  +        assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );
         3927  +
         3928  +        if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
         3929  +          fts3EvalNext(pCsr, pLeft, pRc);
         3930  +        }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
         3931  +          fts3EvalNext(pCsr, pRight, pRc);
         3932  +        }else{
         3933  +          fts3EvalNext(pCsr, pLeft, pRc);
         3934  +          fts3EvalNext(pCsr, pRight, pRc);
         3935  +        }
         3936  +
         3937  +        pExpr->bEof = (pLeft->bEof && pRight->bEof);
         3938  +        iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
         3939  +        if( pRight->bEof || (pLeft->bEof==0 &&  iCmp<0) ){
         3940  +          pExpr->iDocid = pLeft->iDocid;
         3941  +        }else{
         3942  +          pExpr->iDocid = pRight->iDocid;
         3943  +        }
         3944  +
         3945  +        break;
         3946  +      }
         3947  +
         3948  +      case FTSQUERY_NOT: {
         3949  +        Fts3Expr *pLeft = pExpr->pLeft;
         3950  +        Fts3Expr *pRight = pExpr->pRight;
         3951  +
         3952  +        if( pRight->bStart==0 ){
         3953  +          fts3EvalNext(pCsr, pRight, pRc);
         3954  +          assert( *pRc!=SQLITE_OK || pRight->bStart );
         3955  +        }
         3956  +
         3957  +        fts3EvalNext(pCsr, pLeft, pRc);
         3958  +        if( pLeft->bEof==0 ){
         3959  +          while( !*pRc 
         3960  +              && !pRight->bEof 
         3961  +              && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 
         3962  +          ){
         3963  +            fts3EvalNext(pCsr, pRight, pRc);
         3964  +          }
         3965  +        }
         3966  +        pExpr->iDocid = pLeft->iDocid;
         3967  +        pExpr->bEof = pLeft->bEof;
         3968  +        break;
         3969  +      }
         3970  +
         3971  +      default: {
         3972  +        Fts3Phrase *pPhrase = pExpr->pPhrase;
         3973  +        fts3EvalZeroPoslist(pPhrase);
         3974  +        *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
         3975  +        pExpr->iDocid = pPhrase->doclist.iDocid;
         3976  +        break;
         3977  +      }
         3978  +    }
         3979  +  }
         3980  +}
         3981  +
         3982  +static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){
         3983  +  int bHit = 1;
         3984  +  if( *pRc==SQLITE_OK ){
         3985  +    switch( pExpr->eType ){
         3986  +      case FTSQUERY_NEAR:
         3987  +      case FTSQUERY_AND:
         3988  +        bHit = (
         3989  +            fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
         3990  +         && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
         3991  +         && fts3EvalNearTest(pExpr, pRc)
         3992  +        );
         3993  +
         3994  +        /* If the NEAR expression does not match any rows, zero the doclist for 
         3995  +        ** all phrases involved in the NEAR. This is because the snippet(),
         3996  +        ** offsets() and matchinfo() functions are not supposed to recognize 
         3997  +        ** any instances of phrases that are part of unmatched NEAR queries. 
         3998  +        ** For example if this expression:
         3999  +        **
         4000  +        **    ... MATCH 'a OR (b NEAR c)'
         4001  +        **
         4002  +        ** is matched against a row containing:
         4003  +        **
         4004  +        **        'a b d e'
         4005  +        **
         4006  +        ** then any snippet() should ony highlight the "a" term, not the "b"
         4007  +        ** (as "b" is part of a non-matching NEAR clause).
         4008  +        */
         4009  +        if( bHit==0 
         4010  +         && pExpr->eType==FTSQUERY_NEAR 
         4011  +         && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
         4012  +        ){
         4013  +          Fts3Expr *p;
         4014  +          for(p=pExpr; p->pPhrase==0; p=p->pLeft){
         4015  +            if( p->pRight->iDocid==pCsr->iPrevId ){
         4016  +              fts3EvalZeroPoslist(p->pRight->pPhrase);
         4017  +            }
         4018  +          }
         4019  +          if( p->iDocid==pCsr->iPrevId ){
         4020  +            fts3EvalZeroPoslist(p->pPhrase);
         4021  +          }
         4022  +        }
         4023  +
         4024  +        break;
         4025  +
         4026  +      case FTSQUERY_OR: {
         4027  +        int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc);
         4028  +        int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc);
         4029  +        bHit = bHit1 || bHit2;
         4030  +        break;
         4031  +      }
         4032  +
         4033  +      case FTSQUERY_NOT:
         4034  +        bHit = (
         4035  +            fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
         4036  +         && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
         4037  +        );
         4038  +        break;
         4039  +
         4040  +      default: {
         4041  +        if( pCsr->pDeferred 
         4042  +         && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred)
         4043  +        ){
         4044  +          Fts3Phrase *pPhrase = pExpr->pPhrase;
         4045  +          assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 );
         4046  +          if( pExpr->bDeferred ){
         4047  +            fts3EvalZeroPoslist(pPhrase);
         4048  +          }
         4049  +          *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase);
         4050  +          bHit = (pPhrase->doclist.pList!=0);
         4051  +          pExpr->iDocid = pCsr->iPrevId;
         4052  +        }else{
         4053  +          bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId);
         4054  +        }
         4055  +        break;
         4056  +      }
         4057  +    }
         4058  +  }
         4059  +  return bHit;
         4060  +}
         4061  +
         4062  +/*
         4063  +** Return 1 if both of the following are true:
         4064  +**
         4065  +**   1. *pRc is SQLITE_OK when this function returns, and
         4066  +**
         4067  +**   2. After scanning the current FTS table row for the deferred tokens,
         4068  +**      it is determined that the row does not match the query.
         4069  +**
         4070  +** Or, if no error occurs and it seems the current row does match the FTS
         4071  +** query, return 0.
         4072  +*/
         4073  +static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){
         4074  +  int rc = *pRc;
         4075  +  int bMiss = 0;
         4076  +  if( rc==SQLITE_OK ){
         4077  +    if( pCsr->pDeferred ){
         4078  +      rc = fts3CursorSeek(0, pCsr);
         4079  +      if( rc==SQLITE_OK ){
         4080  +        rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
         4081  +      }
         4082  +    }
         4083  +    bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc));
         4084  +    sqlite3Fts3FreeDeferredDoclists(pCsr);
         4085  +    *pRc = rc;
         4086  +  }
         4087  +  return (rc==SQLITE_OK && bMiss);
         4088  +}
         4089  +
         4090  +/*
         4091  +** Advance to the next document that matches the FTS expression in
         4092  +** Fts3Cursor.pExpr.
         4093  +*/
         4094  +int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){
         4095  +  int rc = SQLITE_OK;             /* Return Code */
         4096  +  Fts3Expr *pExpr = pCsr->pExpr;
         4097  +  assert( pCsr->isEof==0 );
         4098  +  if( pExpr==0 ){
         4099  +    pCsr->isEof = 1;
         4100  +  }else{
         4101  +    do {
         4102  +      if( pCsr->isRequireSeek==0 ){
         4103  +        sqlite3_reset(pCsr->pStmt);
         4104  +      }
         4105  +      assert( sqlite3_data_count(pCsr->pStmt)==0 );
         4106  +      fts3EvalNext(pCsr, pExpr, &rc);
         4107  +      pCsr->isEof = pExpr->bEof;
         4108  +      pCsr->isRequireSeek = 1;
         4109  +      pCsr->isMatchinfoNeeded = 1;
         4110  +      pCsr->iPrevId = pExpr->iDocid;
         4111  +    }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) );
         4112  +  }
         4113  +  return rc;
         4114  +}
         4115  +
         4116  +/*
         4117  +** Restart interation for expression pExpr so that the next call to
         4118  +** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental 
         4119  +** loading or merging of phrase doclists for this iteration.
         4120  +**
         4121  +** If *pRc is other than SQLITE_OK when this function is called, it is
         4122  +** a no-op. If an error occurs within this function, *pRc is set to an
         4123  +** SQLite error code before returning.
         4124  +*/
         4125  +static void fts3EvalRestart(
         4126  +  Fts3Cursor *pCsr,
         4127  +  Fts3Expr *pExpr,
         4128  +  int *pRc
         4129  +){
         4130  +  if( pExpr && *pRc==SQLITE_OK ){
         4131  +    Fts3Phrase *pPhrase = pExpr->pPhrase;
         4132  +
         4133  +    if( pPhrase ){
         4134  +      fts3EvalZeroPoslist(pPhrase);
         4135  +      if( pPhrase->bIncr ){
         4136  +        sqlite3Fts3EvalPhraseCleanup(pPhrase);
         4137  +        memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
         4138  +        *pRc = sqlite3Fts3EvalStart(pCsr, pExpr, 0);
         4139  +      }else{
         4140  +        pPhrase->doclist.pNextDocid = 0;
         4141  +        pPhrase->doclist.iDocid = 0;
         4142  +      }
         4143  +    }
         4144  +
         4145  +    pExpr->iDocid = 0;
         4146  +    pExpr->bEof = 0;
         4147  +    pExpr->bStart = 0;
         4148  +
         4149  +    fts3EvalRestart(pCsr, pExpr->pLeft, pRc);
         4150  +    fts3EvalRestart(pCsr, pExpr->pRight, pRc);
         4151  +  }
         4152  +}
         4153  +
         4154  +/*
         4155  +** After allocating the Fts3Expr.aMI[] array for each phrase in the 
         4156  +** expression rooted at pExpr, the cursor iterates through all rows matched
         4157  +** by pExpr, calling this function for each row. This function increments
         4158  +** the values in Fts3Expr.aMI[] according to the position-list currently
         4159  +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase 
         4160  +** expression nodes.
         4161  +*/
         4162  +static void fts3EvalUpdateCounts(Fts3Expr *pExpr){
         4163  +  if( pExpr ){
         4164  +    Fts3Phrase *pPhrase = pExpr->pPhrase;
         4165  +    if( pPhrase && pPhrase->doclist.pList ){
         4166  +      int iCol = 0;
         4167  +      char *p = pPhrase->doclist.pList;
         4168  +
         4169  +      assert( *p );
         4170  +      while( 1 ){
         4171  +        u8 c = 0;
         4172  +        int iCnt = 0;
         4173  +        while( 0xFE & (*p | c) ){
         4174  +          if( (c&0x80)==0 ) iCnt++;
         4175  +          c = *p++ & 0x80;
         4176  +        }
         4177  +
         4178  +        /* aMI[iCol*3 + 1] = Number of occurrences
         4179  +        ** aMI[iCol*3 + 2] = Number of rows containing at least one instance
         4180  +        */
         4181  +        pExpr->aMI[iCol*3 + 1] += iCnt;
         4182  +        pExpr->aMI[iCol*3 + 2] += (iCnt>0);
         4183  +        if( *p==0x00 ) break;
         4184  +        p++;
         4185  +        p += sqlite3Fts3GetVarint32(p, &iCol);
         4186  +      }
         4187  +    }
         4188  +
         4189  +    fts3EvalUpdateCounts(pExpr->pLeft);
         4190  +    fts3EvalUpdateCounts(pExpr->pRight);
         4191  +  }
         4192  +}
         4193  +
         4194  +/*
         4195  +** Expression pExpr must be of type FTSQUERY_PHRASE.
         4196  +**
         4197  +** If it is not already allocated and populated, this function allocates and
         4198  +** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part
         4199  +** of a NEAR expression, then it also allocates and populates the same array
         4200  +** for all other phrases that are part of the NEAR expression.
         4201  +**
         4202  +** SQLITE_OK is returned if the aMI[] array is successfully allocated and
         4203  +** populated. Otherwise, if an error occurs, an SQLite error code is returned.
         4204  +*/
         4205  +static int fts3EvalGatherStats(
         4206  +  Fts3Cursor *pCsr,               /* Cursor object */
         4207  +  Fts3Expr *pExpr                 /* FTSQUERY_PHRASE expression */
         4208  +){
         4209  +  int rc = SQLITE_OK;             /* Return code */
         4210  +
         4211  +  assert( pExpr->eType==FTSQUERY_PHRASE );
         4212  +  if( pExpr->aMI==0 ){
         4213  +    Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
         4214  +    Fts3Expr *pRoot;                /* Root of NEAR expression */
         4215  +    Fts3Expr *p;                    /* Iterator used for several purposes */
         4216  +
         4217  +    sqlite3_int64 iPrevId = pCsr->iPrevId;
         4218  +    sqlite3_int64 iDocid;
         4219  +    u8 bEof;
         4220  +
         4221  +    /* Find the root of the NEAR expression */
         4222  +    pRoot = pExpr;
         4223  +    while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){
         4224  +      pRoot = pRoot->pParent;
         4225  +    }
         4226  +    iDocid = pRoot->iDocid;
         4227  +    bEof = pRoot->bEof;
         4228  +    assert( pRoot->bStart );
         4229  +
         4230  +    /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */
         4231  +    for(p=pRoot; p; p=p->pLeft){
         4232  +      Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight);
         4233  +      assert( pE->aMI==0 );
         4234  +      pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32));
         4235  +      if( !pE->aMI ) return SQLITE_NOMEM;
         4236  +      memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
         4237  +    }
         4238  +
         4239  +    fts3EvalRestart(pCsr, pRoot, &rc);
         4240  +
         4241  +    while( pCsr->isEof==0 && rc==SQLITE_OK ){
         4242  +
         4243  +      do {
         4244  +        /* Ensure the %_content statement is reset. */
         4245  +        if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt);
         4246  +        assert( sqlite3_data_count(pCsr->pStmt)==0 );
         4247  +
         4248  +        /* Advance to the next document */
         4249  +        fts3EvalNext(pCsr, pRoot, &rc);
         4250  +        pCsr->isEof = pRoot->bEof;
         4251  +        pCsr->isRequireSeek = 1;
         4252  +        pCsr->isMatchinfoNeeded = 1;
         4253  +        pCsr->iPrevId = pRoot->iDocid;
         4254  +      }while( pCsr->isEof==0 
         4255  +           && pRoot->eType==FTSQUERY_NEAR 
         4256  +           && fts3EvalLoadDeferred(pCsr, &rc) 
         4257  +      );
         4258  +
         4259  +      if( rc==SQLITE_OK && pCsr->isEof==0 ){
         4260  +        fts3EvalUpdateCounts(pRoot);
         4261  +      }
         4262  +    }
         4263  +
         4264  +    pCsr->isEof = 0;
         4265  +    pCsr->iPrevId = iPrevId;
         4266  +
         4267  +    if( bEof ){
         4268  +      pRoot->bEof = bEof;
         4269  +    }else{
         4270  +      /* Caution: pRoot may iterate through docids in ascending or descending
         4271  +      ** order. For this reason, even though it seems more defensive, the 
         4272  +      ** do loop can not be written:
         4273  +      **
         4274  +      **   do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK );
         4275  +      */
         4276  +      fts3EvalRestart(pCsr, pRoot, &rc);
         4277  +      do {
         4278  +        fts3EvalNext(pCsr, pRoot, &rc);
         4279  +        assert( pRoot->bEof==0 );
         4280  +      }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
         4281  +      fts3EvalLoadDeferred(pCsr, &rc);
         4282  +    }
         4283  +  }
         4284  +  return rc;
         4285  +}
         4286  +
         4287  +/*
         4288  +** This function is used by the matchinfo() module to query a phrase 
         4289  +** expression node for the following information:
         4290  +**
         4291  +**   1. The total number of occurrences of the phrase in each column of 
         4292  +**      the FTS table (considering all rows), and
         4293  +**
         4294  +**   2. For each column, the number of rows in the table for which the
         4295  +**      column contains at least one instance of the phrase.
         4296  +**
         4297  +** If no error occurs, SQLITE_OK is returned and the values for each column
         4298  +** written into the array aiOut as follows:
         4299  +**
         4300  +**   aiOut[iCol*3 + 1] = Number of occurrences
         4301  +**   aiOut[iCol*3 + 2] = Number of rows containing at least one instance
         4302  +**
         4303  +** Caveats:
         4304  +**
         4305  +**   * If a phrase consists entirely of deferred tokens, then all output 
         4306  +**     values are set to the number of documents in the table. In other
         4307  +**     words we assume that very common tokens occur exactly once in each 
         4308  +**     column of each row of the table.
         4309  +**
         4310  +**   * If a phrase contains some deferred tokens (and some non-deferred 
         4311  +**     tokens), count the potential occurrence identified by considering
         4312  +**     the non-deferred tokens instead of actual phrase occurrences.
         4313  +**
         4314  +**   * If the phrase is part of a NEAR expression, then only phrase instances
         4315  +**     that meet the NEAR constraint are included in the counts.
         4316  +*/
         4317  +int sqlite3Fts3EvalPhraseStats(
         4318  +  Fts3Cursor *pCsr,               /* FTS cursor handle */
         4319  +  Fts3Expr *pExpr,                /* Phrase expression */
         4320  +  u32 *aiOut                      /* Array to write results into (see above) */
         4321  +){
         4322  +  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
         4323  +  int rc = SQLITE_OK;
         4324  +  int iCol;
         4325  +
         4326  +  if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){
         4327  +    assert( pCsr->nDoc>0 );
         4328  +    for(iCol=0; iCol<pTab->nColumn; iCol++){
         4329  +      aiOut[iCol*3 + 1] = pCsr->nDoc;
         4330  +      aiOut[iCol*3 + 2] = pCsr->nDoc;
         4331  +    }
         4332  +  }else{
         4333  +    rc = fts3EvalGatherStats(pCsr, pExpr);
         4334  +    if( rc==SQLITE_OK ){
         4335  +      assert( pExpr->aMI );
         4336  +      for(iCol=0; iCol<pTab->nColumn; iCol++){
         4337  +        aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1];
         4338  +        aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2];
         4339  +      }
         4340  +    }
         4341  +  }
         4342  +
         4343  +  return rc;
         4344  +}
         4345  +
         4346  +/*
         4347  +** The expression pExpr passed as the second argument to this function
         4348  +** must be of type FTSQUERY_PHRASE. 
         4349  +**
         4350  +** The returned value is either NULL or a pointer to a buffer containing
         4351  +** a position-list indicating the occurrences of the phrase in column iCol
         4352  +** of the current row. 
         4353  +**
         4354  +** More specifically, the returned buffer contains 1 varint for each 
         4355  +** occurence of the phrase in the column, stored using the normal (delta+2) 
         4356  +** compression and is terminated by either an 0x01 or 0x00 byte. For example,
         4357  +** if the requested column contains "a b X c d X X" and the position-list
         4358  +** for 'X' is requested, the buffer returned may contain:
         4359  +**
         4360  +**     0x04 0x05 0x03 0x01   or   0x04 0x05 0x03 0x00
         4361  +**
         4362  +** This function works regardless of whether or not the phrase is deferred,
         4363  +** incremental, or neither.
         4364  +*/
         4365  +char *sqlite3Fts3EvalPhrasePoslist(
         4366  +  Fts3Cursor *pCsr,               /* FTS3 cursor object */
         4367  +  Fts3Expr *pExpr,                /* Phrase to return doclist for */
         4368  +  int iCol                        /* Column to return position list for */
         4369  +){
         4370  +  Fts3Phrase *pPhrase = pExpr->pPhrase;
         4371  +  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
         4372  +  char *pIter = pPhrase->doclist.pList;
         4373  +  int iThis;
         4374  +
         4375  +  assert( iCol>=0 && iCol<pTab->nColumn );
         4376  +  if( !pIter 
         4377  +   || pExpr->bEof 
         4378  +   || pExpr->iDocid!=pCsr->iPrevId
         4379  +   || (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol) 
         4380  +  ){
         4381  +    return 0;
         4382  +  }
         4383  +
         4384  +  assert( pPhrase->doclist.nList>0 );
         4385  +  if( *pIter==0x01 ){
         4386  +    pIter++;
         4387  +    pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
         4388  +  }else{
         4389  +    iThis = 0;
         4390  +  }
         4391  +  while( iThis<iCol ){
         4392  +    fts3ColumnlistCopy(0, &pIter);
         4393  +    if( *pIter==0x00 ) return 0;
         4394  +    pIter++;
         4395  +    pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
         4396  +  }
         4397  +
         4398  +  return ((iCol==iThis)?pIter:0);
         4399  +}
         4400  +
         4401  +/*
         4402  +** Free all components of the Fts3Phrase structure that were allocated by
         4403  +** the eval module. Specifically, this means to free:
         4404  +**
         4405  +**   * the contents of pPhrase->doclist, and
         4406  +**   * any Fts3MultiSegReader objects held by phrase tokens.
         4407  +*/
         4408  +void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){
         4409  +  if( pPhrase ){
         4410  +    int i;
         4411  +    sqlite3_free(pPhrase->doclist.aAll);
         4412  +    fts3EvalZeroPoslist(pPhrase);
         4413  +    memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
         4414  +    for(i=0; i<pPhrase->nToken; i++){
         4415  +      fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr);
         4416  +      pPhrase->aToken[i].pSegcsr = 0;
         4417  +    }
         4418  +  }
         4419  +}
         4420  +
  3843   4421   #endif

Changes to ext/fts3/fts3Int.h.

    43     43   /*
    44     44   ** Macro to return the number of elements in an array. SQLite has a
    45     45   ** similar macro called ArraySize(). Use a different name to avoid
    46     46   ** a collision when building an amalgamation with built-in FTS3.
    47     47   */
    48     48   #define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
    49     49   
           50  +
           51  +#ifndef MIN
           52  +# define MIN(x,y) ((x)<(y)?(x):(y))
           53  +#endif
           54  +
    50     55   /*
    51     56   ** Maximum length of a varint encoded integer. The varint format is different
    52     57   ** from that used by SQLite, so the maximum length is 10, not 9.
    53     58   */
    54     59   #define FTS3_VARINT_MAX 10
    55     60   
           61  +/*
           62  +** FTS4 virtual tables may maintain multiple indexes - one index of all terms
           63  +** in the document set and zero or more prefix indexes. All indexes are stored
           64  +** as one or more b+-trees in the %_segments and %_segdir tables. 
           65  +**
           66  +** It is possible to determine which index a b+-tree belongs to based on the
           67  +** value stored in the "%_segdir.level" column. Given this value L, the index
           68  +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with
           69  +** level values between 0 and 1023 (inclusive) belong to index 0, all levels
           70  +** between 1024 and 2047 to index 1, and so on.
           71  +**
           72  +** It is considered impossible for an index to use more than 1024 levels. In 
           73  +** theory though this may happen, but only after at least 
           74  +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables.
           75  +*/
           76  +#define FTS3_SEGDIR_MAXLEVEL      1024
           77  +#define FTS3_SEGDIR_MAXLEVEL_STR "1024"
           78  +
    56     79   /*
    57     80   ** The testcase() macro is only used by the amalgamation.  If undefined,
    58     81   ** make it a no-op.
    59     82   */
    60     83   #ifndef testcase
    61     84   # define testcase(X)
    62     85   #endif
................................................................................
   120    143   
   121    144   typedef struct Fts3Table Fts3Table;
   122    145   typedef struct Fts3Cursor Fts3Cursor;
   123    146   typedef struct Fts3Expr Fts3Expr;
   124    147   typedef struct Fts3Phrase Fts3Phrase;
   125    148   typedef struct Fts3PhraseToken Fts3PhraseToken;
   126    149   
          150  +typedef struct Fts3Doclist Fts3Doclist;
   127    151   typedef struct Fts3SegFilter Fts3SegFilter;
   128    152   typedef struct Fts3DeferredToken Fts3DeferredToken;
   129    153   typedef struct Fts3SegReader Fts3SegReader;
   130         -typedef struct Fts3SegReaderCursor Fts3SegReaderCursor;
          154  +typedef struct Fts3MultiSegReader Fts3MultiSegReader;
   131    155   
   132    156   /*
   133    157   ** A connection to a fulltext index is an instance of the following
   134    158   ** structure. The xCreate and xConnect methods create an instance
   135    159   ** of this structure and xDestroy and xDisconnect free that instance.
   136    160   ** All other methods receive a pointer to the structure as one of their
   137    161   ** arguments.
................................................................................
   144    168     int nColumn;                    /* number of named columns in virtual table */
   145    169     char **azColumn;                /* column names.  malloced */
   146    170     sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
   147    171   
   148    172     /* Precompiled statements used by the implementation. Each of these 
   149    173     ** statements is run and reset within a single virtual table API call. 
   150    174     */
   151         -  sqlite3_stmt *aStmt[24];
          175  +  sqlite3_stmt *aStmt[27];
   152    176   
   153    177     char *zReadExprlist;
   154    178     char *zWriteExprlist;
   155    179   
   156    180     int nNodeSize;                  /* Soft limit for node size */
   157    181     u8 bHasStat;                    /* True if %_stat table exists */
   158    182     u8 bHasDocsize;                 /* True if %_docsize table exists */
          183  +  u8 bDescIdx;                    /* True if doclists are in reverse order */
   159    184     int nPgsz;                      /* Page size for host database */
   160    185     char *zSegmentsTbl;             /* Name of %_segments table */
   161    186     sqlite3_blob *pSegments;        /* Blob handle open on %_segments table */
   162    187   
   163         -  /* The following hash table is used to buffer pending index updates during
          188  +  /* TODO: Fix the first paragraph of this comment.
          189  +  **
          190  +  ** The following hash table is used to buffer pending index updates during
   164    191     ** transactions. Variable nPendingData estimates the memory size of the 
   165    192     ** pending data, including hash table overhead, but not malloc overhead. 
   166    193     ** When nPendingData exceeds nMaxPendingData, the buffer is flushed 
   167    194     ** automatically. Variable iPrevDocid is the docid of the most recently
   168    195     ** inserted record.
          196  +  **
          197  +  ** A single FTS4 table may have multiple full-text indexes. For each index
          198  +  ** there is an entry in the aIndex[] array. Index 0 is an index of all the
          199  +  ** terms that appear in the document set. Each subsequent index in aIndex[]
          200  +  ** is an index of prefixes of a specific length.
   169    201     */
   170         -  int nMaxPendingData;
   171         -  int nPendingData;
   172         -  sqlite_int64 iPrevDocid;
   173         -  Fts3Hash pendingTerms;
          202  +  int nIndex;                     /* Size of aIndex[] */
          203  +  struct Fts3Index {
          204  +    int nPrefix;                  /* Prefix length (0 for main terms index) */
          205  +    Fts3Hash hPending;            /* Pending terms table for this index */
          206  +  } *aIndex;
          207  +  int nMaxPendingData;            /* Max pending data before flush to disk */
          208  +  int nPendingData;               /* Current bytes of pending data */
          209  +  sqlite_int64 iPrevDocid;        /* Docid of most recently inserted document */
   174    210   
   175    211   #if defined(SQLITE_DEBUG)
   176    212     /* State variables used for validating that the transaction control
   177    213     ** methods of the virtual table are called at appropriate times.  These
   178    214     ** values do not contribution to the FTS computation; they are used for
   179    215     ** verifying the SQLite core.
   180    216     */
................................................................................
   197    233     Fts3Expr *pExpr;                /* Parsed MATCH query string */
   198    234     int nPhrase;                    /* Number of matchable phrases in query */
   199    235     Fts3DeferredToken *pDeferred;   /* Deferred search tokens, if any */
   200    236     sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
   201    237     char *pNextId;                  /* Pointer into the body of aDoclist */
   202    238     char *aDoclist;                 /* List of docids for full-text queries */
   203    239     int nDoclist;                   /* Size of buffer at aDoclist */
   204         -  int desc;                       /* True to sort in descending order */
          240  +  u8 bDesc;                       /* True to sort in descending order */
   205    241     int eEvalmode;                  /* An FTS3_EVAL_XX constant */
   206    242     int nRowAvg;                    /* Average size of database rows, in pages */
          243  +  int nDoc;                       /* Documents in table */
   207    244   
   208    245     int isMatchinfoNeeded;          /* True when aMatchinfo[] needs filling in */
   209    246     u32 *aMatchinfo;                /* Information about most recent match */
   210    247     int nMatchinfo;                 /* Number of elements in aMatchinfo[] */
   211    248     char *zMatchinfo;               /* Matchinfo specification */
   212    249   };
   213    250   
................................................................................
   230    267   ** indicating that all columns should be searched,
   231    268   ** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4.
   232    269   */
   233    270   #define FTS3_FULLSCAN_SEARCH 0    /* Linear scan of %_content table */
   234    271   #define FTS3_DOCID_SEARCH    1    /* Lookup by rowid on %_content table */
   235    272   #define FTS3_FULLTEXT_SEARCH 2    /* Full-text index search */
   236    273   
          274  +
          275  +struct Fts3Doclist {
          276  +  char *aAll;                    /* Array containing doclist (or NULL) */
          277  +  int nAll;                      /* Size of a[] in bytes */
          278  +  char *pNextDocid;              /* Pointer to next docid */
          279  +
          280  +  sqlite3_int64 iDocid;          /* Current docid (if pList!=0) */
          281  +  int bFreeList;                 /* True if pList should be sqlite3_free()d */
          282  +  char *pList;                   /* Pointer to position list following iDocid */
          283  +  int nList;                     /* Length of position list */
          284  +} doclist;
          285  +
   237    286   /*
   238    287   ** A "phrase" is a sequence of one or more tokens that must match in
   239    288   ** sequence.  A single token is the base case and the most common case.
   240    289   ** For a sequence of tokens contained in double-quotes (i.e. "one two three")
   241    290   ** nToken will be the number of tokens in the string.
   242         -**
   243         -** The nDocMatch and nMatch variables contain data that may be used by the
   244         -** matchinfo() function. They are populated when the full-text index is 
   245         -** queried for hits on the phrase. If one or more tokens in the phrase
   246         -** are deferred, the nDocMatch and nMatch variables are populated based
   247         -** on the assumption that the 
   248    291   */
   249    292   struct Fts3PhraseToken {
   250    293     char *z;                        /* Text of the token */
   251    294     int n;                          /* Number of bytes in buffer z */
   252    295     int isPrefix;                   /* True if token ends with a "*" character */
          296  +
          297  +  /* Variables above this point are populated when the expression is
          298  +  ** parsed (by code in fts3_expr.c). Below this point the variables are
          299  +  ** used when evaluating the expression. */
   253    300     int bFulltext;                  /* True if full-text index was used */
   254         -  Fts3SegReaderCursor *pSegcsr;   /* Segment-reader for this token */
   255    301     Fts3DeferredToken *pDeferred;   /* Deferred token object for this token */
          302  +  Fts3MultiSegReader *pSegcsr;    /* Segment-reader for this token */
   256    303   };
   257    304   
   258    305   struct Fts3Phrase {
   259         -  /* Variables populated by fts3_expr.c when parsing a MATCH expression */
          306  +  /* Cache of doclist for this phrase. */
          307  +  Fts3Doclist doclist;
          308  +  int bIncr;                 /* True if doclist is loaded incrementally */
          309  +
          310  +  /* Variables below this point are populated by fts3_expr.c when parsing 
          311  +  ** a MATCH expression. Everything above is part of the evaluation phase. 
          312  +  */
   260    313     int nToken;                /* Number of tokens in the phrase */
   261    314     int iColumn;               /* Index of column this phrase must match */
   262         -  int isNot;                 /* Phrase prefixed by unary not (-) operator */
   263    315     Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
   264    316   };
   265    317   
   266    318   /*
   267    319   ** A tree of these objects forms the RHS of a MATCH operator.
   268    320   **
   269         -** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded
   270         -** is true, then aDoclist points to a malloced buffer, size nDoclist bytes, 
   271         -** containing the results of the NEAR or phrase query in FTS3 doclist
   272         -** format. As usual, the initial "Length" field found in doclists stored
   273         -** on disk is omitted from this buffer.
          321  +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist 
          322  +** points to a malloced buffer, size nDoclist bytes, containing the results 
          323  +** of this phrase query in FTS3 doclist format. As usual, the initial 
          324  +** "Length" field found in doclists stored on disk is omitted from this 
          325  +** buffer.
          326  +**
          327  +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global
          328  +** matchinfo data. If it is not NULL, it points to an array of size nCol*3,
          329  +** where nCol is the number of columns in the queried FTS table. The array
          330  +** is populated as follows:
          331  +**
          332  +**   aMI[iCol*3 + 0] = Undefined
          333  +**   aMI[iCol*3 + 1] = Number of occurrences
          334  +**   aMI[iCol*3 + 2] = Number of rows containing at least one instance
   274    335   **
   275         -** Variable pCurrent always points to the start of a docid field within
   276         -** aDoclist. Since the doclist is usually scanned in docid order, this can
   277         -** be used to accelerate seeking to the required docid within the doclist.
          336  +** The aMI array is allocated using sqlite3_malloc(). It should be freed 
          337  +** when the expression node is.
   278    338   */
   279    339   struct Fts3Expr {
   280    340     int eType;                 /* One of the FTSQUERY_XXX values defined below */
   281    341     int nNear;                 /* Valid if eType==FTSQUERY_NEAR */
   282    342     Fts3Expr *pParent;         /* pParent->pLeft==this or pParent->pRight==this */
   283    343     Fts3Expr *pLeft;           /* Left operand */
   284    344     Fts3Expr *pRight;          /* Right operand */
   285    345     Fts3Phrase *pPhrase;       /* Valid if eType==FTSQUERY_PHRASE */
   286    346   
   287         -  int isLoaded;              /* True if aDoclist/nDoclist are initialized. */
   288         -  char *aDoclist;            /* Buffer containing doclist */
   289         -  int nDoclist;              /* Size of aDoclist in bytes */
          347  +  /* The following are used by the fts3_eval.c module. */
          348  +  sqlite3_int64 iDocid;      /* Current docid */
          349  +  u8 bEof;                   /* True this expression is at EOF already */
          350  +  u8 bStart;                 /* True if iDocid is valid */
          351  +  u8 bDeferred;              /* True if this expression is entirely deferred */
   290    352   
   291         -  sqlite3_int64 iCurrent;
   292         -  char *pCurrent;
          353  +  u32 *aMI;
   293    354   };
   294    355   
   295    356   /*
   296    357   ** Candidate values for Fts3Query.eType. Note that the order of the first
   297    358   ** four values is in order of precedence when parsing expressions. For 
   298    359   ** example, the following:
   299    360   **
................................................................................
   313    374   /* fts3_write.c */
   314    375   int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
   315    376   int sqlite3Fts3PendingTermsFlush(Fts3Table *);
   316    377   void sqlite3Fts3PendingTermsClear(Fts3Table *);
   317    378   int sqlite3Fts3Optimize(Fts3Table *);
   318    379   int sqlite3Fts3SegReaderNew(int, sqlite3_int64,
   319    380     sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
   320         -int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**);
          381  +int sqlite3Fts3SegReaderPending(
          382  +  Fts3Table*,int,const char*,int,int,Fts3SegReader**);
   321    383   void sqlite3Fts3SegReaderFree(Fts3SegReader *);
   322         -int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
   323         -int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **);
          384  +int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **);
   324    385   int sqlite3Fts3ReadLock(Fts3Table *);
   325         -int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);
          386  +int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*);
   326    387   
   327    388   int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
   328    389   int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
   329    390   
   330    391   void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
   331    392   int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
   332    393   int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
   333    394   void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
   334         -char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);
   335    395   void sqlite3Fts3SegmentsClose(Fts3Table *);
   336    396   
   337         -#define FTS3_SEGCURSOR_PENDING -1
   338         -#define FTS3_SEGCURSOR_ALL     -2
          397  +/* Special values interpreted by sqlite3SegReaderCursor() */
          398  +#define FTS3_SEGCURSOR_PENDING        -1
          399  +#define FTS3_SEGCURSOR_ALL            -2
   339    400   
   340         -int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*);
   341         -int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *);
   342         -void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *);
          401  +int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*);
          402  +int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *);
          403  +void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *);
          404  +
   343    405   int sqlite3Fts3SegReaderCursor(
   344         -    Fts3Table *, int, const char *, int, int, int, Fts3SegReaderCursor *);
          406  +    Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *);
   345    407   
   346    408   /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
   347    409   #define FTS3_SEGMENT_REQUIRE_POS   0x00000001
   348    410   #define FTS3_SEGMENT_IGNORE_EMPTY  0x00000002
   349    411   #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
   350    412   #define FTS3_SEGMENT_PREFIX        0x00000008
   351    413   #define FTS3_SEGMENT_SCAN          0x00000010
................................................................................
   354    416   struct Fts3SegFilter {
   355    417     const char *zTerm;
   356    418     int nTerm;
   357    419     int iCol;
   358    420     int flags;
   359    421   };
   360    422   
   361         -struct Fts3SegReaderCursor {
          423  +struct Fts3MultiSegReader {
   362    424     /* Used internally by sqlite3Fts3SegReaderXXX() calls */
   363    425     Fts3SegReader **apSegment;      /* Array of Fts3SegReader objects */
   364    426     int nSegment;                   /* Size of apSegment array */
   365    427     int nAdvance;                   /* How many seg-readers to advance */
   366    428     Fts3SegFilter *pFilter;         /* Pointer to filter object */
   367    429     char *aBuffer;                  /* Buffer to merge doclists in */
   368    430     int nBuffer;                    /* Allocated size of aBuffer[] in bytes */
   369    431   
   370         -  /* Cost of running this iterator. Used by fts3.c only. */
   371         -  int nCost;
          432  +  int iColFilter;                 /* If >=0, filter for this column */
          433  +
          434  +  /* Used by fts3.c only. */
          435  +  int nCost;                      /* Cost of running iterator */
          436  +  int bLookup;                    /* True if a lookup of a single entry. */
   372    437   
   373    438     /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */
   374    439     char *zTerm;                    /* Pointer to term buffer */
   375    440     int nTerm;                      /* Size of zTerm in bytes */
   376    441     char *aDoclist;                 /* Pointer to doclist buffer */
   377    442     int nDoclist;                   /* Size of aDoclist[] in bytes */
   378    443   };
................................................................................
   379    444   
   380    445   /* fts3.c */
   381    446   int sqlite3Fts3PutVarint(char *, sqlite3_int64);
   382    447   int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
   383    448   int sqlite3Fts3GetVarint32(const char *, int *);
   384    449   int sqlite3Fts3VarintLen(sqlite3_uint64);
   385    450   void sqlite3Fts3Dequote(char *);
          451  +void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
   386    452   
   387         -char *sqlite3Fts3FindPositions(Fts3Cursor *, Fts3Expr *, sqlite3_int64, int);
   388         -int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
   389         -int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *);
   390         -int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);
          453  +int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
   391    454   
   392    455   /* fts3_tokenizer.c */
   393    456   const char *sqlite3Fts3NextToken(const char *, int *);
   394    457   int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
   395    458   int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, 
   396    459       sqlite3_tokenizer **, char **
   397    460   );
................................................................................
   412    475   #ifdef SQLITE_TEST
   413    476   int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
   414    477   int sqlite3Fts3InitTerm(sqlite3 *db);
   415    478   #endif
   416    479   
   417    480   /* fts3_aux.c */
   418    481   int sqlite3Fts3InitAux(sqlite3 *db);
          482  +
          483  +int sqlite3Fts3TermSegReaderCursor(
          484  +  Fts3Cursor *pCsr,               /* Virtual table cursor handle */
          485  +  const char *zTerm,              /* Term to query for */
          486  +  int nTerm,                      /* Size of zTerm in bytes */
          487  +  int isPrefix,                   /* True for a prefix search */
          488  +  Fts3MultiSegReader **ppSegcsr   /* OUT: Allocated seg-reader cursor */
          489  +);
          490  +
          491  +void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
          492  +
          493  +int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int);
          494  +int sqlite3Fts3EvalNext(Fts3Cursor *pCsr);
          495  +
          496  +int sqlite3Fts3MsrIncrStart(
          497  +    Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
          498  +int sqlite3Fts3MsrIncrNext(
          499  +    Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
          500  +char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); 
          501  +int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
          502  +
          503  +int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);
          504  +
   419    505   
   420    506   #endif /* _FTSINT_H */

Changes to ext/fts3/fts3_aux.c.

    24     24   struct Fts3auxTable {
    25     25     sqlite3_vtab base;              /* Base class used by SQLite core */
    26     26     Fts3Table *pFts3Tab;
    27     27   };
    28     28   
    29     29   struct Fts3auxCursor {
    30     30     sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
    31         -  Fts3SegReaderCursor csr;        /* Must be right after "base" */
           31  +  Fts3MultiSegReader csr;        /* Must be right after "base" */
    32     32     Fts3SegFilter filter;
    33     33     char *zStop;
    34     34     int nStop;                      /* Byte-length of string zStop */
    35     35     int isEof;                      /* True if cursor is at EOF */
    36     36     sqlite3_int64 iRowid;           /* Current rowid */
    37     37   
    38     38     int iCol;                       /* Current value of 'col' column */
................................................................................
    92     92     if( !p ) return SQLITE_NOMEM;
    93     93     memset(p, 0, nByte);
    94     94   
    95     95     p->pFts3Tab = (Fts3Table *)&p[1];
    96     96     p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
    97     97     p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
    98     98     p->pFts3Tab->db = db;
           99  +  p->pFts3Tab->nIndex = 1;
    99    100   
   100    101     memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
   101    102     memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
   102    103     sqlite3Fts3Dequote((char *)p->pFts3Tab->zName);
   103    104   
   104    105     *ppVtab = (sqlite3_vtab *)p;
   105    106     return SQLITE_OK;
................................................................................
   372    373     if( idxNum&FTS4AUX_LE_CONSTRAINT ){
   373    374       int iIdx = (idxNum&FTS4AUX_GE_CONSTRAINT) ? 1 : 0;
   374    375       pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx]));
   375    376       pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]);
   376    377       if( pCsr->zStop==0 ) return SQLITE_NOMEM;
   377    378     }
   378    379   
   379         -  rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL,
          380  +  rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL,
   380    381         pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr
   381    382     );
   382    383     if( rc==SQLITE_OK ){
   383    384       rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
   384    385     }
   385    386   
   386    387     if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor);

Changes to ext/fts3/fts3_expr.c.

    77     77   */
    78     78   #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
    79     79   
    80     80   #include "fts3Int.h"
    81     81   #include <string.h>
    82     82   #include <assert.h>
    83     83   
           84  +/*
           85  +** isNot:
           86  +**   This variable is used by function getNextNode(). When getNextNode() is
           87  +**   called, it sets ParseContext.isNot to true if the 'next node' is a 
           88  +**   FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the
           89  +**   FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to
           90  +**   zero.
           91  +*/
    84     92   typedef struct ParseContext ParseContext;
    85     93   struct ParseContext {
    86     94     sqlite3_tokenizer *pTokenizer;      /* Tokenizer module */
    87     95     const char **azCol;                 /* Array of column names for fts3 table */
    88     96     int nCol;                           /* Number of entries in azCol[] */
    89     97     int iDefaultCol;                    /* Default column to query */
           98  +  int isNot;                          /* True if getNextNode() sees a unary - */
    90     99     sqlite3_context *pCtx;              /* Write error message here */
    91    100     int nNest;                          /* Number of nested brackets */
    92    101   };
    93    102   
    94    103   /*
    95    104   ** This function is equivalent to the standard isspace() function. 
    96    105   **
................................................................................
   168    177           memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken);
   169    178   
   170    179           if( iEnd<n && z[iEnd]=='*' ){
   171    180             pRet->pPhrase->aToken[0].isPrefix = 1;
   172    181             iEnd++;
   173    182           }
   174    183           if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
   175         -          pRet->pPhrase->isNot = 1;
          184  +          pParse->isNot = 1;
   176    185           }
   177    186         }
   178    187         nConsumed = iEnd;
   179    188       }
   180    189   
   181    190       pModule->xClose(pCursor);
   182    191     }
................................................................................
   220    229     sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
   221    230     int rc;
   222    231     Fts3Expr *p = 0;
   223    232     sqlite3_tokenizer_cursor *pCursor = 0;
   224    233     char *zTemp = 0;
   225    234     int nTemp = 0;
   226    235   
          236  +  const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
          237  +  int nToken = 0;
          238  +
          239  +  /* The final Fts3Expr data structure, including the Fts3Phrase,
          240  +  ** Fts3PhraseToken structures token buffers are all stored as a single 
          241  +  ** allocation so that the expression can be freed with a single call to
          242  +  ** sqlite3_free(). Setting this up requires a two pass approach.
          243  +  **
          244  +  ** The first pass, in the block below, uses a tokenizer cursor to iterate
          245  +  ** through the tokens in the expression. This pass uses fts3ReallocOrFree()
          246  +  ** to assemble data in two dynamic buffers:
          247  +  **
          248  +  **   Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase
          249  +  **             structure, followed by the array of Fts3PhraseToken 
          250  +  **             structures. This pass only populates the Fts3PhraseToken array.
          251  +  **
          252  +  **   Buffer zTemp: Contains copies of all tokens.
          253  +  **
          254  +  ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below,
          255  +  ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase
          256  +  ** structures.
          257  +  */
   227    258     rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
   228    259     if( rc==SQLITE_OK ){
   229    260       int ii;
   230    261       pCursor->pTokenizer = pTokenizer;
   231    262       for(ii=0; rc==SQLITE_OK; ii++){
   232         -      const char *zToken;
   233         -      int nToken, iBegin, iEnd, iPos;
   234         -      rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
          263  +      const char *zByte;
          264  +      int nByte, iBegin, iEnd, iPos;
          265  +      rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos);
   235    266         if( rc==SQLITE_OK ){
   236         -        int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
   237         -        p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken));
   238         -        zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
   239         -        if( !p || !zTemp ){
   240         -          goto no_mem;
   241         -        }
   242         -        if( ii==0 ){
   243         -          memset(p, 0, nByte);
   244         -          p->pPhrase = (Fts3Phrase *)&p[1];
   245         -        }
   246         -        p->pPhrase = (Fts3Phrase *)&p[1];
   247         -        memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken));
   248         -        p->pPhrase->nToken = ii+1;
   249         -        p->pPhrase->aToken[ii].n = nToken;
   250         -        memcpy(&zTemp[nTemp], zToken, nToken);
   251         -        nTemp += nToken;
   252         -        if( iEnd<nInput && zInput[iEnd]=='*' ){
   253         -          p->pPhrase->aToken[ii].isPrefix = 1;
   254         -        }else{
   255         -          p->pPhrase->aToken[ii].isPrefix = 0;
   256         -        }
          267  +        Fts3PhraseToken *pToken;
          268  +
          269  +        p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken));
          270  +        if( !p ) goto no_mem;
          271  +
          272  +        zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte);
          273  +        if( !zTemp ) goto no_mem;
          274  +
          275  +        assert( nToken==ii );
          276  +        pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii];
          277  +        memset(pToken, 0, sizeof(Fts3PhraseToken));
          278  +
          279  +        memcpy(&zTemp[nTemp], zByte, nByte);
          280  +        nTemp += nByte;
          281  +
          282  +        pToken->n = nByte;
          283  +        pToken->isPrefix = (iEnd<nInput && zInput[iEnd]=='*');
          284  +        nToken = ii+1;
   257    285         }
   258    286       }
   259    287   
   260    288       pModule->xClose(pCursor);
   261    289       pCursor = 0;
   262    290     }
   263    291   
   264    292     if( rc==SQLITE_DONE ){
   265    293       int jj;
   266         -    char *zNew = NULL;
   267         -    int nNew = 0;
   268         -    int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
   269         -    nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken);
   270         -    p = fts3ReallocOrFree(p, nByte + nTemp);
   271         -    if( !p ){
   272         -      goto no_mem;
   273         -    }
   274         -    if( zTemp ){
   275         -      zNew = &(((char *)p)[nByte]);
   276         -      memcpy(zNew, zTemp, nTemp);
   277         -    }else{
   278         -      memset(p, 0, nByte+nTemp);
   279         -    }
          294  +    char *zBuf = 0;
          295  +
          296  +    p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp);
          297  +    if( !p ) goto no_mem;
          298  +    memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p);
          299  +    p->eType = FTSQUERY_PHRASE;
   280    300       p->pPhrase = (Fts3Phrase *)&p[1];
          301  +    p->pPhrase->iColumn = pParse->iDefaultCol;
          302  +    p->pPhrase->nToken = nToken;
          303  +
          304  +    zBuf = (char *)&p->pPhrase->aToken[nToken];
          305  +    memcpy(zBuf, zTemp, nTemp);
          306  +    sqlite3_free(zTemp);
          307  +
   281    308       for(jj=0; jj<p->pPhrase->nToken; jj++){
   282         -      p->pPhrase->aToken[jj].z = &zNew[nNew];
   283         -      nNew += p->pPhrase->aToken[jj].n;
          309  +      p->pPhrase->aToken[jj].z = zBuf;
          310  +      zBuf += p->pPhrase->aToken[jj].n;
   284    311       }
   285         -    sqlite3_free(zTemp);
   286         -    p->eType = FTSQUERY_PHRASE;
   287         -    p->pPhrase->iColumn = pParse->iDefaultCol;
   288    312       rc = SQLITE_OK;
   289    313     }
   290    314   
   291    315     *ppExpr = p;
   292    316     return rc;
   293    317   no_mem:
   294    318   
................................................................................
   336    360     int iCol;
   337    361     int iColLen;
   338    362     int rc;
   339    363     Fts3Expr *pRet = 0;
   340    364   
   341    365     const char *zInput = z;
   342    366     int nInput = n;
          367  +
          368  +  pParse->isNot = 0;
   343    369   
   344    370     /* Skip over any whitespace before checking for a keyword, an open or
   345    371     ** close bracket, or a quoted string. 
   346    372     */
   347    373     while( nInput>0 && fts3isspace(*zInput) ){
   348    374       nInput--;
   349    375       zInput++;
................................................................................
   555    581       Fts3Expr *p = 0;
   556    582       int nByte = 0;
   557    583       rc = getNextNode(pParse, zIn, nIn, &p, &nByte);
   558    584       if( rc==SQLITE_OK ){
   559    585         int isPhrase;
   560    586   
   561    587         if( !sqlite3_fts3_enable_parentheses 
   562         -       && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot 
          588  +       && p->eType==FTSQUERY_PHRASE && pParse->isNot 
   563    589         ){
   564    590           /* Create an implicit NOT operator. */
   565    591           Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
   566    592           if( !pNot ){
   567    593             sqlite3Fts3ExprFree(p);
   568    594             rc = SQLITE_NOMEM;
   569    595             goto exprparse_out;
................................................................................
   573    599           if( pNotBranch ){
   574    600             pNot->pLeft = pNotBranch;
   575    601           }
   576    602           pNotBranch = pNot;
   577    603           p = pPrev;
   578    604         }else{
   579    605           int eType = p->eType;
   580         -        assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot );
   581    606           isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);
   582    607   
   583    608           /* The isRequirePhrase variable is set to true if a phrase or
   584    609           ** an expression contained in parenthesis is required. If a
   585    610           ** binary operator (AND, OR, NOT or NEAR) is encounted when
   586    611           ** isRequirePhrase is set, this is a syntax error.
   587    612           */
................................................................................
   736    761   }
   737    762   
   738    763   /*
   739    764   ** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse().
   740    765   */
   741    766   void sqlite3Fts3ExprFree(Fts3Expr *p){
   742    767     if( p ){
          768  +    assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 );
   743    769       sqlite3Fts3ExprFree(p->pLeft);
   744    770       sqlite3Fts3ExprFree(p->pRight);
   745         -    sqlite3_free(p->aDoclist);
          771  +    sqlite3Fts3EvalPhraseCleanup(p->pPhrase);
          772  +    sqlite3_free(p->aMI);
   746    773       sqlite3_free(p);
   747    774     }
   748    775   }
   749    776   
   750    777   /****************************************************************************
   751    778   *****************************************************************************
   752    779   ** Everything after this point is just test code.
................................................................................
   796    823   */
   797    824   static char *exprToString(Fts3Expr *pExpr, char *zBuf){
   798    825     switch( pExpr->eType ){
   799    826       case FTSQUERY_PHRASE: {
   800    827         Fts3Phrase *pPhrase = pExpr->pPhrase;
   801    828         int i;
   802    829         zBuf = sqlite3_mprintf(
   803         -          "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot);
          830  +          "%zPHRASE %d 0", zBuf, pPhrase->iColumn);
   804    831         for(i=0; zBuf && i<pPhrase->nToken; i++){
   805    832           zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, 
   806    833               pPhrase->aToken[i].n, pPhrase->aToken[i].z,
   807    834               (pPhrase->aToken[i].isPrefix?"+":"")
   808    835           );
   809    836         }
   810    837         return zBuf;

Changes to ext/fts3/fts3_snippet.c.

   172    172     int (*x)(Fts3Expr*,int,void*),  /* Callback function to invoke for phrases */
   173    173     void *pCtx                      /* Second argument to pass to callback */
   174    174   ){
   175    175     int iPhrase = 0;                /* Variable used as the phrase counter */
   176    176     return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
   177    177   }
   178    178   
   179         -/*
   180         -** The argument to this function is always a phrase node. Its doclist 
   181         -** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes
   182         -** to the left of this one in the query tree have already been loaded.
   183         -**
   184         -** If this phrase node is part of a series of phrase nodes joined by 
   185         -** NEAR operators (and is not the left-most of said series), then elements are
   186         -** removed from the phrases doclist consistent with the NEAR restriction. If
   187         -** required, elements may be removed from the doclists of phrases to the
   188         -** left of this one that are part of the same series of NEAR operator 
   189         -** connected phrases.
   190         -**
   191         -** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK.
   192         -*/
   193         -static int fts3ExprNearTrim(Fts3Expr *pExpr){
   194         -  int rc = SQLITE_OK;
   195         -  Fts3Expr *pParent = pExpr->pParent;
   196         -
   197         -  assert( pExpr->eType==FTSQUERY_PHRASE );
   198         -  while( rc==SQLITE_OK
   199         -   && pParent 
   200         -   && pParent->eType==FTSQUERY_NEAR 
   201         -   && pParent->pRight==pExpr 
   202         -  ){
   203         -    /* This expression (pExpr) is the right-hand-side of a NEAR operator. 
   204         -    ** Find the expression to the left of the same operator.
   205         -    */
   206         -    int nNear = pParent->nNear;
   207         -    Fts3Expr *pLeft = pParent->pLeft;
   208         -
   209         -    if( pLeft->eType!=FTSQUERY_PHRASE ){
   210         -      assert( pLeft->eType==FTSQUERY_NEAR );
   211         -      assert( pLeft->pRight->eType==FTSQUERY_PHRASE );
   212         -      pLeft = pLeft->pRight;
   213         -    }
   214         -
   215         -    rc = sqlite3Fts3ExprNearTrim(pLeft, pExpr, nNear);
   216         -
   217         -    pExpr = pLeft;
   218         -    pParent = pExpr->pParent;
   219         -  }
   220         -
   221         -  return rc;
   222         -}
   223         -
   224    179   /*
   225    180   ** This is an fts3ExprIterate() callback used while loading the doclists
   226    181   ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
   227    182   ** fts3ExprLoadDoclists().
   228    183   */
   229    184   static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
   230    185     int rc = SQLITE_OK;
          186  +  Fts3Phrase *pPhrase = pExpr->pPhrase;
   231    187     LoadDoclistCtx *p = (LoadDoclistCtx *)ctx;
   232    188   
   233    189     UNUSED_PARAMETER(iPhrase);
   234    190   
   235    191     p->nPhrase++;
   236         -  p->nToken += pExpr->pPhrase->nToken;
   237         -
   238         -  if( pExpr->isLoaded==0 ){
   239         -    rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr);
   240         -    pExpr->isLoaded = 1;
   241         -    if( rc==SQLITE_OK ){
   242         -      rc = fts3ExprNearTrim(pExpr);
   243         -    }
   244         -  }
          192  +  p->nToken += pPhrase->nToken;
   245    193   
   246    194     return rc;
   247    195   }
   248    196   
   249    197   /*
   250    198   ** Load the doclists for each phrase in the query associated with FTS3 cursor
   251    199   ** pCsr. 
................................................................................
   411    359   static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
   412    360     SnippetIter *p = (SnippetIter *)ctx;
   413    361     SnippetPhrase *pPhrase = &p->aPhrase[iPhrase];
   414    362     char *pCsr;
   415    363   
   416    364     pPhrase->nToken = pExpr->pPhrase->nToken;
   417    365   
   418         -  pCsr = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->pCsr->iPrevId, p->iCol);
          366  +  pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
   419    367     if( pCsr ){
   420    368       int iFirst = 0;
   421    369       pPhrase->pList = pCsr;
   422    370       fts3GetDeltaPosition(&pCsr, &iFirst);
   423    371       pPhrase->pHead = pCsr;
   424    372       pPhrase->pTail = pCsr;
   425    373       pPhrase->iHead = iFirst;
................................................................................
   768    716       if( !c ) nEntry++;
   769    717     }
   770    718   
   771    719     *ppCollist = pEnd;
   772    720     return nEntry;
   773    721   }
   774    722   
   775         -static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){
   776         -  char *pCsr = *pp;
   777         -  while( *pCsr ){
   778         -    int nHit;
   779         -    sqlite3_int64 iCol = 0;
   780         -    if( *pCsr==0x01 ){
   781         -      pCsr++;
   782         -      pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
   783         -    }
   784         -    nHit = fts3ColumnlistCount(&pCsr);
   785         -    assert( nHit>0 );
   786         -    if( isGlobal ){
   787         -      aOut[iCol*3+1]++;
   788         -    }
   789         -    aOut[iCol*3] += nHit;
   790         -  }
   791         -  pCsr++;
   792         -  *pp = pCsr;
   793         -}
   794         -
   795    723   /*
   796    724   ** fts3ExprIterate() callback used to collect the "global" matchinfo stats
   797    725   ** for a single query. 
   798    726   **
   799    727   ** fts3ExprIterate() callback to load the 'global' elements of a
   800    728   ** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements 
   801    729   ** of the matchinfo array that are constant for all rows returned by the 
................................................................................
   821    749   */
   822    750   static int fts3ExprGlobalHitsCb(
   823    751     Fts3Expr *pExpr,                /* Phrase expression node */
   824    752     int iPhrase,                    /* Phrase number (numbered from zero) */
   825    753     void *pCtx                      /* Pointer to MatchInfo structure */
   826    754   ){
   827    755     MatchInfo *p = (MatchInfo *)pCtx;
   828         -  Fts3Cursor *pCsr = p->pCursor;
   829         -  char *pIter;
   830         -  char *pEnd;
   831         -  char *pFree = 0;
   832         -  u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol];
   833         -
   834         -  assert( pExpr->isLoaded );
   835         -  assert( pExpr->eType==FTSQUERY_PHRASE );
   836         -
   837         -  if( pCsr->pDeferred ){
   838         -    Fts3Phrase *pPhrase = pExpr->pPhrase;
   839         -    int ii;
   840         -    for(ii=0; ii<pPhrase->nToken; ii++){
   841         -      if( pPhrase->aToken[ii].bFulltext ) break;
   842         -    }
   843         -    if( ii<pPhrase->nToken ){
   844         -      int nFree = 0;
   845         -      int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
   846         -      if( rc!=SQLITE_OK ) return rc;
   847         -      pIter = pFree;
   848         -      pEnd = &pFree[nFree];
   849         -    }else{
   850         -      int iCol;                   /* Column index */
   851         -      for(iCol=0; iCol<p->nCol; iCol++){
   852         -        aOut[iCol*3 + 1] = (u32)p->nDoc;
   853         -        aOut[iCol*3 + 2] = (u32)p->nDoc;
   854         -      }
   855         -      return SQLITE_OK;
   856         -    }
   857         -  }else{
   858         -    pIter = pExpr->aDoclist;
   859         -    pEnd = &pExpr->aDoclist[pExpr->nDoclist];
   860         -  }
   861         -
   862         -  /* Fill in the global hit count matrix row for this phrase. */
   863         -  while( pIter<pEnd ){
   864         -    while( *pIter++ & 0x80 );      /* Skip past docid. */
   865         -    fts3LoadColumnlistCounts(&pIter, &aOut[1], 1);
   866         -  }
   867         -
   868         -  sqlite3_free(pFree);
   869         -  return SQLITE_OK;
          756  +  return sqlite3Fts3EvalPhraseStats(
          757  +      p->pCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol]
          758  +  );
   870    759   }
   871    760   
   872    761   /*
   873    762   ** fts3ExprIterate() callback used to collect the "local" part of the
   874    763   ** FTS3_MATCHINFO_HITS array. The local stats are those elements of the 
   875    764   ** array that are different for each row returned by the query.
   876    765   */
................................................................................
   879    768     int iPhrase,                    /* Phrase number */
   880    769     void *pCtx                      /* Pointer to MatchInfo structure */
   881    770   ){
   882    771     MatchInfo *p = (MatchInfo *)pCtx;
   883    772     int iStart = iPhrase * p->nCol * 3;
   884    773     int i;
   885    774   
   886         -  for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0;
   887         -
   888         -  if( pExpr->aDoclist ){
          775  +  for(i=0; i<p->nCol; i++){
   889    776       char *pCsr;
   890         -
   891         -    pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1);
          777  +    pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i);
   892    778       if( pCsr ){
   893         -      fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0);
          779  +      p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr);
          780  +    }else{
          781  +      p->aMatchinfo[iStart+i*3] = 0;
   894    782       }
   895    783     }
   896    784   
   897    785     return SQLITE_OK;
   898    786   }
   899    787   
   900    788   static int fts3MatchinfoCheck(
................................................................................
   972    860   ** iterating through a multi-column position-list corresponding to the
   973    861   ** hits for a single phrase on a single row in order to calculate the
   974    862   ** values for a matchinfo() FTS3_MATCHINFO_LCS request.
   975    863   */
   976    864   typedef struct LcsIterator LcsIterator;
   977    865   struct LcsIterator {
   978    866     Fts3Expr *pExpr;                /* Pointer to phrase expression */
   979         -  char *pRead;                    /* Cursor used to iterate through aDoclist */
   980    867     int iPosOffset;                 /* Tokens count up to end of this phrase */
   981         -  int iCol;                       /* Current column number */
          868  +  char *pRead;                    /* Cursor used to iterate through aDoclist */
   982    869     int iPos;                       /* Current position */
   983    870   };
   984    871   
   985    872   /* 
   986    873   ** If LcsIterator.iCol is set to the following value, the iterator has
   987    874   ** finished iterating through all offsets for all columns.
   988    875   */
................................................................................
  1005    892   */
  1006    893   static int fts3LcsIteratorAdvance(LcsIterator *pIter){
  1007    894     char *pRead = pIter->pRead;
  1008    895     sqlite3_int64 iRead;
  1009    896     int rc = 0;
  1010    897   
  1011    898     pRead += sqlite3Fts3GetVarint(pRead, &iRead);
  1012         -  if( iRead==0 ){
  1013         -    pIter->iCol = LCS_ITERATOR_FINISHED;
          899  +  if( iRead==0 || iRead==1 ){
          900  +    pRead = 0;
  1014    901       rc = 1;
  1015    902     }else{
  1016         -    if( iRead==1 ){
  1017         -      pRead += sqlite3Fts3GetVarint(pRead, &iRead);
  1018         -      pIter->iCol = (int)iRead;
  1019         -      pIter->iPos = pIter->iPosOffset;
  1020         -      pRead += sqlite3Fts3GetVarint(pRead, &iRead);
  1021         -      rc = 1;
  1022         -    }
  1023    903       pIter->iPos += (int)(iRead-2);
  1024    904     }
  1025    905   
  1026    906     pIter->pRead = pRead;
  1027    907     return rc;
  1028    908   }
  1029    909     
................................................................................
  1047    927     /* Allocate and populate the array of LcsIterator objects. The array
  1048    928     ** contains one element for each matchable phrase in the query.
  1049    929     **/
  1050    930     aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase);
  1051    931     if( !aIter ) return SQLITE_NOMEM;
  1052    932     memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
  1053    933     (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
          934  +
  1054    935     for(i=0; i<pInfo->nPhrase; i++){
  1055    936       LcsIterator *pIter = &aIter[i];
  1056    937       nToken -= pIter->pExpr->pPhrase->nToken;
  1057    938       pIter->iPosOffset = nToken;
  1058         -    pIter->pRead = sqlite3Fts3FindPositions(pCsr,pIter->pExpr,pCsr->iPrevId,-1);
  1059         -    if( pIter->pRead ){
  1060         -      pIter->iPos = pIter->iPosOffset;
  1061         -      fts3LcsIteratorAdvance(&aIter[i]);
  1062         -    }else{
  1063         -      pIter->iCol = LCS_ITERATOR_FINISHED;
  1064         -    }
  1065    939     }
  1066    940   
  1067    941     for(iCol=0; iCol<pInfo->nCol; iCol++){
  1068    942       int nLcs = 0;                 /* LCS value for this column */
  1069    943       int nLive = 0;                /* Number of iterators in aIter not at EOF */
  1070    944   
  1071         -    /* Loop through the iterators in aIter[]. Set nLive to the number of
  1072         -    ** iterators that point to a position-list corresponding to column iCol.
  1073         -    */
  1074    945       for(i=0; i<pInfo->nPhrase; i++){
  1075         -      assert( aIter[i].iCol>=iCol );
  1076         -      if( aIter[i].iCol==iCol ) nLive++;
          946  +      LcsIterator *pIt = &aIter[i];
          947  +      pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol);
          948  +      if( pIt->pRead ){
          949  +        pIt->iPos = pIt->iPosOffset;
          950  +        fts3LcsIteratorAdvance(&aIter[i]);
          951  +        nLive++;
          952  +      }
  1077    953       }
  1078    954   
  1079         -    /* The following loop runs until all iterators in aIter[] have finished
  1080         -    ** iterating through positions in column iCol. Exactly one of the 
  1081         -    ** iterators is advanced each time the body of the loop is run.
  1082         -    */
  1083    955       while( nLive>0 ){
  1084    956         LcsIterator *pAdv = 0;      /* The iterator to advance by one position */
  1085    957         int nThisLcs = 0;           /* LCS for the current iterator positions */
  1086    958   
  1087    959         for(i=0; i<pInfo->nPhrase; i++){
  1088    960           LcsIterator *pIter = &aIter[i];
  1089         -        if( iCol!=pIter->iCol ){  
          961  +        if( pIter->pRead==0 ){
  1090    962             /* This iterator is already at EOF for this column. */
  1091    963             nThisLcs = 0;
  1092    964           }else{
  1093    965             if( pAdv==0 || pIter->iPos<pAdv->iPos ){
  1094    966               pAdv = pIter;
  1095    967             }
  1096    968             if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){
................................................................................
  1422   1294     TermOffsetCtx *p = (TermOffsetCtx *)ctx;
  1423   1295     int nTerm;                      /* Number of tokens in phrase */
  1424   1296     int iTerm;                      /* For looping through nTerm phrase terms */
  1425   1297     char *pList;                    /* Pointer to position list for phrase */
  1426   1298     int iPos = 0;                   /* First position in position-list */
  1427   1299   
  1428   1300     UNUSED_PARAMETER(iPhrase);
  1429         -  pList = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->iDocid, p->iCol);
         1301  +  pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
  1430   1302     nTerm = pExpr->pPhrase->nToken;
  1431   1303     if( pList ){
  1432   1304       fts3GetDeltaPosition(&pList, &iPos);
  1433   1305       assert( iPos>=0 );
  1434   1306     }
  1435   1307   
  1436   1308     for(iTerm=0; iTerm<nTerm; iTerm++){

Changes to ext/fts3/fts3_term.c.

    23     23   #include <assert.h>
    24     24   
    25     25   typedef struct Fts3termTable Fts3termTable;
    26     26   typedef struct Fts3termCursor Fts3termCursor;
    27     27   
    28     28   struct Fts3termTable {
    29     29     sqlite3_vtab base;              /* Base class used by SQLite core */
           30  +  int iIndex;                     /* Index for Fts3Table.aIndex[] */
    30     31     Fts3Table *pFts3Tab;
    31     32   };
    32     33   
    33     34   struct Fts3termCursor {
    34     35     sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
    35         -  Fts3SegReaderCursor csr;        /* Must be right after "base" */
           36  +  Fts3MultiSegReader csr;        /* Must be right after "base" */
    36     37     Fts3SegFilter filter;
    37     38   
    38     39     int isEof;                      /* True if cursor is at EOF */
    39     40     char *pNext;
    40     41   
    41     42     sqlite3_int64 iRowid;           /* Current 'rowid' value */
    42     43     sqlite3_int64 iDocid;           /* Current 'docid' value */
................................................................................
    52     53   /*
    53     54   ** This function does all the work for both the xConnect and xCreate methods.
    54     55   ** These tables have no persistent representation of their own, so xConnect
    55     56   ** and xCreate are identical operations.
    56     57   */
    57     58   static int fts3termConnectMethod(
    58     59     sqlite3 *db,                    /* Database connection */
    59         -  void *pUnused,                  /* Unused */
           60  +  void *pCtx,                     /* Non-zero for an fts4prefix table */
    60     61     int argc,                       /* Number of elements in argv array */
    61     62     const char * const *argv,       /* xCreate/xConnect argument array */
    62     63     sqlite3_vtab **ppVtab,          /* OUT: New sqlite3_vtab object */
    63     64     char **pzErr                    /* OUT: sqlite3_malloc'd error message */
    64     65   ){
    65     66     char const *zDb;                /* Name of database (e.g. "main") */
    66     67     char const *zFts3;              /* Name of fts3 table */
    67     68     int nDb;                        /* Result of strlen(zDb) */
    68     69     int nFts3;                      /* Result of strlen(zFts3) */
    69     70     int nByte;                      /* Bytes of space to allocate here */
    70     71     int rc;                         /* value returned by declare_vtab() */
    71     72     Fts3termTable *p;                /* Virtual table object to return */
           73  +  int iIndex = 0;
    72     74   
    73         -  UNUSED_PARAMETER(pUnused);
           75  +  if( argc==5 ){
           76  +    iIndex = atoi(argv[4]);
           77  +    argc--;
           78  +  }
    74     79   
    75     80     /* The user should specify a single argument - the name of an fts3 table. */
    76     81     if( argc!=4 ){
    77     82       *pzErr = sqlite3_mprintf(
    78     83           "wrong number of arguments to fts4term constructor"
    79     84       );
    80     85       return SQLITE_ERROR;
................................................................................
    93     98     if( !p ) return SQLITE_NOMEM;
    94     99     memset(p, 0, nByte);
    95    100   
    96    101     p->pFts3Tab = (Fts3Table *)&p[1];
    97    102     p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
    98    103     p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
    99    104     p->pFts3Tab->db = db;
          105  +  p->pFts3Tab->nIndex = iIndex+1;
          106  +  p->iIndex = iIndex;
   100    107   
   101    108     memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
   102    109     memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
   103    110     sqlite3Fts3Dequote((char *)p->pFts3Tab->zName);
   104    111   
   105    112     *ppVtab = (sqlite3_vtab *)p;
   106    113     return SQLITE_OK;
................................................................................
   240    247     sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
   241    248     int idxNum,                     /* Strategy index */
   242    249     const char *idxStr,             /* Unused */
   243    250     int nVal,                       /* Number of elements in apVal */
   244    251     sqlite3_value **apVal           /* Arguments for the indexing scheme */
   245    252   ){
   246    253     Fts3termCursor *pCsr = (Fts3termCursor *)pCursor;
   247         -  Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab;
          254  +  Fts3termTable *p = (Fts3termTable *)pCursor->pVtab;
          255  +  Fts3Table *pFts3 = p->pFts3Tab;
   248    256     int rc;
   249    257   
   250    258     UNUSED_PARAMETER(nVal);
   251    259     UNUSED_PARAMETER(idxNum);
   252    260     UNUSED_PARAMETER(idxStr);
   253    261     UNUSED_PARAMETER(apVal);
   254    262   
................................................................................
   258    266     testcase(pCsr->filter.zTerm);
   259    267     sqlite3Fts3SegReaderFinish(&pCsr->csr);
   260    268     memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr);
   261    269   
   262    270     pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
   263    271     pCsr->filter.flags |= FTS3_SEGMENT_SCAN;
   264    272   
   265         -  rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL,
          273  +  rc = sqlite3Fts3SegReaderCursor(pFts3, p->iIndex, FTS3_SEGCURSOR_ALL,
   266    274         pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr
   267    275     );
   268    276     if( rc==SQLITE_OK ){
   269    277       rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
   270    278     }
   271    279     if( rc==SQLITE_OK ){
   272    280       rc = fts3termNextMethod(pCursor);

Added ext/fts3/fts3_test.c.

            1  +/*
            2  +** 2011 Jun 13
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** This file is not part of the production FTS code. It is only used for
           14  +** testing. It contains a Tcl command that can be used to test if a document
           15  +** matches an FTS NEAR expression.
           16  +*/
           17  +
           18  +#include <tcl.h>
           19  +#include <string.h>
           20  +#include <assert.h>
           21  +
           22  +#define NM_MAX_TOKEN 12
           23  +
           24  +typedef struct NearPhrase NearPhrase;
           25  +typedef struct NearDocument NearDocument;
           26  +typedef struct NearToken NearToken;
           27  +
           28  +struct NearDocument {
           29  +  int nToken;                     /* Length of token in bytes */
           30  +  NearToken *aToken;              /* Token array */
           31  +};
           32  +
           33  +struct NearToken {
           34  +  int n;                          /* Length of token in bytes */
           35  +  const char *z;                  /* Pointer to token string */
           36  +};
           37  +
           38  +struct NearPhrase {
           39  +  int nNear;                      /* Preceding NEAR value */
           40  +  int nToken;                     /* Number of tokens in this phrase */
           41  +  NearToken aToken[NM_MAX_TOKEN]; /* Array of tokens in this phrase */
           42  +};
           43  +
           44  +static int nm_phrase_match(
           45  +  NearPhrase *p,
           46  +  NearToken *aToken
           47  +){
           48  +  int ii;
           49  +
           50  +  for(ii=0; ii<p->nToken; ii++){
           51  +    NearToken *pToken = &p->aToken[ii];
           52  +    if( pToken->n>0 && pToken->z[pToken->n-1]=='*' ){
           53  +      if( aToken[ii].n<(pToken->n-1) ) return 0;
           54  +      if( memcmp(aToken[ii].z, pToken->z, pToken->n-1) ) return 0;
           55  +    }else{
           56  +      if( aToken[ii].n!=pToken->n ) return 0;
           57  +      if( memcmp(aToken[ii].z, pToken->z, pToken->n) ) return 0;
           58  +    }
           59  +  }
           60  +
           61  +  return 1;
           62  +}
           63  +
           64  +static int nm_near_chain(
           65  +  int iDir,                       /* Direction to iterate through aPhrase[] */
           66  +  NearDocument *pDoc,             /* Document to match against */
           67  +  int iPos,                       /* Position at which iPhrase was found */
           68  +  int nPhrase,                    /* Size of phrase array */
           69  +  NearPhrase *aPhrase,            /* Phrase array */
           70  +  int iPhrase                     /* Index of phrase found */
           71  +){
           72  +  int iStart;
           73  +  int iStop;
           74  +  int ii;
           75  +  int nNear;
           76  +  int iPhrase2;
           77  +  NearPhrase *p;
           78  +  NearPhrase *pPrev;
           79  +
           80  +  assert( iDir==1 || iDir==-1 );
           81  +
           82  +  if( iDir==1 ){
           83  +    if( (iPhrase+1)==nPhrase ) return 1;
           84  +    nNear = aPhrase[iPhrase+1].nNear;
           85  +  }else{
           86  +    if( iPhrase==0 ) return 1;
           87  +    nNear = aPhrase[iPhrase].nNear;
           88  +  }
           89  +  pPrev = &aPhrase[iPhrase];
           90  +  iPhrase2 = iPhrase+iDir;
           91  +  p = &aPhrase[iPhrase2];
           92  +
           93  +  iStart = iPos - nNear - p->nToken;
           94  +  iStop = iPos + nNear + pPrev->nToken;
           95  +
           96  +  if( iStart<0 ) iStart = 0;
           97  +  if( iStop > pDoc->nToken - p->nToken ) iStop = pDoc->nToken - p->nToken;
           98  +
           99  +  for(ii=iStart; ii<=iStop; ii++){
          100  +    if( nm_phrase_match(p, &pDoc->aToken[ii]) ){
          101  +      if( nm_near_chain(iDir, pDoc, ii, nPhrase, aPhrase, iPhrase2) ) return 1;
          102  +    }
          103  +  }
          104  +
          105  +  return 0;
          106  +}
          107  +
          108  +static int nm_match_count(
          109  +  NearDocument *pDoc,             /* Document to match against */
          110  +  int nPhrase,                    /* Size of phrase array */
          111  +  NearPhrase *aPhrase,            /* Phrase array */
          112  +  int iPhrase                     /* Index of phrase to count matches for */
          113  +){
          114  +  int nOcc = 0;
          115  +  int ii;
          116  +  NearPhrase *p = &aPhrase[iPhrase];
          117  +
          118  +  for(ii=0; ii<(pDoc->nToken + 1 - p->nToken); ii++){
          119  +    if( nm_phrase_match(p, &pDoc->aToken[ii]) ){
          120  +      /* Test forward NEAR chain (i>iPhrase) */
          121  +      if( 0==nm_near_chain(1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue;
          122  +
          123  +      /* Test reverse NEAR chain (i<iPhrase) */
          124  +      if( 0==nm_near_chain(-1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue;
          125  +
          126  +      /* This is a real match. Increment the counter. */
          127  +      nOcc++;
          128  +    }
          129  +  } 
          130  +
          131  +  return nOcc;
          132  +}
          133  +
          134  +/*
          135  +** Tclcmd: fts3_near_match DOCUMENT EXPR ?OPTIONS?
          136  +*/
          137  +static int fts3_near_match_cmd(
          138  +  ClientData clientData,
          139  +  Tcl_Interp *interp,
          140  +  int objc,
          141  +  Tcl_Obj *CONST objv[]
          142  +){
          143  +  int nTotal = 0;
          144  +  int rc;
          145  +  int ii;
          146  +  int nPhrase;
          147  +  NearPhrase *aPhrase = 0;
          148  +  NearDocument doc = {0, 0};
          149  +  Tcl_Obj **apDocToken;
          150  +  Tcl_Obj *pRet;
          151  +  Tcl_Obj *pPhrasecount = 0;
          152  +  
          153  +  Tcl_Obj **apExprToken;
          154  +  int nExprToken;
          155  +
          156  +  /* Must have 3 or more arguments. */
          157  +  if( objc<3 || (objc%2)==0 ){
          158  +    Tcl_WrongNumArgs(interp, 1, objv, "DOCUMENT EXPR ?OPTION VALUE?...");
          159  +    rc = TCL_ERROR;
          160  +    goto near_match_out;
          161  +  }
          162  +
          163  +  for(ii=3; ii<objc; ii+=2){
          164  +    enum NM_enum { NM_PHRASECOUNTS };
          165  +    struct TestnmSubcmd {
          166  +      char *zName;
          167  +      enum NM_enum eOpt;
          168  +    } aOpt[] = {
          169  +      { "-phrasecountvar", NM_PHRASECOUNTS },
          170  +      { 0, 0 }
          171  +    };
          172  +    int iOpt;
          173  +    if( Tcl_GetIndexFromObjStruct(
          174  +        interp, objv[ii], aOpt, sizeof(aOpt[0]), "option", 0, &iOpt) 
          175  +    ){
          176  +      return TCL_ERROR;
          177  +    }
          178  +
          179  +    switch( aOpt[iOpt].eOpt ){
          180  +      case NM_PHRASECOUNTS:
          181  +        pPhrasecount = objv[ii+1];
          182  +        break;
          183  +    }
          184  +  }
          185  +
          186  +  rc = Tcl_ListObjGetElements(interp, objv[1], &doc.nToken, &apDocToken);
          187  +  if( rc!=TCL_OK ) goto near_match_out;
          188  +  doc.aToken = (NearToken *)ckalloc(doc.nToken*sizeof(NearToken));
          189  +  for(ii=0; ii<doc.nToken; ii++){
          190  +    doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &doc.aToken[ii].n);
          191  +  }
          192  +
          193  +  rc = Tcl_ListObjGetElements(interp, objv[2], &nExprToken, &apExprToken);
          194  +  if( rc!=TCL_OK ) goto near_match_out;
          195  +
          196  +  nPhrase = (nExprToken + 1) / 2;
          197  +  aPhrase = (NearPhrase *)ckalloc(nPhrase * sizeof(NearPhrase));
          198  +  memset(aPhrase, 0, nPhrase * sizeof(NearPhrase));
          199  +  for(ii=0; ii<nPhrase; ii++){
          200  +    Tcl_Obj *pPhrase = apExprToken[ii*2];
          201  +    Tcl_Obj **apToken;
          202  +    int nToken;
          203  +    int jj;
          204  +
          205  +    rc = Tcl_ListObjGetElements(interp, pPhrase, &nToken, &apToken);
          206  +    if( rc!=TCL_OK ) goto near_match_out;
          207  +    if( nToken>NM_MAX_TOKEN ){
          208  +      Tcl_AppendResult(interp, "Too many tokens in phrase", 0);
          209  +      rc = TCL_ERROR;
          210  +      goto near_match_out;
          211  +    }
          212  +    for(jj=0; jj<nToken; jj++){
          213  +      NearToken *pT = &aPhrase[ii].aToken[jj];
          214  +      pT->z = Tcl_GetStringFromObj(apToken[jj], &pT->n);
          215  +    }
          216  +    aPhrase[ii].nToken = nToken;
          217  +  }
          218  +  for(ii=1; ii<nPhrase; ii++){
          219  +    Tcl_Obj *pNear = apExprToken[2*ii-1];
          220  +    int nNear;
          221  +    rc = Tcl_GetIntFromObj(interp, pNear, &nNear);
          222  +    if( rc!=TCL_OK ) goto near_match_out;
          223  +    aPhrase[ii].nNear = nNear;
          224  +  }
          225  +
          226  +  pRet = Tcl_NewObj();
          227  +  Tcl_IncrRefCount(pRet);
          228  +  for(ii=0; ii<nPhrase; ii++){
          229  +    int nOcc = nm_match_count(&doc, nPhrase, aPhrase, ii);
          230  +    Tcl_ListObjAppendElement(interp, pRet, Tcl_NewIntObj(nOcc));
          231  +    nTotal += nOcc;
          232  +  }
          233  +  if( pPhrasecount ){
          234  +    Tcl_ObjSetVar2(interp, pPhrasecount, 0, pRet, 0);
          235  +  }
          236  +  Tcl_DecrRefCount(pRet);
          237  +  Tcl_SetObjResult(interp, Tcl_NewBooleanObj(nTotal>0));
          238  +
          239  + near_match_out: 
          240  +  ckfree((char *)aPhrase);
          241  +  ckfree((char *)doc.aToken);
          242  +  return rc;
          243  +}
          244  +
          245  +int Sqlitetestfts3_Init(Tcl_Interp *interp){
          246  +  Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);
          247  +  return TCL_OK;
          248  +}
          249  +

Changes to ext/fts3/fts3_write.c.

    32     32   **
    33     33   ** This means that if we have a pointer into a buffer containing node data,
    34     34   ** it is always safe to read up to two varints from it without risking an
    35     35   ** overread, even if the node data is corrupted.
    36     36   */
    37     37   #define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2)
    38     38   
           39  +/*
           40  +** Under certain circumstances, b-tree nodes (doclists) can be loaded into
           41  +** memory incrementally instead of all at once. This can be a big performance
           42  +** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext()
           43  +** method before retrieving all query results (as may happen, for example,
           44  +** if a query has a LIMIT clause).
           45  +**
           46  +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD 
           47  +** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes.
           48  +** The code is written so that the hard lower-limit for each of these values 
           49  +** is 1. Clearly such small values would be inefficient, but can be useful 
           50  +** for testing purposes.
           51  +**
           52  +** TODO: Add a test interface to modify these "constants" from a script for
           53  +** this purpose.
           54  +*/
           55  +#define FTS3_NODE_CHUNKSIZE (4*1024) 
           56  +#define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4)
           57  +/* #define FTS3_NODE_CHUNKSIZE 1 */
           58  +/* #define FTS3_NODE_CHUNK_THRESHOLD 1 */
           59  +
    39     60   typedef struct PendingList PendingList;
    40     61   typedef struct SegmentNode SegmentNode;
    41     62   typedef struct SegmentWriter SegmentWriter;
    42     63   
    43     64   /*
    44         -** Data structure used while accumulating terms in the pending-terms hash
    45         -** table. The hash table entry maps from term (a string) to a malloc'd
    46         -** instance of this structure.
           65  +** An instance of the following data structure is used to build doclists
           66  +** incrementally. See function fts3PendingListAppend() for details.
    47     67   */
    48     68   struct PendingList {
    49     69     int nData;
    50     70     char *aData;
    51     71     int nSpace;
    52     72     sqlite3_int64 iLastDocid;
    53     73     sqlite3_int64 iLastCol;
................................................................................
    70     90   ** a contiguous set of segment b-tree leaf nodes. Although the details of
    71     91   ** this structure are only manipulated by code in this file, opaque handles
    72     92   ** of type Fts3SegReader* are also used by code in fts3.c to iterate through
    73     93   ** terms when querying the full-text index. See functions:
    74     94   **
    75     95   **   sqlite3Fts3SegReaderNew()
    76     96   **   sqlite3Fts3SegReaderFree()
    77         -**   sqlite3Fts3SegReaderCost()
    78     97   **   sqlite3Fts3SegReaderIterate()
    79     98   **
    80     99   ** Methods used to manipulate Fts3SegReader structures:
    81    100   **
    82    101   **   fts3SegReaderNext()
    83    102   **   fts3SegReaderFirstDocid()
    84    103   **   fts3SegReaderNextDocid()
................................................................................
    89    108     sqlite3_int64 iStartBlock;      /* Rowid of first leaf block to traverse */
    90    109     sqlite3_int64 iLeafEndBlock;    /* Rowid of final leaf block to traverse */
    91    110     sqlite3_int64 iEndBlock;        /* Rowid of final block in segment (or 0) */
    92    111     sqlite3_int64 iCurrentBlock;    /* Current leaf block (or 0) */
    93    112   
    94    113     char *aNode;                    /* Pointer to node data (or NULL) */
    95    114     int nNode;                      /* Size of buffer at aNode (or 0) */
          115  +  int nPopulate;                  /* If >0, bytes of buffer aNode[] loaded */
          116  +  sqlite3_blob *pBlob;            /* If not NULL, blob handle to read node */
          117  +
    96    118     Fts3HashElem **ppNextElem;
    97    119   
    98    120     /* Variables set by fts3SegReaderNext(). These may be read directly
    99    121     ** by the caller. They are valid from the time SegmentReaderNew() returns
   100    122     ** until SegmentReaderNext() returns something other than SQLITE_OK
   101    123     ** (i.e. SQLITE_DONE).
   102    124     */
   103    125     int nTerm;                      /* Number of bytes in current term */
   104    126     char *zTerm;                    /* Pointer to current term */
   105    127     int nTermAlloc;                 /* Allocated size of zTerm buffer */
   106    128     char *aDoclist;                 /* Pointer to doclist of current entry */
   107    129     int nDoclist;                   /* Size of doclist in current entry */
   108    130   
   109         -  /* The following variables are used to iterate through the current doclist */
          131  +  /* The following variables are used by fts3SegReaderNextDocid() to iterate 
          132  +  ** through the current doclist (aDoclist/nDoclist).
          133  +  */
   110    134     char *pOffsetList;
          135  +  int nOffsetList;                /* For descending pending seg-readers only */
   111    136     sqlite3_int64 iDocid;
   112    137   };
   113    138   
   114    139   #define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0)
   115    140   #define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1])
   116    141   
   117    142   /*
................................................................................
   141    166   ** the interior part of the segment b+-tree structures (everything except
   142    167   ** the leaf nodes). These functions and type are only ever used by code
   143    168   ** within the fts3SegWriterXXX() family of functions described above.
   144    169   **
   145    170   **   fts3NodeAddTerm()
   146    171   **   fts3NodeWrite()
   147    172   **   fts3NodeFree()
          173  +**
          174  +** When a b+tree is written to the database (either as a result of a merge
          175  +** or the pending-terms table being flushed), leaves are written into the 
          176  +** database file as soon as they are completely populated. The interior of
          177  +** the tree is assembled in memory and written out only once all leaves have
          178  +** been populated and stored. This is Ok, as the b+-tree fanout is usually
          179  +** very large, meaning that the interior of the tree consumes relatively 
          180  +** little memory.
   148    181   */
   149    182   struct SegmentNode {
   150    183     SegmentNode *pParent;           /* Parent node (or NULL for root node) */
   151    184     SegmentNode *pRight;            /* Pointer to right-sibling */
   152    185     SegmentNode *pLeftmost;         /* Pointer to left-most node of this depth */
   153    186     int nEntry;                     /* Number of terms written to node so far */
   154    187     char *zTerm;                    /* Pointer to previous term buffer */
................................................................................
   171    204   #define SQL_DELETE_ALL_STAT            6
   172    205   #define SQL_SELECT_CONTENT_BY_ROWID    7
   173    206   #define SQL_NEXT_SEGMENT_INDEX         8
   174    207   #define SQL_INSERT_SEGMENTS            9
   175    208   #define SQL_NEXT_SEGMENTS_ID          10
   176    209   #define SQL_INSERT_SEGDIR             11
   177    210   #define SQL_SELECT_LEVEL              12
   178         -#define SQL_SELECT_ALL_LEVEL          13
          211  +#define SQL_SELECT_LEVEL_RANGE        13
   179    212   #define SQL_SELECT_LEVEL_COUNT        14
   180         -#define SQL_SELECT_SEGDIR_COUNT_MAX   15
   181         -#define SQL_DELETE_SEGDIR_BY_LEVEL    16
          213  +#define SQL_SELECT_SEGDIR_MAX_LEVEL   15
          214  +#define SQL_DELETE_SEGDIR_LEVEL       16
   182    215   #define SQL_DELETE_SEGMENTS_RANGE     17
   183    216   #define SQL_CONTENT_INSERT            18
   184    217   #define SQL_DELETE_DOCSIZE            19
   185    218   #define SQL_REPLACE_DOCSIZE           20
   186    219   #define SQL_SELECT_DOCSIZE            21
   187    220   #define SQL_SELECT_DOCTOTAL           22
   188    221   #define SQL_REPLACE_DOCTOTAL          23
          222  +
          223  +#define SQL_SELECT_ALL_PREFIX_LEVEL   24
          224  +#define SQL_DELETE_ALL_TERMS_SEGDIR   25
          225  +
          226  +#define SQL_DELETE_SEGDIR_RANGE       26
   189    227   
   190    228   /*
   191    229   ** This function is used to obtain an SQLite prepared statement handle
   192    230   ** for the statement identified by the second argument. If successful,
   193    231   ** *pp is set to the requested statement handle and SQLITE_OK returned.
   194    232   ** Otherwise, an SQLite error code is returned and *pp is set to 0.
   195    233   **
................................................................................
   218    256   /* 10 */  "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)",
   219    257   /* 11 */  "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",
   220    258   
   221    259             /* Return segments in order from oldest to newest.*/ 
   222    260   /* 12 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
   223    261               "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
   224    262   /* 13 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
   225         -            "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC",
          263  +            "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?"
          264  +            "ORDER BY level DESC, idx ASC",
   226    265   
   227    266   /* 14 */  "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
   228         -/* 15 */  "SELECT count(*), max(level) FROM %Q.'%q_segdir'",
          267  +/* 15 */  "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
   229    268   
   230    269   /* 16 */  "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
   231    270   /* 17 */  "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
   232    271   /* 18 */  "INSERT INTO %Q.'%q_content' VALUES(%s)",
   233    272   /* 19 */  "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
   234    273   /* 20 */  "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
   235    274   /* 21 */  "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
   236    275   /* 22 */  "SELECT value FROM %Q.'%q_stat' WHERE id=0",
   237    276   /* 23 */  "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
          277  +/* 24 */  "",
          278  +/* 25 */  "",
          279  +
          280  +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
          281  +
   238    282     };
   239    283     int rc = SQLITE_OK;
   240    284     sqlite3_stmt *pStmt;
   241    285   
   242    286     assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
   243    287     assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
   244    288     
................................................................................
   386    430   **
   387    431   **   0: idx
   388    432   **   1: start_block
   389    433   **   2: leaves_end_block
   390    434   **   3: end_block
   391    435   **   4: root
   392    436   */
   393         -int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){
          437  +int sqlite3Fts3AllSegdirs(
          438  +  Fts3Table *p,                   /* FTS3 table */
          439  +  int iIndex,                     /* Index for p->aIndex[] */
          440  +  int iLevel,                     /* Level to select */
          441  +  sqlite3_stmt **ppStmt           /* OUT: Compiled statement */
          442  +){
   394    443     int rc;
   395    444     sqlite3_stmt *pStmt = 0;
          445  +
          446  +  assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 );
          447  +  assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
          448  +  assert( iIndex>=0 && iIndex<p->nIndex );
          449  +
   396    450     if( iLevel<0 ){
   397         -    rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0);
          451  +    /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */
          452  +    rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0);
          453  +    if( rc==SQLITE_OK ){ 
          454  +      sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
          455  +      sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1);
          456  +    }
   398    457     }else{
          458  +    /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */
   399    459       rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
   400         -    if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel);
          460  +    if( rc==SQLITE_OK ){ 
          461  +      sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL);
          462  +    }
   401    463     }
   402    464     *ppStmt = pStmt;
   403    465     return rc;
   404    466   }
   405    467   
   406    468   
   407    469   /*
................................................................................
   507    569     *pRc = rc;
   508    570     if( p!=*pp ){
   509    571       *pp = p;
   510    572       return 1;
   511    573     }
   512    574     return 0;
   513    575   }
          576  +
          577  +/*
          578  +** Free a PendingList object allocated by fts3PendingListAppend().
          579  +*/
          580  +static void fts3PendingListDelete(PendingList *pList){
          581  +  sqlite3_free(pList);
          582  +}
          583  +
          584  +/*
          585  +** Add an entry to one of the pending-terms hash tables.
          586  +*/
          587  +static int fts3PendingTermsAddOne(
          588  +  Fts3Table *p,
          589  +  int iCol,
          590  +  int iPos,
          591  +  Fts3Hash *pHash,                /* Pending terms hash table to add entry to */
          592  +  const char *zToken,
          593  +  int nToken
          594  +){
          595  +  PendingList *pList;
          596  +  int rc = SQLITE_OK;
          597  +
          598  +  pList = (PendingList *)fts3HashFind(pHash, zToken, nToken);
          599  +  if( pList ){
          600  +    p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
          601  +  }
          602  +  if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
          603  +    if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){
          604  +      /* Malloc failed while inserting the new entry. This can only 
          605  +      ** happen if there was no previous entry for this token.
          606  +      */
          607  +      assert( 0==fts3HashFind(pHash, zToken, nToken) );
          608  +      sqlite3_free(pList);
          609  +      rc = SQLITE_NOMEM;
          610  +    }
          611  +  }
          612  +  if( rc==SQLITE_OK ){
          613  +    p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
          614  +  }
          615  +  return rc;
          616  +}
   514    617   
   515    618   /*
   516    619   ** Tokenize the nul-terminated string zText and add all tokens to the
   517    620   ** pending-terms hash-table. The docid used is that currently stored in
   518    621   ** p->iPrevDocid, and the column is specified by argument iCol.
   519    622   **
   520    623   ** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
................................................................................
   556    659     }
   557    660     pCsr->pTokenizer = pTokenizer;
   558    661   
   559    662     xNext = pModule->xNext;
   560    663     while( SQLITE_OK==rc
   561    664         && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos))
   562    665     ){
   563         -    PendingList *pList;
   564         - 
          666  +    int i;
   565    667       if( iPos>=nWord ) nWord = iPos+1;
   566    668   
   567    669       /* Positions cannot be negative; we use -1 as a terminator internally.
   568    670       ** Tokens must have a non-zero length.
   569    671       */
   570    672       if( iPos<0 || !zToken || nToken<=0 ){
   571    673         rc = SQLITE_ERROR;
   572    674         break;
   573    675       }
   574    676   
   575         -    pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken);
   576         -    if( pList ){
   577         -      p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
   578         -    }
   579         -    if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
   580         -      if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){
   581         -        /* Malloc failed while inserting the new entry. This can only 
   582         -        ** happen if there was no previous entry for this token.
   583         -        */
   584         -        assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) );
   585         -        sqlite3_free(pList);
   586         -        rc = SQLITE_NOMEM;
   587         -      }
   588         -    }
   589         -    if( rc==SQLITE_OK ){
   590         -      p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
          677  +    /* Add the term to the terms index */
          678  +    rc = fts3PendingTermsAddOne(
          679  +        p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken
          680  +    );
          681  +    
          682  +    /* Add the term to each of the prefix indexes that it is not too 
          683  +    ** short for. */
          684  +    for(i=1; rc==SQLITE_OK && i<p->nIndex; i++){
          685  +      struct Fts3Index *pIndex = &p->aIndex[i];
          686  +      if( nToken<pIndex->nPrefix ) continue;
          687  +      rc = fts3PendingTermsAddOne(
          688  +          p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix
          689  +      );
   591    690       }
   592    691     }
   593    692   
   594    693     pModule->xClose(pCsr);
   595    694     *pnWord = nWord;
   596    695     return (rc==SQLITE_DONE ? SQLITE_OK : rc);
   597    696   }
................................................................................
   613    712       if( rc!=SQLITE_OK ) return rc;
   614    713     }
   615    714     p->iPrevDocid = iDocid;
   616    715     return SQLITE_OK;
   617    716   }
   618    717   
   619    718   /*
   620         -** Discard the contents of the pending-terms hash table. 
          719  +** Discard the contents of the pending-terms hash tables. 
   621    720   */
   622    721   void sqlite3Fts3PendingTermsClear(Fts3Table *p){
   623         -  Fts3HashElem *pElem;
   624         -  for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
   625         -    sqlite3_free(fts3HashData(pElem));
          722  +  int i;
          723  +  for(i=0; i<p->nIndex; i++){
          724  +    Fts3HashElem *pElem;
          725  +    Fts3Hash *pHash = &p->aIndex[i].hPending;
          726  +    for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){
          727  +      PendingList *pList = (PendingList *)fts3HashData(pElem);
          728  +      fts3PendingListDelete(pList);
          729  +    }
          730  +    fts3HashClear(pHash);
   626    731     }
   627         -  fts3HashClear(&p->pendingTerms);
   628    732     p->nPendingData = 0;
   629    733   }
   630    734   
   631    735   /*
   632    736   ** This function is called by the xUpdate() method as part of an INSERT
   633    737   ** operation. It adds entries for each term in the new record to the
   634    738   ** pendingTerms hash table.
................................................................................
   776    880     *pRC = rc;
   777    881   }
   778    882   
   779    883   /*
   780    884   ** Forward declaration to account for the circular dependency between
   781    885   ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx().
   782    886   */
   783         -static int fts3SegmentMerge(Fts3Table *, int);
          887  +static int fts3SegmentMerge(Fts3Table *, int, int);
   784    888   
   785    889   /* 
   786    890   ** This function allocates a new level iLevel index in the segdir table.
   787    891   ** Usually, indexes are allocated within a level sequentially starting
   788    892   ** with 0, so the allocated index is one greater than the value returned
   789    893   ** by:
   790    894   **
................................................................................
   793    897   ** However, if there are already FTS3_MERGE_COUNT indexes at the requested
   794    898   ** level, they are merged into a single level (iLevel+1) segment and the 
   795    899   ** allocated index is 0.
   796    900   **
   797    901   ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK
   798    902   ** returned. Otherwise, an SQLite error code is returned.
   799    903   */
   800         -static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){
          904  +static int fts3AllocateSegdirIdx(
          905  +  Fts3Table *p, 
          906  +  int iIndex,                     /* Index for p->aIndex */
          907  +  int iLevel, 
          908  +  int *piIdx
          909  +){
   801    910     int rc;                         /* Return Code */
   802    911     sqlite3_stmt *pNextIdx;         /* Query for next idx at level iLevel */
   803    912     int iNext = 0;                  /* Result of query pNextIdx */
   804    913   
   805    914     /* Set variable iNext to the next available segdir index at level iLevel. */
   806    915     rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0);
   807    916     if( rc==SQLITE_OK ){
   808         -    sqlite3_bind_int(pNextIdx, 1, iLevel);
          917  +    sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel);
   809    918       if( SQLITE_ROW==sqlite3_step(pNextIdx) ){
   810    919         iNext = sqlite3_column_int(pNextIdx, 0);
   811    920       }
   812    921       rc = sqlite3_reset(pNextIdx);
   813    922     }
   814    923   
   815    924     if( rc==SQLITE_OK ){
   816    925       /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already
   817    926       ** full, merge all segments in level iLevel into a single iLevel+1
   818    927       ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
   819    928       ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
   820    929       */
   821    930       if( iNext>=FTS3_MERGE_COUNT ){
   822         -      rc = fts3SegmentMerge(p, iLevel);
          931  +      rc = fts3SegmentMerge(p, iIndex, iLevel);
   823    932         *piIdx = 0;
   824    933       }else{
   825    934         *piIdx = iNext;
   826    935       }
   827    936     }
   828    937   
   829    938     return rc;
................................................................................
   856    965   ** method (xFilter etc.) that may directly or indirectly call this function
   857    966   ** must call sqlite3Fts3SegmentsClose() before returning.
   858    967   */
   859    968   int sqlite3Fts3ReadBlock(
   860    969     Fts3Table *p,                   /* FTS3 table handle */
   861    970     sqlite3_int64 iBlockid,         /* Access the row with blockid=$iBlockid */
   862    971     char **paBlob,                  /* OUT: Blob data in malloc'd buffer */
   863         -  int *pnBlob                     /* OUT: Size of blob data */
          972  +  int *pnBlob,                    /* OUT: Size of blob data */
          973  +  int *pnLoad                     /* OUT: Bytes actually loaded */
   864    974   ){
   865    975     int rc;                         /* Return code */
   866    976   
   867    977     /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */
   868    978     assert( pnBlob);
   869    979   
   870    980     if( p->pSegments ){
................................................................................
   877    987       rc = sqlite3_blob_open(
   878    988          p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments
   879    989       );
   880    990     }
   881    991   
   882    992     if( rc==SQLITE_OK ){
   883    993       int nByte = sqlite3_blob_bytes(p->pSegments);
          994  +    *pnBlob = nByte;
   884    995       if( paBlob ){
   885    996         char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING);
   886    997         if( !aByte ){
   887    998           rc = SQLITE_NOMEM;
   888    999         }else{
         1000  +        if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){
         1001  +          nByte = FTS3_NODE_CHUNKSIZE;
         1002  +          *pnLoad = nByte;
         1003  +        }
   889   1004           rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0);
   890   1005           memset(&aByte[nByte], 0, FTS3_NODE_PADDING);
   891   1006           if( rc!=SQLITE_OK ){
   892   1007             sqlite3_free(aByte);
   893   1008             aByte = 0;
   894   1009           }
   895   1010         }
   896   1011         *paBlob = aByte;
   897   1012       }
   898         -    *pnBlob = nByte;
   899   1013     }
   900   1014   
   901   1015     return rc;
   902   1016   }
   903   1017   
   904   1018   /*
   905   1019   ** Close the blob handle at p->pSegments, if it is open. See comments above
   906   1020   ** the sqlite3Fts3ReadBlock() function for details.
   907   1021   */
   908   1022   void sqlite3Fts3SegmentsClose(Fts3Table *p){
   909   1023     sqlite3_blob_close(p->pSegments);
   910   1024     p->pSegments = 0;
   911   1025   }
         1026  +    
         1027  +static int fts3SegReaderIncrRead(Fts3SegReader *pReader){
         1028  +  int nRead;                      /* Number of bytes to read */
         1029  +  int rc;                         /* Return code */
         1030  +
         1031  +  nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE);
         1032  +  rc = sqlite3_blob_read(
         1033  +      pReader->pBlob, 
         1034  +      &pReader->aNode[pReader->nPopulate],
         1035  +      nRead,
         1036  +      pReader->nPopulate
         1037  +  );
         1038  +
         1039  +  if( rc==SQLITE_OK ){
         1040  +    pReader->nPopulate += nRead;
         1041  +    memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING);
         1042  +    if( pReader->nPopulate==pReader->nNode ){
         1043  +      sqlite3_blob_close(pReader->pBlob);
         1044  +      pReader->pBlob = 0;
         1045  +      pReader->nPopulate = 0;
         1046  +    }
         1047  +  }
         1048  +  return rc;
         1049  +}
         1050  +
         1051  +static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){
         1052  +  int rc = SQLITE_OK;
         1053  +  assert( !pReader->pBlob 
         1054  +       || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode])
         1055  +  );
         1056  +  while( pReader->pBlob && rc==SQLITE_OK 
         1057  +     &&  (pFrom - pReader->aNode + nByte)>pReader->nPopulate
         1058  +  ){
         1059  +    rc = fts3SegReaderIncrRead(pReader);
         1060  +  }
         1061  +  return rc;
         1062  +}
   912   1063   
   913   1064   /*
   914   1065   ** Move the iterator passed as the first argument to the next term in the
   915   1066   ** segment. If successful, SQLITE_OK is returned. If there is no next term,
   916   1067   ** SQLITE_DONE. Otherwise, an SQLite error code.
   917   1068   */
   918         -static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
         1069  +static int fts3SegReaderNext(
         1070  +  Fts3Table *p, 
         1071  +  Fts3SegReader *pReader,
         1072  +  int bIncr
         1073  +){
         1074  +  int rc;                         /* Return code of various sub-routines */
   919   1075     char *pNext;                    /* Cursor variable */
   920   1076     int nPrefix;                    /* Number of bytes in term prefix */
   921   1077     int nSuffix;                    /* Number of bytes in term suffix */
   922   1078   
   923   1079     if( !pReader->aDoclist ){
   924   1080       pNext = pReader->aNode;
   925   1081     }else{
   926   1082       pNext = &pReader->aDoclist[pReader->nDoclist];
   927   1083     }
   928   1084   
   929   1085     if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
   930         -    int rc;                       /* Return code from Fts3ReadBlock() */
   931   1086   
   932   1087       if( fts3SegReaderIsPending(pReader) ){
   933   1088         Fts3HashElem *pElem = *(pReader->ppNextElem);
   934   1089         if( pElem==0 ){
   935   1090           pReader->aNode = 0;
   936   1091         }else{
   937   1092           PendingList *pList = (PendingList *)fts3HashData(pElem);
................................................................................
   943   1098           assert( pReader->aNode );
   944   1099         }
   945   1100         return SQLITE_OK;
   946   1101       }
   947   1102   
   948   1103       if( !fts3SegReaderIsRootOnly(pReader) ){
   949   1104         sqlite3_free(pReader->aNode);
         1105  +      sqlite3_blob_close(pReader->pBlob);
         1106  +      pReader->pBlob = 0;
   950   1107       }
   951   1108       pReader->aNode = 0;
   952   1109   
   953   1110       /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf 
   954   1111       ** blocks have already been traversed.  */
   955   1112       assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock );
   956   1113       if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){
   957   1114         return SQLITE_OK;
   958   1115       }
   959   1116   
   960   1117       rc = sqlite3Fts3ReadBlock(
   961         -        p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode
         1118  +        p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, 
         1119  +        (bIncr ? &pReader->nPopulate : 0)
   962   1120       );
   963   1121       if( rc!=SQLITE_OK ) return rc;
         1122  +    assert( pReader->pBlob==0 );
         1123  +    if( bIncr && pReader->nPopulate<pReader->nNode ){
         1124  +      pReader->pBlob = p->pSegments;
         1125  +      p->pSegments = 0;
         1126  +    }
   964   1127       pNext = pReader->aNode;
   965   1128     }
         1129  +
         1130  +  assert( !fts3SegReaderIsPending(pReader) );
         1131  +
         1132  +  rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
         1133  +  if( rc!=SQLITE_OK ) return rc;
   966   1134     
   967   1135     /* Because of the FTS3_NODE_PADDING bytes of padding, the following is 
   968         -  ** safe (no risk of overread) even if the node data is corrupted.  
   969         -  */
         1136  +  ** safe (no risk of overread) even if the node data is corrupted. */
   970   1137     pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
   971   1138     pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
   972   1139     if( nPrefix<0 || nSuffix<=0 
   973   1140      || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 
   974   1141     ){
   975   1142       return SQLITE_CORRUPT_VTAB;
   976   1143     }
................................................................................
   980   1147       char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
   981   1148       if( !zNew ){
   982   1149         return SQLITE_NOMEM;
   983   1150       }
   984   1151       pReader->zTerm = zNew;
   985   1152       pReader->nTermAlloc = nNew;
   986   1153     }
         1154  +
         1155  +  rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX);
         1156  +  if( rc!=SQLITE_OK ) return rc;
         1157  +
   987   1158     memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
   988   1159     pReader->nTerm = nPrefix+nSuffix;
   989   1160     pNext += nSuffix;
   990   1161     pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
   991   1162     pReader->aDoclist = pNext;
   992   1163     pReader->pOffsetList = 0;
   993   1164   
   994   1165     /* Check that the doclist does not appear to extend past the end of the
   995   1166     ** b-tree node. And that the final byte of the doclist is 0x00. If either 
   996   1167     ** of these statements is untrue, then the data structure is corrupt.
   997   1168     */
   998   1169     if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 
   999         -   || pReader->aDoclist[pReader->nDoclist-1]
         1170  +   || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
  1000   1171     ){
  1001   1172       return SQLITE_CORRUPT_VTAB;
  1002   1173     }
  1003   1174     return SQLITE_OK;
  1004   1175   }
  1005   1176   
  1006   1177   /*
  1007   1178   ** Set the SegReader to point to the first docid in the doclist associated
  1008   1179   ** with the current term.
  1009   1180   */
  1010         -static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){
  1011         -  int n;
         1181  +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){
         1182  +  int rc = SQLITE_OK;
  1012   1183     assert( pReader->aDoclist );
  1013   1184     assert( !pReader->pOffsetList );
  1014         -  n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
  1015         -  pReader->pOffsetList = &pReader->aDoclist[n];
         1185  +  if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){
         1186  +    u8 bEof = 0;
         1187  +    pReader->iDocid = 0;
         1188  +    pReader->nOffsetList = 0;
         1189  +    sqlite3Fts3DoclistPrev(0,
         1190  +        pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, 
         1191  +        &pReader->iDocid, &pReader->nOffsetList, &bEof
         1192  +    );
         1193  +  }else{
         1194  +    rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX);
         1195  +    if( rc==SQLITE_OK ){
         1196  +      int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
         1197  +      pReader->pOffsetList = &pReader->aDoclist[n];
         1198  +    }
         1199  +  }
         1200  +  return rc;
  1016   1201   }
  1017   1202   
  1018   1203   /*
  1019   1204   ** Advance the SegReader to point to the next docid in the doclist
  1020   1205   ** associated with the current term.
  1021   1206   ** 
  1022   1207   ** If arguments ppOffsetList and pnOffsetList are not NULL, then 
  1023   1208   ** *ppOffsetList is set to point to the first column-offset list
  1024   1209   ** in the doclist entry (i.e. immediately past the docid varint).
  1025   1210   ** *pnOffsetList is set to the length of the set of column-offset
  1026   1211   ** lists, not including the nul-terminator byte. For example:
  1027   1212   */
  1028         -static void fts3SegReaderNextDocid(
  1029         -  Fts3SegReader *pReader,
  1030         -  char **ppOffsetList,
  1031         -  int *pnOffsetList
         1213  +static int fts3SegReaderNextDocid(
         1214  +  Fts3Table *pTab,
         1215  +  Fts3SegReader *pReader,         /* Reader to advance to next docid */
         1216  +  char **ppOffsetList,            /* OUT: Pointer to current position-list */
         1217  +  int *pnOffsetList               /* OUT: Length of *ppOffsetList in bytes */
  1032   1218   ){
         1219  +  int rc = SQLITE_OK;
  1033   1220     char *p = pReader->pOffsetList;
  1034   1221     char c = 0;
  1035   1222   
  1036         -  /* Pointer p currently points at the first byte of an offset list. The
  1037         -  ** following two lines advance it to point one byte past the end of
  1038         -  ** the same offset list.
  1039         -  */
  1040         -  while( *p | c ) c = *p++ & 0x80;
  1041         -  p++;
  1042         -
  1043         -  /* If required, populate the output variables with a pointer to and the
  1044         -  ** size of the previous offset-list.
  1045         -  */
  1046         -  if( ppOffsetList ){
  1047         -    *ppOffsetList = pReader->pOffsetList;
  1048         -    *pnOffsetList = (int)(p - pReader->pOffsetList - 1);
  1049         -  }
  1050         -
  1051         -  /* If there are no more entries in the doclist, set pOffsetList to
  1052         -  ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
  1053         -  ** Fts3SegReader.pOffsetList to point to the next offset list before
  1054         -  ** returning.
  1055         -  */
  1056         -  if( p>=&pReader->aDoclist[pReader->nDoclist] ){
  1057         -    pReader->pOffsetList = 0;
  1058         -  }else{
  1059         -    sqlite3_int64 iDelta;
  1060         -    pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
  1061         -    pReader->iDocid += iDelta;
  1062         -  }
  1063         -}
  1064         -
  1065         -/*
  1066         -** This function is called to estimate the amount of data that will be 
  1067         -** loaded from the disk If SegReaderIterate() is called on this seg-reader,
  1068         -** in units of average document size.
  1069         -** 
  1070         -** This can be used as follows: If the caller has a small doclist that 
  1071         -** contains references to N documents, and is considering merging it with
  1072         -** a large doclist (size X "average documents"), it may opt not to load
  1073         -** the large doclist if X>N.
  1074         -*/
  1075         -int sqlite3Fts3SegReaderCost(
  1076         -  Fts3Cursor *pCsr,               /* FTS3 cursor handle */
  1077         -  Fts3SegReader *pReader,         /* Segment-reader handle */
  1078         -  int *pnCost                     /* IN/OUT: Number of bytes read */
         1223  +  assert( p );
         1224  +
         1225  +  if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){
         1226  +    /* A pending-terms seg-reader for an FTS4 table that uses order=desc.
         1227  +    ** Pending-terms doclists are always built up in ascending order, so
         1228  +    ** we have to iterate through them backwards here. */
         1229  +    u8 bEof = 0;
         1230  +    if( ppOffsetList ){
         1231  +      *ppOffsetList = pReader->pOffsetList;
         1232  +      *pnOffsetList = pReader->nOffsetList - 1;
         1233  +    }
         1234  +    sqlite3Fts3DoclistPrev(0,
         1235  +        pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid,
         1236  +        &pReader->nOffsetList, &bEof
         1237  +    );
         1238  +    if( bEof ){
         1239  +      pReader->pOffsetList = 0;
         1240  +    }else{
         1241  +      pReader->pOffsetList = p;
         1242  +    }
         1243  +  }else{
         1244  +
         1245  +    /* Pointer p currently points at the first byte of an offset list. The
         1246  +    ** following block advances it to point one byte past the end of
         1247  +    ** the same offset list. */
         1248  +    while( 1 ){
         1249  +  
         1250  +      /* The following line of code (and the "p++" below the while() loop) is
         1251  +      ** normally all that is required to move pointer p to the desired 
         1252  +      ** position. The exception is if this node is being loaded from disk
         1253  +      ** incrementally and pointer "p" now points to the first byte passed
         1254  +      ** the populated part of pReader->aNode[].
         1255  +      */
         1256  +      while( *p | c ) c = *p++ & 0x80;
         1257  +      assert( *p==0 );
         1258  +  
         1259  +      if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break;
         1260  +      rc = fts3SegReaderIncrRead(pReader);
         1261  +      if( rc!=SQLITE_OK ) return rc;
         1262  +    }
         1263  +    p++;
         1264  +  
         1265  +    /* If required, populate the output variables with a pointer to and the
         1266  +    ** size of the previous offset-list.
         1267  +    */
         1268  +    if( ppOffsetList ){
         1269  +      *ppOffsetList = pReader->pOffsetList;
         1270  +      *pnOffsetList = (int)(p - pReader->pOffsetList - 1);
         1271  +    }
         1272  +  
         1273  +    /* If there are no more entries in the doclist, set pOffsetList to
         1274  +    ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
         1275  +    ** Fts3SegReader.pOffsetList to point to the next offset list before
         1276  +    ** returning.
         1277  +    */
         1278  +    if( p>=&pReader->aDoclist[pReader->nDoclist] ){
         1279  +      pReader->pOffsetList = 0;
         1280  +    }else{
         1281  +      rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX);
         1282  +      if( rc==SQLITE_OK ){
         1283  +        sqlite3_int64 iDelta;
         1284  +        pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
         1285  +        if( pTab->bDescIdx ){
         1286  +          pReader->iDocid -= iDelta;
         1287  +        }else{
         1288  +          pReader->iDocid += iDelta;
         1289  +        }
         1290  +      }
         1291  +    }
         1292  +  }
         1293  +
         1294  +  return SQLITE_OK;
         1295  +}
         1296  +
         1297  +
         1298  +int sqlite3Fts3MsrOvfl(
         1299  +  Fts3Cursor *pCsr, 
         1300  +  Fts3MultiSegReader *pMsr,
         1301  +  int *pnOvfl
  1079   1302   ){
  1080   1303     Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
  1081         -  int rc = SQLITE_OK;             /* Return code */
  1082         -  int nCost = 0;                  /* Cost in bytes to return */
  1083         -  int pgsz = p->nPgsz;            /* Database page size */
  1084         -
  1085         -  /* If this seg-reader is reading the pending-terms table, or if all data
  1086         -  ** for the segment is stored on the root page of the b-tree, then the cost
  1087         -  ** is zero. In this case all required data is already in main memory.
  1088         -  */
  1089         -  if( p->bHasStat 
  1090         -   && !fts3SegReaderIsPending(pReader) 
  1091         -   && !fts3SegReaderIsRootOnly(pReader) 
  1092         -  ){
  1093         -    int nBlob = 0;
  1094         -    sqlite3_int64 iBlock;
  1095         -
  1096         -    if( pCsr->nRowAvg==0 ){
  1097         -      /* The average document size, which is required to calculate the cost
  1098         -      ** of each doclist, has not yet been determined. Read the required 
  1099         -      ** data from the %_stat table to calculate it.
  1100         -      **
  1101         -      ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 
  1102         -      ** varints, where nCol is the number of columns in the FTS3 table.
  1103         -      ** The first varint is the number of documents currently stored in
  1104         -      ** the table. The following nCol varints contain the total amount of
  1105         -      ** data stored in all rows of each column of the table, from left
  1106         -      ** to right.
  1107         -      */
  1108         -      sqlite3_stmt *pStmt;
  1109         -      sqlite3_int64 nDoc = 0;
  1110         -      sqlite3_int64 nByte = 0;
  1111         -      const char *pEnd;
  1112         -      const char *a;
  1113         -
  1114         -      rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
  1115         -      if( rc!=SQLITE_OK ) return rc;
  1116         -      a = sqlite3_column_blob(pStmt, 0);
  1117         -      assert( a );
  1118         -
  1119         -      pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
  1120         -      a += sqlite3Fts3GetVarint(a, &nDoc);
  1121         -      while( a<pEnd ){
  1122         -        a += sqlite3Fts3GetVarint(a, &nByte);
  1123         -      }
  1124         -      if( nDoc==0 || nByte==0 ){
  1125         -        sqlite3_reset(pStmt);
  1126         -        return SQLITE_CORRUPT_VTAB;
  1127         -      }
  1128         -
  1129         -      pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz);
  1130         -      assert( pCsr->nRowAvg>0 ); 
  1131         -      rc = sqlite3_reset(pStmt);
  1132         -      if( rc!=SQLITE_OK ) return rc;
  1133         -    }
  1134         -
  1135         -    /* Assume that a blob flows over onto overflow pages if it is larger
  1136         -    ** than (pgsz-35) bytes in size (the file-format documentation
  1137         -    ** confirms this).
  1138         -    */
  1139         -    for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){
  1140         -      rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob);
  1141         -      if( rc!=SQLITE_OK ) break;
  1142         -      if( (nBlob+35)>pgsz ){
  1143         -        int nOvfl = (nBlob + 34)/pgsz;
  1144         -        nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg);
  1145         -      }
  1146         -    }
  1147         -  }
  1148         -
  1149         -  *pnCost += nCost;
         1304  +  int nOvfl = 0;
         1305  +  int ii;
         1306  +  int rc = SQLITE_OK;
         1307  +  int pgsz = p->nPgsz;
         1308  +
         1309  +  assert( p->bHasStat );
         1310  +  assert( pgsz>0 );
         1311  +
         1312  +  for(ii=0; rc==SQLITE_OK && ii<pMsr->nSegment; ii++){
         1313  +    Fts3SegReader *pReader = pMsr->apSegment[ii];
         1314  +    if( !fts3SegReaderIsPending(pReader) 
         1315  +     && !fts3SegReaderIsRootOnly(pReader) 
         1316  +    ){
         1317  +      int jj;
         1318  +      for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){
         1319  +        int nBlob;
         1320  +        rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0);
         1321  +        if( rc!=SQLITE_OK ) break;
         1322  +        if( (nBlob+35)>pgsz ){
         1323  +          nOvfl += (nBlob + 34)/pgsz;
         1324  +        }
         1325  +      }
         1326  +    }
         1327  +  }
         1328  +  *pnOvfl = nOvfl;
  1150   1329     return rc;
  1151   1330   }
  1152   1331   
  1153   1332   /*
  1154   1333   ** Free all allocations associated with the iterator passed as the 
  1155   1334   ** second argument.
  1156   1335   */
  1157   1336   void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
  1158   1337     if( pReader && !fts3SegReaderIsPending(pReader) ){
  1159   1338       sqlite3_free(pReader->zTerm);
  1160   1339       if( !fts3SegReaderIsRootOnly(pReader) ){
  1161   1340         sqlite3_free(pReader->aNode);
         1341  +      sqlite3_blob_close(pReader->pBlob);
  1162   1342       }
  1163   1343     }
  1164   1344     sqlite3_free(pReader);
  1165   1345   }
  1166   1346   
  1167   1347   /*
  1168   1348   ** Allocate a new SegReader object.
................................................................................
  1231   1411     }
  1232   1412     return c;
  1233   1413   }
  1234   1414   
  1235   1415   /*
  1236   1416   ** This function is used to allocate an Fts3SegReader that iterates through
  1237   1417   ** a subset of the terms stored in the Fts3Table.pendingTerms array.
         1418  +**
         1419  +** If the isPrefixIter parameter is zero, then the returned SegReader iterates
         1420  +** through each term in the pending-terms table. Or, if isPrefixIter is
         1421  +** non-zero, it iterates through each term and its prefixes. For example, if
         1422  +** the pending terms hash table contains the terms "sqlite", "mysql" and
         1423  +** "firebird", then the iterator visits the following 'terms' (in the order
         1424  +** shown):
         1425  +**
         1426  +**   f fi fir fire fireb firebi firebir firebird
         1427  +**   m my mys mysq mysql
         1428  +**   s sq sql sqli sqlit sqlite
         1429  +**
         1430  +** Whereas if isPrefixIter is zero, the terms visited are:
         1431  +**
         1432  +**   firebird mysql sqlite
  1238   1433   */
  1239   1434   int sqlite3Fts3SegReaderPending(
  1240   1435     Fts3Table *p,                   /* Virtual table handle */
         1436  +  int iIndex,                     /* Index for p->aIndex */
  1241   1437     const char *zTerm,              /* Term to search for */
  1242   1438     int nTerm,                      /* Size of buffer zTerm */
  1243         -  int isPrefix,                   /* True for a term-prefix query */
         1439  +  int bPrefix,                    /* True for a prefix iterator */
  1244   1440     Fts3SegReader **ppReader        /* OUT: SegReader for pending-terms */
  1245   1441   ){
  1246   1442     Fts3SegReader *pReader = 0;     /* Fts3SegReader object to return */
  1247   1443     Fts3HashElem **aElem = 0;       /* Array of term hash entries to scan */
  1248   1444     int nElem = 0;                  /* Size of array at aElem */
  1249   1445     int rc = SQLITE_OK;             /* Return Code */
         1446  +  Fts3Hash *pHash;
  1250   1447   
  1251         -  if( isPrefix ){
         1448  +  pHash = &p->aIndex[iIndex].hPending;
         1449  +  if( bPrefix ){
  1252   1450       int nAlloc = 0;               /* Size of allocated array at aElem */
  1253   1451       Fts3HashElem *pE = 0;         /* Iterator variable */
  1254   1452   
  1255         -    for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){
         1453  +    for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){
  1256   1454         char *zKey = (char *)fts3HashKey(pE);
  1257   1455         int nKey = fts3HashKeysize(pE);
  1258   1456         if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){
  1259   1457           if( nElem==nAlloc ){
  1260   1458             Fts3HashElem **aElem2;
  1261   1459             nAlloc += 16;
  1262   1460             aElem2 = (Fts3HashElem **)sqlite3_realloc(
................................................................................
  1265   1463             if( !aElem2 ){
  1266   1464               rc = SQLITE_NOMEM;
  1267   1465               nElem = 0;
  1268   1466               break;
  1269   1467             }
  1270   1468             aElem = aElem2;
  1271   1469           }
         1470  +
  1272   1471           aElem[nElem++] = pE;
  1273   1472         }
  1274   1473       }
  1275   1474   
  1276   1475       /* If more than one term matches the prefix, sort the Fts3HashElem
  1277   1476       ** objects in term order using qsort(). This uses the same comparison
  1278   1477       ** callback as is used when flushing terms to disk.
  1279   1478       */
  1280   1479       if( nElem>1 ){
  1281   1480         qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm);
  1282   1481       }
  1283   1482   
  1284   1483     }else{
  1285         -    Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm);
         1484  +    /* The query is a simple term lookup that matches at most one term in
         1485  +    ** the index. All that is required is a straight hash-lookup. */
         1486  +    Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm);
  1286   1487       if( pE ){
  1287   1488         aElem = &pE;
  1288   1489         nElem = 1;
  1289   1490       }
  1290   1491     }
  1291   1492   
  1292   1493     if( nElem>0 ){
................................................................................
  1298   1499         memset(pReader, 0, nByte);
  1299   1500         pReader->iIdx = 0x7FFFFFFF;
  1300   1501         pReader->ppNextElem = (Fts3HashElem **)&pReader[1];
  1301   1502         memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *));
  1302   1503       }
  1303   1504     }
  1304   1505   
  1305         -  if( isPrefix ){
         1506  +  if( bPrefix ){
  1306   1507       sqlite3_free(aElem);
  1307   1508     }
  1308   1509     *ppReader = pReader;
  1309   1510     return rc;
  1310   1511   }
  1311   1512   
  1312   1513   /*
................................................................................
  1358   1559     int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
  1359   1560     if( rc==0 ){
  1360   1561       if( pLhs->iDocid==pRhs->iDocid ){
  1361   1562         rc = pRhs->iIdx - pLhs->iIdx;
  1362   1563       }else{
  1363   1564         rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1;
  1364   1565       }
         1566  +  }
         1567  +  assert( pLhs->aNode && pRhs->aNode );
         1568  +  return rc;
         1569  +}
         1570  +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){
         1571  +  int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
         1572  +  if( rc==0 ){
         1573  +    if( pLhs->iDocid==pRhs->iDocid ){
         1574  +      rc = pRhs->iIdx - pLhs->iIdx;
         1575  +    }else{
         1576  +      rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1;
         1577  +    }
  1365   1578     }
  1366   1579     assert( pLhs->aNode && pRhs->aNode );
  1367   1580     return rc;
  1368   1581   }
  1369   1582   
  1370   1583   /*
  1371   1584   ** Compare the term that the Fts3SegReader object passed as the first argument
................................................................................
  1910   2123       }
  1911   2124       rc = sqlite3_reset(pStmt);
  1912   2125     }
  1913   2126     return rc;
  1914   2127   }
  1915   2128   
  1916   2129   /*
  1917         -** Set *pnSegment to the total number of segments in the database. Set
  1918         -** *pnMax to the largest segment level in the database (segment levels
  1919         -** are stored in the 'level' column of the %_segdir table).
         2130  +** Set *pnMax to the largest segment level in the database for the index
         2131  +** iIndex.
         2132  +**
         2133  +** Segment levels are stored in the 'level' column of the %_segdir table.
  1920   2134   **
  1921   2135   ** Return SQLITE_OK if successful, or an SQLite error code if not.
  1922   2136   */
  1923         -static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){
         2137  +static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){
  1924   2138     sqlite3_stmt *pStmt;
  1925   2139     int rc;
         2140  +  assert( iIndex>=0 && iIndex<p->nIndex );
  1926   2141   
  1927         -  rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0);
         2142  +  /* Set pStmt to the compiled version of:
         2143  +  **
         2144  +  **   SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?
         2145  +  **
         2146  +  ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR).
         2147  +  */
         2148  +  rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0);
  1928   2149     if( rc!=SQLITE_OK ) return rc;
         2150  +  sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
         2151  +  sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1);
  1929   2152     if( SQLITE_ROW==sqlite3_step(pStmt) ){
  1930         -    *pnSegment = sqlite3_column_int(pStmt, 0);
  1931         -    *pnMax = sqlite3_column_int(pStmt, 1);
         2153  +    *pnMax = sqlite3_column_int(pStmt, 0);
  1932   2154     }
  1933   2155     return sqlite3_reset(pStmt);
  1934   2156   }
  1935   2157   
  1936   2158   /*
  1937   2159   ** This function is used after merging multiple segments into a single large
  1938   2160   ** segment to delete the old, now redundant, segment b-trees. Specifically,
................................................................................
  1945   2167   **   2) deletes all %_segdir entries with level iLevel, or all %_segdir
  1946   2168   **      entries regardless of level if (iLevel<0).
  1947   2169   **
  1948   2170   ** SQLITE_OK is returned if successful, otherwise an SQLite error code.
  1949   2171   */
  1950   2172   static int fts3DeleteSegdir(
  1951   2173     Fts3Table *p,                   /* Virtual table handle */
         2174  +  int iIndex,                     /* Index for p->aIndex */
  1952   2175     int iLevel,                     /* Level of %_segdir entries to delete */
  1953   2176     Fts3SegReader **apSegment,      /* Array of SegReader objects */
  1954   2177     int nReader                     /* Size of array apSegment */
  1955   2178   ){
  1956   2179     int rc;                         /* Return Code */
  1957   2180     int i;                          /* Iterator variable */
  1958   2181     sqlite3_stmt *pDelete;          /* SQL statement to delete rows */
................................................................................
  1967   2190         rc = sqlite3_reset(pDelete);
  1968   2191       }
  1969   2192     }
  1970   2193     if( rc!=SQLITE_OK ){
  1971   2194       return rc;
  1972   2195     }
  1973   2196   
         2197  +  assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL );
  1974   2198     if( iLevel==FTS3_SEGCURSOR_ALL ){
  1975         -    fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
  1976         -  }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
  1977         -    sqlite3Fts3PendingTermsClear(p);
         2199  +    rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0);
         2200  +    if( rc==SQLITE_OK ){
         2201  +      sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
         2202  +      sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1);
         2203  +    }
  1978   2204     }else{
  1979         -    assert( iLevel>=0 );
  1980         -    rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0);
         2205  +    rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0);
  1981   2206       if( rc==SQLITE_OK ){
  1982         -      sqlite3_bind_int(pDelete, 1, iLevel);
  1983         -      sqlite3_step(pDelete);
  1984         -      rc = sqlite3_reset(pDelete);
         2207  +      sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel);
  1985   2208       }
  1986   2209     }
         2210  +
         2211  +  if( rc==SQLITE_OK ){
         2212  +    sqlite3_step(pDelete);
         2213  +    rc = sqlite3_reset(pDelete);
         2214  +  }
  1987   2215   
  1988   2216     return rc;
  1989   2217   }
  1990   2218   
  1991   2219   /*
  1992   2220   ** When this function is called, buffer *ppList (size *pnList bytes) contains 
  1993   2221   ** a position list that may (or may not) feature multiple columns. This
................................................................................
  2026   2254       p = &pList[1];
  2027   2255       p += sqlite3Fts3GetVarint32(p, &iCurrent);
  2028   2256     }
  2029   2257   
  2030   2258     *ppList = pList;
  2031   2259     *pnList = nList;
  2032   2260   }
         2261  +
         2262  +int sqlite3Fts3MsrIncrStart(
         2263  +  Fts3Table *p,                   /* Virtual table handle */
         2264  +  Fts3MultiSegReader *pCsr,       /* Cursor object */
         2265  +  int iCol,                       /* Column to match on. */
         2266  +  const char *zTerm,              /* Term to iterate through a doclist for */
         2267  +  int nTerm                       /* Number of bytes in zTerm */
         2268  +){
         2269  +  int i;
         2270  +  int nSegment = pCsr->nSegment;
         2271  +  int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
         2272  +    p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
         2273  +  );
         2274  +
         2275  +  assert( pCsr->pFilter==0 );
         2276  +  assert( zTerm && nTerm>0 );
         2277  +
         2278  +  /* Advance each segment iterator until it points to the term zTerm/nTerm. */
         2279  +  for(i=0; i<nSegment; i++){
         2280  +    Fts3SegReader *pSeg = pCsr->apSegment[i];
         2281  +    do {
         2282  +      int rc = fts3SegReaderNext(p, pSeg, 1);
         2283  +      if( rc!=SQLITE_OK ) return rc;
         2284  +    }while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
         2285  +  }
         2286  +  fts3SegReaderSort(pCsr->apSegment, nSegment, nSegment, fts3SegReaderCmp);
         2287  +
         2288  +  /* Determine how many of the segments actually point to zTerm/nTerm. */
         2289  +  for(i=0; i<nSegment; i++){
         2290  +    Fts3SegReader *pSeg = pCsr->apSegment[i];
         2291  +    if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){
         2292  +      break;
         2293  +    }
         2294  +  }
         2295  +  pCsr->nAdvance = i;
         2296  +
         2297  +  /* Advance each of the segments to point to the first docid. */
         2298  +  for(i=0; i<pCsr->nAdvance; i++){
         2299  +    int rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]);
         2300  +    if( rc!=SQLITE_OK ) return rc;
         2301  +  }
         2302  +  fts3SegReaderSort(pCsr->apSegment, i, i, xCmp);
         2303  +
         2304  +  assert( iCol<0 || iCol<p->nColumn );
         2305  +  pCsr->iColFilter = iCol;
         2306  +
         2307  +  return SQLITE_OK;
         2308  +}
         2309  +
         2310  +int sqlite3Fts3MsrIncrNext(
         2311  +  Fts3Table *p,                   /* Virtual table handle */
         2312  +  Fts3MultiSegReader *pMsr,       /* Multi-segment-reader handle */
         2313  +  sqlite3_int64 *piDocid,         /* OUT: Docid value */
         2314  +  char **paPoslist,               /* OUT: Pointer to position list */
         2315  +  int *pnPoslist                  /* OUT: Size of position list in bytes */
         2316  +){
         2317  +  int nMerge = pMsr->nAdvance;
         2318  +  Fts3SegReader **apSegment = pMsr->apSegment;
         2319  +  int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
         2320  +    p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
         2321  +  );
         2322  +
         2323  +  if( nMerge==0 ){
         2324  +    *paPoslist = 0;
         2325  +    return SQLITE_OK;
         2326  +  }
         2327  +
         2328  +  while( 1 ){
         2329  +    Fts3SegReader *pSeg;
         2330  +    pSeg = pMsr->apSegment[0];
         2331  +
         2332  +    if( pSeg->pOffsetList==0 ){
         2333  +      *paPoslist = 0;
         2334  +      break;
         2335  +    }else{
         2336  +      int rc;
         2337  +      char *pList;
         2338  +      int nList;
         2339  +      int j;
         2340  +      sqlite3_int64 iDocid = apSegment[0]->iDocid;
         2341  +
         2342  +      rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList);
         2343  +      j = 1;
         2344  +      while( rc==SQLITE_OK 
         2345  +        && j<nMerge
         2346  +        && apSegment[j]->pOffsetList
         2347  +        && apSegment[j]->iDocid==iDocid
         2348  +      ){
         2349  +        rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0);
         2350  +        j++;
         2351  +      }
         2352  +      if( rc!=SQLITE_OK ) return rc;
         2353  +      fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp);
         2354  +
         2355  +      if( pMsr->iColFilter>=0 ){
         2356  +        fts3ColumnFilter(pMsr->iColFilter, &pList, &nList);
         2357  +      }
         2358  +
         2359  +      if( nList>0 ){
         2360  +        *piDocid = iDocid;
         2361  +        *paPoslist = pList;
         2362  +        *pnPoslist = nList;
         2363  +        break;
         2364  +      }
         2365  +    }
         2366  +    
         2367  +  }
         2368  +
         2369  +  return SQLITE_OK;
         2370  +}
  2033   2371   
  2034   2372   int sqlite3Fts3SegReaderStart(
  2035   2373     Fts3Table *p,                   /* Virtual table handle */
  2036         -  Fts3SegReaderCursor *pCsr,      /* Cursor object */
         2374  +  Fts3MultiSegReader *pCsr,       /* Cursor object */
  2037   2375     Fts3SegFilter *pFilter          /* Restrictions on range of iteration */
  2038   2376   ){
  2039   2377     int i;
  2040   2378   
  2041   2379     /* Initialize the cursor object */
  2042   2380     pCsr->pFilter = pFilter;
  2043   2381   
................................................................................
  2048   2386     ** b-tree leaf nodes contain more than one term.
  2049   2387     */
  2050   2388     for(i=0; i<pCsr->nSegment; i++){
  2051   2389       int nTerm = pFilter->nTerm;
  2052   2390       const char *zTerm = pFilter->zTerm;
  2053   2391       Fts3SegReader *pSeg = pCsr->apSegment[i];
  2054   2392       do {
  2055         -      int rc = fts3SegReaderNext(p, pSeg);
         2393  +      int rc = fts3SegReaderNext(p, pSeg, 0);
  2056   2394         if( rc!=SQLITE_OK ) return rc;
  2057   2395       }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
  2058   2396     }
  2059   2397     fts3SegReaderSort(
  2060   2398         pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp);
  2061   2399   
  2062   2400     return SQLITE_OK;
  2063   2401   }
  2064   2402   
  2065   2403   int sqlite3Fts3SegReaderStep(
  2066   2404     Fts3Table *p,                   /* Virtual table handle */
  2067         -  Fts3SegReaderCursor *pCsr       /* Cursor object */
         2405  +  Fts3MultiSegReader *pCsr        /* Cursor object */
  2068   2406   ){
  2069   2407     int rc = SQLITE_OK;
  2070   2408   
  2071   2409     int isIgnoreEmpty =  (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY);
  2072   2410     int isRequirePos =   (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS);
  2073   2411     int isColFilter =    (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER);
  2074   2412     int isPrefix =       (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX);
  2075   2413     int isScan =         (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN);
  2076   2414   
  2077   2415     Fts3SegReader **apSegment = pCsr->apSegment;
  2078   2416     int nSegment = pCsr->nSegment;
  2079   2417     Fts3SegFilter *pFilter = pCsr->pFilter;
         2418  +  int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
         2419  +    p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
         2420  +  );
  2080   2421   
  2081   2422     if( pCsr->nSegment==0 ) return SQLITE_OK;
  2082   2423   
  2083   2424     do {
  2084   2425       int nMerge;
  2085   2426       int i;
  2086   2427     
  2087   2428       /* Advance the first pCsr->nAdvance entries in the apSegment[] array
  2088   2429       ** forward. Then sort the list in order of current term again.  
  2089   2430       */
  2090   2431       for(i=0; i<pCsr->nAdvance; i++){
  2091         -      rc = fts3SegReaderNext(p, apSegment[i]);
         2432  +      rc = fts3SegReaderNext(p, apSegment[i], 0);
  2092   2433         if( rc!=SQLITE_OK ) return rc;
  2093   2434       }
  2094   2435       fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp);
  2095   2436       pCsr->nAdvance = 0;
  2096   2437   
  2097   2438       /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */
  2098   2439       assert( rc==SQLITE_OK );
................................................................................
  2123   2464           && apSegment[nMerge]->nTerm==pCsr->nTerm 
  2124   2465           && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm)
  2125   2466       ){
  2126   2467         nMerge++;
  2127   2468       }
  2128   2469   
  2129   2470       assert( isIgnoreEmpty || (isRequirePos && !isColFilter) );
  2130         -    if( nMerge==1 && !isIgnoreEmpty ){
         2471  +    if( nMerge==1 
         2472  +     && !isIgnoreEmpty 
         2473  +     && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0)
         2474  +    ){
  2131   2475         pCsr->aDoclist = apSegment[0]->aDoclist;
  2132   2476         pCsr->nDoclist = apSegment[0]->nDoclist;
  2133   2477         rc = SQLITE_ROW;
  2134   2478       }else{
  2135   2479         int nDoclist = 0;           /* Size of doclist */
  2136   2480         sqlite3_int64 iPrev = 0;    /* Previous docid stored in doclist */
  2137   2481   
  2138   2482         /* The current term of the first nMerge entries in the array
  2139   2483         ** of Fts3SegReader objects is the same. The doclists must be merged
  2140   2484         ** and a single term returned with the merged doclist.
  2141   2485         */
  2142   2486         for(i=0; i<nMerge; i++){
  2143         -        fts3SegReaderFirstDocid(apSegment[i]);
         2487  +        fts3SegReaderFirstDocid(p, apSegment[i]);
  2144   2488         }
  2145         -      fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp);
         2489  +      fts3SegReaderSort(apSegment, nMerge, nMerge, xCmp);
  2146   2490         while( apSegment[0]->pOffsetList ){
  2147   2491           int j;                    /* Number of segments that share a docid */
  2148   2492           char *pList;
  2149   2493           int nList;
  2150   2494           int nByte;
  2151   2495           sqlite3_int64 iDocid = apSegment[0]->iDocid;
  2152         -        fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
         2496  +        fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList);
  2153   2497           j = 1;
  2154   2498           while( j<nMerge
  2155   2499               && apSegment[j]->pOffsetList
  2156   2500               && apSegment[j]->iDocid==iDocid
  2157   2501           ){
  2158         -          fts3SegReaderNextDocid(apSegment[j], 0, 0);
         2502  +          fts3SegReaderNextDocid(p, apSegment[j], 0, 0);
  2159   2503             j++;
  2160   2504           }
  2161   2505   
  2162   2506           if( isColFilter ){
  2163   2507             fts3ColumnFilter(pFilter->iCol, &pList, &nList);
  2164   2508           }
  2165   2509   
  2166   2510           if( !isIgnoreEmpty || nList>0 ){
  2167         -          nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0);
         2511  +
         2512  +          /* Calculate the 'docid' delta value to write into the merged 
         2513  +          ** doclist. */
         2514  +          sqlite3_int64 iDelta;
         2515  +          if( p->bDescIdx && nDoclist>0 ){
         2516  +            iDelta = iPrev - iDocid;
         2517  +          }else{
         2518  +            iDelta = iDocid - iPrev;
         2519  +          }
         2520  +          assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) );
         2521  +          assert( nDoclist>0 || iDelta==iDocid );
         2522  +
         2523  +          nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0);
  2168   2524             if( nDoclist+nByte>pCsr->nBuffer ){
  2169   2525               char *aNew;
  2170   2526               pCsr->nBuffer = (nDoclist+nByte)*2;
  2171   2527               aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer);
  2172   2528               if( !aNew ){
  2173   2529                 return SQLITE_NOMEM;
  2174   2530               }
  2175   2531               pCsr->aBuffer = aNew;
  2176   2532             }
  2177         -          nDoclist += sqlite3Fts3PutVarint(
  2178         -              &pCsr->aBuffer[nDoclist], iDocid-iPrev
  2179         -          );
         2533  +          nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta);
  2180   2534             iPrev = iDocid;
  2181   2535             if( isRequirePos ){
  2182   2536               memcpy(&pCsr->aBuffer[nDoclist], pList, nList);
  2183   2537               nDoclist += nList;
  2184   2538               pCsr->aBuffer[nDoclist++] = '\0';
  2185   2539             }
  2186   2540           }
  2187   2541   
  2188         -        fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp);
         2542  +        fts3SegReaderSort(apSegment, nMerge, j, xCmp);
  2189   2543         }
  2190   2544         if( nDoclist>0 ){
  2191   2545           pCsr->aDoclist = pCsr->aBuffer;
  2192   2546           pCsr->nDoclist = nDoclist;
  2193   2547           rc = SQLITE_ROW;
  2194   2548         }
  2195   2549       }
  2196   2550       pCsr->nAdvance = nMerge;
  2197   2551     }while( rc==SQLITE_OK );
  2198   2552   
  2199   2553     return rc;
  2200   2554   }
         2555  +
  2201   2556   
  2202   2557   void sqlite3Fts3SegReaderFinish(
  2203         -  Fts3SegReaderCursor *pCsr       /* Cursor object */
         2558  +  Fts3MultiSegReader *pCsr       /* Cursor object */
  2204   2559   ){
  2205   2560     if( pCsr ){
  2206   2561       int i;
  2207   2562       for(i=0; i<pCsr->nSegment; i++){
  2208   2563         sqlite3Fts3SegReaderFree(pCsr->apSegment[i]);
  2209   2564       }
  2210   2565       sqlite3_free(pCsr->apSegment);
................................................................................
  2223   2578   ** currently present in the database.
  2224   2579   **
  2225   2580   ** If this function is called with iLevel<0, but there is only one
  2226   2581   ** segment in the database, SQLITE_DONE is returned immediately. 
  2227   2582   ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, 
  2228   2583   ** an SQLite error code is returned.
  2229   2584   */
  2230         -static int fts3SegmentMerge(Fts3Table *p, int iLevel){
         2585  +static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){
  2231   2586     int rc;                         /* Return code */
  2232   2587     int iIdx = 0;                   /* Index of new segment */
  2233         -  int iNewLevel = 0;              /* Level to create new segment at */
         2588  +  int iNewLevel = 0;              /* Level/index to create new segment at */
  2234   2589     SegmentWriter *pWriter = 0;     /* Used to write the new, merged, segment */
  2235   2590     Fts3SegFilter filter;           /* Segment term filter condition */
  2236         -  Fts3SegReaderCursor csr;        /* Cursor to iterate through level(s) */
         2591  +  Fts3MultiSegReader csr;        /* Cursor to iterate through level(s) */
         2592  +  int bIgnoreEmpty = 0;           /* True to ignore empty segments */
  2237   2593   
  2238         -  rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, 0, &csr);
         2594  +  assert( iLevel==FTS3_SEGCURSOR_ALL
         2595  +       || iLevel==FTS3_SEGCURSOR_PENDING
         2596  +       || iLevel>=0
         2597  +  );
         2598  +  assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
         2599  +  assert( iIndex>=0 && iIndex<p->nIndex );
         2600  +
         2601  +  rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr);
  2239   2602     if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished;
  2240   2603   
  2241   2604     if( iLevel==FTS3_SEGCURSOR_ALL ){
  2242   2605       /* This call is to merge all segments in the database to a single
  2243   2606       ** segment. The level of the new segment is equal to the the numerically 
  2244         -    ** greatest segment level currently present in the database. The index
  2245         -    ** of the new segment is always 0.  */
  2246         -    int nDummy; /* TODO: Remove this */
         2607  +    ** greatest segment level currently present in the database for this
         2608  +    ** index. The idx of the new segment is always 0.  */
  2247   2609       if( csr.nSegment==1 ){
  2248   2610         rc = SQLITE_DONE;
  2249   2611         goto finished;
  2250   2612       }
  2251         -    rc = fts3SegmentCountMax(p, &nDummy, &iNewLevel);
         2613  +    rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel);
         2614  +    bIgnoreEmpty = 1;
         2615  +
         2616  +  }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
         2617  +    iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL; 
         2618  +    rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx);
  2252   2619     }else{
  2253         -    /* This call is to merge all segments at level iLevel. Find the next
         2620  +    /* This call is to merge all segments at level iLevel. find the next
  2254   2621       ** available segment index at level iLevel+1. The call to
  2255   2622       ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to 
  2256   2623       ** a single iLevel+2 segment if necessary.  */
  2257         -    iNewLevel = iLevel+1;
  2258         -    rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx);
         2624  +    rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx);
         2625  +    iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1;
  2259   2626     }
  2260   2627     if( rc!=SQLITE_OK ) goto finished;
  2261   2628     assert( csr.nSegment>0 );
  2262         -  assert( iNewLevel>=0 );
         2629  +  assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) );
         2630  +  assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) );
  2263   2631   
  2264   2632     memset(&filter, 0, sizeof(Fts3SegFilter));
  2265   2633     filter.flags = FTS3_SEGMENT_REQUIRE_POS;
  2266         -  filter.flags |= (iLevel==FTS3_SEGCURSOR_ALL ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
         2634  +  filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
  2267   2635   
  2268   2636     rc = sqlite3Fts3SegReaderStart(p, &csr, &filter);
  2269   2637     while( SQLITE_OK==rc ){
  2270   2638       rc = sqlite3Fts3SegReaderStep(p, &csr);
  2271   2639       if( rc!=SQLITE_ROW ) break;
  2272   2640       rc = fts3SegWriterAdd(p, &pWriter, 1, 
  2273   2641           csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
  2274   2642     }
  2275   2643     if( rc!=SQLITE_OK ) goto finished;
  2276   2644     assert( pWriter );
  2277   2645   
  2278         -  rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment);
  2279         -  if( rc!=SQLITE_OK ) goto finished;
         2646  +  if( iLevel!=FTS3_SEGCURSOR_PENDING ){
         2647  +    rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment);
         2648  +    if( rc!=SQLITE_OK ) goto finished;
         2649  +  }
  2280   2650     rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
  2281   2651   
  2282   2652    finished:
  2283   2653     fts3SegWriterFree(pWriter);
  2284   2654     sqlite3Fts3SegReaderFinish(&csr);
  2285   2655     return rc;
  2286   2656   }
  2287   2657   
  2288   2658   
  2289   2659   /* 
  2290         -** Flush the contents of pendingTerms to a level 0 segment.
         2660  +** Flush the contents of pendingTerms to level 0 segments.
  2291   2661   */
  2292   2662   int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
  2293         -  return fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING);
         2663  +  int rc = SQLITE_OK;
         2664  +  int i;
         2665  +  for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
         2666  +    rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING);
         2667  +    if( rc==SQLITE_DONE ) rc = SQLITE_OK;
         2668  +  }
         2669  +  sqlite3Fts3PendingTermsClear(p);
         2670  +  return rc;
  2294   2671   }
  2295   2672   
  2296   2673   /*
  2297   2674   ** Encode N integers as varints into a blob.
  2298   2675   */
  2299   2676   static void fts3EncodeIntArray(
  2300   2677     int N,             /* The number of integers to encode */
................................................................................
  2436   2813       return;
  2437   2814     }
  2438   2815     sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC);
  2439   2816     sqlite3_step(pStmt);
  2440   2817     *pRC = sqlite3_reset(pStmt);
  2441   2818     sqlite3_free(a);
  2442   2819   }
         2820  +
         2821  +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
         2822  +  int i;
         2823  +  int bSeenDone = 0;
         2824  +  int rc = SQLITE_OK;
         2825  +  for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
         2826  +    rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL);
         2827  +    if( rc==SQLITE_DONE ){
         2828  +      bSeenDone = 1;
         2829  +      rc = SQLITE_OK;
         2830  +    }
         2831  +  }
         2832  +  sqlite3Fts3SegmentsClose(p);
         2833  +  sqlite3Fts3PendingTermsClear(p);
         2834  +
         2835  +  return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc;
         2836  +}
  2443   2837   
  2444   2838   /*
  2445   2839   ** Handle a 'special' INSERT of the form:
  2446   2840   **
  2447   2841   **   "INSERT INTO tbl(tbl) VALUES(<expr>)"
  2448   2842   **
  2449   2843   ** Argument pVal contains the result of <expr>. Currently the only 
................................................................................
  2453   2847     int rc;                         /* Return Code */
  2454   2848     const char *zVal = (const char *)sqlite3_value_text(pVal);
  2455   2849     int nVal = sqlite3_value_bytes(pVal);
  2456   2850   
  2457   2851     if( !zVal ){
  2458   2852       return SQLITE_NOMEM;
  2459   2853     }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
  2460         -    rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL);
  2461         -    if( rc==SQLITE_DONE ){
  2462         -      rc = SQLITE_OK;
  2463         -    }else{
  2464         -      sqlite3Fts3PendingTermsClear(p);
  2465         -    }
         2854  +    rc = fts3DoOptimize(p, 0);
  2466   2855   #ifdef SQLITE_TEST
  2467   2856     }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
  2468   2857       p->nNodeSize = atoi(&zVal[9]);
  2469   2858       rc = SQLITE_OK;
  2470   2859     }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
  2471   2860       p->nMaxPendingData = atoi(&zVal[11]);
  2472   2861       rc = SQLITE_OK;
  2473   2862   #endif
  2474   2863     }else{
  2475   2864       rc = SQLITE_ERROR;
  2476   2865     }
  2477   2866   
  2478         -  sqlite3Fts3SegmentsClose(p);
  2479   2867     return rc;
  2480   2868   }
  2481   2869   
  2482         -/*
  2483         -** Return the deferred doclist associated with deferred token pDeferred.
  2484         -** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already
  2485         -** been called to allocate and populate the doclist.
  2486         -*/
  2487         -char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){
  2488         -  if( pDeferred->pList ){
  2489         -    *pnByte = pDeferred->pList->nData;
  2490         -    return pDeferred->pList->aData;
  2491         -  }
  2492         -  *pnByte = 0;
  2493         -  return 0;
  2494         -}
  2495         -
  2496         -/*
  2497         -** Helper fucntion for FreeDeferredDoclists(). This function removes all
  2498         -** references to deferred doclists from within the tree of Fts3Expr 
  2499         -** structures headed by 
  2500         -*/
  2501         -static void fts3DeferredDoclistClear(Fts3Expr *pExpr){
  2502         -  if( pExpr ){
  2503         -    fts3DeferredDoclistClear(pExpr->pLeft);
  2504         -    fts3DeferredDoclistClear(pExpr->pRight);
  2505         -    if( pExpr->isLoaded ){
  2506         -      sqlite3_free(pExpr->aDoclist);
  2507         -      pExpr->isLoaded = 0;
  2508         -      pExpr->aDoclist = 0;
  2509         -      pExpr->nDoclist = 0;
  2510         -      pExpr->pCurrent = 0;
  2511         -      pExpr->iCurrent = 0;
  2512         -    }
  2513         -  }
  2514         -}
  2515         -
  2516   2870   /*
  2517   2871   ** Delete all cached deferred doclists. Deferred doclists are cached
  2518   2872   ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function.
  2519   2873   */
  2520   2874   void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){
  2521   2875     Fts3DeferredToken *pDef;
  2522   2876     for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){
  2523         -    sqlite3_free(pDef->pList);
         2877  +    fts3PendingListDelete(pDef->pList);
  2524   2878       pDef->pList = 0;
  2525   2879     }
  2526         -  if( pCsr->pDeferred ){
  2527         -    fts3DeferredDoclistClear(pCsr->pExpr);
  2528         -  }
  2529   2880   }
  2530   2881   
  2531   2882   /*
  2532   2883   ** Free all entries in the pCsr->pDeffered list. Entries are added to 
  2533   2884   ** this list using sqlite3Fts3DeferToken().
  2534   2885   */
  2535   2886   void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){
  2536   2887     Fts3DeferredToken *pDef;
  2537   2888     Fts3DeferredToken *pNext;
  2538   2889     for(pDef=pCsr->pDeferred; pDef; pDef=pNext){
  2539   2890       pNext = pDef->pNext;
  2540         -    sqlite3_free(pDef->pList);
         2891  +    fts3PendingListDelete(pDef->pList);
  2541   2892       sqlite3_free(pDef);
  2542   2893     }
  2543   2894     pCsr->pDeferred = 0;
  2544   2895   }
  2545   2896   
  2546   2897   /*
  2547   2898   ** Generate deferred-doclists for all tokens in the pCsr->pDeferred list
................................................................................
  2597   2948           rc = fts3PendingListAppendVarint(&pDef->pList, 0);
  2598   2949         }
  2599   2950       }
  2600   2951     }
  2601   2952   
  2602   2953     return rc;
  2603   2954   }
         2955  +
         2956  +int sqlite3Fts3DeferredTokenList(
         2957  +  Fts3DeferredToken *p, 
         2958  +  char **ppData, 
         2959  +  int *pnData
         2960  +){
         2961  +  char *pRet;
         2962  +  int nSkip;
         2963  +  sqlite3_int64 dummy;
         2964  +
         2965  +  *ppData = 0;
         2966  +  *pnData = 0;
         2967  +
         2968  +  if( p->pList==0 ){
         2969  +    return SQLITE_OK;
         2970  +  }
         2971  +
         2972  +  pRet = (char *)sqlite3_malloc(p->pList->nData);
         2973  +  if( !pRet ) return SQLITE_NOMEM;
         2974  +
         2975  +  nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy);
         2976  +  *pnData = p->pList->nData - nSkip;
         2977  +  *ppData = pRet;
         2978  +  
         2979  +  memcpy(pRet, &p->pList->aData[nSkip], *pnData);
         2980  +  return SQLITE_OK;
         2981  +}
  2604   2982   
  2605   2983   /*
  2606   2984   ** Add an entry for token pToken to the pCsr->pDeferred list.
  2607   2985   */
  2608   2986   int sqlite3Fts3DeferToken(
  2609   2987     Fts3Cursor *pCsr,               /* Fts3 table cursor */
  2610   2988     Fts3PhraseToken *pToken,        /* Token to defer */
................................................................................
  2672   3050     sqlite3_value **apVal,          /* Array of arguments */
  2673   3051     sqlite_int64 *pRowid            /* OUT: The affected (or effected) rowid */
  2674   3052   ){
  2675   3053     Fts3Table *p = (Fts3Table *)pVtab;
  2676   3054     int rc = SQLITE_OK;             /* Return Code */
  2677   3055     int isRemove = 0;               /* True for an UPDATE or DELETE */
  2678   3056     sqlite3_int64 iRemove = 0;      /* Rowid removed by UPDATE or DELETE */
  2679         -  u32 *aSzIns;                    /* Sizes of inserted documents */
         3057  +  u32 *aSzIns = 0;                /* Sizes of inserted documents */
  2680   3058     u32 *aSzDel;                    /* Sizes of deleted documents */
  2681   3059     int nChng = 0;                  /* Net change in number of documents */
  2682   3060     int bInsertDone = 0;
  2683   3061   
  2684   3062     assert( p->pSegments==0 );
  2685   3063   
  2686   3064     /* Check for a "special" INSERT operation. One of the form:
................................................................................
  2687   3065     **
  2688   3066     **   INSERT INTO xyz(xyz) VALUES('command');
  2689   3067     */
  2690   3068     if( nArg>1 
  2691   3069      && sqlite3_value_type(apVal[0])==SQLITE_NULL 
  2692   3070      && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL 
  2693   3071     ){
  2694         -    return fts3SpecialInsert(p, apVal[p->nColumn+2]);
         3072  +    rc = fts3SpecialInsert(p, apVal[p->nColumn+2]);
         3073  +    goto update_out;
  2695   3074     }
  2696   3075   
  2697   3076     /* Allocate space to hold the change in document sizes */
  2698   3077     aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 );
  2699         -  if( aSzIns==0 ) return SQLITE_NOMEM;
         3078  +  if( aSzIns==0 ){
         3079  +    rc = SQLITE_NOMEM;
         3080  +    goto update_out;
         3081  +  }
  2700   3082     aSzDel = &aSzIns[p->nColumn+1];
  2701   3083     memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2);
  2702   3084   
  2703   3085     /* If this is an INSERT operation, or an UPDATE that modifies the rowid
  2704   3086     ** value, then this operation requires constraint handling.
  2705   3087     **
  2706   3088     ** If the on-conflict mode is REPLACE, this means that the existing row
................................................................................
  2742   3124         }else{
  2743   3125           rc = fts3InsertData(p, apVal, pRowid);
  2744   3126           bInsertDone = 1;
  2745   3127         }
  2746   3128       }
  2747   3129     }
  2748   3130     if( rc!=SQLITE_OK ){
  2749         -    sqlite3_free(aSzIns);
  2750         -    return rc;
         3131  +    goto update_out;
  2751   3132     }
  2752   3133   
  2753   3134     /* If this is a DELETE or UPDATE operation, remove the old record. */
  2754   3135     if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
  2755   3136       assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
  2756   3137       rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
  2757   3138       isRemove = 1;
................................................................................
  2776   3157       nChng++;
  2777   3158     }
  2778   3159   
  2779   3160     if( p->bHasStat ){
  2780   3161       fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
  2781   3162     }
  2782   3163   
         3164  + update_out:
  2783   3165     sqlite3_free(aSzIns);
  2784   3166     sqlite3Fts3SegmentsClose(p);
  2785   3167     return rc;
  2786   3168   }
  2787   3169   
  2788   3170   /* 
  2789   3171   ** Flush any data in the pending-terms hash table to disk. If successful,
................................................................................
  2790   3172   ** merge all segments in the database (including the new segment, if 
  2791   3173   ** there was any data to flush) into a single segment. 
  2792   3174   */
  2793   3175   int sqlite3Fts3Optimize(Fts3Table *p){
  2794   3176     int rc;
  2795   3177     rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0);
  2796   3178     if( rc==SQLITE_OK ){
  2797         -    rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL);
  2798         -    if( rc==SQLITE_OK ){
  2799         -      rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
  2800         -      if( rc==SQLITE_OK ){
  2801         -        sqlite3Fts3PendingTermsClear(p);
  2802         -      }
         3179  +    rc = fts3DoOptimize(p, 1);
         3180  +    if( rc==SQLITE_OK || rc==SQLITE_DONE ){
         3181  +      int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
         3182  +      if( rc2!=SQLITE_OK ) rc = rc2;
  2803   3183       }else{
  2804   3184         sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0);
  2805   3185         sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
  2806   3186       }
  2807   3187     }
  2808   3188     sqlite3Fts3SegmentsClose(p);
  2809   3189     return rc;
  2810   3190   }
  2811   3191   
  2812   3192   #endif

Changes to main.mk.

   216    216     parse.h \
   217    217     sqlite3.h
   218    218   
   219    219   
   220    220   # Source code to the test files.
   221    221   #
   222    222   TESTSRC = \
          223  +  $(TOP)/ext/fts3/fts3_test.c \
   223    224     $(TOP)/src/test1.c \
   224    225     $(TOP)/src/test2.c \
   225    226     $(TOP)/src/test3.c \
   226    227     $(TOP)/src/test4.c \
   227    228     $(TOP)/src/test5.c \
   228    229     $(TOP)/src/test6.c \
   229    230     $(TOP)/src/test7.c \

Changes to src/tclsqlite.c.

  3580   3580       extern int Sqlitetestrtree_Init(Tcl_Interp*);
  3581   3581       extern int Sqlitequota_Init(Tcl_Interp*);
  3582   3582       extern int Sqlitemultiplex_Init(Tcl_Interp*);
  3583   3583       extern int SqliteSuperlock_Init(Tcl_Interp*);
  3584   3584       extern int SqlitetestSyscall_Init(Tcl_Interp*);
  3585   3585       extern int Sqlitetestfuzzer_Init(Tcl_Interp*);
  3586   3586       extern int Sqlitetestwholenumber_Init(Tcl_Interp*);
         3587  +
         3588  +#ifdef SQLITE_ENABLE_FTS3
         3589  +    extern int Sqlitetestfts3_Init(Tcl_Interp *interp);
         3590  +#endif
  3587   3591   
  3588   3592   #ifdef SQLITE_ENABLE_ZIPVFS
  3589   3593       extern int Zipvfs_Init(Tcl_Interp*);
  3590   3594       Zipvfs_Init(interp);
  3591   3595   #endif
  3592   3596   
  3593   3597       Sqliteconfig_Init(interp);
................................................................................
  3620   3624       Sqlitetestrtree_Init(interp);
  3621   3625       Sqlitequota_Init(interp);
  3622   3626       Sqlitemultiplex_Init(interp);
  3623   3627       SqliteSuperlock_Init(interp);
  3624   3628       SqlitetestSyscall_Init(interp);
  3625   3629       Sqlitetestfuzzer_Init(interp);
  3626   3630       Sqlitetestwholenumber_Init(interp);
         3631  +
         3632  +#ifdef SQLITE_ENABLE_FTS3
         3633  +    Sqlitetestfts3_Init(interp);
         3634  +#endif
  3627   3635   
  3628   3636       Tcl_CreateObjCommand(interp,"load_testfixture_extensions",init_all_cmd,0,0);
  3629   3637   
  3630   3638   #ifdef SQLITE_SSE
  3631   3639       Sqlitetestsse_Init(interp);
  3632   3640   #endif
  3633   3641     }

Added test/fts3auto.test.

            1  +# 2011 June 10
            2  +#
            3  +#    May you do good and not evil.
            4  +#    May you find forgiveness for yourself and forgive others.
            5  +#    May you share freely, never taking more than you give.
            6  +#
            7  +#***********************************************************************
            8  +#
            9  +
           10  +set testdir [file dirname $argv0]
           11  +source $testdir/tester.tcl
           12  +
           13  +# If this build does not include FTS3, skip the tests in this file.
           14  +#
           15  +ifcapable !fts3 { finish_test ; return }
           16  +source $testdir/fts3_common.tcl
           17  +source $testdir/malloc_common.tcl
           18  +
           19  +set testprefix fts3auto
           20  +set sfep $sqlite_fts3_enable_parentheses
           21  +set sqlite_fts3_enable_parentheses 1
           22  +
           23  +#--------------------------------------------------------------------------
           24  +# Start of Tcl infrastructure used by tests. The entry points are:
           25  +#
           26  +#   do_fts3query_test
           27  +#   fts3_make_deferrable
           28  +#   fts3_zero_long_segments 
           29  +#
           30  +
           31  +#
           32  +#    do_fts3query_test TESTNAME ?OPTIONS? TABLE MATCHEXPR
           33  +#
           34  +# This proc runs several test cases on FTS3/4 table $TABLE using match
           35  +# expression $MATCHEXPR. All documents in $TABLE must be formatted so that
           36  +# they can be "tokenized" using the Tcl list commands (llength, lindex etc.).
           37  +# The name and column names used by $TABLE must not require any quoting or
           38  +# escaping when used in SQL statements.
           39  +#
           40  +# $MATCHINFO may be any expression accepted by the FTS4 MATCH operator, 
           41  +# except that the "<column-name>:token" syntax is not supported. Tcl list
           42  +# commands are used to tokenize the expression. Any parenthesis must appear
           43  +# either as separate list elements, or as the first (for opening) or last
           44  +# (for closing) character of a list element. i.e. the expression "(a OR b)c"
           45  +# will not be parsed correctly, but "( a OR b) c" will.
           46  +#
           47  +# Available OPTIONS are:
           48  +#
           49  +#     -deferred TOKENLIST
           50  +#
           51  +# If the "deferred" option is supplied, it is passed a list of tokens that
           52  +# are deferred by FTS and result in the relevant matchinfo() stats being an
           53  +# approximation. 
           54  +#
           55  +set sqlite_fts3_enable_parentheses 1
           56  +proc do_fts3query_test {tn args} {
           57  +
           58  +  set nArg [llength $args]
           59  +  if {$nArg < 2 || ($nArg % 2)} {
           60  +    set cmd do_fts3query_test
           61  +    error "wrong # args: should be \"$cmd ?-deferred LIST? TABLE MATCHEXPR\""
           62  +  }
           63  +  set tbl   [lindex $args [expr $nArg-2]]
           64  +  set match [lindex $args [expr $nArg-1]]
           65  +  set deferred [list]
           66  +
           67  +  foreach {k v} [lrange $args 0 [expr $nArg-3]] {
           68  +    switch -- $k {
           69  +      -deferred {
           70  +        set deferred $v
           71  +      }
           72  +      default {
           73  +        error "bad option \"$k\": must be -deferred"
           74  +      }
           75  +    }
           76  +  }
           77  +
           78  +  get_near_results $tbl $match $deferred aMatchinfo
           79  +
           80  +  set matchinfo_asc [list]
           81  +  foreach docid [lsort -integer -incr [array names aMatchinfo]] {
           82  +    lappend matchinfo_asc $docid $aMatchinfo($docid)
           83  +  }
           84  +  set matchinfo_desc [list]
           85  +  foreach docid [lsort -integer -decr [array names aMatchinfo]] {
           86  +    lappend matchinfo_desc $docid $aMatchinfo($docid)
           87  +  }
           88  +
           89  +  set title "(\"$match\" -> [llength [array names aMatchinfo]] rows)"
           90  +
           91  +  do_execsql_test $tn$title.1 "
           92  +    SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC
           93  +  " [lsort -integer -incr [array names aMatchinfo]] 
           94  +
           95  +  do_execsql_test $tn$title.2 "
           96  +    SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid DESC
           97  +  " [lsort -integer -decr [array names aMatchinfo]] 
           98  +
           99  +  do_execsql_test $tn$title.3 "
          100  +    SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl 
          101  +    WHERE $tbl MATCH '$match' ORDER BY docid DESC
          102  +  " $matchinfo_desc
          103  +
          104  +  do_execsql_test $tn$title.4 "
          105  +    SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl 
          106  +    WHERE $tbl MATCH '$match' ORDER BY docid ASC
          107  +  " $matchinfo_asc
          108  +}
          109  +
          110  +#    fts3_make_deferrable TABLE TOKEN
          111  +#
          112  +proc fts3_make_deferrable {tbl token} {
          113  +
          114  +  set stmt [sqlite3_prepare db "SELECT * FROM $tbl" -1 dummy]
          115  +  set name [sqlite3_column_name $stmt 0]
          116  +  sqlite3_finalize $stmt
          117  +
          118  +  set nRow [db one "SELECT count(*) FROM $tbl"]
          119  +  set pgsz [db one "PRAGMA page_size"]
          120  +  execsql BEGIN
          121  +  for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} {
          122  +    set doc [string repeat "$token " 100]
          123  +    execsql "INSERT INTO $tbl ($name) VALUES(\$doc)"
          124  +  }
          125  +  execsql "INSERT INTO $tbl ($name) VALUES('aaaaaaa ${token}aaaaa')"
          126  +  execsql COMMIT
          127  +
          128  +  return [expr $nRow*$pgsz]
          129  +}
          130  +
          131  +#    fts3_zero_long_segments TABLE ?LIMIT?
          132  +#
          133  +proc fts3_zero_long_segments {tbl limit} {
          134  +  execsql " 
          135  +    UPDATE ${tbl}_segments 
          136  +    SET block = zeroblob(length(block)) 
          137  +    WHERE length(block)>$limit
          138  +  "
          139  +  return [db changes]
          140  +}
          141  +
          142  +
          143  +proc mit {blob} {
          144  +  set scan(littleEndian) i*
          145  +  set scan(bigEndian) I*
          146  +  binary scan $blob $scan($::tcl_platform(byteOrder)) r
          147  +  return $r
          148  +}
          149  +db func mit mit
          150  +
          151  +proc fix_near_expr {expr} { 
          152  +  set out [list]
          153  +  lappend out [lindex $expr 0]
          154  +  foreach {a b} [lrange $expr 1 end] {
          155  +    if {[string match -nocase near $a]}   { set a 10 }
          156  +    if {[string match -nocase near/* $a]} { set a [string range $a 5 end] }
          157  +    lappend out $a
          158  +    lappend out $b
          159  +  }
          160  +  return $out
          161  +}
          162  +
          163  +proc get_single_near_results {tbl expr deferred arrayvar nullvar} {
          164  +  upvar $arrayvar aMatchinfo
          165  +  upvar $nullvar nullentry
          166  +  catch {array unset aMatchinfo}
          167  +
          168  +  set expr [fix_near_expr $expr]
          169  +
          170  +  # Calculate the expected results using [fts3_near_match]. The following
          171  +  # loop populates the "hits" and "counts" arrays as follows:
          172  +  # 
          173  +  #   1. For each document in the table that matches the NEAR expression,
          174  +  #      hits($docid) is set to 1. The set of docids that match the expression
          175  +  #      can therefore be found using [array names hits].
          176  +  #
          177  +  #   2. For each column of each document in the table, counts($docid,$iCol)
          178  +  #      is set to the -phrasecountvar output.
          179  +  #
          180  +  set res [list]
          181  +  catch { array unset hits }
          182  +  db eval "SELECT docid, * FROM $tbl" d {
          183  +    set iCol 0
          184  +    foreach col [lrange $d(*) 1 end] {
          185  +      set docid $d(docid)
          186  +      set hit [fts3_near_match $d($col) $expr -p counts($docid,$iCol)]
          187  +      if {$hit} { set hits($docid) 1 }
          188  +      incr iCol
          189  +    }
          190  +  }
          191  +  set nPhrase [expr ([llength $expr]+1)/2]
          192  +  set nCol $iCol
          193  +
          194  +  # This block populates the nHit and nDoc arrays. For each phrase/column
          195  +  # in the query/table, array elements are set as follows:
          196  +  #
          197  +  #   nHit($iPhrase,$iCol) - Total number of hits for phrase $iPhrase in 
          198  +  #                          column $iCol.
          199  +  #
          200  +  #   nDoc($iPhrase,$iCol) - Number of documents with at least one hit for
          201  +  #                          phrase $iPhrase in column $iCol.
          202  +  #
          203  +  for {set iPhrase 0} {$iPhrase < $nPhrase} {incr iPhrase} {
          204  +    for {set iCol 0} {$iCol < $nCol} {incr iCol} {
          205  +      set nHit($iPhrase,$iCol) 0
          206  +      set nDoc($iPhrase,$iCol) 0
          207  +    }
          208  +  }
          209  +  foreach key [array names counts] {
          210  +    set iCol [lindex [split $key ,] 1]
          211  +    set iPhrase 0
          212  +    foreach c $counts($key) {
          213  +      if {$c>0} { incr nDoc($iPhrase,$iCol) 1 }
          214  +      incr nHit($iPhrase,$iCol) $c
          215  +      incr iPhrase
          216  +    }
          217  +  }
          218  +
          219  +  if {[llength $deferred] && [llength $expr]==1} {
          220  +    set phrase [lindex $expr 0]
          221  +    set rewritten [list]
          222  +    set partial 0
          223  +    foreach tok $phrase {
          224  +      if {[lsearch $deferred $tok]>=0} {
          225  +        lappend rewritten *
          226  +      } else {
          227  +        lappend rewritten $tok
          228  +        set partial 1
          229  +      }
          230  +    }
          231  +    if {$partial==0} {
          232  +      set tblsize [db one "SELECT count(*) FROM $tbl"]
          233  +      for {set iCol 0} {$iCol < $nCol} {incr iCol} {
          234  +        set nHit(0,$iCol) $tblsize
          235  +        set nDoc(0,$iCol) $tblsize
          236  +      }
          237  +    } elseif {$rewritten != $phrase} {
          238  +      while {[lindex $rewritten end] == "*"} {
          239  +        set rewritten [lrange $rewritten 0 end-1]
          240  +      }
          241  +      while {[lindex $rewritten 0] == "*"} {
          242  +        set rewritten [lrange $rewritten 1 end]
          243  +      }
          244  +      get_single_near_results $tbl [list $rewritten] {} aRewrite nullentry
          245  +      foreach docid [array names hits] {
          246  +        set aMatchinfo($docid) $aRewrite($docid)
          247  +      }
          248  +      return
          249  +    }
          250  +  }
          251  +
          252  +  # Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to
          253  +  # contain the output of matchinfo('x') for the document.
          254  +  #
          255  +  foreach docid [array names hits] {
          256  +    set mi [list]
          257  +    for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
          258  +      for {set iCol 0} {$iCol<$nCol} {incr iCol} {
          259  +        lappend mi [lindex $counts($docid,$iCol) $iPhrase]
          260  +        lappend mi $nHit($iPhrase,$iCol)
          261  +        lappend mi $nDoc($iPhrase,$iCol)
          262  +      }
          263  +    }
          264  +    set aMatchinfo($docid) $mi
          265  +  }
          266  +
          267  +  # Set up the nullentry output.
          268  +  #
          269  +  set nullentry [list]
          270  +  for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
          271  +    for {set iCol 0} {$iCol<$nCol} {incr iCol} {
          272  +      lappend nullentry 0 $nHit($iPhrase,$iCol) $nDoc($iPhrase,$iCol)
          273  +    }
          274  +  }
          275  +}
          276  +
          277  +
          278  +proc matching_brackets {expr} {
          279  +  if {[string range $expr 0 0]!="(" || [string range $expr end end] !=")"} { 
          280  +    return 0 
          281  +  }
          282  +
          283  +  set iBracket 1
          284  +  set nExpr [string length $expr]
          285  +  for {set i 1} {$iBracket && $i < $nExpr} {incr i} {
          286  +    set c [string range $expr $i $i]
          287  +    if {$c == "("} {incr iBracket}
          288  +    if {$c == ")"} {incr iBracket -1}
          289  +  }
          290  +
          291  +  return [expr ($iBracket==0 && $i==$nExpr)]
          292  +}
          293  +
          294  +proc get_near_results {tbl expr deferred arrayvar {nullvar ""}} {
          295  +  upvar $arrayvar aMatchinfo
          296  +  if {$nullvar != ""} { upvar $nullvar nullentry }
          297  +
          298  +  set expr [string trim $expr]
          299  +  while { [matching_brackets $expr] } {
          300  +    set expr [string trim [string range $expr 1 end-1]]
          301  +  }
          302  +
          303  +  set prec(NOT) 1
          304  +  set prec(AND) 2
          305  +  set prec(OR)  3
          306  +
          307  +  set currentprec 0
          308  +  set iBracket 0
          309  +  set expr_length [llength $expr]
          310  +  for {set i 0} {$i < $expr_length} {incr i} {
          311  +    set op [lindex $expr $i]
          312  +    if {$iBracket==0 && [info exists prec($op)] && $prec($op)>=$currentprec } {
          313  +      set opidx $i
          314  +      set currentprec $prec($op)
          315  +    } else {
          316  +      for {set j 0} {$j < [string length $op]} {incr j} {
          317  +        set c [string range $op $j $j]
          318  +        if {$c == "("} { incr iBracket +1 }
          319  +        if {$c == ")"} { incr iBracket -1 }
          320  +      }
          321  +    }
          322  +  }
          323  +  if {$iBracket!=0} { error "mismatched brackets in: $expr" }
          324  +
          325  +  if {[info exists opidx]==0} {
          326  +    get_single_near_results $tbl $expr $deferred aMatchinfo nullentry
          327  +  } else {
          328  +    set eLeft  [lrange $expr 0 [expr $opidx-1]]
          329  +    set eRight [lrange $expr [expr $opidx+1] end]
          330  +
          331  +    get_near_results $tbl $eLeft  $deferred aLeft  nullleft
          332  +    get_near_results $tbl $eRight $deferred aRight nullright
          333  +
          334  +    switch -- [lindex $expr $opidx] {
          335  +      "NOT" {
          336  +        foreach hit [array names aLeft] {
          337  +          if {0==[info exists aRight($hit)]} {
          338  +            set aMatchinfo($hit) $aLeft($hit)
          339  +          }
          340  +        }
          341  +        set nullentry $nullleft
          342  +      }
          343  +
          344  +      "AND" {
          345  +        foreach hit [array names aLeft] {
          346  +          if {[info exists aRight($hit)]} {
          347  +            set aMatchinfo($hit) [concat $aLeft($hit) $aRight($hit)]
          348  +          }
          349  +        }
          350  +        set nullentry [concat $nullleft $nullright]
          351  +      }
          352  +
          353  +      "OR" {
          354  +        foreach hit [array names aLeft] {
          355  +          if {[info exists aRight($hit)]} {
          356  +            set aMatchinfo($hit) [concat $aLeft($hit) $aRight($hit)]
          357  +            unset aRight($hit)
          358  +          } else {
          359  +            set aMatchinfo($hit) [concat $aLeft($hit) $nullright]
          360  +          }
          361  +        }
          362  +        foreach hit [array names aRight] {
          363  +          set aMatchinfo($hit) [concat $nullleft $aRight($hit)]
          364  +        }
          365  +
          366  +        set nullentry [concat $nullleft $nullright]
          367  +      }
          368  +    }
          369  +  }
          370  +}
          371  +
          372  +
          373  +# End of test procs. Actual tests are below this line.
          374  +#--------------------------------------------------------------------------
          375  +
          376  +#--------------------------------------------------------------------------
          377  +# The following test cases - fts3auto-1.* - focus on testing the Tcl 
          378  +# command [fts3_near_match], which is used by other tests in this file.
          379  +#
          380  +proc test_fts3_near_match {tn doc expr res} {
          381  +  fts3_near_match $doc $expr -phrasecountvar p
          382  +  uplevel do_test [list $tn] [list [list set {} $p]] [list $res]
          383  +}
          384  +
          385  +test_fts3_near_match 1.1.1 {a b c a b} a                   {2}
          386  +test_fts3_near_match 1.1.2 {a b c a b} {a 5 b 6 c}         {2 2 1}
          387  +test_fts3_near_match 1.1.3 {a b c a b} {"a b"}             {2}
          388  +test_fts3_near_match 1.1.4 {a b c a b} {"b c"}             {1}
          389  +test_fts3_near_match 1.1.5 {a b c a b} {"c c"}             {0}
          390  +
          391  +test_fts3_near_match 1.2.1 "a b c d e f g" {b 2 f}         {0 0}
          392  +test_fts3_near_match 1.2.2 "a b c d e f g" {b 3 f}         {1 1}
          393  +test_fts3_near_match 1.2.3 "a b c d e f g" {f 2 b}         {0 0}
          394  +test_fts3_near_match 1.2.4 "a b c d e f g" {f 3 b}         {1 1}
          395  +test_fts3_near_match 1.2.5 "a b c d e f g" {"a b" 2 "f g"} {0 0}
          396  +test_fts3_near_match 1.2.6 "a b c d e f g" {"a b" 3 "f g"} {1 1}
          397  +
          398  +set A "a b c d e f g h i j k l m n o p q r s t u v w x y z"
          399  +test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"}      {0 0 0}
          400  +test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"}      {1 1 1}
          401  +
          402  +#--------------------------------------------------------------------------
          403  +# Test cases fts3auto-2.* run some simple tests using the 
          404  +# [do_fts3query_test] proc.
          405  +#
          406  +foreach {tn create} {
          407  +  1    "fts4(a, b)"
          408  +  2    "fts4(a, b, order=DESC)"
          409  +  3    "fts4(a, b, order=ASC)"
          410  +  4    "fts4(a, b, prefix=1)"
          411  +  5    "fts4(a, b, order=DESC, prefix=1)"
          412  +  6    "fts4(a, b, order=ASC, prefix=1)"
          413  +} {
          414  +  do_test 2.$tn.1 {
          415  +    catchsql { DROP TABLE t1 }
          416  +    execsql  "CREATE VIRTUAL TABLE t1 USING $create"
          417  +    for {set i 0} {$i<32} {incr i} {
          418  +      set doc [list]
          419  +      if {$i&0x01} {lappend doc one}
          420  +      if {$i&0x02} {lappend doc two}
          421  +      if {$i&0x04} {lappend doc three}
          422  +      if {$i&0x08} {lappend doc four}
          423  +      if {$i&0x10} {lappend doc five}
          424  +      execsql { INSERT INTO t1 VALUES($doc, null) }
          425  +    }
          426  +  } {}
          427  +
          428  +  foreach {tn2 expr} {
          429  +    1     {one}
          430  +    2     {one NEAR/1 five}
          431  +    3     {t*}
          432  +    4     {t* NEAR/0 five}
          433  +    5     {o* NEAR/1 f*}
          434  +    6     {one NEAR five NEAR two NEAR four NEAR three}
          435  +    7     {one NEAR xyz}
          436  +    8     {one OR two}
          437  +    9     {one AND two}
          438  +    10    {one NOT two}
          439  +    11    {one AND two OR three}
          440  +    12    {three OR one AND two}
          441  +    13    {(three OR one) AND two}
          442  +    14    {(three OR one) AND two NOT (five NOT four)}
          443  +    15    {"one two"}
          444  +    16    {"one two" NOT "three four"}
          445  +  } {
          446  +    do_fts3query_test 2.$tn.2.$tn2 t1 $expr
          447  +  }
          448  +}
          449  +
          450  +#--------------------------------------------------------------------------
          451  +# Some test cases involving deferred tokens.
          452  +#
          453  +
          454  +foreach {tn create} {
          455  +  1    "fts4(x)"
          456  +  2    "fts4(x, order=DESC)"
          457  +} {
          458  +  catchsql { DROP TABLE t1 }
          459  +  execsql  "CREATE VIRTUAL TABLE t1 USING $create"
          460  +  do_execsql_test 3.$tn.1 {
          461  +    INSERT INTO t1(docid, x) VALUES(-2, 'a b c d e f g h i j k');
          462  +    INSERT INTO t1(docid, x) VALUES(-1, 'b c d e f g h i j k a');
          463  +    INSERT INTO t1(docid, x) VALUES(0, 'c d e f g h i j k a b');
          464  +    INSERT INTO t1(docid, x) VALUES(1, 'd e f g h i j k a b c');
          465  +    INSERT INTO t1(docid, x) VALUES(2, 'e f g h i j k a b c d');
          466  +    INSERT INTO t1(docid, x) VALUES(3, 'f g h i j k a b c d e');
          467  +    INSERT INTO t1(docid, x) VALUES(4, 'a c e g i k');
          468  +    INSERT INTO t1(docid, x) VALUES(5, 'a d g j');
          469  +    INSERT INTO t1(docid, x) VALUES(6, 'c a b');
          470  +  }
          471  +
          472  +  set limit [fts3_make_deferrable t1 c]
          473  +
          474  +  do_fts3query_test 3.$tn.2.1 t1 {a OR c}
          475  +
          476  +  do_test 3.$tn.3 { 
          477  +    fts3_zero_long_segments t1 $limit 
          478  +  } {1}
          479  +
          480  +  foreach {tn2 expr def} {
          481  +    1     {a NEAR c}            {}
          482  +    2     {a AND c}             c
          483  +    3     {"a c"}               c
          484  +    4     {"c a"}               c
          485  +    5     {"a c" NEAR/1 g}      {}
          486  +    6     {"a c" NEAR/0 g}      {}
          487  +  } {
          488  +    do_fts3query_test 3.$tn.4.$tn2 -deferred $def t1 $expr
          489  +  }
          490  +}
          491  +
          492  +#--------------------------------------------------------------------------
          493  +# 
          494  +foreach {tn create} {
          495  +  1    "fts4(x, y)"
          496  +  2    "fts4(x, y, order=DESC)"
          497  +  3    "fts4(x, y, order=DESC, prefix=2)"
          498  +} {
          499  +
          500  +  execsql [subst {
          501  +    DROP TABLE t1;
          502  +    CREATE VIRTUAL TABLE t1 USING $create;
          503  +    INSERT INTO t1 VALUES('one two five four five', '');
          504  +    INSERT INTO t1 VALUES('', 'one two five four five');
          505  +    INSERT INTO t1 VALUES('one two', 'five four five');
          506  +  }]
          507  +
          508  +  do_fts3query_test 4.$tn.1.1 t1 {one AND five}
          509  +  do_fts3query_test 4.$tn.1.2 t1 {one NEAR five}
          510  +  do_fts3query_test 4.$tn.1.3 t1 {one NEAR/1 five}
          511  +  do_fts3query_test 4.$tn.1.4 t1 {one NEAR/2 five}
          512  +  do_fts3query_test 4.$tn.1.5 t1 {one NEAR/3 five}
          513  +
          514  +  do_test 4.$tn.2 { 
          515  +    set limit [fts3_make_deferrable t1 five]
          516  +    execsql { INSERT INTO t1(t1) VALUES('optimize') }
          517  +    expr {[fts3_zero_long_segments t1 $limit]>0}
          518  +  } {1}
          519  +
          520  +  do_fts3query_test 4.$tn.3.1 -deferred five t1 {one AND five}
          521  +  do_fts3query_test 4.$tn.3.2 -deferred five t1 {one NEAR five}
          522  +  do_fts3query_test 4.$tn.3.3 -deferred five t1 {one NEAR/1 five}
          523  +  do_fts3query_test 4.$tn.3.4 -deferred five t1 {one NEAR/2 five}
          524  +  do_fts3query_test 4.$tn.3.5 -deferred five t1 {one NEAR/3 five}
          525  +
          526  +  do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*}
          527  +  do_fts3query_test 4.$tn.4.2 -deferred fi* t1 {on* NEAR fi*}
          528  +  do_fts3query_test 4.$tn.4.3 -deferred fi* t1 {on* NEAR/1 fi*}
          529  +  do_fts3query_test 4.$tn.4.4 -deferred fi* t1 {on* NEAR/2 fi*}
          530  +  do_fts3query_test 4.$tn.4.5 -deferred fi* t1 {on* NEAR/3 fi*}
          531  +}
          532  +
          533  +set sqlite_fts3_enable_parentheses $sfep
          534  +finish_test
          535  +

Changes to test/fts3defer.test.

    16     16   ifcapable !fts3 {
    17     17     finish_test
    18     18     return
    19     19   }
    20     20   
    21     21   set sqlite_fts3_enable_parentheses 1
    22     22   
           23  +set fts3_simple_deferred_tokens_only 1
           24  +
    23     25   set ::testprefix fts3defer
    24     26   
    25     27   #--------------------------------------------------------------------------
    26     28   # Test cases fts3defer-1.* are the "warm body" cases. The database contains
    27     29   # one row with 15000 instances of the token "a". This makes the doclist for
    28     30   # "a" so large that FTS3 will avoid loading it in most cases.
    29     31   #
................................................................................
   253    255   
   254    256     do_select_test 1.1 {
   255    257       SELECT rowid FROM t1 WHERE t1 MATCH 'jk xnxhf'
   256    258     } {13 29 40 47 48 52 63 92}
   257    259     do_select_test 1.2 {
   258    260       SELECT rowid FROM t1 WHERE t1 MATCH 'jk eh'
   259    261     } {100}
   260         -if {$tn==3} breakpoint
   261    262     do_select_test 1.3 {
   262    263       SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf'
   263    264     } {7 70 98}
   264    265     do_select_test 1.4 {
   265    266       SELECT rowid FROM t1 WHERE t1 MATCH 'duszemmzl jk'
   266    267     } {3 5 8 10 13 18 20 23 32 37 41 43 55 60 65 67 72 74 76 81 94 96 97}
   267    268     do_select_test 1.5 {
................................................................................
   278    279     } {68 100}
   279    280     do_select_test 1.9 {
   280    281       SELECT rowid FROM t1 WHERE t1 MATCH 'zm ubwrfqnbjf'
   281    282     } {7 70 98}
   282    283     do_select_test 1.10 {
   283    284       SELECT rowid FROM t1 WHERE t1 MATCH 'z* vgsld'
   284    285     } {10 13 17 31 35 51 58 88 89 90 93 100}
   285         -  do_select_test 1.11 {
   286         -    SELECT rowid FROM t1 
   287         -    WHERE t1 MATCH '(
   288         -      zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR 
   289         -      zk OR zkhdvkw OR zm OR zsmhnf
   290         -    ) vgsld'
   291         -  } {10 13 17 31 35 51 58 88 89 90 93 100}
          286  +
          287  +  if { $fts3_simple_deferred_tokens_only==0 } {
          288  +    do_select_test 1.11 {
          289  +      SELECT rowid FROM t1 
          290  +      WHERE t1 MATCH '(
          291  +        zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR 
          292  +        zk OR zkhdvkw OR zm OR zsmhnf
          293  +      ) vgsld'
          294  +    } {10 13 17 31 35 51 58 88 89 90 93 100}
          295  +  }
   292    296   
   293    297     do_select_test 2.1 {
   294    298       SELECT rowid FROM t1 WHERE t1 MATCH '"zm agmckuiu"'
   295    299     } {3 24 52 53}
   296    300     do_select_test 2.2 {
   297    301       SELECT rowid FROM t1 WHERE t1 MATCH '"zm zf"'
   298    302     } {33 53 75 88 101}
................................................................................
   360    364     # The following block of tests runs normally with FTS3 or FTS4 without the
   361    365     # long doclists zeroed. And with OOM-injection for FTS4 with long doclists
   362    366     # zeroed. Change this by messing with the [set dmt_modes] commands above.
   363    367     #
   364    368     foreach DO_MALLOC_TEST $dmt_modes {
   365    369       
   366    370       # Phrase search.
          371  +    #
   367    372       do_select_test 5.$DO_MALLOC_TEST.1 {
   368    373         SELECT rowid FROM t1 WHERE t1 MATCH '"jk mjpavjuhw"'
   369    374       } {8 15 36 64 67 72}
   370    375   
   371    376       # Multiple tokens search.
   372    377       do_select_test 5.$DO_MALLOC_TEST.2 {
   373    378         SELECT rowid FROM t1 WHERE t1 MATCH 'duszemmzl zm'
................................................................................
   412    417     } {10}
   413    418     do_select_test 6.2.1 {
   414    419       SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk"'
   415    420     } {8}
   416    421     do_select_test 6.2.2 {
   417    422       SELECT rowid FROM t1 WHERE t1 MATCH '"zm azavwm"'
   418    423     } {15 26 92 96}
   419         -  do_select_test 6.2.3 {
   420         -    SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"'
   421         -  } {8 15 26 92 96}
          424  +  if {$fts3_simple_deferred_tokens_only==0} {
          425  +    do_select_test 6.2.3 {
          426  +      SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"'
          427  +    } {8 15 26 92 96}
          428  +  }
   422    429   }
   423    430   
   424    431   set testprefix fts3defer
   425    432   
   426    433   do_execsql_test 3.1 {
   427    434     CREATE VIRTUAL TABLE x1 USING fts4(a, b);
   428    435     INSERT INTO x1 VALUES('a b c', 'd e f');

Changes to test/fts3defer2.test.

    44     44     INSERT INTO t1(t1) VALUES('optimize');
    45     45   }
    46     46   do_execsql_test 1.1.4 {
    47     47     SELECT count(*) FROM t1_segments WHERE length(block)>10000;
    48     48     UPDATE t1_segments SET block = zeroblob(length(block)) WHERE length(block)>10000;
    49     49   } {2}
    50     50   
           51  +do_execsql_test 1.2.0 {
           52  +  SELECT content FROM t1 WHERE t1 MATCH 'f (e a)';
           53  +} {{a b c d e f a x y}}
           54  +
    51     55   do_execsql_test 1.2.1 {
    52     56     SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
    53     57   } {{a b c d e f a x y}}
    54     58   
    55     59   do_execsql_test 1.2.2 {
    56     60     SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
    57     61     FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
    58     62   } [list                              \
    59     63      {a b c d [e] [f] [a] x y}         \
    60     64      {0 1 8 1 0 0 10 1 0 2 12 1}       \
    61         -   [list 3 1   1 1 1   1 8 8   1 8 8   8 5001 9]
           65  +   [list 3 1   1 1 1   1 1 1   1 1 1   8 5001 9]
    62     66   ]
    63     67   
    64     68   do_execsql_test 1.2.3 {
    65     69     SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
    66     70     FROM t1 WHERE t1 MATCH 'f (e NEAR/3 a)';
    67     71   } [list                                 \
    68     72      {[a] b c d [e] [f] [a] x y}          \
    69     73      {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1}  \
    70         -   [list 3 1   1 1 1   1 8 8   2 8 8   8 5001 9]
           74  +   [list 3 1   1 1 1   1 1 1   2 2 1   8 5001 9]
    71     75   ]
    72     76   
    73     77   do_execsql_test 1.3.1 { DROP TABLE t1 }
    74     78   
    75     79   #-----------------------------------------------------------------------------
    76     80   # Test cases fts3defer2-2.* focus specifically on the matchinfo function.
    77     81   # 
................................................................................
    95     99     do_execsql_test 2.2.$tn.1 {
    96    100       SELECT mit(matchinfo(t2, 'pcxnal')) FROM t2 WHERE t2 MATCH 'a b';
    97    101     } [list                                          \
    98    102       [list 2 1  1 54 54  1 3 3  54 372 8]        \
    99    103       [list 2 1  1 54 54  1 3 3  54 372 7]        \
   100    104     ]
   101    105   
   102         -  set sqlite_fts3_enable_parentheses 1
   103    106     do_execsql_test 2.2.$tn.2 {
          107  +    SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g z';
          108  +  } [list                                       \
          109  +    [list 1 2 2  1 54 54]                       \
          110  +  ]
          111  +
          112  +  set sqlite_fts3_enable_parentheses 1
          113  +  do_execsql_test 2.2.$tn.3 {
   104    114       SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g OR (g z)';
   105    115     } [list                                       \
   106    116       [list 1 2 2  1 2 2   1 54 54]               \
   107    117       [list 1 2 2  1 2 2   0 54 54]               \
   108    118     ]
   109    119     set sqlite_fts3_enable_parentheses 0
   110    120   }

Changes to test/fts3matchinfo.test.

    64     64     CREATE VIRTUAL TABLE t3 USING fts3(mtchinfo=fts3);
    65     65     INSERT INTO t3(mtchinfo) VALUES('Beside the lake, beneath the trees');
    66     66     SELECT mtchinfo FROM t3;
    67     67   } {{Beside the lake, beneath the trees}}
    68     68   
    69     69   do_execsql_test 3.2 {
    70     70     CREATE VIRTUAL TABLE xx USING FTS4;
           71  +}
           72  +do_execsql_test 3.3 {
    71     73     SELECT * FROM xx WHERE xx MATCH 'abc';
           74  +}
           75  +do_execsql_test 3.4 {
    72     76     SELECT * FROM xx WHERE xx MATCH 'a b c';
    73     77   }
    74     78   
    75     79   
    76     80   #--------------------------------------------------------------------------
    77     81   # Proc [do_matchinfo_test] is used to test the FTSX matchinfo() function.
    78     82   #
................................................................................
   236    240   do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'}       { s {3} }
   237    241   do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
   238    242   do_matchinfo_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
   239    243   do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1} }
   240    244   
   241    245   do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')";
   242    246   
   243         -do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { 
   244         -  x {{5 8 2   5 5 5} {3 8 2   3 5 5}}
   245         -  s {2 1} 
          247  +# It used to be that the second 'a' token would be deferred. That doesn't
          248  +# work any longer.
          249  +if 0 {
          250  +  do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { 
          251  +    x {{5 8 2   5 5 5} {3 8 2   3 5 5}}
          252  +    s {2 1} 
          253  +  }
   246    254   }
   247    255   
   248    256   do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'}         { s {2} }
   249    257   do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'}       { s {3} }
   250    258   do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
   251    259   do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
   252    260   do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1 1} }

Added test/fts3prefix.test.

            1  +# 2011 May 04
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS3 module.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +set testprefix fts3prefix
           18  +
           19  +ifcapable !fts3 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +# This proc tests that the prefixes index appears to represent the same content
           25  +# as the terms index.
           26  +#
           27  +proc fts3_terms_and_prefixes {db tbl prefixlengths} {
           28  +
           29  +  set iIndex 0
           30  +  foreach len $prefixlengths {
           31  +    incr iIndex
           32  +    $db eval {
           33  +      DROP TABLE IF EXISTS fts3check1;
           34  +      DROP TABLE IF EXISTS fts3check2;
           35  +    }
           36  +    $db eval "CREATE VIRTUAL TABLE fts3check1 USING fts4term($tbl, 0);"
           37  +    $db eval "CREATE VIRTUAL TABLE fts3check2 USING fts4term($tbl, $iIndex);"
           38  +
           39  +    $db eval {
           40  +      DROP TABLE IF EXISTS temp.terms;
           41  +      DROP TABLE IF EXISTS temp.prefixes;
           42  +      CREATE TEMP TABLE terms AS SELECT * FROM fts3check1;
           43  +      CREATE TEMP TABLE prefixes AS SELECT * FROM fts3check2;
           44  +      CREATE INDEX temp.idx ON prefixes(term);
           45  +      DROP TABLE fts3check1;
           46  +      DROP TABLE fts3check2;
           47  +    }
           48  +
           49  +    set nExpect 0
           50  +    $db eval { SELECT term, docid, col, pos FROM temp.terms } a {
           51  +      if {[string length $a(term)]<$len} continue
           52  +      incr nExpect
           53  +      set prefix [string range $a(term) 0 [expr $len-1]]
           54  +      set r [$db one { 
           55  +        SELECT count(*) FROM temp.prefixes WHERE 
           56  +        term = $prefix AND docid = $a(docid) AND col = $a(col) AND pos = $a(pos)
           57  +      }]
           58  +      if {$r != 1} {
           59  +        error "$t, $a(docid), $a(col), $a(pos)"
           60  +      }
           61  +    }
           62  +
           63  +    set nCount [$db one {SELECT count(*) FROM temp.prefixes}]
           64  +    if {$nCount != $nExpect} {
           65  +      error "prefixes.count(*) is $nCount expected $nExpect"
           66  +    }
           67  +  
           68  +    execsql { DROP TABLE temp.prefixes }
           69  +    execsql { DROP TABLE temp.terms }
           70  +
           71  +    set list [list]
           72  +    $db eval "
           73  +      SELECT sum( 1 << (16*(level%1024)) ) AS total, (level/1024) AS tree 
           74  +      FROM ${tbl}_segdir GROUP BY tree
           75  +    " {
           76  +      lappend list [list $total $tree]
           77  +    }
           78  +
           79  +    if { [lsort -integer -index 0 $list] != [lsort -integer -index 1 $list] } {
           80  +      error "inconsistent tree structures: $list"
           81  +    }
           82  +  }
           83  +
           84  +  return ""
           85  +}
           86  +proc fts3_tap_test {tn db tbl lens} {
           87  +  uplevel [list do_test $tn [list fts3_terms_and_prefixes $db $tbl $lens] ""]
           88  +}
           89  +
           90  +#-------------------------------------------------------------------------
           91  +# Test cases 1.* are a sanity check. They test that the prefixes index is
           92  +# being constructed correctly for the simplest possible case.
           93  +#
           94  +do_execsql_test 1.1 {
           95  +  CREATE VIRTUAL TABLE t1 USING fts4(prefix='1,3,6');
           96  +
           97  +  CREATE VIRTUAL TABLE p1 USING fts4term(t1, 1);
           98  +  CREATE VIRTUAL TABLE p2 USING fts4term(t1, 2);
           99  +  CREATE VIRTUAL TABLE p3 USING fts4term(t1, 3);
          100  +  CREATE VIRTUAL TABLE terms USING fts4term(t1);
          101  +}
          102  +do_execsql_test 1.2 {
          103  +  INSERT INTO t1 VALUES('sqlite mysql firebird');
          104  +}
          105  +do_execsql_test 1.3.1 { SELECT term FROM p1 } {f m s}
          106  +do_execsql_test 1.3.2 { SELECT term FROM p2 } {fir mys sql}
          107  +do_execsql_test 1.3.3 { SELECT term FROM p3 } {firebi sqlite}
          108  +do_execsql_test 1.4 {
          109  +  SELECT term FROM terms;
          110  +} {firebird mysql sqlite}
          111  +
          112  +fts3_tap_test 1.5 db t1 {1 3 6}
          113  +
          114  +#-------------------------------------------------------------------------
          115  +# A slightly more complicated dataset. This test also verifies that DELETE
          116  +# operations do not corrupt the prefixes index.
          117  +#
          118  +do_execsql_test 2.1 {
          119  +  INSERT INTO t1 VALUES('FTS3 and FTS4 are an SQLite virtual table modules');
          120  +  INSERT INTO t1 VALUES('that allows users to perform full-text searches on');
          121  +  INSERT INTO t1 VALUES('a set of documents. The most common (and');
          122  +  INSERT INTO t1 VALUES('effective) way to describe full-text searches is');
          123  +  INSERT INTO t1 VALUES('"what Google, Yahoo and Altavista do with');
          124  +  INSERT INTO t1 VALUES('documents placed on the World Wide Web". Users');
          125  +  INSERT INTO t1 VALUES('input a term, or series of terms, perhaps');
          126  +  INSERT INTO t1 VALUES('connected by a binary operator or grouped together');
          127  +  INSERT INTO t1 VALUES('into a phrase, and the full-text query system');
          128  +  INSERT INTO t1 VALUES('finds the set of documents that best matches those');
          129  +  INSERT INTO t1 VALUES('terms considering the operators and groupings the');
          130  +  INSERT INTO t1 VALUES('user has specified. This article describes the');
          131  +  INSERT INTO t1 VALUES('deployment and usage of FTS3 and FTS4.');
          132  +  INSERT INTO t1 VALUES('FTS1 and FTS2 are obsolete full-text search');
          133  +  INSERT INTO t1 VALUES('modules for SQLite. There are known issues with');
          134  +  INSERT INTO t1 VALUES('these older modules and their use should be');
          135  +  INSERT INTO t1 VALUES('avoided. Portions of the original FTS3 code were');
          136  +  INSERT INTO t1 VALUES('contributed to the SQLite project by Scott Hess of');
          137  +  INSERT INTO t1 VALUES('Google. It is now developed and maintained as part');
          138  +  INSERT INTO t1 VALUES('of SQLite. ');
          139  +}
          140  +fts3_tap_test 2.2 db t1 {1 3 6}
          141  +do_execsql_test 2.3 { DELETE FROM t1 WHERE docid%2; }
          142  +fts3_tap_test 2.4 db t1 {1 3 6}
          143  +
          144  +do_execsql_test 2.5 { INSERT INTO t1(t1) VALUES('optimize') }
          145  +fts3_tap_test 2.6 db t1 {1 3 6}
          146  +
          147  +do_execsql_test 3.1 {
          148  +  CREATE VIRTUAL TABLE t2 USING fts4(prefix='1,2,3');
          149  +  INSERT INTO t2 VALUES('On 12 September the wind direction turned and');
          150  +  INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the');
          151  +  INSERT INTO t2 VALUES('fleet was forced to take shelter at');
          152  +  INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind');
          153  +  INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet');
          154  +  INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey');
          155  +  INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved');
          156  +  INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he');
          157  +  INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of');
          158  +  INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland');
          159  +  INSERT INTO t2 VALUES('and waited for Harold''s return from the north.');
          160  +  INSERT INTO t2 VALUES('On 12 September the wind direction turned and');
          161  +  INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the');
          162  +  INSERT INTO t2 VALUES('fleet was forced to take shelter at');
          163  +  INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind');
          164  +  INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet');
          165  +  INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey');
          166  +  INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved');
          167  +  INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he');
          168  +  INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of');
          169  +  INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland');
          170  +  INSERT INTO t2 VALUES('and waited for Harold''s return from the north.');
          171  +}
          172  +
          173  +fts3_tap_test 3.2 db t2 {1 2 3}
          174  +do_execsql_test 3.3 { SELECT optimize(t2) FROM t2 LIMIT 1 } {{Index optimized}}
          175  +fts3_tap_test 3.4 db t2 {1 2 3}
          176  +
          177  +
          178  +#-------------------------------------------------------------------------
          179  +# Simple tests for reading the prefix-index.
          180  +#
          181  +do_execsql_test 4.1 {
          182  +  CREATE VIRTUAL TABLE t3 USING fts4(prefix="1,4");
          183  +  INSERT INTO t3 VALUES('one two three');
          184  +  INSERT INTO t3 VALUES('four five six');
          185  +  INSERT INTO t3 VALUES('seven eight nine');
          186  +}
          187  +do_execsql_test 4.2 {
          188  +  SELECT * FROM t3 WHERE t3 MATCH 'f*'
          189  +} {{four five six}}
          190  +do_execsql_test 4.3 {
          191  +  SELECT * FROM t3 WHERE t3 MATCH 'four*'
          192  +} {{four five six}}
          193  +do_execsql_test 4.4 {
          194  +  SELECT * FROM t3 WHERE t3 MATCH 's*'
          195  +} {{four five six} {seven eight nine}}
          196  +do_execsql_test 4.5 {
          197  +  SELECT * FROM t3 WHERE t3 MATCH 'sev*'
          198  +} {{seven eight nine}}
          199  +do_execsql_test 4.6 {
          200  +  SELECT * FROM t3 WHERE t3 MATCH 'one*'
          201  +} {{one two three}}
          202  +
          203  +finish_test

Changes to test/fts3rnd.test.

   158    158   
   159    159     #lsort -uniq -integer $ret
   160    160     set ret
   161    161   }
   162    162   
   163    163   # This [proc] is used to test the FTS3 matchinfo() function.
   164    164   # 
   165         -proc simple_token_matchinfo {zToken} {
          165  +proc simple_token_matchinfo {zToken bDesc} {
   166    166   
   167    167     set nDoc(0) 0
   168    168     set nDoc(1) 0
   169    169     set nDoc(2) 0
   170    170     set nHit(0) 0
   171    171     set nHit(1) 0
   172    172     set nHit(2) 0
   173    173   
          174  +  set dir -inc
          175  +  if {$bDesc} { set dir -dec }
   174    176   
   175    177     foreach key [array names ::t1] {
   176    178       set value $::t1($key)
   177    179       set a($key) [list]
   178    180       foreach i {0 1 2} col $value {
   179    181         set hit [llength [lsearch -all $col $zToken]]
   180    182         lappend a($key) $hit
   181    183         incr nHit($i) $hit
   182    184         if {$hit>0} { incr nDoc($i) }
   183    185       }
   184    186     }
   185    187   
   186    188     set ret [list]
   187         -  foreach docid [lsort -integer [array names a]] {
          189  +  foreach docid [lsort -integer $dir [array names a]] {
   188    190       if { [lindex [lsort -integer $a($docid)] end] } {
   189    191         set matchinfo [list 1 3]
   190    192         foreach i {0 1 2} hit $a($docid) {
   191    193           lappend matchinfo $hit $nHit($i) $nDoc($i)
   192    194         }
   193    195         lappend ret $docid $matchinfo
   194    196       }
................................................................................
   258    260   proc mit {blob} {
   259    261     set scan(littleEndian) i*
   260    262     set scan(bigEndian) I*
   261    263     binary scan $blob $scan($::tcl_platform(byteOrder)) r
   262    264     return $r
   263    265   }
   264    266   db func mit mit
   265         -
   266    267   set sqlite_fts3_enable_parentheses 1
   267    268   
   268         -foreach nodesize {50 500 1000 2000} {
          269  +proc do_orderbydocid_test {tn sql res} {
          270  +  uplevel [list do_select_test $tn.asc "$sql ORDER BY docid ASC" $res]
          271  +  uplevel [list do_select_test $tn.desc "$sql ORDER BY docid DESC" \
          272  +    [lsort -int -dec $res]
          273  +  ]
          274  +}
          275  +
          276  +set NUM_TRIALS 100
          277  +
          278  +foreach {nodesize order} {
          279  +  50    DESC
          280  +  50    ASC
          281  +  500   ASC
          282  +  1000  DESC
          283  +  2000  ASC
          284  +} {
   269    285     catch { array unset ::t1 }
          286  +  set testname "$nodesize/$order"
   270    287   
   271    288     # Create the FTS3 table. Populate it (and the Tcl array) with 100 rows.
   272    289     #
   273    290     db transaction {
   274    291       catchsql { DROP TABLE t1 }
   275         -    execsql "CREATE VIRTUAL TABLE t1 USING fts3(a, b, c)"
          292  +    execsql "CREATE VIRTUAL TABLE t1 USING fts4(a, b, c, order=$order)"
   276    293       execsql "INSERT INTO t1(t1) VALUES('nodesize=$nodesize')"
   277    294       for {set i 0} {$i < 100} {incr i} { insert_row $i }
   278    295     }
   279    296     
   280         -  for {set iTest 1} {$iTest <= 100} {incr iTest} {
          297  +  for {set iTest 1} {$iTest <= $NUM_TRIALS} {incr iTest} {
   281    298       catchsql COMMIT
   282    299   
   283    300       set DO_MALLOC_TEST 0
   284    301       set nRep 10
   285    302       if {$iTest==100 && $nodesize==50} { 
   286    303         set DO_MALLOC_TEST 1 
   287    304         set nRep 2
   288    305       }
          306  +
          307  +    set ::testprefix fts3rnd-1.$testname.$iTest
   289    308     
   290    309       # Delete one row, update one row and insert one row.
   291    310       #
   292    311       set rows [array names ::t1]
   293    312       set nRow [llength $rows]
   294    313       set iUpdate [lindex $rows [expr {int(rand()*$nRow)}]]
   295    314       set iDelete $iUpdate
................................................................................
   303    322       execsql BEGIN
   304    323         insert_row $iInsert
   305    324         update_row $iUpdate
   306    325         delete_row $iDelete
   307    326       if {0==($iTest%2)} { execsql COMMIT }
   308    327   
   309    328       if {0==($iTest%2)} { 
   310         -      do_test fts3rnd-1.$nodesize.$iTest.0 { fts3_integrity_check t1 } ok 
          329  +      #do_test 0 { fts3_integrity_check t1 } ok 
   311    330       }
   312    331   
   313    332       # Pick 10 terms from the vocabulary. Check that the results of querying
   314    333       # the database for the set of documents containing each of these terms
   315    334       # is the same as the result obtained by scanning the contents of the Tcl 
   316    335       # array for each term.
   317    336       #
   318    337       for {set i 0} {$i < 10} {incr i} {
   319    338         set term [random_term]
   320         -      do_select_test fts3rnd-1.$nodesize.$iTest.1.$i {
          339  +      do_select_test 1.$i.asc {
          340  +        SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term
          341  +        ORDER BY docid ASC
          342  +      } [simple_token_matchinfo $term 0]
          343  +      do_select_test 1.$i.desc {
   321    344           SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term
   322         -      } [simple_token_matchinfo $term]
          345  +        ORDER BY docid DESC
          346  +      } [simple_token_matchinfo $term 1]
   323    347       }
   324    348   
   325    349       # This time, use the first two characters of each term as a term prefix
   326    350       # to query for. Test that querying the Tcl array produces the same results
   327    351       # as querying the FTS3 table for the prefix.
   328    352       #
   329    353       for {set i 0} {$i < $nRep} {incr i} {
   330    354         set prefix [string range [random_term] 0 end-1]
   331    355         set match "${prefix}*"
   332         -      do_select_test fts3rnd-1.$nodesize.$iTest.2.$i {
          356  +      do_orderbydocid_test 2.$i {
   333    357           SELECT docid FROM t1 WHERE t1 MATCH $match
   334    358         } [simple_phrase $match]
   335    359       }
   336    360   
   337    361       # Similar to the above, except for phrase queries.
   338    362       #
   339    363       for {set i 0} {$i < $nRep} {incr i} {
   340    364         set term [list [random_term] [random_term]]
   341    365         set match "\"$term\""
   342         -      do_select_test fts3rnd-1.$nodesize.$iTest.3.$i {
          366  +      do_orderbydocid_test 3.$i {
   343    367           SELECT docid FROM t1 WHERE t1 MATCH $match
   344    368         } [simple_phrase $term]
   345    369       }
   346    370   
   347    371       # Three word phrases.
   348    372       #
   349    373       for {set i 0} {$i < $nRep} {incr i} {
   350    374         set term [list [random_term] [random_term] [random_term]]
   351    375         set match "\"$term\""
   352         -      do_select_test fts3rnd-1.$nodesize.$iTest.4.$i {
          376  +      do_orderbydocid_test 4.$i {
   353    377           SELECT docid FROM t1 WHERE t1 MATCH $match
   354    378         } [simple_phrase $term]
   355    379       }
   356    380   
   357    381       # Three word phrases made up of term-prefixes.
   358    382       #
   359    383       for {set i 0} {$i < $nRep} {incr i} {
   360    384         set    query "[string range [random_term] 0 end-1]* "
   361    385         append query "[string range [random_term] 0 end-1]* "
   362    386         append query "[string range [random_term] 0 end-1]*"
   363    387   
   364    388         set match "\"$query\""
   365         -      do_select_test fts3rnd-1.$nodesize.$iTest.5.$i {
          389  +      do_orderbydocid_test 5.$i {
   366    390           SELECT docid FROM t1 WHERE t1 MATCH $match
   367    391         } [simple_phrase $query]
   368    392       }
   369    393   
   370         -    # A NEAR query with terms as the arguments.
          394  +    # A NEAR query with terms as the arguments:
          395  +    #
          396  +    #     ... MATCH '$term1 NEAR $term2' ...
   371    397       #
   372    398       for {set i 0} {$i < $nRep} {incr i} {
   373    399         set terms [list [random_term] [random_term]]
   374    400         set match [join $terms " NEAR "]
   375         -      do_select_test fts3rnd-1.$nodesize.$iTest.6.$i {
          401  +      do_orderbydocid_test 6.$i {
   376    402           SELECT docid FROM t1 WHERE t1 MATCH $match 
   377    403         } [simple_near $terms 10]
   378    404       }
   379    405   
   380    406       # A 3-way NEAR query with terms as the arguments.
   381    407       #
   382    408       for {set i 0} {$i < $nRep} {incr i} {
   383    409         set terms [list [random_term] [random_term] [random_term]]
   384    410         set nNear 11
   385    411         set match [join $terms " NEAR/$nNear "]
   386         -      do_select_test fts3rnd-1.$nodesize.$iTest.7.$i {
          412  +      do_orderbydocid_test 7.$i {
   387    413           SELECT docid FROM t1 WHERE t1 MATCH $match
   388    414         } [simple_near $terms $nNear]
   389    415       }
   390    416       
   391    417       # Set operations on simple term queries.
   392    418       #
   393    419       foreach {tn op proc} {
................................................................................
   395    421         9  NOT setop_not
   396    422         10 AND setop_and
   397    423       } {
   398    424         for {set i 0} {$i < $nRep} {incr i} {
   399    425           set term1 [random_term]
   400    426           set term2 [random_term]
   401    427           set match "$term1 $op $term2"
   402         -        do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i {
          428  +        do_orderbydocid_test $tn.$i {
   403    429             SELECT docid FROM t1 WHERE t1 MATCH $match
   404    430           } [$proc [simple_phrase $term1] [simple_phrase $term2]]
   405    431         }
   406    432       }
   407    433    
   408    434       # Set operations on NEAR queries.
   409    435       #
   410    436       foreach {tn op proc} {
   411         -      8  OR  setop_or
   412         -      9  NOT setop_not
   413         -      10 AND setop_and
          437  +      11 OR  setop_or
          438  +      12 NOT setop_not
          439  +      13 AND setop_and
   414    440       } {
   415    441         for {set i 0} {$i < $nRep} {incr i} {
   416    442           set term1 [random_term]
   417    443           set term2 [random_term]
   418    444           set term3 [random_term]
   419    445           set term4 [random_term]
   420    446           set match "$term1 NEAR $term2 $op $term3 NEAR $term4"
   421         -        do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i {
          447  +        do_orderbydocid_test $tn.$i {
   422    448             SELECT docid FROM t1 WHERE t1 MATCH $match
   423    449           } [$proc                                  \
   424    450               [simple_near [list $term1 $term2] 10] \
   425    451               [simple_near [list $term3 $term4] 10]
   426    452             ]
   427    453         }
   428    454       }
   429    455   
   430    456       catchsql COMMIT
   431    457     }
   432    458   }
   433    459   
   434    460   finish_test

Changes to test/fts3sort.test.

    17     17   
    18     18   # If SQLITE_ENABLE_FTS3 is defined, omit this file.
    19     19   ifcapable !fts3 {
    20     20     finish_test
    21     21     return
    22     22   }
    23     23   
    24         -set testprefix fts3sort
    25     24   
    26         -proc build_database {nRow} {
           25  +proc build_database {nRow param} {
    27     26     db close
    28     27     forcedelete test.db
    29     28     sqlite3 db test.db
    30     29   
    31     30     set vocab [list    aa ab ac   ba bb bc    ca cb cc   da]
    32     31     expr srand(0)
    33     32   
    34         -  execsql { CREATE VIRTUAL TABLE t1 USING fts4 }
           33  +  execsql "CREATE VIRTUAL TABLE t1 USING fts4($param)"
    35     34     for {set i 0} {$i < $nRow} {incr i} {
    36     35       set v [expr int(rand()*1000000)]
    37     36       set doc [list]
    38     37       for {set div 1} {$div < 1000000} {set div [expr $div*10]} {
    39     38         lappend doc [lindex $vocab [expr ($v/$div) % 10]]
    40     39       }
    41     40       execsql { INSERT INTO t1 VALUES($doc) }
    42     41     }
    43     42   }
    44     43   
    45         -set nRow 1000
    46         -do_test 1.0 {
    47         -  build_database $nRow
    48         -  execsql { SELECT count(*) FROM t1 }
    49         -} $nRow
           44  +set testprefix fts3sort
           45  +
           46  +unset -nocomplain CONTROL
           47  +foreach {t param} {
           48  +  1     ""
           49  +  2     "order=asc"
           50  +  3     "order=desc"
           51  +} {
           52  +
           53  +  set testprefix fts3sort-1.$t
    50     54   
    51         -foreach {tn query} {
           55  +  set nRow 1000
           56  +  do_test 1.0 {
           57  +    build_database $nRow $param
           58  +    execsql { SELECT count(*) FROM t1 }
           59  +  } $nRow
           60  +  
           61  +  foreach {tn query} {
    52     62     1   "SELECT docid, * FROM t1"
    53     63     2   "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa'"
    54     64     3   "SELECT docid, * FROM t1 WHERE t1 MATCH 'a*'"
    55     65     4   "SELECT docid, quote(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'a*'"
    56     66     5   "SELECT docid, quote(matchinfo(t1,'pcnxals')) FROM t1 WHERE t1 MATCH 'b*'"
    57     67     6   "SELECT docid, * FROM t1 WHERE t1 MATCH 'a* b* c*'"
    58     68     7   "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa OR da'"
    59     69     8   "SELECT docid, * FROM t1 WHERE t1 MATCH 'nosuchtoken'"
    60     70     9   "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR da'"
    61     71     10  "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR nosuchtoken'"
    62         -} {
    63         -
    64         -  unset -nocomplain A B C D
    65         -  set A_list [list]
    66         -  set B_list [list]
    67         -  set C_list [list]
    68         -  set D_list [list]
           72  +  11  "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa NEAR bb'"
           73  +  12  "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH '\"aa bb\"'"
           74  +  13  "SELECT docid, content FROM t1 WHERE t1 MATCH 'aa NEAR/2 bb NEAR/3 cc'"
           75  +  14  "SELECT docid, content FROM t1 WHERE t1 MATCH '\"aa bb cc\"'"
           76  +  } {
           77  +  
           78  +    unset -nocomplain A B C D
           79  +    set A_list [list]
           80  +    set B_list [list]
           81  +    set C_list [list]
           82  +    set D_list [list]
           83  +  
           84  +    unset -nocomplain X
           85  +    db eval "$query ORDER BY rowid ASC"  X  { 
           86  +      set A($X(docid)) [array get X] 
           87  +      lappend A_list $X(docid)
           88  +    }
           89  +    unset -nocomplain X
           90  +    db eval "$query ORDER BY rowid DESC" X  { 
           91  +      set B($X(docid)) [array get X] 
           92  +      lappend B_list $X(docid)
           93  +    }
           94  +    unset -nocomplain X
           95  +    db eval "$query ORDER BY docid ASC"  X  { 
           96  +      set C($X(docid)) [array get X] 
           97  +      lappend C_list $X(docid)
           98  +    }
           99  +    unset -nocomplain X
          100  +    db eval "$query ORDER BY docid DESC" X  { 
          101  +      set D($X(docid)) [array get X] 
          102  +      lappend D_list $X(docid)
          103  +    }
          104  +  
          105  +    do_test $tn.1 { set A_list } [lsort -integer -increasing $A_list]
          106  +    do_test $tn.2 { set B_list } [lsort -integer -decreasing $B_list]
          107  +    do_test $tn.3 { set C_list } [lsort -integer -increasing $C_list]
          108  +    do_test $tn.4 { set D_list } [lsort -integer -decreasing $D_list]
          109  +  
          110  +    unset -nocomplain DATA
          111  +    unset -nocomplain X
          112  +    db eval "$query" X  { 
          113  +      set DATA($X(docid)) [array get X] 
          114  +    }
          115  +  
          116  +    do_test $tn.5 { lsort [array get A] } [lsort [array get DATA]]
          117  +    do_test $tn.6 { lsort [array get B] } [lsort [array get DATA]]
          118  +    do_test $tn.7 { lsort [array get C] } [lsort [array get DATA]]
          119  +    do_test $tn.8 { lsort [array get D] } [lsort [array get DATA]]
    69    120   
    70         -  unset -nocomplain X
    71         -  db eval "$query ORDER BY rowid ASC"  X  { 
    72         -    set A($X(docid)) [array get X] 
    73         -    lappend A_list $X(docid)
    74         -  }
    75         -  unset -nocomplain X
    76         -  db eval "$query ORDER BY rowid DESC" X  { 
    77         -    set B($X(docid)) [array get X] 
    78         -    lappend B_list $X(docid)
    79         -  }
    80         -  unset -nocomplain X
    81         -  db eval "$query ORDER BY docid ASC"  X  { 
    82         -    set C($X(docid)) [array get X] 
    83         -    lappend C_list $X(docid)
          121  +    if {[info exists CONTROL($tn)]} {
          122  +      do_test $tn.9 { set CONTROL($tn) } [lsort [array get DATA]]
          123  +    } else {
          124  +      set CONTROL($tn) [lsort [array get DATA]]
          125  +    }
    84    126     }
    85         -  unset -nocomplain X
    86         -  db eval "$query ORDER BY docid DESC" X  { 
    87         -    set D($X(docid)) [array get X] 
    88         -    lappend D_list $X(docid)
    89         -  }
          127  +}
          128  +unset -nocomplain CONTROL
          129  +
          130  +set testprefix fts3sort
    90    131   
    91         -  do_test 1.$tn.1 { set A_list } [lsort -integer -increasing $A_list]
    92         -  do_test 1.$tn.2 { set B_list } [lsort -integer -decreasing $B_list]
    93         -  do_test 1.$tn.3 { set C_list } [lsort -integer -increasing $C_list]
    94         -  do_test 1.$tn.4 { set D_list } [lsort -integer -decreasing $D_list]
    95         -
    96         -  unset -nocomplain DATA
    97         -  unset -nocomplain X
    98         -  db eval "$query" X  { 
    99         -    set DATA($X(docid)) [array get X] 
   100         -  }
   101         -
   102         -  do_test 1.$tn.5 { lsort [array get A] } [lsort [array get DATA]]
   103         -  do_test 1.$tn.6 { lsort [array get B] } [lsort [array get DATA]]
   104         -  do_test 1.$tn.7 { lsort [array get C] } [lsort [array get DATA]]
   105         -  do_test 1.$tn.8 { lsort [array get D] } [lsort [array get DATA]]
          132  +#-------------------------------------------------------------------------
          133  +# Tests for parsing the "order=asc" and "order=desc" directives.
          134  +#
          135  +foreach {tn param res} {
          136  +  1 "order=asc"             {0 {}}
          137  +  2 "order=desc"            {0 {}}
          138  +  3 "order=dec"             {1 {unrecognized order: dec}}
          139  +  4 "order=xxx, order=asc"  {1 {unrecognized order: xxx}}
          140  +  5 "order=desc, order=asc" {0 {}}
          141  +} {
          142  +  execsql { DROP TABLE IF EXISTS t1 }
          143  +  do_catchsql_test 2.1.$tn "
          144  +    CREATE VIRTUAL TABLE t1 USING fts4(a, b, $param)
          145  +  " $res
   106    146   }
          147  +
          148  +do_execsql_test 2.2 {
          149  +  BEGIN;
          150  +    CREATE VIRTUAL TABLE t2 USING fts4(order=desc);
          151  +    INSERT INTO t2 VALUES('aa bb');
          152  +    INSERT INTO t2 VALUES('bb cc');
          153  +    INSERT INTO t2 VALUES('cc aa');
          154  +    SELECT docid FROM t2 WHERE t2 MATCH 'aa';
          155  +  END;
          156  +} {3 1}
          157  +do_execsql_test 2.3 {
          158  +  SELECT docid FROM t2 WHERE t2 MATCH 'aa';
          159  +} {3 1}
   107    160   
   108    161   finish_test
          162  +

Changes to test/hook.test.

   270    270         SELECT * FROM t1 EXCEPT SELECT * FROM t3;
   271    271         SELECT * FROM t1 ORDER BY b;
   272    272         SELECT * FROM t1 GROUP BY b;
   273    273       }
   274    274       set ::update_hook
   275    275     } [list]
   276    276   }
          277  +
          278  +do_test hook-4.4 {
          279  +  execsql {
          280  +    CREATE TABLE t4(a UNIQUE, b);
          281  +    INSERT INTO t4 VALUES(1, 'a');
          282  +    INSERT INTO t4 VALUES(2, 'b');
          283  +  }
          284  +  set ::update_hook [list]
          285  +  execsql {
          286  +    REPLACE INTO t4 VALUES(1, 'c');
          287  +  }
          288  +  set ::update_hook
          289  +} [list INSERT main t4 3 ]
          290  +do_execsql_test hook-4.4.1 {
          291  +  SELECT * FROM t4 ORDER BY a;
          292  +} {1 c 2 b}
          293  +do_test hook-4.4.2 {
          294  +  set ::update_hook [list]
          295  +  execsql {
          296  +    PRAGMA recursive_triggers = on;
          297  +    REPLACE INTO t4 VALUES(1, 'd');
          298  +  }
          299  +  set ::update_hook
          300  +} [list INSERT main t4 4 ]
          301  +do_execsql_test hook-4.4.3 {
          302  +  SELECT * FROM t4 ORDER BY a;
          303  +} {1 d 2 b}
          304  +
   277    305   db update_hook {}
   278    306   #
   279    307   #----------------------------------------------------------------------------
   280    308   
   281    309   #----------------------------------------------------------------------------
   282    310   # Test the rollback-hook. The rollback-hook is a bit more complicated than
   283    311   # either the commit or update hooks because a rollback can happen 

Changes to test/permutations.test.

   175    175   } -files {
   176    176     fts3aa.test fts3ab.test fts3ac.test fts3ad.test fts3ae.test
   177    177     fts3af.test fts3ag.test fts3ah.test fts3ai.test fts3aj.test
   178    178     fts3ak.test fts3al.test fts3am.test fts3an.test fts3ao.test
   179    179     fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test
   180    180     fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test 
   181    181     fts3near.test fts3query.test fts3shared.test fts3snippet.test 
          182  +  fts3sort.test
   182    183   
   183    184     fts3fault.test fts3malloc.test fts3matchinfo.test
   184    185   
   185         -  fts3aux1.test fts3comp1.test
          186  +  fts3aux1.test fts3comp1.test fts3auto.test
   186    187   }
   187    188   
   188    189   
   189    190   lappend ::testsuitelist xxx
   190    191   #-------------------------------------------------------------------------
   191    192   # Define the coverage related test suites:
   192    193   #

Changes to test/tester.tcl.

   370    370     } {
   371    371       set testname "${::testprefix}-$testname"
   372    372     }
   373    373   }
   374    374       
   375    375   proc do_execsql_test {testname sql {result {}}} {
   376    376     fix_testname testname
   377         -  uplevel do_test $testname [list "execsql {$sql}"] [list [list {*}$result]]
          377  +  uplevel do_test [list $testname] [list "execsql {$sql}"] [list [list {*}$result]]
   378    378   }
   379    379   proc do_catchsql_test {testname sql result} {
   380    380     fix_testname testname
   381         -  uplevel do_test $testname [list "catchsql {$sql}"] [list $result]
          381  +  uplevel do_test [list $testname] [list "catchsql {$sql}"] [list $result]
   382    382   }
   383    383   proc do_eqp_test {name sql res} {
   384    384     uplevel do_execsql_test $name [list "EXPLAIN QUERY PLAN $sql"] [list $res]
   385    385   }
   386    386   
   387    387   #-------------------------------------------------------------------------
   388    388   #   Usage: do_select_tests PREFIX ?SWITCHES? TESTLIST

Changes to test/trace2.test.

   137    137       "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)"
   138    138     }
   139    139   
   140    140     do_trace_test 2.3 {
   141    141       INSERT INTO x1(x1) VALUES('optimize');
   142    142     } {
   143    143       "INSERT INTO x1(x1) VALUES('optimize');"
   144         -    "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' ORDER BY level DESC, idx ASC"
   145         -    "-- SELECT count(*), max(level) FROM 'main'.'x1_segdir'"
          144  +    "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?ORDER BY level DESC, idx ASC"
          145  +    "-- SELECT max(level) FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?"
   146    146       "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)"
   147         -    "-- DELETE FROM 'main'.'x1_segdir'"
          147  +    "-- DELETE FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?"
   148    148       "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)"
   149    149     }
   150    150   }
   151    151   
   152    152   finish_test