/ Check-in [c6d9f7d8]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge fts4aux branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: c6d9f7d8c48e1ff405e1c1d98a166974fc829f58
User & Date: dan 2011-02-02 04:40:07
Context
2011-02-02
16:34
Merge in the blocking-checkpoint enhancement, including the new sqlite3_wal_checkpoint_v2() interface and the PRAGMA wal_checkpoint(full) statement. check-in: bac7342c user: drh tags: trunk
04:40
Merge fts4aux branch. check-in: c6d9f7d8 user: dan tags: trunk
04:26
Add missing file fts3_aux.c. Closed-Leaf check-in: 0147d973 user: dan tags: fts4aux
2011-02-01
00:04
Version 3.7.5 check-in: ed759d5a user: drh tags: trunk, release, version-3.7.5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

   444    444     assert( p->pSegments==0 );
   445    445   
   446    446     /* Free any prepared statements held */
   447    447     for(i=0; i<SizeofArray(p->aStmt); i++){
   448    448       sqlite3_finalize(p->aStmt[i]);
   449    449     }
   450    450     sqlite3_free(p->zSegmentsTbl);
          451  +  sqlite3_free(p->zReadExprlist);
          452  +  sqlite3_free(p->zWriteExprlist);
   451    453   
   452    454     /* Invoke the tokenizer destructor to free the tokenizer. */
   453    455     p->pTokenizer->pModule->xDestroy(p->pTokenizer);
   454    456   
   455    457     sqlite3_free(p);
   456    458     return SQLITE_OK;
   457    459   }
................................................................................
   660    662     zValue = sqlite3_mprintf("%s", &zCsr[1]);
   661    663     if( zValue ){
   662    664       sqlite3Fts3Dequote(zValue);
   663    665     }
   664    666     *pzValue = zValue;
   665    667     return 1;
   666    668   }
          669  +
          670  +/*
          671  +** Append the output of a printf() style formatting to an existing string.
          672  +*/
          673  +static void fts3Appendf(
          674  +  int *pRc,                       /* IN/OUT: Error code */
          675  +  char **pz,                      /* IN/OUT: Pointer to string buffer */
          676  +  const char *zFormat,            /* Printf format string to append */
          677  +  ...                             /* Arguments for printf format string */
          678  +){
          679  +  if( *pRc==SQLITE_OK ){
          680  +    va_list ap;
          681  +    char *z;
          682  +    va_start(ap, zFormat);
          683  +    z = sqlite3_vmprintf(zFormat, ap);
          684  +    if( z && *pz ){
          685  +      char *z2 = sqlite3_mprintf("%s%s", *pz, z);
          686  +      sqlite3_free(z);
          687  +      z = z2;
          688  +    }
          689  +    if( z==0 ) *pRc = SQLITE_NOMEM;
          690  +    sqlite3_free(*pz);
          691  +    *pz = z;
          692  +  }
          693  +}
          694  +
          695  +/*
          696  +** Return a list of comma separated SQL expressions that could be used
          697  +** in a SELECT statement such as the following:
          698  +**
          699  +**     SELECT <list of expressions> FROM %_content AS x ...
          700  +**
          701  +** to return the docid, followed by each column of text data in order
          702  +** from left to write. If parameter zFunc is not NULL, then instead of
          703  +** being returned directly each column of text data is passed to an SQL
          704  +** function named zFunc first. For example, if zFunc is "unzip" and the
          705  +** table has the three user-defined columns "a", "b", and "c", the following
          706  +** string is returned:
          707  +**
          708  +**     "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c')"
          709  +**
          710  +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It
          711  +** is the responsibility of the caller to eventually free it.
          712  +**
          713  +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and
          714  +** a NULL pointer is returned). Otherwise, if an OOM error is encountered
          715  +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If
          716  +** no error occurs, *pRc is left unmodified.
          717  +*/
          718  +static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){
          719  +  char *zRet = 0;
          720  +  int i;
          721  +  if( !zFunc ) zFunc = "";
          722  +  fts3Appendf(pRc, &zRet, "docid");
          723  +  for(i=0; i<p->nColumn; i++){
          724  +    fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunc, i, p->azColumn[i]);
          725  +  }
          726  +  return zRet;
          727  +}
          728  +
          729  +/*
          730  +** Return a list of N comma separated question marks, where N is the number
          731  +** of columns in the %_content table (one for the docid plus one for each
          732  +** user-defined text column).
          733  +**
          734  +** If argument zFunc is not NULL, then all but the first question mark
          735  +** is preceded by zFunc and an open bracket, and followed by a closed
          736  +** bracket. For example, if zFunc is "zip" and the FTS3 table has three 
          737  +** user-defined text columns, the following string is returned:
          738  +**
          739  +**     "?, zip(?), zip(?), zip(?)"
          740  +**
          741  +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It
          742  +** is the responsibility of the caller to eventually free it.
          743  +**
          744  +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and
          745  +** a NULL pointer is returned). Otherwise, if an OOM error is encountered
          746  +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If
          747  +** no error occurs, *pRc is left unmodified.
          748  +*/
          749  +static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){
          750  +  char *zRet = 0;
          751  +  int i;
          752  +  if( !zFunc ) zFunc = "";
          753  +  fts3Appendf(pRc, &zRet, "?");
          754  +  for(i=0; i<p->nColumn; i++){
          755  +    fts3Appendf(pRc, &zRet, ",%s(?)", zFunc);
          756  +  }
          757  +  return zRet;
          758  +}
   667    759   
   668    760   /*
   669    761   ** This function is the implementation of both the xConnect and xCreate
   670    762   ** methods of the FTS3 virtual table.
   671    763   **
   672    764   ** The argv[] array contains the following:
   673    765   **
................................................................................
   697    789     int nDb;                        /* Bytes required to hold database name */
   698    790     int nName;                      /* Bytes required to hold table name */
   699    791     int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
   700    792     int bNoDocsize = 0;             /* True to omit %_docsize table */
   701    793     const char **aCol;              /* Array of column names */
   702    794     sqlite3_tokenizer *pTokenizer = 0;        /* Tokenizer for this table */
   703    795   
          796  +  char *zCompress = 0;
          797  +  char *zUncompress = 0;
          798  +
   704    799     assert( strlen(argv[0])==4 );
   705    800     assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
   706    801          || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
   707    802     );
   708    803   
   709    804     nDb = (int)strlen(argv[1]) + 1;
   710    805     nName = (int)strlen(argv[2]) + 1;
................................................................................
   747    842         if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){
   748    843           if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){
   749    844             bNoDocsize = 1;
   750    845           }else{
   751    846             *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal);
   752    847             rc = SQLITE_ERROR;
   753    848           }
          849  +      }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){
          850  +        zCompress = zVal;
          851  +        zVal = 0;
          852  +      }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){
          853  +        zUncompress = zVal;
          854  +        zVal = 0;
   754    855         }else{
   755    856           *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
   756    857           rc = SQLITE_ERROR;
   757    858         }
   758    859         sqlite3_free(zVal);
   759    860       }
   760    861   
................................................................................
   820    921       memcpy(zCsr, z, n);
   821    922       zCsr[n] = '\0';
   822    923       sqlite3Fts3Dequote(zCsr);
   823    924       p->azColumn[iCol] = zCsr;
   824    925       zCsr += n+1;
   825    926       assert( zCsr <= &((char *)p)[nByte] );
   826    927     }
          928  +
          929  +  if( (zCompress==0)!=(zUncompress==0) ){
          930  +    char const *zMissing = (zCompress==0 ? "compress" : "uncompress");
          931  +    rc = SQLITE_ERROR;
          932  +    *pzErr = sqlite3_mprintf("missing %s parameter", zMissing);
          933  +  }
          934  +  p->zReadExprlist = fts3ReadExprList(p, zUncompress, &rc);
          935  +  p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc);
          936  +  if( rc!=SQLITE_OK ) goto fts3_init_out;
   827    937   
   828    938     /* If this is an xCreate call, create the underlying tables in the 
   829    939     ** database. TODO: For xConnect(), it could verify that said tables exist.
   830    940     */
   831    941     if( isCreate ){
   832    942       rc = fts3CreateTables(p);
   833    943     }
................................................................................
   838    948     */
   839    949     fts3DatabasePageSize(&rc, p);
   840    950   
   841    951     /* Declare the table schema to SQLite. */
   842    952     fts3DeclareVtab(&rc, p);
   843    953   
   844    954   fts3_init_out:
   845         -
          955  +  sqlite3_free(zCompress);
          956  +  sqlite3_free(zUncompress);
   846    957     sqlite3_free((void *)aCol);
   847    958     if( rc!=SQLITE_OK ){
   848    959       if( p ){
   849    960         fts3DisconnectMethod((sqlite3_vtab *)p);
   850    961       }else if( pTokenizer ){
   851    962         pTokenizer->pModule->xDestroy(pTokenizer);
   852    963       }
................................................................................
  1931   2042       if( !*ppOut ) return SQLITE_NOMEM;
  1932   2043       sqlite3Fts3PutVarint(*ppOut, docid);
  1933   2044     }
  1934   2045   
  1935   2046     return SQLITE_OK;
  1936   2047   }
  1937   2048   
  1938         -/*
  1939         -** An Fts3SegReaderArray is used to store an array of Fts3SegReader objects.
  1940         -** Elements are added to the array using fts3SegReaderArrayAdd(). 
  1941         -*/
  1942         -struct Fts3SegReaderArray {
  1943         -  int nSegment;                   /* Number of valid entries in apSegment[] */
  1944         -  int nAlloc;                     /* Allocated size of apSegment[] */
  1945         -  int nCost;                      /* The cost of executing SegReaderIterate() */
  1946         -  Fts3SegReader *apSegment[1];    /* Array of seg-reader objects */
  1947         -};
  1948         -
  1949         -
  1950         -/*
  1951         -** Free an Fts3SegReaderArray object. Also free all seg-readers in the
  1952         -** array (using sqlite3Fts3SegReaderFree()).
  1953         -*/
  1954         -static void fts3SegReaderArrayFree(Fts3SegReaderArray *pArray){
  1955         -  if( pArray ){
  1956         -    int i;
  1957         -    for(i=0; i<pArray->nSegment; i++){
  1958         -      sqlite3Fts3SegReaderFree(pArray->apSegment[i]);
  1959         -    }
  1960         -    sqlite3_free(pArray);
  1961         -  }
  1962         -}
  1963         -
  1964         -static int fts3SegReaderArrayAdd(
  1965         -  Fts3SegReaderArray **ppArray, 
  1966         -  Fts3SegReader *pNew
  1967         -){
  1968         -  Fts3SegReaderArray *pArray = *ppArray;
  1969         -
  1970         -  if( !pArray || pArray->nAlloc==pArray->nSegment ){
  1971         -    int nNew = (pArray ? pArray->nAlloc+16 : 16);
  1972         -    pArray = (Fts3SegReaderArray *)sqlite3_realloc(pArray, 
  1973         -        sizeof(Fts3SegReaderArray) + (nNew-1) * sizeof(Fts3SegReader*)
  1974         -    );
  1975         -    if( !pArray ){
  1976         -      sqlite3Fts3SegReaderFree(pNew);
  1977         -      return SQLITE_NOMEM;
  1978         -    }
  1979         -    if( nNew==16 ){
  1980         -      pArray->nSegment = 0;
  1981         -      pArray->nCost = 0;
  1982         -    }
  1983         -    pArray->nAlloc = nNew;
  1984         -    *ppArray = pArray;
  1985         -  }
  1986         -
  1987         -  pArray->apSegment[pArray->nSegment++] = pNew;
  1988         -  return SQLITE_OK;
  1989         -}
  1990         -
  1991         -static int fts3TermSegReaderArray(
         2049  +int sqlite3Fts3SegReaderCursor(
         2050  +  Fts3Table *p,                   /* FTS3 table handle */
         2051  +  int iLevel,                     /* Level of segments to scan */
         2052  +  const char *zTerm,              /* Term to query for */
         2053  +  int nTerm,                      /* Size of zTerm in bytes */
         2054  +  int isPrefix,                   /* True for a prefix search */
         2055  +  Fts3SegReaderCursor *pCsr       /* Cursor object to populate */
         2056  +){
         2057  +  int rc = SQLITE_OK;
         2058  +  int rc2;
         2059  +  int iAge = 0;
         2060  +  sqlite3_stmt *pStmt = 0;
         2061  +  Fts3SegReader *pPending = 0;
         2062  +
         2063  +  assert( iLevel==FTS3_SEGCURSOR_ALL 
         2064  +      ||  iLevel==FTS3_SEGCURSOR_PENDING 
         2065  +      ||  iLevel>=0
         2066  +  );
         2067  +  assert( FTS3_SEGCURSOR_PENDING<0 );
         2068  +  assert( FTS3_SEGCURSOR_ALL<0 );
         2069  +  assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) );
         2070  +
         2071  +  memset(pCsr, 0, sizeof(Fts3SegReaderCursor));
         2072  +
         2073  +  /* If iLevel is less than 0, include a seg-reader for the pending-terms. */
         2074  +  if( iLevel<0 ){
         2075  +    rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pPending);
         2076  +    if( rc==SQLITE_OK && pPending ){
         2077  +      int nByte = (sizeof(Fts3SegReader *) * 16);
         2078  +      pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte);
         2079  +      if( pCsr->apSegment==0 ){
         2080  +        rc = SQLITE_NOMEM;
         2081  +      }else{
         2082  +        pCsr->apSegment[0] = pPending;
         2083  +        pCsr->nSegment = 1;
         2084  +        pPending = 0;
         2085  +      }
         2086  +    }
         2087  +  }
         2088  +
         2089  +  if( iLevel!=FTS3_SEGCURSOR_PENDING ){
         2090  +    if( rc==SQLITE_OK ){
         2091  +      rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt);
         2092  +    }
         2093  +    while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
         2094  +
         2095  +      /* Read the values returned by the SELECT into local variables. */
         2096  +      sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1);
         2097  +      sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2);
         2098  +      sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3);
         2099  +      int nRoot = sqlite3_column_bytes(pStmt, 4);
         2100  +      char const *zRoot = sqlite3_column_blob(pStmt, 4);
         2101  +
         2102  +      /* If nSegment is a multiple of 16 the array needs to be extended. */
         2103  +      if( (pCsr->nSegment%16)==0 ){
         2104  +        Fts3SegReader **apNew;
         2105  +        int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
         2106  +        apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte);
         2107  +        if( !apNew ){
         2108  +          rc = SQLITE_NOMEM;
         2109  +          goto finished;
         2110  +        }
         2111  +        pCsr->apSegment = apNew;
         2112  +      }
         2113  +
         2114  +      /* If zTerm is not NULL, and this segment is not stored entirely on its
         2115  +      ** root node, the range of leaves scanned can be reduced. Do this. */
         2116  +      if( iStartBlock && zTerm ){
         2117  +        sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0);
         2118  +        rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi);
         2119  +        if( rc!=SQLITE_OK ) goto finished;
         2120  +        if( isPrefix==0 ) iLeavesEndBlock = iStartBlock;
         2121  +      }
         2122  +  
         2123  +      rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock,
         2124  +          iEndBlock, zRoot, nRoot, &pCsr->apSegment[pCsr->nSegment]
         2125  +      );
         2126  +      if( rc!=SQLITE_OK ) goto finished;
         2127  +      pCsr->nSegment++;
         2128  +      iAge++;
         2129  +    }
         2130  +  }
         2131  +
         2132  + finished:
         2133  +  rc2 = sqlite3_reset(pStmt);
         2134  +  if( rc==SQLITE_DONE ) rc = rc2;
         2135  +  sqlite3Fts3SegReaderFree(pPending);
         2136  +
         2137  +  return rc;
         2138  +}
         2139  +
         2140  +
         2141  +static int fts3TermSegReaderCursor(
  1992   2142     Fts3Cursor *pCsr,               /* Virtual table cursor handle */
  1993   2143     const char *zTerm,              /* Term to query for */
  1994   2144     int nTerm,                      /* Size of zTerm in bytes */
  1995   2145     int isPrefix,                   /* True for a prefix search */
  1996         -  Fts3SegReaderArray **ppArray    /* OUT: Allocated seg-reader array */
  1997         -){
  1998         -  Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
  1999         -  int rc;                         /* Return code */
  2000         -  Fts3SegReaderArray *pArray = 0; /* Array object to build */
  2001         -  Fts3SegReader *pReader = 0;     /* Seg-reader to add to pArray */ 
  2002         -  sqlite3_stmt *pStmt = 0;        /* SQL statement to scan %_segdir table */
  2003         -  int iAge = 0;                   /* Used to assign ages to segments */
  2004         -
  2005         -  /* Allocate a seg-reader to scan the pending terms, if any. */
  2006         -  rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pReader);
  2007         -  if( rc==SQLITE_OK && pReader ) {
  2008         -    rc = fts3SegReaderArrayAdd(&pArray, pReader);
  2009         -  }
  2010         -
  2011         -  /* Loop through the entire %_segdir table. For each segment, create a
  2012         -  ** Fts3SegReader to iterate through the subset of the segment leaves
  2013         -  ** that may contain a term that matches zTerm/nTerm. For non-prefix
  2014         -  ** searches, this is always a single leaf. For prefix searches, this
  2015         -  ** may be a contiguous block of leaves.
  2016         -  */
  2017         -  if( rc==SQLITE_OK ){
  2018         -    rc = sqlite3Fts3AllSegdirs(p, &pStmt);
  2019         -  }
  2020         -  while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
  2021         -    Fts3SegReader *pNew = 0;
  2022         -    int nRoot = sqlite3_column_bytes(pStmt, 4);
  2023         -    char const *zRoot = sqlite3_column_blob(pStmt, 4);
  2024         -    if( sqlite3_column_int64(pStmt, 1)==0 ){
  2025         -      /* The entire segment is stored on the root node (which must be a
  2026         -      ** leaf). Do not bother inspecting any data in this case, just
  2027         -      ** create a Fts3SegReader to scan the single leaf. 
  2028         -      */
  2029         -      rc = sqlite3Fts3SegReaderNew(iAge, 0, 0, 0, zRoot, nRoot, &pNew);
  2030         -    }else{
  2031         -      sqlite3_int64 i1;           /* First leaf that may contain zTerm */
  2032         -      sqlite3_int64 i2;           /* Final leaf that may contain zTerm */
  2033         -      rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1, (isPrefix?&i2:0));
  2034         -      if( isPrefix==0 ) i2 = i1;
  2035         -      if( rc==SQLITE_OK ){
  2036         -        rc = sqlite3Fts3SegReaderNew(iAge, i1, i2, 0, 0, 0, &pNew);
  2037         -      }
  2038         -    }
  2039         -    assert( (pNew==0)==(rc!=SQLITE_OK) );
  2040         -
  2041         -    /* If a new Fts3SegReader was allocated, add it to the array. */
  2042         -    if( rc==SQLITE_OK ){
  2043         -      rc = fts3SegReaderArrayAdd(&pArray, pNew);
  2044         -    }
  2045         -    if( rc==SQLITE_OK ){
  2046         -      rc = sqlite3Fts3SegReaderCost(pCsr, pNew, &pArray->nCost);
  2047         -    }
  2048         -    iAge++;
  2049         -  }
  2050         -
  2051         -  if( rc==SQLITE_DONE ){
  2052         -    rc = sqlite3_reset(pStmt);
  2053         -  }else{
  2054         -    sqlite3_reset(pStmt);
  2055         -  }
  2056         -  if( rc!=SQLITE_OK ){
  2057         -    fts3SegReaderArrayFree(pArray);
  2058         -    pArray = 0;
  2059         -  }
  2060         -  *ppArray = pArray;
  2061         -  return rc;
         2146  +  Fts3SegReaderCursor **ppSegcsr  /* OUT: Allocated seg-reader cursor */
         2147  +){
         2148  +  Fts3SegReaderCursor *pSegcsr;   /* Object to allocate and return */
         2149  +  int rc = SQLITE_NOMEM;          /* Return code */
         2150  +
         2151  +  pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor));
         2152  +  if( pSegcsr ){
         2153  +    Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
         2154  +    int i;
         2155  +    int nCost = 0;
         2156  +    rc = sqlite3Fts3SegReaderCursor(
         2157  +        p, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, pSegcsr);
         2158  +  
         2159  +    for(i=0; rc==SQLITE_OK && i<pSegcsr->nSegment; i++){
         2160  +      rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost);
         2161  +    }
         2162  +    pSegcsr->nCost = nCost;
         2163  +  }
         2164  +
         2165  +  *ppSegcsr = pSegcsr;
         2166  +  return rc;
         2167  +}
         2168  +
         2169  +static void fts3SegReaderCursorFree(Fts3SegReaderCursor *pSegcsr){
         2170  +  sqlite3Fts3SegReaderFinish(pSegcsr);
         2171  +  sqlite3_free(pSegcsr);
  2062   2172   }
  2063   2173   
  2064   2174   /*
  2065   2175   ** This function retreives the doclist for the specified term (or term
  2066   2176   ** prefix) from the database. 
  2067   2177   **
  2068   2178   ** The returned doclist may be in one of two formats, depending on the 
................................................................................
  2077   2187     Fts3PhraseToken *pTok,          /* Token to query for */
  2078   2188     int iColumn,                    /* Column to query (or -ve for all columns) */
  2079   2189     int isReqPos,                   /* True to include position lists in output */
  2080   2190     int *pnOut,                     /* OUT: Size of buffer at *ppOut */
  2081   2191     char **ppOut                    /* OUT: Malloced result buffer */
  2082   2192   ){
  2083   2193     int rc;                         /* Return code */
  2084         -  Fts3SegReaderArray *pArray;     /* Seg-reader array for this term */
  2085         -  TermSelect tsc;               /* Context object for fts3TermSelectCb() */
  2086         -  Fts3SegFilter filter;         /* Segment term filter configuration */
         2194  +  Fts3SegReaderCursor *pSegcsr;   /* Seg-reader cursor for this term */
         2195  +  TermSelect tsc;                 /* Context object for fts3TermSelectCb() */
         2196  +  Fts3SegFilter filter;           /* Segment term filter configuration */
  2087   2197   
  2088         -  pArray = pTok->pArray;
         2198  +  pSegcsr = pTok->pSegcsr;
  2089   2199     memset(&tsc, 0, sizeof(TermSelect));
  2090   2200     tsc.isReqPos = isReqPos;
  2091   2201   
  2092   2202     filter.flags = FTS3_SEGMENT_IGNORE_EMPTY 
  2093   2203           | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0)
  2094   2204           | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
  2095   2205           | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
  2096   2206     filter.iCol = iColumn;
  2097   2207     filter.zTerm = pTok->z;
  2098   2208     filter.nTerm = pTok->n;
  2099   2209   
  2100         -  rc = sqlite3Fts3SegReaderIterate(p, pArray->apSegment, pArray->nSegment, 
  2101         -      &filter, fts3TermSelectCb, (void *)&tsc
  2102         -  );
         2210  +  rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter);
         2211  +  while( SQLITE_OK==rc
         2212  +      && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) 
         2213  +  ){
         2214  +    rc = fts3TermSelectCb(p, (void *)&tsc, 
         2215  +        pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist
         2216  +    );
         2217  +  }
         2218  +
  2103   2219     if( rc==SQLITE_OK ){
  2104   2220       rc = fts3TermSelectMerge(&tsc);
  2105   2221     }
  2106         -
  2107   2222     if( rc==SQLITE_OK ){
  2108   2223       *ppOut = tsc.aaOutput[0];
  2109   2224       *pnOut = tsc.anOutput[0];
  2110   2225     }else{
  2111   2226       int i;
  2112   2227       for(i=0; i<SizeofArray(tsc.aaOutput); i++){
  2113   2228         sqlite3_free(tsc.aaOutput[i]);
  2114   2229       }
  2115   2230     }
  2116   2231   
  2117         -  fts3SegReaderArrayFree(pArray);
  2118         -  pTok->pArray = 0;
         2232  +  fts3SegReaderCursorFree(pSegcsr);
         2233  +  pTok->pSegcsr = 0;
  2119   2234     return rc;
  2120   2235   }
  2121   2236   
  2122   2237   /*
  2123   2238   ** This function counts the total number of docids in the doclist stored
  2124   2239   ** in buffer aList[], size nList bytes.
  2125   2240   **
................................................................................
  2234   2349     /* If this is an xFilter() evaluation, create a segment-reader for each
  2235   2350     ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo
  2236   2351     ** evaluation, only create segment-readers if there are no Fts3DeferredToken
  2237   2352     ** objects attached to the phrase-tokens.
  2238   2353     */
  2239   2354     for(ii=0; ii<pPhrase->nToken; ii++){
  2240   2355       Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
  2241         -    if( pTok->pArray==0 ){
         2356  +    if( pTok->pSegcsr==0 ){
  2242   2357         if( (pCsr->eEvalmode==FTS3_EVAL_FILTER)
  2243   2358          || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) 
  2244   2359          || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) 
  2245   2360         ){
  2246         -        rc = fts3TermSegReaderArray(
  2247         -            pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray
         2361  +        rc = fts3TermSegReaderCursor(
         2362  +            pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr
  2248   2363           );
  2249   2364           if( rc!=SQLITE_OK ) return rc;
  2250   2365         }
  2251   2366       }
  2252   2367     }
  2253   2368   
  2254   2369     for(ii=0; ii<pPhrase->nToken; ii++){
................................................................................
  2271   2386         pTok = &pPhrase->aToken[iTok];
  2272   2387       }else{
  2273   2388         int nMinCost = 0x7FFFFFFF;
  2274   2389         int jj;
  2275   2390   
  2276   2391         /* Find the remaining token with the lowest cost. */
  2277   2392         for(jj=0; jj<pPhrase->nToken; jj++){
  2278         -        Fts3SegReaderArray *pArray = pPhrase->aToken[jj].pArray;
  2279         -        if( pArray && pArray->nCost<nMinCost ){
         2393  +        Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[jj].pSegcsr;
         2394  +        if( pSegcsr && pSegcsr->nCost<nMinCost ){
  2280   2395             iTok = jj;
  2281         -          nMinCost = pArray->nCost;
         2396  +          nMinCost = pSegcsr->nCost;
  2282   2397           }
  2283   2398         }
  2284   2399         pTok = &pPhrase->aToken[iTok];
  2285   2400   
  2286   2401         /* This branch is taken if it is determined that loading the doclist
  2287   2402         ** for the next token would require more IO than loading all documents
  2288   2403         ** currently identified by doclist pOut/nOut. No further doclists will
................................................................................
  2293   2408           break;
  2294   2409         }
  2295   2410       }
  2296   2411   
  2297   2412       if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){
  2298   2413         rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList);
  2299   2414       }else{
  2300         -      if( pTok->pArray ){
         2415  +      if( pTok->pSegcsr ){
  2301   2416           rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList);
  2302   2417         }
  2303   2418         pTok->bFulltext = 1;
  2304   2419       }
  2305         -    assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pArray==0 );
         2420  +    assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pSegcsr==0 );
  2306   2421       if( rc!=SQLITE_OK ) break;
  2307   2422   
  2308   2423       if( isFirst ){
  2309   2424         pOut = pList;
  2310   2425         nOut = nList;
  2311   2426         if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){
  2312   2427           nDoc = fts3DoclistCountDocids(1, pOut, nOut);
................................................................................
  2476   2591   
  2477   2592     if( pExpr->eType==FTSQUERY_PHRASE ){
  2478   2593       Fts3Phrase *pPhrase = pExpr->pPhrase;
  2479   2594       int ii;
  2480   2595   
  2481   2596       for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){
  2482   2597         Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
  2483         -      if( pTok->pArray==0 ){
  2484         -        rc = fts3TermSegReaderArray(
  2485         -            pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray
         2598  +      if( pTok->pSegcsr==0 ){
         2599  +        rc = fts3TermSegReaderCursor(
         2600  +            pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr
  2486   2601           );
  2487   2602         }
  2488   2603       }
  2489   2604     }else{ 
  2490   2605       rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr);
  2491   2606       if( rc==SQLITE_OK ){
  2492   2607         rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr);
................................................................................
  2502   2617   */
  2503   2618   static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){
  2504   2619     if( pExpr ){
  2505   2620       Fts3Phrase *pPhrase = pExpr->pPhrase;
  2506   2621       if( pPhrase ){
  2507   2622         int kk;
  2508   2623         for(kk=0; kk<pPhrase->nToken; kk++){
  2509         -        fts3SegReaderArrayFree(pPhrase->aToken[kk].pArray);
  2510         -        pPhrase->aToken[kk].pArray = 0;
         2624  +        fts3SegReaderCursorFree(pPhrase->aToken[kk].pSegcsr);
         2625  +        pPhrase->aToken[kk].pSegcsr = 0;
  2511   2626         }
  2512   2627       }
  2513   2628       fts3ExprFreeSegReaders(pExpr->pLeft);
  2514   2629       fts3ExprFreeSegReaders(pExpr->pRight);
  2515   2630     }
  2516   2631   }
  2517   2632   
................................................................................
  2523   2638   static int fts3ExprCost(Fts3Expr *pExpr){
  2524   2639     int nCost;                      /* Return value */
  2525   2640     if( pExpr->eType==FTSQUERY_PHRASE ){
  2526   2641       Fts3Phrase *pPhrase = pExpr->pPhrase;
  2527   2642       int ii;
  2528   2643       nCost = 0;
  2529   2644       for(ii=0; ii<pPhrase->nToken; ii++){
  2530         -      Fts3SegReaderArray *pArray = pPhrase->aToken[ii].pArray;
  2531         -      if( pArray ){
  2532         -        nCost += pPhrase->aToken[ii].pArray->nCost;
  2533         -      }
         2645  +      Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[ii].pSegcsr;
         2646  +      if( pSegcsr ) nCost += pSegcsr->nCost;
  2534   2647       }
  2535   2648     }else{
  2536   2649       nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight);
  2537   2650     }
  2538   2651     return nCost;
  2539   2652   }
  2540   2653   
................................................................................
  2868   2981     sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
  2869   2982     int idxNum,                     /* Strategy index */
  2870   2983     const char *idxStr,             /* Unused */
  2871   2984     int nVal,                       /* Number of elements in apVal */
  2872   2985     sqlite3_value **apVal           /* Arguments for the indexing scheme */
  2873   2986   ){
  2874   2987     const char *azSql[] = {
  2875         -    "SELECT * FROM %Q.'%q_content' WHERE docid = ?", /* non-full-table-scan */
  2876         -    "SELECT * FROM %Q.'%q_content'",                 /* full-table-scan */
         2988  +    "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?", /* non-full-scan */
         2989  +    "SELECT %s FROM %Q.'%q_content' AS x ",                /* full-scan */
  2877   2990     };
  2878   2991     int rc;                         /* Return code */
  2879   2992     char *zSql;                     /* SQL statement used to access %_content */
  2880   2993     Fts3Table *p = (Fts3Table *)pCursor->pVtab;
  2881   2994     Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  2882   2995   
  2883   2996     UNUSED_PARAMETER(idxStr);
................................................................................
  2924   3037     }
  2925   3038   
  2926   3039     /* Compile a SELECT statement for this cursor. For a full-table-scan, the
  2927   3040     ** statement loops through all rows of the %_content table. For a
  2928   3041     ** full-text query or docid lookup, the statement retrieves a single
  2929   3042     ** row by docid.
  2930   3043     */
  2931         -  zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName);
         3044  +  zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH];
         3045  +  zSql = sqlite3_mprintf(zSql, p->zReadExprlist, p->zDb, p->zName);
  2932   3046     if( !zSql ){
  2933   3047       rc = SQLITE_NOMEM;
  2934   3048     }else{
  2935   3049       rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
  2936   3050       sqlite3_free(zSql);
  2937   3051     }
  2938   3052     if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){
................................................................................
  3442   3556     const sqlite3_tokenizer_module *pPorter = 0;
  3443   3557   
  3444   3558   #ifdef SQLITE_ENABLE_ICU
  3445   3559     const sqlite3_tokenizer_module *pIcu = 0;
  3446   3560     sqlite3Fts3IcuTokenizerModule(&pIcu);
  3447   3561   #endif
  3448   3562   
         3563  +  rc = sqlite3Fts3InitAux(db);
         3564  +  if( rc!=SQLITE_OK ) return rc;
         3565  +
  3449   3566     sqlite3Fts3SimpleTokenizerModule(&pSimple);
  3450   3567     sqlite3Fts3PorterTokenizerModule(&pPorter);
  3451   3568   
  3452   3569     /* Allocate and initialise the hash-table used to store tokenizers. */
  3453   3570     pHash = sqlite3_malloc(sizeof(Fts3Hash));
  3454   3571     if( !pHash ){
  3455   3572       rc = SQLITE_NOMEM;

Changes to ext/fts3/fts3Int.h.

   103    103   typedef struct Fts3Expr Fts3Expr;
   104    104   typedef struct Fts3Phrase Fts3Phrase;
   105    105   typedef struct Fts3PhraseToken Fts3PhraseToken;
   106    106   
   107    107   typedef struct Fts3SegFilter Fts3SegFilter;
   108    108   typedef struct Fts3DeferredToken Fts3DeferredToken;
   109    109   typedef struct Fts3SegReader Fts3SegReader;
   110         -typedef struct Fts3SegReaderArray Fts3SegReaderArray;
          110  +typedef struct Fts3SegReaderCursor Fts3SegReaderCursor;
   111    111   
   112    112   /*
   113    113   ** A connection to a fulltext index is an instance of the following
   114    114   ** structure. The xCreate and xConnect methods create an instance
   115    115   ** of this structure and xDestroy and xDisconnect free that instance.
   116    116   ** All other methods receive a pointer to the structure as one of their
   117    117   ** arguments.
................................................................................
   126    126     sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
   127    127   
   128    128     /* Precompiled statements used by the implementation. Each of these 
   129    129     ** statements is run and reset within a single virtual table API call. 
   130    130     */
   131    131     sqlite3_stmt *aStmt[24];
   132    132   
          133  +  char *zReadExprlist;
          134  +  char *zWriteExprlist;
          135  +
   133    136     int nNodeSize;                  /* Soft limit for node size */
   134    137     u8 bHasStat;                    /* True if %_stat table exists */
   135    138     u8 bHasDocsize;                 /* True if %_docsize table exists */
   136    139     int nPgsz;                      /* Page size for host database */
   137    140     char *zSegmentsTbl;             /* Name of %_segments table */
   138    141     sqlite3_blob *pSegments;        /* Blob handle open on %_segments table */
   139    142   
................................................................................
   213    216   ** on the assumption that the 
   214    217   */
   215    218   struct Fts3PhraseToken {
   216    219     char *z;                        /* Text of the token */
   217    220     int n;                          /* Number of bytes in buffer z */
   218    221     int isPrefix;                   /* True if token ends with a "*" character */
   219    222     int bFulltext;                  /* True if full-text index was used */
   220         -  Fts3SegReaderArray *pArray;     /* Segment-reader for this token */
          223  +  Fts3SegReaderCursor *pSegcsr;   /* Segment-reader for this token */
   221    224     Fts3DeferredToken *pDeferred;   /* Deferred token object for this token */
   222    225   };
   223    226   
   224    227   struct Fts3Phrase {
   225    228     /* Variables populated by fts3_expr.c when parsing a MATCH expression */
   226    229     int nToken;                /* Number of tokens in the phrase */
   227    230     int iColumn;               /* Index of column this phrase must match */
................................................................................
   281    284   int sqlite3Fts3PendingTermsFlush(Fts3Table *);
   282    285   void sqlite3Fts3PendingTermsClear(Fts3Table *);
   283    286   int sqlite3Fts3Optimize(Fts3Table *);
   284    287   int sqlite3Fts3SegReaderNew(int, sqlite3_int64,
   285    288     sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
   286    289   int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**);
   287    290   void sqlite3Fts3SegReaderFree(Fts3SegReader *);
   288         -int sqlite3Fts3SegReaderIterate(
   289         -  Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *,
   290         -  int (*)(Fts3Table *, void *, char *, int, char *, int),  void *
   291         -);
   292    291   int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
   293         -int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **);
          292  +int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **);
   294    293   int sqlite3Fts3ReadLock(Fts3Table *);
   295    294   int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);
   296    295   
   297    296   int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
   298    297   int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
   299    298   
   300    299   void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
   301    300   int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
   302    301   int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
   303    302   void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
   304    303   char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);
   305         -
   306    304   void sqlite3Fts3SegmentsClose(Fts3Table *);
   307    305   
          306  +#define FTS3_SEGCURSOR_PENDING -1
          307  +#define FTS3_SEGCURSOR_ALL     -2
          308  +
          309  +int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*);
          310  +int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *);
          311  +void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *);
          312  +int sqlite3Fts3SegReaderCursor(
          313  +    Fts3Table *, int, const char *, int, int, Fts3SegReaderCursor *);
          314  +
   308    315   /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
   309    316   #define FTS3_SEGMENT_REQUIRE_POS   0x00000001
   310    317   #define FTS3_SEGMENT_IGNORE_EMPTY  0x00000002
   311    318   #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
   312    319   #define FTS3_SEGMENT_PREFIX        0x00000008
   313    320   
   314    321   /* Type passed as 4th argument to SegmentReaderIterate() */
   315    322   struct Fts3SegFilter {
   316    323     const char *zTerm;
   317    324     int nTerm;
   318    325     int iCol;
   319    326     int flags;
   320    327   };
          328  +
          329  +struct Fts3SegReaderCursor {
          330  +  /* Used internally by sqlite3Fts3SegReaderXXX() calls */
          331  +  Fts3SegReader **apSegment;      /* Array of Fts3SegReader objects */
          332  +  int nSegment;                   /* Size of apSegment array */
          333  +  int nAdvance;                   /* How many seg-readers to advance */
          334  +  Fts3SegFilter *pFilter;         /* Pointer to filter object */
          335  +  char *aBuffer;                  /* Buffer to merge doclists in */
          336  +  int nBuffer;                    /* Allocated size of aBuffer[] in bytes */
          337  +
          338  +  /* Cost of running this iterator. Used by fts3.c only. */
          339  +  int nCost;
          340  +
          341  +  /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */
          342  +  char *zTerm;                    /* Pointer to term buffer */
          343  +  int nTerm;                      /* Size of zTerm in bytes */
          344  +  char *aDoclist;                 /* Pointer to doclist buffer */
          345  +  int nDoclist;                   /* Size of aDoclist[] in bytes */
          346  +};
   321    347   
   322    348   /* fts3.c */
   323    349   int sqlite3Fts3PutVarint(char *, sqlite3_int64);
   324    350   int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
   325    351   int sqlite3Fts3GetVarint32(const char *, int *);
   326    352   int sqlite3Fts3VarintLen(sqlite3_uint64);
   327    353   void sqlite3Fts3Dequote(char *);
................................................................................
   351    377     char **, int, int, const char *, int, Fts3Expr **
   352    378   );
   353    379   void sqlite3Fts3ExprFree(Fts3Expr *);
   354    380   #ifdef SQLITE_TEST
   355    381   int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
   356    382   #endif
   357    383   
          384  +/* fts3_aux.c */
          385  +int sqlite3Fts3InitAux(sqlite3 *db);
          386  +
   358    387   #endif /* _FTSINT_H */

Added ext/fts3/fts3_aux.c.

            1  +/*
            2  +** 2011 Jan 27
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +*/
           14  +
           15  +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
           16  +
           17  +#include "fts3Int.h"
           18  +#include <string.h>
           19  +#include <assert.h>
           20  +
           21  +typedef struct Fts3auxTable Fts3auxTable;
           22  +typedef struct Fts3auxCursor Fts3auxCursor;
           23  +
           24  +struct Fts3auxTable {
           25  +  sqlite3_vtab base;              /* Base class used by SQLite core */
           26  +  Fts3Table *pFts3Tab;
           27  +};
           28  +
           29  +struct Fts3auxCursor {
           30  +  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
           31  +
           32  +  Fts3SegFilter filter;
           33  +  Fts3SegReaderCursor csr;
           34  +  int isEof;
           35  +
           36  +  sqlite3_int64 iRowid;
           37  +  sqlite3_int64 nDoc;
           38  +  sqlite3_int64 nOcc;
           39  +};
           40  +
           41  +/*
           42  +** Schema of the terms table.
           43  +*/
           44  +#define FTS3_TERMS_SCHEMA "CREATE TABLE x(term, documents, occurrences)"
           45  +
           46  +/*
           47  +** This function does all the work for both the xConnect and xCreate methods.
           48  +** These tables have no persistent representation of their own, so xConnect
           49  +** and xCreate are identical operations.
           50  +*/
           51  +static int fts3auxConnectMethod(
           52  +  sqlite3 *db,                    /* Database connection */
           53  +  void *pUnused,                  /* Unused */
           54  +  int argc,                       /* Number of elements in argv array */
           55  +  const char * const *argv,       /* xCreate/xConnect argument array */
           56  +  sqlite3_vtab **ppVtab,          /* OUT: New sqlite3_vtab object */
           57  +  char **pzErr                    /* OUT: sqlite3_malloc'd error message */
           58  +){
           59  +  char const *zDb;                /* Name of database (e.g. "main") */
           60  +  char const *zFts3;              /* Name of fts3 table */
           61  +  int nDb;                        /* Result of strlen(zDb) */
           62  +  int nFts3;                      /* Result of strlen(zFts3) */
           63  +  int nByte;                      /* Bytes of space to allocate here */
           64  +  int rc;                         /* value returned by declare_vtab() */
           65  +  Fts3auxTable *p;                /* Virtual table object to return */
           66  +
           67  +  /* The user should specify a single argument - the name of an fts3 table. */
           68  +  if( argc!=4 ){
           69  +    *pzErr = sqlite3_mprintf("wrong number of arguments");
           70  +    return SQLITE_ERROR;
           71  +  }
           72  +
           73  +  zDb = argv[1]; 
           74  +  nDb = strlen(zDb);
           75  +  zFts3 = argv[3];
           76  +  nFts3 = strlen(zFts3);
           77  +
           78  +  rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA);
           79  +  if( rc!=SQLITE_OK ) return rc;
           80  +
           81  +  nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
           82  +  p = (Fts3auxTable *)sqlite3_malloc(nByte);
           83  +  if( !p ) return SQLITE_NOMEM;
           84  +  memset(p, 0, nByte);
           85  +
           86  +  p->pFts3Tab = (Fts3Table *)&p[1];
           87  +  p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
           88  +  p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
           89  +  p->pFts3Tab->db = db;
           90  +
           91  +  memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
           92  +  memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
           93  +
           94  +  *ppVtab = (sqlite3_vtab *)p;
           95  +  return SQLITE_OK;
           96  +}
           97  +
           98  +/*
           99  +** This function does the work for both the xDisconnect and xDestroy methods.
          100  +** These tables have no persistent representation of their own, so xDisconnect
          101  +** and xDestroy are identical operations.
          102  +*/
          103  +static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){
          104  +  Fts3auxTable *p = (Fts3auxTable *)pVtab;
          105  +  Fts3Table *pFts3 = p->pFts3Tab;
          106  +  int i;
          107  +
          108  +  /* Free any prepared statements held */
          109  +  for(i=0; i<SizeofArray(pFts3->aStmt); i++){
          110  +    sqlite3_finalize(pFts3->aStmt[i]);
          111  +  }
          112  +  sqlite3_free(pFts3->zSegmentsTbl);
          113  +  sqlite3_free(p);
          114  +  return SQLITE_OK;
          115  +}
          116  +
          117  +/*
          118  +** xBestIndex - Analyze a WHERE and ORDER BY clause.
          119  +*/
          120  +static int fts3auxBestIndexMethod(
          121  +  sqlite3_vtab *pVTab, 
          122  +  sqlite3_index_info *pInfo
          123  +){
          124  +
          125  +  /* This vtab delivers always results in "ORDER BY term ASC" order. */
          126  +  if( pInfo->nOrderBy==1 
          127  +   && pInfo->aOrderBy[0].iColumn==0 
          128  +   && pInfo->aOrderBy[0].desc==0
          129  +  ){
          130  +    pInfo->orderByConsumed = 1;
          131  +  }
          132  +
          133  +  pInfo->estimatedCost = 20000;
          134  +  return SQLITE_OK;
          135  +}
          136  +
          137  +/*
          138  +** xOpen - Open a cursor.
          139  +*/
          140  +static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
          141  +  Fts3auxCursor *pCsr;            /* Pointer to cursor object to return */
          142  +
          143  +  pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor));
          144  +  if( !pCsr ) return SQLITE_NOMEM;
          145  +  memset(pCsr, 0, sizeof(Fts3auxCursor));
          146  +
          147  +  *ppCsr = (sqlite3_vtab_cursor *)pCsr;
          148  +  return SQLITE_OK;
          149  +}
          150  +
          151  +/*
          152  +** xClose - Close a cursor.
          153  +*/
          154  +static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){
          155  +  Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab;
          156  +  Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
          157  +
          158  +  sqlite3Fts3SegmentsClose(pFts3);
          159  +  sqlite3Fts3SegReaderFinish(&pCsr->csr);
          160  +  sqlite3_free(pCsr);
          161  +  return SQLITE_OK;
          162  +}
          163  +
          164  +/*
          165  +** xNext - Advance the cursor to the next row, if any.
          166  +*/
          167  +static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){
          168  +  Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
          169  +  Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab;
          170  +  int rc;
          171  +
          172  +  rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr);
          173  +  if( rc==SQLITE_ROW ){
          174  +    int i;
          175  +    int isIgnore = 1;
          176  +    int nDoclist = pCsr->csr.nDoclist;
          177  +    char *aDoclist = pCsr->csr.aDoclist;
          178  +
          179  +    /* Now count the number of documents and positions in the doclist
          180  +    ** in pCsr->csr.aDoclist[]. Store the number of documents in pCsr->nDoc
          181  +    ** and the number of occurrences in pCsr->nOcc.  */
          182  +    pCsr->nDoc = 0;
          183  +    pCsr->nOcc = 0;
          184  +    i = 0;
          185  +    while( i<nDoclist ){
          186  +      sqlite3_int64 v = 0;
          187  +      i += sqlite3Fts3GetVarint(&aDoclist[i], &v);
          188  +      if( isIgnore ){
          189  +        isIgnore = 0;
          190  +      }else if( v>1 ){
          191  +        pCsr->nOcc++;
          192  +      }else{
          193  +        if( v==0 ) pCsr->nDoc++;
          194  +        isIgnore = 1;
          195  +      }
          196  +    }
          197  +
          198  +    rc = SQLITE_OK;
          199  +    pCsr->iRowid++;
          200  +  }else{
          201  +    pCsr->isEof = 1;
          202  +  }
          203  +  return rc;
          204  +}
          205  +
          206  +/*
          207  +** xFilter - Initialize a cursor to point at the start of its data.
          208  +*/
          209  +static int fts3auxFilterMethod(
          210  +  sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
          211  +  int idxNum,                     /* Strategy index */
          212  +  const char *idxStr,             /* Unused */
          213  +  int nVal,                       /* Number of elements in apVal */
          214  +  sqlite3_value **apVal           /* Arguments for the indexing scheme */
          215  +){
          216  +  Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
          217  +  Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab;
          218  +  int rc;
          219  +
          220  +  sqlite3Fts3SegReaderFinish(&pCsr->csr);
          221  +  memset(&pCsr->csr, 0, sizeof(Fts3SegReaderCursor));
          222  +  pCsr->isEof = 0;
          223  +  pCsr->iRowid = 0;
          224  +  pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
          225  +
          226  +  rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL, 0, 0,1,&pCsr->csr);
          227  +  if( rc==SQLITE_OK ){
          228  +    rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
          229  +  }
          230  +
          231  +  if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor);
          232  +  return rc;
          233  +}
          234  +
          235  +/*
          236  +** xEof - Return true if the cursor is at EOF, or false otherwise.
          237  +*/
          238  +static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){
          239  +  Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
          240  +  return pCsr->isEof;
          241  +}
          242  +
          243  +/*
          244  +** xColumn - Return a column value.
          245  +*/
          246  +static int fts3auxColumnMethod(
          247  +  sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
          248  +  sqlite3_context *pContext,      /* Context for sqlite3_result_xxx() calls */
          249  +  int iCol                        /* Index of column to read value from */
          250  +){
          251  +  Fts3auxCursor *p = (Fts3auxCursor *)pCursor;
          252  +
          253  +  assert( p->isEof==0 );
          254  +  if( iCol==0 ){        /* Column "term" */
          255  +    sqlite3_result_text(pContext, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT);
          256  +  }else if( iCol==1 ){  /* Column "documents" */
          257  +    sqlite3_result_int64(pContext, p->nDoc);
          258  +  }else{                /* Column "occurrences" */
          259  +    sqlite3_result_int64(pContext, p->nOcc);
          260  +  }
          261  +
          262  +  return SQLITE_OK;
          263  +}
          264  +
          265  +/*
          266  +** xRowid - Return the current rowid for the cursor.
          267  +*/
          268  +static int fts3auxRowidMethod(
          269  +  sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
          270  +  sqlite_int64 *pRowid            /* OUT: Rowid value */
          271  +){
          272  +  Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
          273  +  *pRowid = pCsr->iRowid;
          274  +  return SQLITE_OK;
          275  +}
          276  +
          277  +/*
          278  +** Register the fts3aux module with database connection db. Return SQLITE_OK
          279  +** if successful or an error code if sqlite3_create_module() fails.
          280  +*/
          281  +int sqlite3Fts3InitAux(sqlite3 *db){
          282  +  static const sqlite3_module fts3aux_module = {
          283  +     0,                           /* iVersion      */
          284  +     fts3auxConnectMethod,        /* xCreate       */
          285  +     fts3auxConnectMethod,        /* xConnect      */
          286  +     fts3auxBestIndexMethod,      /* xBestIndex    */
          287  +     fts3auxDisconnectMethod,     /* xDisconnect   */
          288  +     fts3auxDisconnectMethod,     /* xDestroy      */
          289  +     fts3auxOpenMethod,           /* xOpen         */
          290  +     fts3auxCloseMethod,          /* xClose        */
          291  +     fts3auxFilterMethod,         /* xFilter       */
          292  +     fts3auxNextMethod,           /* xNext         */
          293  +     fts3auxEofMethod,            /* xEof          */
          294  +     fts3auxColumnMethod,         /* xColumn       */
          295  +     fts3auxRowidMethod,          /* xRowid        */
          296  +     0,                           /* xUpdate       */
          297  +     0,                           /* xBegin        */
          298  +     0,                           /* xSync         */
          299  +     0,                           /* xCommit       */
          300  +     0,                           /* xRollback     */
          301  +     0,                           /* xFindFunction */
          302  +     0                            /* xRename       */
          303  +  };
          304  +  int rc;                         /* Return code */
          305  +
          306  +  rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0);
          307  +  return rc;
          308  +}
          309  +
          310  +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */

Changes to ext/fts3/fts3_snippet.c.

   956    956       if( rc!=SQLITE_OK ) return rc;
   957    957     }
   958    958     pStmt = *ppStmt;
   959    959     assert( sqlite3_data_count(pStmt)==1 );
   960    960   
   961    961     a = sqlite3_column_blob(pStmt, 0);
   962    962     a += sqlite3Fts3GetVarint(a, &nDoc);
          963  +  if( nDoc==0 ) return SQLITE_CORRUPT;
   963    964     *pnDoc = (u32)nDoc;
   964    965   
   965    966     if( paLen ) *paLen = a;
   966    967     return SQLITE_OK;
   967    968   }
   968    969   
   969    970   /*
................................................................................
  1162   1163             sqlite3_int64 nDoc;     /* Number of rows in table */
  1163   1164             const char *a;          /* Aggregate column length array */
  1164   1165   
  1165   1166             rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a);
  1166   1167             if( rc==SQLITE_OK ){
  1167   1168               int iCol;
  1168   1169               for(iCol=0; iCol<pInfo->nCol; iCol++){
         1170  +              u32 iVal;
  1169   1171                 sqlite3_int64 nToken;
  1170   1172                 a += sqlite3Fts3GetVarint(a, &nToken);
  1171         -              pInfo->aMatchinfo[iCol] = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc);
         1173  +              iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc);
         1174  +              pInfo->aMatchinfo[iCol] = iVal;
  1172   1175               }
  1173   1176             }
  1174   1177           }
  1175   1178           break;
  1176   1179   
  1177   1180         case FTS3_MATCHINFO_LENGTH: {
  1178   1181           sqlite3_stmt *pSelectDocsize = 0;

Changes to ext/fts3/fts3_write.c.

   208    208   /* 0  */  "DELETE FROM %Q.'%q_content' WHERE rowid = ?",
   209    209   /* 1  */  "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)",
   210    210   /* 2  */  "DELETE FROM %Q.'%q_content'",
   211    211   /* 3  */  "DELETE FROM %Q.'%q_segments'",
   212    212   /* 4  */  "DELETE FROM %Q.'%q_segdir'",
   213    213   /* 5  */  "DELETE FROM %Q.'%q_docsize'",
   214    214   /* 6  */  "DELETE FROM %Q.'%q_stat'",
   215         -/* 7  */  "SELECT * FROM %Q.'%q_content' WHERE rowid=?",
          215  +/* 7  */  "SELECT %s FROM %Q.'%q_content' AS x WHERE rowid=?",
   216    216   /* 8  */  "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1",
   217    217   /* 9  */  "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
   218    218   /* 10 */  "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)",
   219    219   /* 11 */  "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",
   220    220   
   221    221             /* Return segments in order from oldest to newest.*/ 
   222    222   /* 12 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
................................................................................
   225    225               "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC",
   226    226   
   227    227   /* 14 */  "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
   228    228   /* 15 */  "SELECT count(*), max(level) FROM %Q.'%q_segdir'",
   229    229   
   230    230   /* 16 */  "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
   231    231   /* 17 */  "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
   232         -/* 18 */  "INSERT INTO %Q.'%q_content' VALUES(%z)",
          232  +/* 18 */  "INSERT INTO %Q.'%q_content' VALUES(%s)",
   233    233   /* 19 */  "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
   234    234   /* 20 */  "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
   235    235   /* 21 */  "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
   236    236   /* 22 */  "SELECT value FROM %Q.'%q_stat' WHERE id=0",
   237    237   /* 23 */  "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
   238    238     };
   239    239     int rc = SQLITE_OK;
................................................................................
   242    242     assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
   243    243     assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
   244    244     
   245    245     pStmt = p->aStmt[eStmt];
   246    246     if( !pStmt ){
   247    247       char *zSql;
   248    248       if( eStmt==SQL_CONTENT_INSERT ){
   249         -      int i;                      /* Iterator variable */  
   250         -      char *zVarlist;             /* The "?, ?, ..." string */
   251         -      zVarlist = (char *)sqlite3_malloc(2*p->nColumn+2);
   252         -      if( !zVarlist ){
   253         -        *pp = 0;
   254         -        return SQLITE_NOMEM;
   255         -      }
   256         -      zVarlist[0] = '?';
   257         -      zVarlist[p->nColumn*2+1] = '\0';
   258         -      for(i=1; i<=p->nColumn; i++){
   259         -        zVarlist[i*2-1] = ',';
   260         -        zVarlist[i*2] = '?';
   261         -      }
   262         -      zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, zVarlist);
          249  +      zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist);
          250  +    }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){
          251  +      zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist, p->zDb, p->zName);
   263    252       }else{
   264    253         zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
   265    254       }
   266    255       if( !zSql ){
   267    256         rc = SQLITE_NOMEM;
   268    257       }else{
   269    258         rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
................................................................................
   296    285   
   297    286     rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0);
   298    287     if( rc==SQLITE_OK ){
   299    288       if( eStmt==SQL_SELECT_DOCSIZE ){
   300    289         sqlite3_bind_int64(pStmt, 1, iDocid);
   301    290       }
   302    291       rc = sqlite3_step(pStmt);
   303         -    if( rc!=SQLITE_ROW ){
          292  +    if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){
   304    293         rc = sqlite3_reset(pStmt);
   305    294         if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT;
   306    295         pStmt = 0;
   307    296       }else{
   308    297         rc = SQLITE_OK;
   309    298       }
   310    299     }
................................................................................
   397    386   **
   398    387   **   0: idx
   399    388   **   1: start_block
   400    389   **   2: leaves_end_block
   401    390   **   3: end_block
   402    391   **   4: root
   403    392   */
   404         -int sqlite3Fts3AllSegdirs(Fts3Table *p, sqlite3_stmt **ppStmt){
   405         -  return fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, ppStmt, 0);
          393  +int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){
          394  +  int rc;
          395  +  sqlite3_stmt *pStmt = 0;
          396  +  if( iLevel<0 ){
          397  +    rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0);
          398  +  }else{
          399  +    rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
          400  +    if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel);
          401  +  }
          402  +  *ppStmt = pStmt;
          403  +  return rc;
   406    404   }
   407    405   
   408    406   
   409    407   /*
   410    408   ** Append a single varint to a PendingList buffer. SQLITE_OK is returned
   411    409   ** if successful, or an SQLite error code otherwise.
   412    410   **
................................................................................
  1100   1098         ** the table. The following nCol varints contain the total amount of
  1101   1099         ** data stored in all rows of each column of the table, from left
  1102   1100         ** to right.
  1103   1101         */
  1104   1102         sqlite3_stmt *pStmt;
  1105   1103         sqlite3_int64 nDoc = 0;
  1106   1104         sqlite3_int64 nByte = 0;
         1105  +      const char *pEnd;
  1107   1106         const char *a;
         1107  +
  1108   1108         rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
  1109         -      if( rc ) return rc;
         1109  +      if( rc!=SQLITE_OK ) return rc;
  1110   1110         a = sqlite3_column_blob(pStmt, 0);
  1111         -      if( a ){
  1112         -        const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
  1113         -        a += sqlite3Fts3GetVarint(a, &nDoc);
  1114         -        while( a<pEnd ){
  1115         -          a += sqlite3Fts3GetVarint(a, &nByte);
  1116         -        }
         1111  +      assert( a );
         1112  +
         1113  +      pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
         1114  +      a += sqlite3Fts3GetVarint(a, &nDoc);
         1115  +      while( a<pEnd ){
         1116  +        a += sqlite3Fts3GetVarint(a, &nByte);
  1117   1117         }
  1118   1118         if( nDoc==0 || nByte==0 ){
  1119   1119           sqlite3_reset(pStmt);
  1120   1120           return SQLITE_CORRUPT;
  1121   1121         }
  1122   1122   
  1123   1123         pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz);
................................................................................
  1299   1299     if( isPrefix ){
  1300   1300       sqlite3_free(aElem);
  1301   1301     }
  1302   1302     *ppReader = pReader;
  1303   1303     return rc;
  1304   1304   }
  1305   1305   
  1306         -
  1307         -/*
  1308         -** The second argument to this function is expected to be a statement of
  1309         -** the form:
  1310         -**
  1311         -**   SELECT 
  1312         -**     idx,                  -- col 0
  1313         -**     start_block,          -- col 1
  1314         -**     leaves_end_block,     -- col 2
  1315         -**     end_block,            -- col 3
  1316         -**     root                  -- col 4
  1317         -**   FROM %_segdir ...
  1318         -**
  1319         -** This function allocates and initializes a Fts3SegReader structure to
  1320         -** iterate through the terms stored in the segment identified by the
  1321         -** current row that pStmt is pointing to. 
  1322         -**
  1323         -** If successful, the Fts3SegReader is left pointing to the first term
  1324         -** in the segment and SQLITE_OK is returned. Otherwise, an SQLite error
  1325         -** code is returned.
  1326         -*/
  1327         -static int fts3SegReaderNew(
  1328         -  sqlite3_stmt *pStmt,            /* See above */
  1329         -  int iAge,                       /* Segment "age". */
  1330         -  Fts3SegReader **ppReader        /* OUT: Allocated Fts3SegReader */
  1331         -){
  1332         -  return sqlite3Fts3SegReaderNew(iAge, 
  1333         -      sqlite3_column_int64(pStmt, 1),
  1334         -      sqlite3_column_int64(pStmt, 2),
  1335         -      sqlite3_column_int64(pStmt, 3),
  1336         -      sqlite3_column_blob(pStmt, 4),
  1337         -      sqlite3_column_bytes(pStmt, 4),
  1338         -      ppReader
  1339         -  );
  1340         -}
  1341         -
  1342   1306   /*
  1343   1307   ** Compare the entries pointed to by two Fts3SegReader structures. 
  1344   1308   ** Comparison is as follows:
  1345   1309   **
  1346   1310   **   1) EOF is greater than not EOF.
  1347   1311   **
  1348   1312   **   2) The current terms (if any) are compared using memcmp(). If one
................................................................................
  1939   1903         *pisEmpty = sqlite3_column_int(pStmt, 0);
  1940   1904       }
  1941   1905       rc = sqlite3_reset(pStmt);
  1942   1906     }
  1943   1907     return rc;
  1944   1908   }
  1945   1909   
  1946         -/*
  1947         -** Set *pnSegment to the number of segments of level iLevel in the database.
  1948         -**
  1949         -** Return SQLITE_OK if successful, or an SQLite error code if not.
  1950         -*/
  1951         -static int fts3SegmentCount(Fts3Table *p, int iLevel, int *pnSegment){
  1952         -  sqlite3_stmt *pStmt;
  1953         -  int rc;
  1954         -
  1955         -  assert( iLevel>=0 );
  1956         -  rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_COUNT, &pStmt, 0);
  1957         -  if( rc!=SQLITE_OK ) return rc;
  1958         -  sqlite3_bind_int(pStmt, 1, iLevel);
  1959         -  if( SQLITE_ROW==sqlite3_step(pStmt) ){
  1960         -    *pnSegment = sqlite3_column_int(pStmt, 0);
  1961         -  }
  1962         -  return sqlite3_reset(pStmt);
  1963         -}
  1964         -
  1965   1910   /*
  1966   1911   ** Set *pnSegment to the total number of segments in the database. Set
  1967   1912   ** *pnMax to the largest segment level in the database (segment levels
  1968   1913   ** are stored in the 'level' column of the %_segdir table).
  1969   1914   **
  1970   1915   ** Return SQLITE_OK if successful, or an SQLite error code if not.
  1971   1916   */
................................................................................
  2016   1961         rc = sqlite3_reset(pDelete);
  2017   1962       }
  2018   1963     }
  2019   1964     if( rc!=SQLITE_OK ){
  2020   1965       return rc;
  2021   1966     }
  2022   1967   
  2023         -  if( iLevel>=0 ){
         1968  +  if( iLevel==FTS3_SEGCURSOR_ALL ){
         1969  +    fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
         1970  +  }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
         1971  +    sqlite3Fts3PendingTermsClear(p);
         1972  +  }else{
         1973  +    assert( iLevel>=0 );
  2024   1974       rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0);
  2025   1975       if( rc==SQLITE_OK ){
  2026   1976         sqlite3_bind_int(pDelete, 1, iLevel);
  2027   1977         sqlite3_step(pDelete);
  2028   1978         rc = sqlite3_reset(pDelete);
  2029   1979       }
  2030         -  }else{
  2031         -    fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
  2032   1980     }
  2033   1981   
  2034   1982     return rc;
  2035   1983   }
  2036   1984   
  2037   1985   /*
  2038   1986   ** When this function is called, buffer *ppList (size *pnList bytes) contains 
................................................................................
  2073   2021       p += sqlite3Fts3GetVarint32(p, &iCurrent);
  2074   2022     }
  2075   2023   
  2076   2024     *ppList = pList;
  2077   2025     *pnList = nList;
  2078   2026   }
  2079   2027   
  2080         -/*
  2081         -** sqlite3Fts3SegReaderIterate() callback used when merging multiple 
  2082         -** segments to create a single, larger segment.
  2083         -*/
  2084         -static int fts3MergeCallback(
  2085         -  Fts3Table *p,                   /* FTS3 Virtual table handle */
  2086         -  void *pContext,                 /* Pointer to SegmentWriter* to write with */
  2087         -  char *zTerm,                    /* Term to write to the db */
  2088         -  int nTerm,                      /* Number of bytes in zTerm */
  2089         -  char *aDoclist,                 /* Doclist associated with zTerm */
  2090         -  int nDoclist                    /* Number of bytes in doclist */
         2028  +int sqlite3Fts3SegReaderStart(
         2029  +  Fts3Table *p,                   /* Virtual table handle */
         2030  +  Fts3SegReaderCursor *pCsr,      /* Cursor object */
         2031  +  Fts3SegFilter *pFilter          /* Restrictions on range of iteration */
  2091   2032   ){
  2092         -  SegmentWriter **ppW = (SegmentWriter **)pContext;
  2093         -  return fts3SegWriterAdd(p, ppW, 1, zTerm, nTerm, aDoclist, nDoclist);
  2094         -}
         2033  +  int i;
  2095   2034   
  2096         -/*
  2097         -** sqlite3Fts3SegReaderIterate() callback used when flushing the contents
  2098         -** of the pending-terms hash table to the database.
  2099         -*/
  2100         -static int fts3FlushCallback(
  2101         -  Fts3Table *p,                   /* FTS3 Virtual table handle */
  2102         -  void *pContext,                 /* Pointer to SegmentWriter* to write with */
  2103         -  char *zTerm,                    /* Term to write to the db */
  2104         -  int nTerm,                      /* Number of bytes in zTerm */
  2105         -  char *aDoclist,                 /* Doclist associated with zTerm */
  2106         -  int nDoclist                    /* Number of bytes in doclist */
  2107         -){
  2108         -  SegmentWriter **ppW = (SegmentWriter **)pContext;
  2109         -  return fts3SegWriterAdd(p, ppW, 0, zTerm, nTerm, aDoclist, nDoclist);
  2110         -}
  2111         -
  2112         -/*
  2113         -** This function is used to iterate through a contiguous set of terms 
  2114         -** stored in the full-text index. It merges data contained in one or 
  2115         -** more segments to support this.
  2116         -**
  2117         -** The second argument is passed an array of pointers to SegReader objects
  2118         -** allocated with sqlite3Fts3SegReaderNew(). This function merges the range 
  2119         -** of terms selected by each SegReader. If a single term is present in
  2120         -** more than one segment, the associated doclists are merged. For each
  2121         -** term and (possibly merged) doclist in the merged range, the callback
  2122         -** function xFunc is invoked with its arguments set as follows.
  2123         -**
  2124         -**   arg 0: Copy of 'p' parameter passed to this function
  2125         -**   arg 1: Copy of 'pContext' parameter passed to this function
  2126         -**   arg 2: Pointer to buffer containing term
  2127         -**   arg 3: Size of arg 2 buffer in bytes
  2128         -**   arg 4: Pointer to buffer containing doclist
  2129         -**   arg 5: Size of arg 2 buffer in bytes
  2130         -**
  2131         -** The 4th argument to this function is a pointer to a structure of type
  2132         -** Fts3SegFilter, defined in fts3Int.h. The contents of this structure
  2133         -** further restrict the range of terms that callbacks are made for and
  2134         -** modify the behaviour of this function. See comments above structure
  2135         -** definition for details.
  2136         -*/
  2137         -int sqlite3Fts3SegReaderIterate(
  2138         -  Fts3Table *p,                   /* Virtual table handle */
  2139         -  Fts3SegReader **apSegment,      /* Array of Fts3SegReader objects */
  2140         -  int nSegment,                   /* Size of apSegment array */
  2141         -  Fts3SegFilter *pFilter,         /* Restrictions on range of iteration */
  2142         -  int (*xFunc)(Fts3Table *, void *, char *, int, char *, int),  /* Callback */
  2143         -  void *pContext                  /* Callback context (2nd argument) */
  2144         -){
  2145         -  int i;                          /* Iterator variable */
  2146         -  char *aBuffer = 0;              /* Buffer to merge doclists in */
  2147         -  int nAlloc = 0;                 /* Allocated size of aBuffer buffer */
  2148         -  int rc = SQLITE_OK;             /* Return code */
  2149         -
  2150         -  int isIgnoreEmpty =  (pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY);
  2151         -  int isRequirePos =   (pFilter->flags & FTS3_SEGMENT_REQUIRE_POS);
  2152         -  int isColFilter =    (pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER);
  2153         -  int isPrefix =       (pFilter->flags & FTS3_SEGMENT_PREFIX);
  2154         -
  2155         -  /* If there are zero segments, this function is a no-op. This scenario
  2156         -  ** comes about only when reading from an empty database.
  2157         -  */
  2158         -  if( nSegment==0 ) goto finished;
         2035  +  /* Initialize the cursor object */
         2036  +  pCsr->pFilter = pFilter;
  2159   2037   
  2160   2038     /* If the Fts3SegFilter defines a specific term (or term prefix) to search 
  2161   2039     ** for, then advance each segment iterator until it points to a term of
  2162   2040     ** equal or greater value than the specified term. This prevents many
  2163   2041     ** unnecessary merge/sort operations for the case where single segment
  2164   2042     ** b-tree leaf nodes contain more than one term.
  2165   2043     */
  2166         -  for(i=0; i<nSegment; i++){
         2044  +  for(i=0; i<pCsr->nSegment; i++){
  2167   2045       int nTerm = pFilter->nTerm;
  2168   2046       const char *zTerm = pFilter->zTerm;
  2169         -    Fts3SegReader *pSeg = apSegment[i];
         2047  +    Fts3SegReader *pSeg = pCsr->apSegment[i];
  2170   2048       do {
  2171         -      rc = fts3SegReaderNext(p, pSeg);
  2172         -      if( rc!=SQLITE_OK ) goto finished;
         2049  +      int rc = fts3SegReaderNext(p, pSeg);
         2050  +      if( rc!=SQLITE_OK ) return rc;
  2173   2051       }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
  2174   2052     }
         2053  +  fts3SegReaderSort(
         2054  +      pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp);
  2175   2055   
  2176         -  fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp);
  2177         -  while( apSegment[0]->aNode ){
  2178         -    int nTerm = apSegment[0]->nTerm;
  2179         -    char *zTerm = apSegment[0]->zTerm;
  2180         -    int nMerge = 1;
         2056  +  return SQLITE_OK;
         2057  +}
         2058  +
         2059  +int sqlite3Fts3SegReaderStep(
         2060  +  Fts3Table *p,                   /* Virtual table handle */
         2061  +  Fts3SegReaderCursor *pCsr       /* Cursor object */
         2062  +){
         2063  +  int rc = SQLITE_OK;
         2064  +
         2065  +  int isIgnoreEmpty =  (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY);
         2066  +  int isRequirePos =   (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS);
         2067  +  int isColFilter =    (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER);
         2068  +  int isPrefix =       (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX);
         2069  +
         2070  +  Fts3SegReader **apSegment = pCsr->apSegment;
         2071  +  int nSegment = pCsr->nSegment;
         2072  +  Fts3SegFilter *pFilter = pCsr->pFilter;
         2073  +
         2074  +  if( pCsr->nSegment==0 ) return SQLITE_OK;
         2075  +
         2076  +  do {
         2077  +    int nMerge;
         2078  +    int i;
         2079  +  
         2080  +    /* Advance the first pCsr->nAdvance entries in the apSegment[] array
         2081  +    ** forward. Then sort the list in order of current term again.  
         2082  +    */
         2083  +    for(i=0; i<pCsr->nAdvance; i++){
         2084  +      rc = fts3SegReaderNext(p, apSegment[i]);
         2085  +      if( rc!=SQLITE_OK ) return rc;
         2086  +    }
         2087  +    fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp);
         2088  +    pCsr->nAdvance = 0;
         2089  +
         2090  +    /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */
         2091  +    assert( rc==SQLITE_OK );
         2092  +    if( apSegment[0]->aNode==0 ) break;
         2093  +
         2094  +    pCsr->nTerm = apSegment[0]->nTerm;
         2095  +    pCsr->zTerm = apSegment[0]->zTerm;
  2181   2096   
  2182   2097       /* If this is a prefix-search, and if the term that apSegment[0] points
  2183   2098       ** to does not share a suffix with pFilter->zTerm/nTerm, then all 
  2184   2099       ** required callbacks have been made. In this case exit early.
  2185   2100       **
  2186   2101       ** Similarly, if this is a search for an exact match, and the first term
  2187   2102       ** of segment apSegment[0] is not a match, exit early.
  2188   2103       */
  2189   2104       if( pFilter->zTerm ){
  2190         -      if( nTerm<pFilter->nTerm 
  2191         -       || (!isPrefix && nTerm>pFilter->nTerm)
  2192         -       || memcmp(zTerm, pFilter->zTerm, pFilter->nTerm) 
  2193         -    ){
  2194         -        goto finished;
         2105  +      if( pCsr->nTerm<pFilter->nTerm 
         2106  +       || (!isPrefix && pCsr->nTerm>pFilter->nTerm)
         2107  +       || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) 
         2108  +      ){
         2109  +        break;
  2195   2110         }
  2196   2111       }
  2197   2112   
         2113  +    nMerge = 1;
  2198   2114       while( nMerge<nSegment 
  2199   2115           && apSegment[nMerge]->aNode
  2200         -        && apSegment[nMerge]->nTerm==nTerm 
  2201         -        && 0==memcmp(zTerm, apSegment[nMerge]->zTerm, nTerm)
         2116  +        && apSegment[nMerge]->nTerm==pCsr->nTerm 
         2117  +        && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm)
  2202   2118       ){
  2203   2119         nMerge++;
  2204   2120       }
  2205   2121   
  2206   2122       assert( isIgnoreEmpty || (isRequirePos && !isColFilter) );
  2207   2123       if( nMerge==1 && !isIgnoreEmpty ){
  2208         -      Fts3SegReader *p0 = apSegment[0];
  2209         -      rc = xFunc(p, pContext, zTerm, nTerm, p0->aDoclist, p0->nDoclist);
  2210         -      if( rc!=SQLITE_OK ) goto finished;
         2124  +      pCsr->aDoclist = apSegment[0]->aDoclist;
         2125  +      pCsr->nDoclist = apSegment[0]->nDoclist;
         2126  +      rc = SQLITE_ROW;
  2211   2127       }else{
  2212   2128         int nDoclist = 0;           /* Size of doclist */
  2213   2129         sqlite3_int64 iPrev = 0;    /* Previous docid stored in doclist */
  2214   2130   
  2215   2131         /* The current term of the first nMerge entries in the array
  2216   2132         ** of Fts3SegReader objects is the same. The doclists must be merged
  2217         -      ** and a single term added to the new segment.
         2133  +      ** and a single term returned with the merged doclist.
  2218   2134         */
  2219   2135         for(i=0; i<nMerge; i++){
  2220   2136           fts3SegReaderFirstDocid(apSegment[i]);
  2221   2137         }
  2222   2138         fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp);
  2223   2139         while( apSegment[0]->pOffsetList ){
  2224   2140           int j;                    /* Number of segments that share a docid */
................................................................................
  2238   2154   
  2239   2155           if( isColFilter ){
  2240   2156             fts3ColumnFilter(pFilter->iCol, &pList, &nList);
  2241   2157           }
  2242   2158   
  2243   2159           if( !isIgnoreEmpty || nList>0 ){
  2244   2160             nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0);
  2245         -          if( nDoclist+nByte>nAlloc ){
         2161  +          if( nDoclist+nByte>pCsr->nBuffer ){
  2246   2162               char *aNew;
  2247         -            nAlloc = (nDoclist+nByte)*2;
  2248         -            aNew = sqlite3_realloc(aBuffer, nAlloc);
         2163  +            pCsr->nBuffer = (nDoclist+nByte)*2;
         2164  +            aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer);
  2249   2165               if( !aNew ){
  2250         -              rc = SQLITE_NOMEM;
  2251         -              goto finished;
         2166  +              return SQLITE_NOMEM;
  2252   2167               }
  2253         -            aBuffer = aNew;
         2168  +            pCsr->aBuffer = aNew;
  2254   2169             }
  2255         -          nDoclist += sqlite3Fts3PutVarint(&aBuffer[nDoclist], iDocid-iPrev);
         2170  +          nDoclist += sqlite3Fts3PutVarint(
         2171  +              &pCsr->aBuffer[nDoclist], iDocid-iPrev
         2172  +          );
  2256   2173             iPrev = iDocid;
  2257   2174             if( isRequirePos ){
  2258         -            memcpy(&aBuffer[nDoclist], pList, nList);
         2175  +            memcpy(&pCsr->aBuffer[nDoclist], pList, nList);
  2259   2176               nDoclist += nList;
  2260         -            aBuffer[nDoclist++] = '\0';
         2177  +            pCsr->aBuffer[nDoclist++] = '\0';
  2261   2178             }
  2262   2179           }
  2263   2180   
  2264   2181           fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp);
  2265   2182         }
  2266         -
  2267   2183         if( nDoclist>0 ){
  2268         -        rc = xFunc(p, pContext, zTerm, nTerm, aBuffer, nDoclist);
  2269         -        if( rc!=SQLITE_OK ) goto finished;
         2184  +        pCsr->aDoclist = pCsr->aBuffer;
         2185  +        pCsr->nDoclist = nDoclist;
         2186  +        rc = SQLITE_ROW;
  2270   2187         }
  2271   2188       }
         2189  +    pCsr->nAdvance = nMerge;
         2190  +  }while( rc==SQLITE_OK );
  2272   2191   
  2273         -    /* If there is a term specified to filter on, and this is not a prefix
  2274         -    ** search, return now. The callback that corresponds to the required
  2275         -    ** term (if such a term exists in the index) has already been made.
  2276         -    */
  2277         -    if( pFilter->zTerm && !isPrefix ){
  2278         -      goto finished;
  2279         -    }
         2192  +  return rc;
         2193  +}
  2280   2194   
  2281         -    for(i=0; i<nMerge; i++){
  2282         -      rc = fts3SegReaderNext(p, apSegment[i]);
  2283         -      if( rc!=SQLITE_OK ) goto finished;
         2195  +void sqlite3Fts3SegReaderFinish(
         2196  +  Fts3SegReaderCursor *pCsr       /* Cursor object */
         2197  +){
         2198  +  if( pCsr ){
         2199  +    int i;
         2200  +    for(i=0; i<pCsr->nSegment; i++){
         2201  +      sqlite3Fts3SegReaderFree(pCsr->apSegment[i]);
  2284   2202       }
  2285         -    fts3SegReaderSort(apSegment, nSegment, nMerge, fts3SegReaderCmp);
         2203  +    sqlite3_free(pCsr->apSegment);
         2204  +    sqlite3_free(pCsr->aBuffer);
         2205  +
         2206  +    pCsr->nSegment = 0;
         2207  +    pCsr->apSegment = 0;
         2208  +    pCsr->aBuffer = 0;
  2286   2209     }
  2287         -
  2288         - finished:
  2289         -  sqlite3_free(aBuffer);
  2290         -  return rc;
  2291   2210   }
  2292   2211   
  2293   2212   /*
  2294   2213   ** Merge all level iLevel segments in the database into a single 
  2295   2214   ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a
  2296   2215   ** single segment with a level equal to the numerically largest level 
  2297   2216   ** currently present in the database.
................................................................................
  2298   2217   **
  2299   2218   ** If this function is called with iLevel<0, but there is only one
  2300   2219   ** segment in the database, SQLITE_DONE is returned immediately. 
  2301   2220   ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, 
  2302   2221   ** an SQLite error code is returned.
  2303   2222   */
  2304   2223   static int fts3SegmentMerge(Fts3Table *p, int iLevel){
  2305         -  int i;                          /* Iterator variable */
  2306   2224     int rc;                         /* Return code */
  2307   2225     int iIdx;                       /* Index of new segment */
  2308   2226     int iNewLevel = 0;              /* Level to create new segment at */
  2309         -  sqlite3_stmt *pStmt = 0;
  2310         -  SegmentWriter *pWriter = 0;
  2311         -  int nSegment = 0;               /* Number of segments being merged */
  2312         -  Fts3SegReader **apSegment = 0;  /* Array of Segment iterators */
  2313         -  Fts3SegReader *pPending = 0;    /* Iterator for pending-terms */
         2227  +  SegmentWriter *pWriter = 0;     /* Used to write the new, merged, segment */
  2314   2228     Fts3SegFilter filter;           /* Segment term filter condition */
         2229  +  Fts3SegReaderCursor csr;        /* Cursor to iterate through level(s) */
  2315   2230   
  2316         -  if( iLevel<0 ){
         2231  +  rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, &csr);
         2232  +  if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished;
         2233  +
         2234  +  if( iLevel==FTS3_SEGCURSOR_ALL ){
  2317   2235       /* This call is to merge all segments in the database to a single
  2318   2236       ** segment. The level of the new segment is equal to the the numerically 
  2319   2237       ** greatest segment level currently present in the database. The index
  2320         -    ** of the new segment is always 0.
  2321         -    */
         2238  +    ** of the new segment is always 0.  */
         2239  +    int nDummy; /* TODO: Remove this */
         2240  +    if( csr.nSegment==1 ){
         2241  +      rc = SQLITE_DONE;
         2242  +      goto finished;
         2243  +    }
  2322   2244       iIdx = 0;
  2323         -    rc = sqlite3Fts3SegReaderPending(p, 0, 0, 1, &pPending);
  2324         -    if( rc!=SQLITE_OK ) goto finished;
  2325         -    rc = fts3SegmentCountMax(p, &nSegment, &iNewLevel);
  2326         -    if( rc!=SQLITE_OK ) goto finished;
  2327         -    nSegment += (pPending!=0);
  2328         -    if( nSegment<=1 ){
  2329         -      return SQLITE_DONE;
  2330         -    }
         2245  +    rc = fts3SegmentCountMax(p, &nDummy, &iNewLevel);
  2331   2246     }else{
  2332   2247       /* This call is to merge all segments at level iLevel. Find the next
  2333   2248       ** available segment index at level iLevel+1. The call to
  2334   2249       ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to 
  2335         -    ** a single iLevel+2 segment if necessary.
  2336         -    */
         2250  +    ** a single iLevel+2 segment if necessary.  */
  2337   2251       iNewLevel = iLevel+1;
  2338   2252       rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx);
  2339         -    if( rc!=SQLITE_OK ) goto finished;
  2340         -    rc = fts3SegmentCount(p, iLevel, &nSegment);
  2341         -    if( rc!=SQLITE_OK ) goto finished;
  2342   2253     }
  2343         -  assert( nSegment>0 );
  2344         -  assert( iNewLevel>=0 );
  2345         -
  2346         -  /* Allocate space for an array of pointers to segment iterators. */
  2347         -  apSegment = (Fts3SegReader**)sqlite3_malloc(sizeof(Fts3SegReader *)*nSegment);
  2348         -  if( !apSegment ){
  2349         -    rc = SQLITE_NOMEM;
  2350         -    goto finished;
  2351         -  }
  2352         -  memset(apSegment, 0, sizeof(Fts3SegReader *)*nSegment);
  2353         -
  2354         -  /* Allocate a Fts3SegReader structure for each segment being merged. A 
  2355         -  ** Fts3SegReader stores the state data required to iterate through all 
  2356         -  ** entries on all leaves of a single segment. 
  2357         -  */
  2358         -  assert( SQL_SELECT_LEVEL+1==SQL_SELECT_ALL_LEVEL);
  2359         -  rc = fts3SqlStmt(p, SQL_SELECT_LEVEL+(iLevel<0), &pStmt, 0);
  2360   2254     if( rc!=SQLITE_OK ) goto finished;
  2361         -  sqlite3_bind_int(pStmt, 1, iLevel);
  2362         -  for(i=0; SQLITE_ROW==(sqlite3_step(pStmt)); i++){
  2363         -    rc = fts3SegReaderNew(pStmt, i, &apSegment[i]);
  2364         -    if( rc!=SQLITE_OK ){
  2365         -      goto finished;
  2366         -    }
  2367         -  }
  2368         -  rc = sqlite3_reset(pStmt);
  2369         -  if( pPending ){
  2370         -    apSegment[i] = pPending;
  2371         -    pPending = 0;
  2372         -  }
  2373         -  pStmt = 0;
  2374         -  if( rc!=SQLITE_OK ) goto finished;
         2255  +  assert( csr.nSegment>0 );
         2256  +  assert( iNewLevel>=0 );
  2375   2257   
  2376   2258     memset(&filter, 0, sizeof(Fts3SegFilter));
  2377   2259     filter.flags = FTS3_SEGMENT_REQUIRE_POS;
  2378         -  filter.flags |= (iLevel<0 ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
  2379         -  rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment,
  2380         -      &filter, fts3MergeCallback, (void *)&pWriter
  2381         -  );
         2260  +  filter.flags |= (iLevel==FTS3_SEGCURSOR_ALL ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
         2261  +
         2262  +  rc = sqlite3Fts3SegReaderStart(p, &csr, &filter);
         2263  +  while( SQLITE_OK==rc ){
         2264  +    rc = sqlite3Fts3SegReaderStep(p, &csr);
         2265  +    if( rc!=SQLITE_ROW ) break;
         2266  +    rc = fts3SegWriterAdd(p, &pWriter, 1, 
         2267  +        csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
         2268  +  }
         2269  +  if( rc!=SQLITE_OK ) goto finished;
         2270  +  assert( pWriter );
         2271  +
         2272  +  rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment);
  2382   2273     if( rc!=SQLITE_OK ) goto finished;
  2383         -
  2384         -  rc = fts3DeleteSegdir(p, iLevel, apSegment, nSegment);
  2385         -  if( rc==SQLITE_OK ){
  2386         -    rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
  2387         -  }
         2274  +  rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
  2388   2275   
  2389   2276    finished:
  2390   2277     fts3SegWriterFree(pWriter);
  2391         -  if( apSegment ){
  2392         -    for(i=0; i<nSegment; i++){
  2393         -      sqlite3Fts3SegReaderFree(apSegment[i]);
  2394         -    }
  2395         -    sqlite3_free(apSegment);
  2396         -  }
  2397         -  sqlite3Fts3SegReaderFree(pPending);
  2398         -  sqlite3_reset(pStmt);
         2278  +  sqlite3Fts3SegReaderFinish(&csr);
  2399   2279     return rc;
  2400   2280   }
  2401   2281   
  2402   2282   
  2403   2283   /* 
  2404   2284   ** Flush the contents of pendingTerms to a level 0 segment.
  2405   2285   */
  2406   2286   int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
  2407         -  int rc;                         /* Return Code */
  2408         -  int idx;                        /* Index of new segment created */
  2409         -  SegmentWriter *pWriter = 0;     /* Used to write the segment */
  2410         -  Fts3SegReader *pReader = 0;     /* Used to iterate through the hash table */
  2411         -
  2412         -  /* Allocate a SegReader object to iterate through the contents of the
  2413         -  ** pending-terms table. If an error occurs, or if there are no terms
  2414         -  ** in the pending-terms table, return immediately.
  2415         -  */
  2416         -  rc = sqlite3Fts3SegReaderPending(p, 0, 0, 1, &pReader);
  2417         -  if( rc!=SQLITE_OK || pReader==0 ){
  2418         -    return rc;
  2419         -  }
  2420         -
  2421         -  /* Determine the next index at level 0. If level 0 is already full, this
  2422         -  ** call may merge all existing level 0 segments into a single level 1
  2423         -  ** segment.
  2424         -  */
  2425         -  rc = fts3AllocateSegdirIdx(p, 0, &idx);
  2426         -
  2427         -  /* If no errors have occured, iterate through the contents of the 
  2428         -  ** pending-terms hash table using the Fts3SegReader iterator. The callback
  2429         -  ** writes each term (along with its doclist) to the database via the
  2430         -  ** SegmentWriter handle pWriter.
  2431         -  */
  2432         -  if( rc==SQLITE_OK ){
  2433         -    void *c = (void *)&pWriter;   /* SegReaderIterate() callback context */
  2434         -    Fts3SegFilter f;              /* SegReaderIterate() parameters */
  2435         -
  2436         -    memset(&f, 0, sizeof(Fts3SegFilter));
  2437         -    f.flags = FTS3_SEGMENT_REQUIRE_POS;
  2438         -    rc = sqlite3Fts3SegReaderIterate(p, &pReader, 1, &f, fts3FlushCallback, c);
  2439         -  }
  2440         -  assert( pWriter || rc!=SQLITE_OK );
  2441         -
  2442         -  /* If no errors have occured, flush the SegmentWriter object to the
  2443         -  ** database. Then delete the SegmentWriter and Fts3SegReader objects
  2444         -  ** allocated by this function.
  2445         -  */
  2446         -  if( rc==SQLITE_OK ){
  2447         -    rc = fts3SegWriterFlush(p, pWriter, 0, idx);
  2448         -  }
  2449         -  fts3SegWriterFree(pWriter);
  2450         -  sqlite3Fts3SegReaderFree(pReader);
  2451         -
  2452         -  if( rc==SQLITE_OK ){
  2453         -    sqlite3Fts3PendingTermsClear(p);
  2454         -  }
  2455         -  return rc;
         2287  +  return fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING);
  2456   2288   }
  2457   2289   
  2458   2290   /*
  2459   2291   ** Encode N integers as varints into a blob.
  2460   2292   */
  2461   2293   static void fts3EncodeIntArray(
  2462   2294     int N,             /* The number of integers to encode */
................................................................................
  2615   2447     int rc;                         /* Return Code */
  2616   2448     const char *zVal = (const char *)sqlite3_value_text(pVal);
  2617   2449     int nVal = sqlite3_value_bytes(pVal);
  2618   2450   
  2619   2451     if( !zVal ){
  2620   2452       return SQLITE_NOMEM;
  2621   2453     }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
  2622         -    rc = fts3SegmentMerge(p, -1);
         2454  +    rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL);
  2623   2455       if( rc==SQLITE_DONE ){
  2624   2456         rc = SQLITE_OK;
  2625   2457       }else{
  2626   2458         sqlite3Fts3PendingTermsClear(p);
  2627   2459       }
  2628   2460   #ifdef SQLITE_TEST
  2629   2461     }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
................................................................................
  2873   2705   ** merge all segments in the database (including the new segment, if 
  2874   2706   ** there was any data to flush) into a single segment. 
  2875   2707   */
  2876   2708   int sqlite3Fts3Optimize(Fts3Table *p){
  2877   2709     int rc;
  2878   2710     rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0);
  2879   2711     if( rc==SQLITE_OK ){
  2880         -    rc = fts3SegmentMerge(p, -1);
         2712  +    rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL);
  2881   2713       if( rc==SQLITE_OK ){
  2882   2714         rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
  2883   2715         if( rc==SQLITE_OK ){
  2884   2716           sqlite3Fts3PendingTermsClear(p);
  2885   2717         }
  2886   2718       }else{
  2887   2719         sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0);

Changes to main.mk.

    49     49   TCCX += -I$(TOP)/ext/async
    50     50   
    51     51   # Object files for the SQLite library.
    52     52   #
    53     53   LIBOBJ+= alter.o analyze.o attach.o auth.o \
    54     54            backup.o bitvec.o btmutex.o btree.o build.o \
    55     55            callback.o complete.o ctime.o date.o delete.o expr.o fault.o fkey.o \
    56         -         fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \
           56  +         fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \
    57     57            fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o fts3_write.o \
    58     58            func.o global.o hash.o \
    59     59            icu.o insert.o journal.o legacy.o loadext.o \
    60     60            main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \
    61     61            memjournal.o \
    62     62            mutex.o mutex_noop.o mutex_os2.o mutex_unix.o mutex_w32.o \
    63     63            notify.o opcodes.o os.o os_os2.o os_unix.o os_win.o \
................................................................................
   183    183     $(TOP)/ext/fts2/fts2_tokenizer.h \
   184    184     $(TOP)/ext/fts2/fts2_tokenizer.c \
   185    185     $(TOP)/ext/fts2/fts2_tokenizer1.c
   186    186   SRC += \
   187    187     $(TOP)/ext/fts3/fts3.c \
   188    188     $(TOP)/ext/fts3/fts3.h \
   189    189     $(TOP)/ext/fts3/fts3Int.h \
          190  +  $(TOP)/ext/fts3/fts3_aux.c \
   190    191     $(TOP)/ext/fts3/fts3_expr.c \
   191    192     $(TOP)/ext/fts3/fts3_hash.c \
   192    193     $(TOP)/ext/fts3/fts3_hash.h \
   193    194     $(TOP)/ext/fts3/fts3_icu.c \
   194    195     $(TOP)/ext/fts3/fts3_porter.c \
   195    196     $(TOP)/ext/fts3/fts3_snippet.c \
   196    197     $(TOP)/ext/fts3/fts3_tokenizer.h \
................................................................................
   289    290     $(TOP)/src/vdbeapi.c \
   290    291     $(TOP)/src/vdbeaux.c \
   291    292     $(TOP)/src/vdbe.c \
   292    293     $(TOP)/src/vdbemem.c \
   293    294     $(TOP)/src/where.c \
   294    295     parse.c \
   295    296     $(TOP)/ext/fts3/fts3.c \
          297  +  $(TOP)/ext/fts3/fts3_aux.c \
   296    298     $(TOP)/ext/fts3/fts3_expr.c \
   297    299     $(TOP)/ext/fts3/fts3_tokenizer.c \
   298    300     $(TOP)/ext/fts3/fts3_write.c \
   299    301     $(TOP)/ext/async/sqlite3async.c
   300    302   
   301    303   # Header files used by all library source files.
   302    304   #
................................................................................
   457    459   	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer.c
   458    460   
   459    461   fts2_tokenizer1.o:	$(TOP)/ext/fts2/fts2_tokenizer1.c $(HDR) $(EXTHDR)
   460    462   	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer1.c
   461    463   
   462    464   fts3.o:	$(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR)
   463    465   	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c
          466  +
          467  +fts3_aux.o:	$(TOP)/ext/fts3/fts3_aux.c $(HDR) $(EXTHDR)
          468  +	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_aux.c
   464    469   
   465    470   fts3_expr.o:	$(TOP)/ext/fts3/fts3_expr.c $(HDR) $(EXTHDR)
   466    471   	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c
   467    472   
   468    473   fts3_hash.o:	$(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR)
   469    474   	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c
   470    475   

Changes to src/pager.c.

  2846   2846     PAGER_INCR(pPager->nRead);
  2847   2847     IOTRACE(("PGIN %p %d\n", pPager, pgno));
  2848   2848     PAGERTRACE(("FETCH %d page %d hash(%08x)\n",
  2849   2849                  PAGERID(pPager), pgno, pager_pagehash(pPg)));
  2850   2850   
  2851   2851     return rc;
  2852   2852   }
         2853  +
         2854  +/*
         2855  +** Update the value of the change-counter at offsets 24 and 92 in
         2856  +** the header and the sqlite version number at offset 96.
         2857  +**
         2858  +** This is an unconditional update.  See also the pager_incr_changecounter()
         2859  +** routine which only updates the change-counter if the update is actually
         2860  +** needed, as determined by the pPager->changeCountDone state variable.
         2861  +*/
         2862  +static void pager_write_changecounter(PgHdr *pPg){
         2863  +  u32 change_counter;
         2864  +
         2865  +  /* Increment the value just read and write it back to byte 24. */
         2866  +  change_counter = sqlite3Get4byte((u8*)pPg->pPager->dbFileVers)+1;
         2867  +  put32bits(((char*)pPg->pData)+24, change_counter);
         2868  +
         2869  +  /* Also store the SQLite version number in bytes 96..99 and in
         2870  +  ** bytes 92..95 store the change counter for which the version number
         2871  +  ** is valid. */
         2872  +  put32bits(((char*)pPg->pData)+92, change_counter);
         2873  +  put32bits(((char*)pPg->pData)+96, SQLITE_VERSION_NUMBER);
         2874  +}
  2853   2875   
  2854   2876   #ifndef SQLITE_OMIT_WAL
  2855   2877   /*
  2856   2878   ** This function is invoked once for each page that has already been 
  2857   2879   ** written into the log file when a WAL transaction is rolled back.
  2858   2880   ** Parameter iPg is the page number of said page. The pCtx argument 
  2859   2881   ** is actually a pointer to the Pager structure.
................................................................................
  2917   2939       rc = pagerUndoCallback((void *)pPager, pList->pgno);
  2918   2940       pList = pNext;
  2919   2941     }
  2920   2942   
  2921   2943     return rc;
  2922   2944   }
  2923   2945   
  2924         -
  2925         -/*
  2926         -** Update the value of the change-counter at offsets 24 and 92 in
  2927         -** the header and the sqlite version number at offset 96.
  2928         -**
  2929         -** This is an unconditional update.  See also the pager_incr_changecounter()
  2930         -** routine which only updates the change-counter if the update is actually
  2931         -** needed, as determined by the pPager->changeCountDone state variable.
  2932         -*/
  2933         -static void pager_write_changecounter(PgHdr *pPg){
  2934         -  u32 change_counter;
  2935         -
  2936         -  /* Increment the value just read and write it back to byte 24. */
  2937         -  change_counter = sqlite3Get4byte((u8*)pPg->pPager->dbFileVers)+1;
  2938         -  put32bits(((char*)pPg->pData)+24, change_counter);
  2939         -
  2940         -  /* Also store the SQLite version number in bytes 96..99 and in
  2941         -  ** bytes 92..95 store the change counter for which the version number
  2942         -  ** is valid. */
  2943         -  put32bits(((char*)pPg->pData)+92, change_counter);
  2944         -  put32bits(((char*)pPg->pData)+96, SQLITE_VERSION_NUMBER);
  2945         -}
  2946         -
  2947   2946   /*
  2948   2947   ** This function is a wrapper around sqlite3WalFrames(). As well as logging
  2949   2948   ** the contents of the list of pages headed by pList (connected by pDirty),
  2950   2949   ** this function notifies any active backup processes that the pages have
  2951   2950   ** changed.
  2952   2951   **
  2953   2952   ** The list of pages passed into this routine is always sorted by page number.

Changes to src/sqlite.h.in.

  5530   5530   ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_USED</dt>
  5531   5531   ** <dd>This parameter returns the number of lookaside memory slots currently
  5532   5532   ** checked out.</dd>)^
  5533   5533   **
  5534   5534   ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_HIT</dt>
  5535   5535   ** <dd>This parameter returns the number malloc attempts that were 
  5536   5536   ** satisfied using lookaside memory. Only the high-water value is meaningful;
  5537         -** the current value is always zero.
  5538         -** checked out.</dd>)^
         5537  +** the current value is always zero.)^
  5539   5538   **
  5540   5539   ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE</dt>
  5541   5540   ** <dd>This parameter returns the number malloc attempts that might have
  5542   5541   ** been satisfied using lookaside memory but failed due to the amount of
  5543   5542   ** memory requested being larger than the lookaside slot size.
  5544   5543   ** Only the high-water value is meaningful;
  5545         -** the current value is always zero.
  5546         -** checked out.</dd>)^
         5544  +** the current value is always zero.)^
  5547   5545   **
  5548   5546   ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL</dt>
  5549   5547   ** <dd>This parameter returns the number malloc attempts that might have
  5550   5548   ** been satisfied using lookaside memory but failed due to all lookaside
  5551   5549   ** memory already being in use.
  5552   5550   ** Only the high-water value is meaningful;
  5553         -** the current value is always zero.
  5554         -** checked out.</dd>)^
         5551  +** the current value is always zero.)^
  5555   5552   **
  5556   5553   ** ^(<dt>SQLITE_DBSTATUS_CACHE_USED</dt>
  5557   5554   ** <dd>This parameter returns the approximate number of of bytes of heap
  5558   5555   ** memory used by all pager caches associated with the database connection.)^
  5559   5556   ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0.
  5560   5557   **
  5561   5558   ** ^(<dt>SQLITE_DBSTATUS_SCHEMA_USED</dt>

Added test/fts3aux1.test.

            1  +# 2011 January 27
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS3 module.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +ifcapable !fts3 { finish_test ; return }
           18  +set ::testprefix fts3aux1
           19  +
           20  +do_execsql_test 1.1 {
           21  +  CREATE VIRTUAL TABLE t1 USING fts4;
           22  +  INSERT INTO t1 VALUES('one two three four');
           23  +  INSERT INTO t1 VALUES('three four five six');
           24  +  INSERT INTO t1 VALUES('one three five seven');
           25  +
           26  +  CREATE VIRTUAL TABLE terms USING fts4aux(t1);
           27  +  SELECT * FROM terms;
           28  +} {
           29  +  five  2 2     four  2 2     one   2 2     seven 1 1 
           30  +  six   1 1     three 3 3     two   1 1
           31  +}
           32  +
           33  +do_execsql_test 1.2 {
           34  +  INSERT INTO t1 VALUES('one one one three three three');
           35  +  SELECT * FROM terms;
           36  +} { 
           37  +  five  2 2     four  2 2     one   3 5     seven 1 1 
           38  +  six   1 1     three 4 6     two   1 1
           39  +}
           40  +
           41  +do_execsql_test 1.3 {
           42  +  DELETE FROM t1;
           43  +  SELECT * FROM terms;
           44  +} {}
           45  +
           46  +do_execsql_test 1.4 {
           47  +  INSERT INTO t1 VALUES('a b a b a b a');
           48  +  INSERT INTO t1 SELECT * FROM t1;
           49  +  INSERT INTO t1 SELECT * FROM t1;
           50  +  INSERT INTO t1 SELECT * FROM t1;
           51  +  INSERT INTO t1 SELECT * FROM t1;
           52  +  INSERT INTO t1 SELECT * FROM t1;
           53  +  INSERT INTO t1 SELECT * FROM t1;
           54  +  INSERT INTO t1 SELECT * FROM t1;
           55  +  INSERT INTO t1 SELECT * FROM t1;
           56  +  SELECT * FROM terms;
           57  +} {a 256 1024    b 256 768}
           58  +
           59  +finish_test

Added test/fts3comp1.test.

            1  +# 2011 January 27
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS3 module.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +ifcapable !fts3 { finish_test ; return }
           18  +set ::testprefix fts3comp1
           19  +
           20  +set next_x 0
           21  +proc zip {x} {
           22  + incr ::next_x
           23  + set ::strings($::next_x) $x
           24  + return $::next_x
           25  +}
           26  +proc unzip {x} {
           27  +  return $::strings($x)
           28  +}
           29  +
           30  +db func zip zip
           31  +db func unzip unzip
           32  +
           33  +do_execsql_test 1.0 {
           34  +  CREATE VIRTUAL TABLE t1 USING fts4(
           35  +    a, b, 
           36  +    compress=zip, uncompress=unzip
           37  +  );
           38  +  INSERT INTO t1 VALUES('one two three', 'two four six');
           39  +}
           40  +
           41  +do_execsql_test 1.1 {
           42  +  SELECT a, b FROM t1;
           43  +} {{one two three} {two four six}}
           44  +
           45  +do_execsql_test 1.2 {
           46  +  SELECT c0a, c1b FROM t1_content;
           47  +} {1 2}
           48  +
           49  +do_execsql_test 1.3 {
           50  +  INSERT INTO t1 VALUES('three six nine', 'four eight twelve');
           51  +  SELECT a, b FROM t1;
           52  +} {{one two three} {two four six} {three six nine} {four eight twelve}}
           53  +
           54  +do_execsql_test 1.4 {
           55  +  SELECT c0a, c1b FROM t1_content;
           56  +} {1 2 3 4}
           57  +
           58  +do_execsql_test 1.5 {
           59  +  CREATE VIRTUAL TABLE terms USING fts4aux(t1);
           60  +  SELECT * FROM terms;
           61  +} {
           62  +  eight 1 1    four 2 2    nine 1 1    one 1 1 
           63  +  six 2 2      three 2 2   twelve 1 1  two 1 2
           64  +}
           65  +
           66  +do_execsql_test 1.6 {
           67  +  DELETE FROM t1 WHERE docid = 1;
           68  +  SELECT * FROM terms;
           69  +} {
           70  +  eight 1 1   four 1 1    nine 1 1 
           71  +  six 1 1     three 1 1   twelve 1 1
           72  +}
           73  +
           74  +do_execsql_test 1.7 {
           75  +  SELECT c0a, c1b FROM t1_content;
           76  +} {3 4}
           77  +
           78  +finish_test

Changes to test/fts3corrupt.test.

   126    126     "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
   127    127   ]
   128    128   
   129    129   do_catchsql_test 4.3 {
   130    130     UPDATE t1_segdir SET root = $blob;
   131    131     SELECT rowid FROM t1 WHERE t1 MATCH 'world';
   132    132   } {1 {database disk image is malformed}}
          133  +
          134  +# Test a special kind of corruption, where the %_stat table contains
          135  +# an invalid entry. At one point this could lead to a division-by-zero
          136  +# error in fts4.
          137  +#
          138  +do_execsql_test 5.0 {
          139  +  DROP TABLE t1;
          140  +  CREATE VIRTUAL TABLE t1 USING fts4;
          141  +}
          142  +do_test 5.1 {
          143  +  db func nn nn
          144  +  execsql BEGIN
          145  +  execsql { INSERT INTO t1 VALUES('one') }
          146  +  execsql { INSERT INTO t1 VALUES('two') }
          147  +  execsql { INSERT INTO t1 VALUES('three') }
          148  +  execsql { INSERT INTO t1 VALUES('four') }
          149  +  execsql COMMIT
          150  +} {}
          151  +do_catchsql_test 5.2 {
          152  +  UPDATE t1_stat SET value = X'0000';
          153  +  SELECT matchinfo(t1, 'nxa') FROM t1 WHERE t1 MATCH 't*';
          154  +} {1 {database disk image is malformed}}
          155  +do_catchsql_test 5.3 {
          156  +  UPDATE t1_stat SET value = NULL;
          157  +  SELECT matchinfo(t1, 'nxa') FROM t1 WHERE t1 MATCH 't*';
          158  +} {1 {database disk image is malformed}}
          159  +
   133    160   
   134    161   finish_test
   135    162   

Changes to test/permutations.test.

   177    177     fts3af.test fts3ag.test fts3ah.test fts3ai.test fts3aj.test
   178    178     fts3ak.test fts3al.test fts3am.test fts3an.test fts3ao.test
   179    179     fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test
   180    180     fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test 
   181    181     fts3near.test fts3query.test fts3shared.test fts3snippet.test 
   182    182   
   183    183     fts3fault.test fts3malloc.test fts3matchinfo.test
          184  +
          185  +  fts3aux1.test fts3comp1.test
   184    186   }
   185    187   
   186    188   
   187    189   lappend ::testsuitelist xxx
   188    190   #-------------------------------------------------------------------------
   189    191   # Define the coverage related test suites:
   190    192   #

Changes to test/trace2.test.

   124    124       INSERT INTO x1 VALUES('Wind chill values as low as -13');
   125    125     }
   126    126   
   127    127     do_trace_test 2.2 {
   128    128       INSERT INTO x1 VALUES('North northwest wind between 8 and 14 mph');
   129    129     } {
   130    130       "INSERT INTO x1 VALUES('North northwest wind between 8 and 14 mph');" 
   131         -    "-- INSERT INTO 'main'.'x1_content' VALUES(?,?)" 
          131  +    "-- INSERT INTO 'main'.'x1_content' VALUES(?,(?))" 
   132    132       "-- REPLACE INTO 'main'.'x1_docsize' VALUES(?,?)" 
   133    133       "-- SELECT value FROM 'main'.'x1_stat' WHERE id=0" 
   134    134       "-- REPLACE INTO 'main'.'x1_stat' VALUES(0,?)" 
   135    135       "-- SELECT (SELECT max(idx) FROM 'main'.'x1_segdir' WHERE level = ?) + 1" 
   136    136       "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)"
   137    137       "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)"
   138    138     }
   139    139   
   140    140     do_trace_test 2.3 {
   141    141       INSERT INTO x1(x1) VALUES('optimize');
   142    142     } {
   143    143       "INSERT INTO x1(x1) VALUES('optimize');"
   144         -    "-- SELECT count(*), max(level) FROM 'main'.'x1_segdir'"
   145    144       "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' ORDER BY level DESC, idx ASC"
          145  +    "-- SELECT count(*), max(level) FROM 'main'.'x1_segdir'"
   146    146       "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)"
   147    147       "-- DELETE FROM 'main'.'x1_segdir'"
   148    148       "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)"
   149    149     }
   150    150   }
   151    151   
   152    152   finish_test

Changes to tool/mksqlite3c.tcl.

   290    290      tokenize.c
   291    291      complete.c
   292    292   
   293    293      main.c
   294    294      notify.c
   295    295   
   296    296      fts3.c
          297  +   fts3_aux.c
   297    298      fts3_expr.c
   298    299      fts3_hash.c
   299    300      fts3_porter.c
   300    301      fts3_tokenizer.c
   301    302      fts3_tokenizer1.c
   302    303      fts3_write.c
   303    304      fts3_snippet.c