/ Check-in [2037dba6]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Begin testing fts5 OOM and IO error handling.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 2037dba62fdd995ad15b642abe499a790f5ffe5c
User & Date: dan 2014-12-03 17:27:35
Context
2014-12-18
18:25
Fix various problems in fts5 revealed by fault-injection tests. check-in: e358c3de user: dan tags: fts5
2014-12-03
17:27
Begin testing fts5 OOM and IO error handling. check-in: 2037dba6 user: dan tags: fts5
2014-12-02
20:18
Add a configuration option to remap the "rank" column to an auxiliary fts5 function. check-in: b5f59712 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

   304    304     int argc,                       /* Number of elements in argv array */
   305    305     const char * const *argv,       /* xCreate/xConnect argument array */
   306    306     sqlite3_vtab **ppVTab,          /* Write the resulting vtab structure here */
   307    307     char **pzErr                    /* Write any error message here */
   308    308   ){
   309    309     Fts5Global *pGlobal = (Fts5Global*)pAux;
   310    310     const char **azConfig = (const char**)argv;
   311         -  int rc;                         /* Return code */
          311  +  int rc = SQLITE_OK;             /* Return code */
   312    312     Fts5Config *pConfig;            /* Results of parsing argc/argv */
   313    313     Fts5Table *pTab = 0;            /* New virtual table object */
   314    314   
   315         -  /* Parse the arguments */
   316         -  rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr);
   317         -  assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 );
   318         -
   319         -  /* Allocate the new vtab object */
          315  +  /* Allocate the new vtab object and parse the configuration */
          316  +  pTab = (Fts5Table*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Table));
          317  +  if( rc==SQLITE_OK ){
          318  +    rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr);
          319  +    assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 );
          320  +  }
   320    321     if( rc==SQLITE_OK ){
   321         -    pTab = (Fts5Table*)sqlite3_malloc(sizeof(Fts5Table));
   322         -    if( pTab==0 ){
   323         -      rc = SQLITE_NOMEM;
   324         -    }else{
   325         -      memset(pTab, 0, sizeof(Fts5Table));
   326         -      pTab->pConfig = pConfig;
   327         -      pTab->pGlobal = pGlobal;
   328         -    }
          322  +    pTab->pConfig = pConfig;
          323  +    pTab->pGlobal = pGlobal;
   329    324     }
   330    325   
   331    326     /* Open the index sub-system */
   332    327     if( rc==SQLITE_OK ){
   333    328       rc = sqlite3Fts5IndexOpen(pConfig, bCreate, &pTab->pIndex, pzErr);
   334    329     }
   335    330   

Changes to ext/fts5/fts5Int.h.

   183    183   
   184    184   int sqlite3Fts5PoslistNext64(
   185    185     const u8 *a, int n,             /* Buffer containing poslist */
   186    186     int *pi,                        /* IN/OUT: Offset within a[] */
   187    187     i64 *piOff                      /* IN/OUT: Current offset */
   188    188   );
   189    189   
          190  +/* Malloc utility */
          191  +void *sqlite3Fts5MallocZero(int *pRc, int nByte);
          192  +
   190    193   /*
   191    194   ** End of interface to code in fts5_buffer.c.
   192    195   **************************************************************************/
   193    196   
   194    197   /**************************************************************************
   195    198   ** Interface to code in fts5_index.c. fts5_index.c contains contains code
   196    199   ** to access the data stored in the %_data table.
................................................................................
   221    224   ** }
   222    225   */
   223    226   
   224    227   /*
   225    228   ** Open a new iterator to iterate though all docids that match the 
   226    229   ** specified token or token prefix.
   227    230   */
   228         -Fts5IndexIter *sqlite3Fts5IndexQuery(
          231  +int sqlite3Fts5IndexQuery(
   229    232     Fts5Index *p,                   /* FTS index to query */
   230    233     const char *pToken, int nToken, /* Token (or prefix) to query for */
   231         -  int flags                       /* Mask of FTS5INDEX_QUERY_X flags */
          234  +  int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
          235  +  Fts5IndexIter **ppIter
   232    236   );
   233    237   
   234    238   /*
   235    239   ** Docid list iteration.
   236    240   */
   237         -int  sqlite3Fts5IterEof(Fts5IndexIter*);
   238         -void sqlite3Fts5IterNext(Fts5IndexIter*);
   239         -void sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
   240         -i64  sqlite3Fts5IterRowid(Fts5IndexIter*);
          241  +int sqlite3Fts5IterEof(Fts5IndexIter*);
          242  +int sqlite3Fts5IterNext(Fts5IndexIter*);
          243  +int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
          244  +i64 sqlite3Fts5IterRowid(Fts5IndexIter*);
   241    245   
   242    246   /*
   243    247   ** Obtain the position list that corresponds to the current position.
   244    248   */
   245         -const u8 *sqlite3Fts5IterPoslist(Fts5IndexIter*, int *pn);
          249  +int sqlite3Fts5IterPoslist(Fts5IndexIter*, const u8 **pp, int *pn);
   246    250   
   247    251   /*
   248    252   ** Close an iterator opened by sqlite3Fts5IndexQuery().
   249    253   */
   250    254   void sqlite3Fts5IterClose(Fts5IndexIter*);
   251    255   
   252    256   /*
................................................................................
   255    259   ** times.
   256    260   **
   257    261   ** For an insert, it must be called once for each token in the new document.
   258    262   ** If the operation is a delete, it must be called (at least) once for each
   259    263   ** unique token in the document with an iCol value less than zero. The iPos
   260    264   ** argument is ignored for a delete.
   261    265   */
   262         -void sqlite3Fts5IndexWrite(
          266  +int sqlite3Fts5IndexWrite(
   263    267     Fts5Index *p,                   /* Index to write to */
   264    268     int iCol,                       /* Column token appears in (-ve -> delete) */
   265    269     int iPos,                       /* Position of token within column */
   266    270     const char *pToken, int nToken  /* Token to add or remove to or from index */
   267    271   );
   268    272   
   269    273   /*
   270    274   ** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
   271    275   ** document iDocid.
   272    276   */
   273         -void sqlite3Fts5IndexBeginWrite(
          277  +int sqlite3Fts5IndexBeginWrite(
   274    278     Fts5Index *p,                   /* Index to write to */
   275    279     i64 iDocid                      /* Docid to add or remove data from */
   276    280   );
   277    281   
   278    282   /*
   279    283   ** Flush any data stored in the in-memory hash tables to the database.
   280    284   ** If the bCommit flag is true, also close any open blob handles.
................................................................................
   317    321   
   318    322   /*
   319    323   ** Return the total number of entries read from the %_data table by 
   320    324   ** this connection since it was created.
   321    325   */
   322    326   int sqlite3Fts5IndexReads(Fts5Index *p);
   323    327   
   324         -/* Malloc utility */
   325         -void *sqlite3Fts5MallocZero(int *pRc, int nByte);
   326         -
   327    328   /*
   328    329   ** End of interface to code in fts5_index.c.
   329    330   **************************************************************************/
   330    331   
   331    332   /**************************************************************************
   332    333   ** Interface to code in fts5_hash.c. 
   333    334   */

Changes to ext/fts5/fts5_buffer.c.

   277    277       *pOut++ = z[i];
   278    278     }
   279    279     if( bParen ) *pOut++ = '}';
   280    280   
   281    281     pBuf->n = pOut - pBuf->p;
   282    282     *pOut = '\0';
   283    283   }
          284  +
          285  +void *sqlite3Fts5MallocZero(int *pRc, int nByte){
          286  +  void *pRet = 0;
          287  +  if( *pRc==SQLITE_OK ){
          288  +    pRet = sqlite3_malloc(nByte);
          289  +    if( pRet==0 && nByte>0 ){
          290  +      *pRc = SQLITE_NOMEM;
          291  +    }else{
          292  +      memset(pRet, 0, nByte);
          293  +    }
          294  +  }
          295  +  return pRet;
          296  +}
          297  +
   284    298   
   285    299   

Changes to ext/fts5/fts5_config.c.

   111    111   
   112    112   /*
   113    113   ** Duplicate the string passed as the only argument into a buffer allocated
   114    114   ** by sqlite3_malloc().
   115    115   **
   116    116   ** Return 0 if an OOM error is encountered.
   117    117   */
   118         -static char *fts5Strdup(const char *z){
   119         -  return sqlite3_mprintf("%s", z);
          118  +static char *fts5Strdup(int *pRc, const char *z){
          119  +  char *pRet = 0;
          120  +  if( *pRc==SQLITE_OK ){
          121  +    pRet = sqlite3_mprintf("%s", z);
          122  +    if( pRet==0 ) *pRc = SQLITE_NOMEM;
          123  +  }
          124  +  return pRet;
   120    125   }
   121    126   
   122    127   /*
   123    128   ** Allocate an instance of the default tokenizer ("simple") at 
   124    129   ** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
   125    130   ** code if an error occurs.
   126    131   */
................................................................................
   155    160   
   156    161     *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
   157    162     if( pRet==0 ) return SQLITE_NOMEM;
   158    163     memset(pRet, 0, sizeof(Fts5Config));
   159    164     pRet->db = db;
   160    165     pRet->iCookie = -1;
   161    166   
   162         -  pRet->azCol = (char**)sqlite3_malloc(sizeof(char*) * nArg);
   163         -  pRet->zDb = fts5Strdup(azArg[1]);
   164         -  pRet->zName = fts5Strdup(azArg[2]);
   165         -  if( sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
   166         -    *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
   167         -    rc = SQLITE_ERROR;
   168         -  }else if( pRet->azCol==0 || pRet->zDb==0 || pRet->zName==0 ){
   169         -    rc = SQLITE_NOMEM;
   170         -  }else{
   171         -    int i;
   172         -    for(i=3; rc==SQLITE_OK && i<nArg; i++){
   173         -      char *zDup = fts5Strdup(azArg[i]);
   174         -      if( zDup==0 ){
   175         -        rc = SQLITE_NOMEM;
   176         -      }else{
   177         -
   178         -        /* Check if this is a special directive - "cmd=arg" */
   179         -        if( zDup[0]!='"' && zDup[0]!='\'' && zDup[0]!='[' && zDup[0]!='`' ){
   180         -          char *p = zDup;
   181         -          while( *p && *p!='=' ) p++;
   182         -          if( *p ){
   183         -            char *zArg = &p[1];
   184         -            *p = '\0';
   185         -            sqlite3Fts5Dequote(zArg);
   186         -            rc = fts5ConfigParseSpecial(pRet, zDup, zArg, pzErr);
   187         -            sqlite3_free(zDup);
   188         -            zDup = 0;
          167  +  pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
          168  +  pRet->zDb = fts5Strdup(&rc, azArg[1]);
          169  +  pRet->zName = fts5Strdup(&rc, azArg[2]);
          170  +  if( rc==SQLITE_OK ){
          171  +    if( sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
          172  +      *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
          173  +      rc = SQLITE_ERROR;
          174  +    }else{
          175  +      int i;
          176  +      for(i=3; rc==SQLITE_OK && i<nArg; i++){
          177  +        char *zDup = fts5Strdup(&rc, azArg[i]);
          178  +        if( zDup ){
          179  +          /* Check if this is a special directive - "cmd=arg" */
          180  +          if( zDup[0]!='"' && zDup[0]!='\'' && zDup[0]!='[' && zDup[0]!='`' ){
          181  +            char *p = zDup;
          182  +            while( *p && *p!='=' ) p++;
          183  +            if( *p ){
          184  +              char *zArg = &p[1];
          185  +              *p = '\0';
          186  +              sqlite3Fts5Dequote(zArg);
          187  +              rc = fts5ConfigParseSpecial(pRet, zDup, zArg, pzErr);
          188  +              sqlite3_free(zDup);
          189  +              zDup = 0;
          190  +            }
   189    191             }
   190         -        }
   191    192   
   192         -        /* If it is not a special directive, it must be a column name. In
   193         -        ** this case, check that it is not the reserved column name "rank". */
   194         -        if( zDup ){
   195         -          sqlite3Fts5Dequote(zDup);
   196         -          pRet->azCol[pRet->nCol++] = zDup;
   197         -          if( sqlite3_stricmp(zDup, FTS5_RANK_NAME)==0 ){
   198         -            *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zDup);
   199         -            rc = SQLITE_ERROR;
          193  +          /* If it is not a special directive, it must be a column name. In
          194  +           ** this case, check that it is not the reserved column name "rank". */
          195  +          if( zDup ){
          196  +            sqlite3Fts5Dequote(zDup);
          197  +            pRet->azCol[pRet->nCol++] = zDup;
          198  +            if( sqlite3_stricmp(zDup, FTS5_RANK_NAME)==0 ){
          199  +              *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zDup);
          200  +              rc = SQLITE_ERROR;
          201  +            }
   200    202             }
   201    203           }
   202    204         }
   203    205       }
   204    206     }
   205    207   
   206    208     if( rc==SQLITE_OK && pRet->pTok==0 ){

Changes to ext/fts5/fts5_expr.c.

   382    382       aIter = (Fts5PoslistReader*)sqlite3_malloc(nByte);
   383    383       if( !aIter ) return SQLITE_NOMEM;
   384    384     }
   385    385   
   386    386     /* Initialize a term iterator for each term in the phrase */
   387    387     for(i=0; i<pPhrase->nTerm; i++){
   388    388       int n;
   389         -    const u8 *a = sqlite3Fts5IterPoslist(pPhrase->aTerm[i].pIter, &n);
   390         -    if( sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]) ) goto ismatch_out;
          389  +    const u8 *a;
          390  +    rc = sqlite3Fts5IterPoslist(pPhrase->aTerm[i].pIter, &a, &n);
          391  +    if( rc || sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]) ){
          392  +      goto ismatch_out;
          393  +    }
   391    394     }
   392    395   
   393    396     while( 1 ){
   394    397       int bMatch;
   395    398       i64 iPos = aIter[0].iPos;
   396    399       do {
   397    400         bMatch = 1;
................................................................................
   572    575   ** set output variable *pbEof to true before returning.
   573    576   */
   574    577   static int fts5ExprNearAdvanceAll(
   575    578     Fts5Expr *pExpr,                /* Expression pPhrase belongs to */
   576    579     Fts5ExprNearset *pNear,         /* Near object to advance iterators of */
   577    580     int *pbEof                      /* OUT: Set to true if phrase at EOF */
   578    581   ){
   579         -  int rc = SQLITE_OK;             /* Return code */
   580    582     int i, j;                       /* Phrase and token index, respectively */
   581    583   
   582    584     for(i=0; i<pNear->nPhrase; i++){
   583    585       Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
   584    586       for(j=0; j<pPhrase->nTerm; j++){
   585    587         Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
   586         -      sqlite3Fts5IterNext(pIter);
   587         -      if( sqlite3Fts5IterEof(pIter) ){
          588  +      int rc = sqlite3Fts5IterNext(pIter);
          589  +      if( rc || sqlite3Fts5IterEof(pIter) ){
   588    590           *pbEof = 1;
   589    591           return rc;
   590    592         }
   591    593       }
   592    594     }
   593    595   
   594         -  return rc;
          596  +  return SQLITE_OK;
   595    597   }
   596    598   
   597    599   /*
   598    600   ** Advance iterator pIter until it points to a value equal to or smaller
   599    601   ** than the initial value of *piMin. If this means the iterator points
   600    602   ** to a value smaller than *piMin, update *piMin to the new smallest value.
   601    603   **
................................................................................
   707    709     while( 1 ){
   708    710       int i;
   709    711   
   710    712       /* Advance the iterators until they all point to the same rowid */
   711    713       rc = fts5ExprNearNextRowidMatch(pExpr, pNode, bFromValid, iFrom);
   712    714       if( pNode->bEof || rc!=SQLITE_OK ) break;
   713    715   
   714         -    for(i=0; i<pNear->nPhrase; i++){
          716  +    for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
   715    717         Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
   716    718         if( pPhrase->nTerm>1 || pNear->iCol>=0 ){
   717    719           int bMatch = 0;
   718    720           rc = fts5ExprPhraseIsMatch(pExpr, pNear->iCol, pPhrase, &bMatch);
   719         -        if( rc!=SQLITE_OK ) return rc;
   720    721           if( bMatch==0 ) break;
   721    722         }else{
   722    723           int n;
   723         -        const u8 *a = sqlite3Fts5IterPoslist(pPhrase->aTerm[0].pIter, &n);
          724  +        const u8 *a;
          725  +        rc = sqlite3Fts5IterPoslist(pPhrase->aTerm[0].pIter, &a, &n);
   724    726           fts5BufferSet(&rc, &pPhrase->poslist, n, a);
   725    727         }
   726    728       }
   727    729   
   728         -    if( i==pNear->nPhrase ){
          730  +    if( rc==SQLITE_OK && i==pNear->nPhrase ){
   729    731         int bMatch = 1;
   730    732         if( pNear->nPhrase>1 ){
   731    733           rc = fts5ExprNearIsMatch(pNear, &bMatch);
   732    734         }
   733    735         if( rc!=SQLITE_OK || bMatch ) break;
   734    736       }
   735    737   
   736    738       /* If control flows to here, then the current rowid is not a match.
   737    739       ** Advance all term iterators in all phrases to the next rowid. */
   738         -    rc = fts5ExprNearAdvanceAll(pExpr, pNear, &pNode->bEof);
          740  +    if( rc==SQLITE_OK ){
          741  +      rc = fts5ExprNearAdvanceAll(pExpr, pNear, &pNode->bEof);
          742  +    }
   739    743       if( pNode->bEof || rc!=SQLITE_OK ) break;
   740    744     }
   741    745   
   742    746     return rc;
   743    747   }
   744    748   
   745    749   /*
................................................................................
   751    755     Fts5Expr *pExpr,
   752    756     Fts5ExprNode *pNode
   753    757   ){
   754    758     Fts5ExprNearset *pNear = pNode->pNear;
   755    759     Fts5ExprTerm *pTerm;
   756    760     Fts5ExprPhrase *pPhrase;
   757    761     int i, j;
          762  +  int rc = SQLITE_OK;
   758    763   
   759         -  for(i=0; i<pNear->nPhrase; i++){
          764  +  for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
   760    765       pPhrase = pNear->apPhrase[i];
   761    766       for(j=0; j<pPhrase->nTerm; j++){
   762    767         pTerm = &pPhrase->aTerm[j];
   763         -      pTerm->pIter = sqlite3Fts5IndexQuery(
          768  +      rc = sqlite3Fts5IndexQuery(
   764    769             pExpr->pIndex, pTerm->zTerm, strlen(pTerm->zTerm),
   765    770             (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
   766         -          (pExpr->bAsc ? FTS5INDEX_QUERY_ASC : 0)
          771  +          (pExpr->bAsc ? FTS5INDEX_QUERY_ASC : 0),
          772  +          &pTerm->pIter
   767    773         );
   768    774         if( pTerm->pIter && sqlite3Fts5IterEof(pTerm->pIter) ){
   769    775           pNode->bEof = 1;
   770         -        return SQLITE_OK;
          776  +        break;
   771    777         }
   772    778       }
   773    779     }
   774    780   
   775         -  return SQLITE_OK;
          781  +  return rc;
   776    782   }
   777    783   
   778    784   /* fts5ExprNodeNext() calls fts5ExprNodeNextMatch(). And vice-versa. */
   779    785   static int fts5ExprNodeNextMatch(Fts5Expr*, Fts5ExprNode*, int, i64);
   780    786   
   781    787   /*
   782    788   ** Compare the values currently indicated by the two nodes as follows:

Changes to ext/fts5/fts5_index.c.

   258    258   #ifdef SQLITE_DEBUG
   259    259   static int fts5Corrupt() { return SQLITE_CORRUPT_VTAB; }
   260    260   # define FTS5_CORRUPT fts5Corrupt()
   261    261   #else
   262    262   # define FTS5_CORRUPT SQLITE_CORRUPT_VTAB
   263    263   #endif
   264    264   
   265         -#ifdef SQLITE_DEBUG
   266         -static int fts5MissingData() { return 0; }
   267         -#else
   268         -# define fts5MissingData() 
   269         -#endif
   270         -
   271    265   
   272    266   typedef struct Fts5BtreeIter Fts5BtreeIter;
   273    267   typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel;
   274    268   typedef struct Fts5ChunkIter Fts5ChunkIter;
   275    269   typedef struct Fts5Data Fts5Data;
   276    270   typedef struct Fts5DlidxIter Fts5DlidxIter;
   277    271   typedef struct Fts5MultiSegIter Fts5MultiSegIter;
................................................................................
   587    581     Fts5Buffer term;                /* Current term */
   588    582     int iLeaf;                      /* Leaf containing terms >= current term */
   589    583     int nEmpty;                     /* Number of "empty" leaves following iLeaf */
   590    584     int bEof;                       /* Set to true at EOF */
   591    585     int bDlidx;                     /* True if there exists a dlidx */
   592    586   };
   593    587   
   594         -
   595         -/*
   596         -** Decode a segment-data rowid from the %_data table. This function is
   597         -** the opposite of macro FTS5_SEGMENT_ROWID().
   598         -*/
   599         -static void fts5DecodeRowid(
   600         -  i64 iRowid,                     /* Rowid from %_data table */
   601         -  int *piIdx,                     /* OUT: Index */
   602         -  int *piSegid,                   /* OUT: Segment id */
   603         -  int *piHeight,                  /* OUT: Height */
   604         -  int *piPgno                     /* OUT: Page number */
   605         -){
   606         -  *piPgno = (int)(iRowid & (((i64)1 << FTS5_DATA_PAGE_B) - 1));
   607         -  iRowid >>= FTS5_DATA_PAGE_B;
   608         -
   609         -  *piHeight = (int)(iRowid & (((i64)1 << FTS5_DATA_HEIGHT_B) - 1));
   610         -  iRowid >>= FTS5_DATA_HEIGHT_B;
   611         -
   612         -  *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1));
   613         -  iRowid >>= FTS5_DATA_ID_B;
   614         -
   615         -  *piIdx = (int)(iRowid & (((i64)1 << FTS5_DATA_IDX_B) - 1));
   616         -}
   617         -
   618         -static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
   619         -  int iIdx,iSegid,iHeight,iPgno;  /* Rowid compenents */
   620         -  fts5DecodeRowid(iKey, &iIdx, &iSegid, &iHeight, &iPgno);
   621         -
   622         -  if( iSegid==0 ){
   623         -    if( iKey==FTS5_AVERAGES_ROWID ){
   624         -      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(averages) ");
   625         -    }else{
   626         -      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
   627         -          "{structure idx=%d}", (int)(iKey-10)
   628         -      );
   629         -    }
   630         -  }
   631         -  else if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){
   632         -    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(dlidx idx=%d segid=%d pgno=%d)",
   633         -        iIdx, iSegid, iPgno
   634         -    );
   635         -  }else{
   636         -    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(idx=%d segid=%d h=%d pgno=%d)",
   637         -        iIdx, iSegid, iHeight, iPgno
   638         -    );
   639         -  }
   640         -}
   641         -
   642         -static void fts5DebugStructure(
   643         -  int *pRc,                       /* IN/OUT: error code */
   644         -  Fts5Buffer *pBuf,
   645         -  Fts5Structure *p
   646         -){
   647         -  int iLvl, iSeg;                 /* Iterate through levels, segments */
   648         -
   649         -  for(iLvl=0; iLvl<p->nLevel; iLvl++){
   650         -    Fts5StructureLevel *pLvl = &p->aLevel[iLvl];
   651         -    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
   652         -        " {lvl=%d nMerge=%d", iLvl, pLvl->nMerge
   653         -    );
   654         -    for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
   655         -      Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
   656         -      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
   657         -          " {id=%d h=%d leaves=%d..%d}", pSeg->iSegid, pSeg->nHeight, 
   658         -          pSeg->pgnoFirst, pSeg->pgnoLast
   659         -      );
   660         -    }
   661         -    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
   662         -  }
   663         -}
   664         -
   665         -
   666    588   
   667    589   static void fts5PutU16(u8 *aOut, u16 iVal){
   668    590     aOut[0] = (iVal>>8);
   669    591     aOut[1] = (iVal&0xFF);
   670    592   }
   671    593   
   672    594   static u16 fts5GetU16(const u8 *aIn){
................................................................................
   684    606     assert( p->rc==SQLITE_OK );
   685    607     pRet = sqlite3_malloc(nByte);
   686    608     if( pRet==0 ){
   687    609       p->rc = SQLITE_NOMEM;
   688    610     }else{
   689    611       memset(pRet, 0, nByte);
   690    612     }
   691         -  return pRet;
   692         -}
   693         -
   694         -void *sqlite3Fts5MallocZero(int *pRc, int nByte){
   695         -  void *pRet = 0;
   696         -  if( *pRc==SQLITE_OK ){
   697         -    pRet = sqlite3_malloc(nByte);
   698         -    if( pRet==0 && nByte>0 ){
   699         -      *pRc = SQLITE_NOMEM;
   700         -    }else{
   701         -      memset(pRet, 0, nByte);
   702         -    }
   703         -  }
   704    613     return pRet;
   705    614   }
   706    615   
   707    616   /*
   708    617   ** Compare the contents of the pLeft buffer with the pRight/nRight blob.
   709    618   **
   710    619   ** Return -ve if pLeft is smaller than pRight, 0 if they are equal or
................................................................................
   790    699       if( p->pReader==0 ){
   791    700         Fts5Config *pConfig = p->pConfig;
   792    701         rc = sqlite3_blob_open(pConfig->db, 
   793    702             pConfig->zDb, p->zDataTbl, "block", iRowid, 0, &p->pReader
   794    703         );
   795    704       }
   796    705   
   797         -    if( rc ) fts5MissingData();
   798         -
   799    706       if( rc==SQLITE_OK ){
   800    707         int nByte = sqlite3_blob_bytes(p->pReader);
   801    708         if( pBuf ){
   802    709           fts5BufferZero(pBuf);
   803    710           fts5BufferGrow(&rc, pBuf, nByte);
   804    711           rc = sqlite3_blob_read(p->pReader, pBuf->p, nByte, 0);
   805    712           if( rc==SQLITE_OK ) pBuf->n = nByte;
................................................................................
  2471   2378     if( p->rc==SQLITE_OK ){
  2472   2379       p->rc = sqlite3Fts5HashWrite(
  2473   2380           p->apHash[iIdx], p->iWriteRowid, iCol, iPos, pToken, nToken
  2474   2381       );
  2475   2382     }
  2476   2383   }
  2477   2384   
  2478         -/*
  2479         -** Insert or remove data to or from the index. Each time a document is 
  2480         -** added to or removed from the index, this function is called one or more
  2481         -** times.
  2482         -**
  2483         -** For an insert, it must be called once for each token in the new document.
  2484         -** If the operation is a delete, it must be called (at least) once for each
  2485         -** unique token in the document with an iCol value less than zero. The iPos
  2486         -** argument is ignored for a delete.
  2487         -*/
  2488         -void sqlite3Fts5IndexWrite(
  2489         -  Fts5Index *p,                   /* Index to write to */
  2490         -  int iCol,                       /* Column token appears in (-ve -> delete) */
  2491         -  int iPos,                       /* Position of token within column */
  2492         -  const char *pToken, int nToken  /* Token to add or remove to or from index */
  2493         -){
  2494         -  int i;                          /* Used to iterate through indexes */
  2495         -  Fts5Config *pConfig = p->pConfig;
  2496         -
  2497         -  /* If an error has already occured this call is a no-op. */
  2498         -  if( p->rc!=SQLITE_OK ) return;
  2499         -
  2500         -  /* Allocate hash tables if they have not already been allocated */
  2501         -  if( p->apHash==0 ){
  2502         -    int nHash = pConfig->nPrefix + 1;
  2503         -    p->apHash = (Fts5Hash**)fts5IdxMalloc(p, sizeof(Fts5Hash*) * nHash);
  2504         -    for(i=0; p->rc==SQLITE_OK && i<nHash; i++){
  2505         -      p->rc = sqlite3Fts5HashNew(&p->apHash[i], &p->nPendingData);
  2506         -    }
  2507         -  }
  2508         -
  2509         -  /* Add the new token to the main terms hash table. And to each of the
  2510         -  ** prefix hash tables that it is large enough for. */
  2511         -  fts5AddTermToHash(p, 0, iCol, iPos, pToken, nToken);
  2512         -  for(i=0; i<pConfig->nPrefix; i++){
  2513         -    if( nToken>=pConfig->aPrefix[i] ){
  2514         -      fts5AddTermToHash(p, i+1, iCol, iPos, pToken, pConfig->aPrefix[i]);
  2515         -    }
  2516         -  }
  2517         -}
  2518         -
  2519   2385   /*
  2520   2386   ** Allocate a new segment-id for the structure pStruct.
  2521   2387   **
  2522   2388   ** If an error has already occurred, this function is a no-op. 0 is 
  2523   2389   ** returned in this case.
  2524   2390   */
  2525   2391   static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
................................................................................
  3338   3204     /* Flush the terms and each prefix index to disk */
  3339   3205     for(i=0; i<=pConfig->nPrefix; i++){
  3340   3206       fts5FlushOneHash(p, i, &nLeaf);
  3341   3207     }
  3342   3208     p->nPendingData = 0;
  3343   3209   }
  3344   3210   
  3345         -/*
  3346         -** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain
  3347         -** to the document with rowid iRowid.
  3348         -*/
  3349         -void sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){
  3350         -  if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){
  3351         -    fts5IndexFlush(p);
  3352         -  }
  3353         -  p->iWriteRowid = iRowid;
  3354         -}
  3355         -
  3356         -/*
  3357         -** Commit data to disk.
  3358         -*/
  3359         -int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
  3360         -  fts5IndexFlush(p);
  3361         -  if( bCommit ) fts5CloseReader(p);
  3362         -  return p->rc;
  3363         -}
  3364         -
  3365         -/*
  3366         -** Discard any data stored in the in-memory hash tables. Do not write it
  3367         -** to the database. Additionally, assume that the contents of the %_data
  3368         -** table may have changed on disk. So any in-memory caches of %_data 
  3369         -** records must be invalidated.
  3370         -*/
  3371         -int sqlite3Fts5IndexRollback(Fts5Index *p){
  3372         -  fts5CloseReader(p);
  3373         -  fts5IndexDiscardData(p);
  3374         -  return SQLITE_OK;
  3375         -}
  3376         -
  3377         -/*
  3378         -** Open a new Fts5Index handle. If the bCreate argument is true, create
  3379         -** and initialize the underlying %_data table.
  3380         -**
  3381         -** If successful, set *pp to point to the new object and return SQLITE_OK.
  3382         -** Otherwise, set *pp to NULL and return an SQLite error code.
  3383         -*/
  3384         -int sqlite3Fts5IndexOpen(
  3385         -  Fts5Config *pConfig, 
  3386         -  int bCreate, 
  3387         -  Fts5Index **pp,
  3388         -  char **pzErr
  3389         -){
  3390         -  int rc = SQLITE_OK;
  3391         -  Fts5Index *p;                   /* New object */
  3392         -
  3393         -  *pp = p = (Fts5Index*)sqlite3_malloc(sizeof(Fts5Index));
  3394         -  if( !p ) return SQLITE_NOMEM;
  3395         -
  3396         -  memset(p, 0, sizeof(Fts5Index));
  3397         -  p->pConfig = pConfig;
  3398         -  p->nCrisisMerge = FTS5_CRISIS_MERGE;
  3399         -  p->nWorkUnit = FTS5_WORK_UNIT;
  3400         -  p->nMaxPendingData = 1024*1024;
  3401         -  p->zDataTbl = sqlite3_mprintf("%s_data", pConfig->zName);
  3402         -  if( p->zDataTbl==0 ){
  3403         -    rc = SQLITE_NOMEM;
  3404         -  }else if( bCreate ){
  3405         -    int i;
  3406         -    Fts5Structure s;
  3407         -    rc = sqlite3Fts5CreateTable(
  3408         -        pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr
  3409         -    );
  3410         -    if( rc==SQLITE_OK ){
  3411         -      memset(&s, 0, sizeof(Fts5Structure));
  3412         -      for(i=0; i<pConfig->nPrefix+1; i++){
  3413         -        fts5StructureWrite(p, i, &s);
  3414         -      }
  3415         -      rc = p->rc;
  3416         -    }
  3417         -    sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0);
  3418         -  }
  3419         -
  3420         -  if( rc ){
  3421         -    sqlite3Fts5IndexClose(p, 0);
  3422         -    *pp = 0;
  3423         -  }
  3424         -  return rc;
  3425         -}
  3426         -
  3427         -/*
  3428         -** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen().
  3429         -*/
  3430         -int sqlite3Fts5IndexClose(Fts5Index *p, int bDestroy){
  3431         -  int rc = SQLITE_OK;
  3432         -  if( bDestroy ){
  3433         -    rc = sqlite3Fts5DropTable(p->pConfig, "data");
  3434         -  }
  3435         -  assert( p->pReader==0 );
  3436         -  sqlite3_finalize(p->pWriter);
  3437         -  sqlite3_finalize(p->pDeleter);
  3438         -  if( p->apHash ){
  3439         -    int i;
  3440         -    for(i=0; i<=p->pConfig->nPrefix; i++){
  3441         -      sqlite3Fts5HashFree(p->apHash[i]);
  3442         -    }
  3443         -    sqlite3_free(p->apHash);
  3444         -  }
  3445         -  sqlite3_free(p->zDataTbl);
  3446         -  sqlite3_free(p);
  3447         -  return rc;
  3448         -}
  3449         -
  3450   3211   /*
  3451   3212   ** Return a simple checksum value based on the arguments.
  3452   3213   */
  3453   3214   static u64 fts5IndexEntryCksum(
  3454   3215     i64 iRowid, 
  3455   3216     int iCol, 
  3456   3217     int iPos, 
................................................................................
  3458   3219     int nTerm
  3459   3220   ){
  3460   3221     int i;
  3461   3222     u64 ret = iRowid;
  3462   3223     ret += (ret<<3) + iCol;
  3463   3224     ret += (ret<<3) + iPos;
  3464   3225     for(i=0; i<nTerm; i++) ret += (ret<<3) + pTerm[i];
  3465         -  return ret;
  3466         -}
  3467         -
  3468         -/*
  3469         -** Calculate and return a checksum that is the XOR of the index entry
  3470         -** checksum of all entries that would be generated by the token specified
  3471         -** by the final 5 arguments.
  3472         -*/
  3473         -u64 sqlite3Fts5IndexCksum(
  3474         -  Fts5Config *pConfig,            /* Configuration object */
  3475         -  i64 iRowid,                     /* Document term appears in */
  3476         -  int iCol,                       /* Column term appears in */
  3477         -  int iPos,                       /* Position term appears in */
  3478         -  const char *pTerm, int nTerm    /* Term at iPos */
  3479         -){
  3480         -  u64 ret = 0;                    /* Return value */
  3481         -  int iIdx;                       /* For iterating through indexes */
  3482         -
  3483         -  for(iIdx=0; iIdx<=pConfig->nPrefix; iIdx++){
  3484         -    int n = ((iIdx==pConfig->nPrefix) ? nTerm : pConfig->aPrefix[iIdx]);
  3485         -    if( n<=nTerm ){
  3486         -      ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, pTerm, n);
  3487         -    }
  3488         -  }
  3489         -
  3490   3226     return ret;
  3491   3227   }
  3492   3228   
  3493   3229   static void fts5BtreeIterInit(
  3494   3230     Fts5Index *p, 
  3495   3231     int iIdx,
  3496   3232     Fts5StructureSegment *pSeg, 
................................................................................
  3731   3467     if( p->rc==SQLITE_OK && iter.iLeaf!=pSeg->pgnoLast ){
  3732   3468       p->rc = FTS5_CORRUPT;
  3733   3469     }
  3734   3470   
  3735   3471     fts5BtreeIterFree(&iter);
  3736   3472   }
  3737   3473   
  3738         -/*
  3739         -** Run internal checks to ensure that the FTS index (a) is internally 
  3740         -** consistent and (b) contains entries for which the XOR of the checksums
  3741         -** as calculated by fts5IndexEntryCksum() is cksum.
  3742         -**
  3743         -** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
  3744         -** checksum does not match. Return SQLITE_OK if all checks pass without
  3745         -** error, or some other SQLite error code if another error (e.g. OOM)
  3746         -** occurs.
  3747         -*/
  3748         -int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
  3749         -  Fts5Config *pConfig = p->pConfig;
  3750         -  int iIdx;                       /* Used to iterate through indexes */
  3751         -  int rc;                         /* Return code */
  3752         -  u64 cksum2 = 0;                 /* Checksum based on contents of indexes */
  3753         -
  3754         -  /* Check that the checksum of the index matches the argument checksum */
  3755         -  for(iIdx=0; iIdx<=pConfig->nPrefix; iIdx++){
  3756         -    Fts5MultiSegIter *pIter;
  3757         -    Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
  3758         -    for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, -1, 0, &pIter);
  3759         -        fts5MultiIterEof(p, pIter)==0;
  3760         -        fts5MultiIterNext(p, pIter, 0, 0)
  3761         -    ){
  3762         -      Fts5PosIter sPos;           /* Used to iterate through position list */
  3763         -      int n;                      /* Size of term in bytes */
  3764         -      i64 iRowid = fts5MultiIterRowid(pIter);
  3765         -      char *z = (char*)fts5MultiIterTerm(pIter, &n);
  3766         -
  3767         -      for(fts5PosIterInit(p, pIter, &sPos);
  3768         -          fts5PosIterEof(p, &sPos)==0;
  3769         -          fts5PosIterNext(p, &sPos)
  3770         -      ){
  3771         -        cksum2 ^= fts5IndexEntryCksum(iRowid, sPos.iCol, sPos.iPos, z, n);
  3772         -#if 0
  3773         -        fprintf(stdout, "rowid=%d ", (int)iRowid);
  3774         -        fprintf(stdout, "term=%.*s ", n, z);
  3775         -        fprintf(stdout, "col=%d ", sPos.iCol);
  3776         -        fprintf(stdout, "off=%d\n", sPos.iPos);
  3777         -        fflush(stdout);
  3778         -#endif
  3779         -      }
  3780         -    }
  3781         -    fts5MultiIterFree(p, pIter);
  3782         -    fts5StructureRelease(pStruct);
  3783         -  }
  3784         -  rc = p->rc;
  3785         -  if( rc==SQLITE_OK && cksum!=cksum2 ) rc = FTS5_CORRUPT;
  3786         -
  3787         -  /* Check that the internal nodes of each segment match the leaves */
  3788         -  for(iIdx=0; rc==SQLITE_OK && iIdx<=pConfig->nPrefix; iIdx++){
  3789         -    Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
  3790         -    if( pStruct ){
  3791         -      int iLvl, iSeg;
  3792         -      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
  3793         -        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
  3794         -          Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
  3795         -          fts5IndexIntegrityCheckSegment(p, iIdx, pSeg);
  3796         -        }
  3797         -      }
  3798         -    }
  3799         -    fts5StructureRelease(pStruct);
  3800         -    rc = p->rc;
  3801         -  }
  3802         -
  3803         -  return rc;
  3804         -}
  3805         -
  3806         -/*
  3807         -** This is part of the fts5_decode() debugging aid.
  3808         -**
  3809         -** Arguments pBlob/nBlob contain a serialized Fts5Structure object. This
  3810         -** function appends a human-readable representation of the same object
  3811         -** to the buffer passed as the second argument. 
  3812         -*/
  3813         -static void fts5DecodeStructure(
  3814         -  int *pRc,                       /* IN/OUT: error code */
  3815         -  Fts5Buffer *pBuf,
  3816         -  const u8 *pBlob, int nBlob
  3817         -){
  3818         -  int rc;                         /* Return code */
  3819         -  Fts5Structure *p = 0;           /* Decoded structure object */
  3820         -
  3821         -  rc = fts5StructureDecode(pBlob, nBlob, 0, &p);
  3822         -  if( rc!=SQLITE_OK ){
  3823         -    *pRc = rc;
  3824         -    return;
  3825         -  }
  3826         -
  3827         -  fts5DebugStructure(pRc, pBuf, p);
  3828         -  fts5StructureRelease(p);
  3829         -}
  3830         -
  3831         -/*
  3832         -** Buffer (a/n) is assumed to contain a list of serialized varints. Read
  3833         -** each varint and append its string representation to buffer pBuf. Return
  3834         -** after either the input buffer is exhausted or a 0 value is read.
  3835         -**
  3836         -** The return value is the number of bytes read from the input buffer.
  3837         -*/
  3838         -static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
  3839         -  int iOff = 0;
  3840         -  while( iOff<n ){
  3841         -    int iVal;
  3842         -    iOff += getVarint32(&a[iOff], iVal);
  3843         -    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %d", iVal);
  3844         -  }
  3845         -  return iOff;
  3846         -}
  3847         -
  3848         -/*
  3849         -** The start of buffer (a/n) contains the start of a doclist. The doclist
  3850         -** may or may not finish within the buffer. This function appends a text
  3851         -** representation of the part of the doclist that is present to buffer
  3852         -** pBuf. 
  3853         -**
  3854         -** The return value is the number of bytes read from the input buffer.
  3855         -*/
  3856         -static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
  3857         -  i64 iDocid;
  3858         -  int iOff = 0;
  3859         -
  3860         -  if( iOff<n ){
  3861         -    iOff += sqlite3GetVarint(&a[iOff], (u64*)&iDocid);
  3862         -    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
  3863         -  }
  3864         -  while( iOff<n ){
  3865         -    int nPos;
  3866         -    iOff += getVarint32(&a[iOff], nPos);
  3867         -    iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos));
  3868         -    if( iOff<n ){
  3869         -      i64 iDelta;
  3870         -      iOff += sqlite3GetVarint(&a[iOff], (u64*)&iDelta);
  3871         -      if( iDelta==0 ) return iOff;
  3872         -      iDocid -= iDelta;
  3873         -      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
  3874         -    }
  3875         -  }
  3876         -
  3877         -  return iOff;
  3878         -}
  3879         -
  3880         -/*
  3881         -** The implementation of user-defined scalar function fts5_decode().
  3882         -*/
  3883         -static void fts5DecodeFunction(
  3884         -  sqlite3_context *pCtx,          /* Function call context */
  3885         -  int nArg,                       /* Number of args (always 2) */
  3886         -  sqlite3_value **apVal           /* Function arguments */
  3887         -){
  3888         -  i64 iRowid;                     /* Rowid for record being decoded */
  3889         -  int iIdx,iSegid,iHeight,iPgno;  /* Rowid components */
  3890         -  const u8 *a; int n;             /* Record to decode */
  3891         -  Fts5Buffer s;                   /* Build up text to return here */
  3892         -  int rc = SQLITE_OK;             /* Return code */
  3893         -
  3894         -  assert( nArg==2 );
  3895         -  memset(&s, 0, sizeof(Fts5Buffer));
  3896         -  iRowid = sqlite3_value_int64(apVal[0]);
  3897         -  n = sqlite3_value_bytes(apVal[1]);
  3898         -  a = sqlite3_value_blob(apVal[1]);
  3899         -  fts5DecodeRowid(iRowid, &iIdx, &iSegid, &iHeight, &iPgno);
  3900         -
  3901         -  fts5DebugRowid(&rc, &s, iRowid);
  3902         -  if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){
  3903         -    int i = 0;
  3904         -    i64 iPrev;
  3905         -    if( n>0 ){
  3906         -      i = getVarint(&a[i], (u64*)&iPrev);
  3907         -      sqlite3Fts5BufferAppendPrintf(&rc, &s, " %lld", iPrev);
  3908         -    }
  3909         -    while( i<n ){
  3910         -      i64 iVal;
  3911         -      i += getVarint(&a[i], (u64*)&iVal);
  3912         -      if( iVal==0 ){
  3913         -        sqlite3Fts5BufferAppendPrintf(&rc, &s, " x");
  3914         -      }else{
  3915         -        iPrev = iPrev - iVal;
  3916         -        sqlite3Fts5BufferAppendPrintf(&rc, &s, " %lld", iPrev);
  3917         -      }
  3918         -    }
  3919         -
  3920         -  }else
  3921         -  if( iSegid==0 ){
  3922         -    if( iRowid==FTS5_AVERAGES_ROWID ){
  3923         -      /* todo */
  3924         -    }else{
  3925         -      fts5DecodeStructure(&rc, &s, a, n);
  3926         -    }
  3927         -  }else{
  3928         -
  3929         -    Fts5Buffer term;
  3930         -    memset(&term, 0, sizeof(Fts5Buffer));
  3931         -
  3932         -    if( iHeight==0 ){
  3933         -      int iTermOff = 0;
  3934         -      int iRowidOff = 0;
  3935         -      int iOff;
  3936         -      int nKeep = 0;
  3937         -
  3938         -      iRowidOff = fts5GetU16(&a[0]);
  3939         -      iTermOff = fts5GetU16(&a[2]);
  3940         -
  3941         -      if( iRowidOff ){
  3942         -        iOff = iRowidOff;
  3943         -      }else if( iTermOff ){
  3944         -        iOff = iTermOff;
  3945         -      }else{
  3946         -        iOff = n;
  3947         -      }
  3948         -      fts5DecodePoslist(&rc, &s, &a[4], iOff-4);
  3949         -
  3950         -
  3951         -      assert( iRowidOff==0 || iOff==iRowidOff );
  3952         -      if( iRowidOff ){
  3953         -        iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff);
  3954         -      }
  3955         -
  3956         -      assert( iTermOff==0 || iOff==iTermOff );
  3957         -      while( iOff<n ){
  3958         -        int nByte;
  3959         -        iOff += getVarint32(&a[iOff], nByte);
  3960         -        term.n= nKeep;
  3961         -        fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff]);
  3962         -        iOff += nByte;
  3963         -
  3964         -        sqlite3Fts5BufferAppendPrintf(
  3965         -            &rc, &s, " term=%.*s", term.n, (const char*)term.p
  3966         -        );
  3967         -        iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff);
  3968         -        if( iOff<n ){
  3969         -          iOff += getVarint32(&a[iOff], nKeep);
  3970         -        }
  3971         -      }
  3972         -      fts5BufferFree(&term);
  3973         -    }else{
  3974         -      Fts5NodeIter ss;
  3975         -      for(fts5NodeIterInit(a, n, &ss); ss.aData; fts5NodeIterNext(&rc, &ss)){
  3976         -        if( ss.term.n==0 ){
  3977         -          sqlite3Fts5BufferAppendPrintf(&rc, &s, " left=%d", ss.iChild);
  3978         -        }else{
  3979         -          sqlite3Fts5BufferAppendPrintf(&rc,&s, " \"%.*s\"", 
  3980         -              ss.term.n, ss.term.p
  3981         -          );
  3982         -        }
  3983         -        if( ss.nEmpty ){
  3984         -          sqlite3Fts5BufferAppendPrintf(&rc, &s, " empty=%d%s", ss.nEmpty,
  3985         -              ss.bDlidx ? "*" : ""
  3986         -          );
  3987         -        }
  3988         -      }
  3989         -      fts5NodeIterFree(&ss);
  3990         -    }
  3991         -  }
  3992         -  
  3993         -  if( rc==SQLITE_OK ){
  3994         -    sqlite3_result_text(pCtx, (const char*)s.p, s.n, SQLITE_TRANSIENT);
  3995         -  }else{
  3996         -    sqlite3_result_error_code(pCtx, rc);
  3997         -  }
  3998         -  fts5BufferFree(&s);
  3999         -}
  4000         -
  4001         -/*
  4002         -** This is called as part of registering the FTS5 module with database
  4003         -** connection db. It registers several user-defined scalar functions useful
  4004         -** with FTS5.
  4005         -**
  4006         -** If successful, SQLITE_OK is returned. If an error occurs, some other
  4007         -** SQLite error code is returned instead.
  4008         -*/
  4009         -int sqlite3Fts5IndexInit(sqlite3 *db){
  4010         -  int rc = sqlite3_create_function(
  4011         -      db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0
  4012         -  );
  4013         -  return rc;
  4014         -}
  4015         -
  4016   3474   /*
  4017   3475   ** Iterator pMulti currently points to a valid entry (not EOF). This
  4018   3476   ** function appends a copy of the position-list of the entry pMulti 
  4019   3477   ** currently points to to buffer pBuf.
  4020   3478   **
  4021   3479   ** If an error occurs, an error code is left in p->rc. It is assumed
  4022   3480   ** no error has already occurred when this function is called.
................................................................................
  4254   3712         fts5DoclistIterInit(&doclist, bAsc, pIter->pDoclist);
  4255   3713       }
  4256   3714     }
  4257   3715   
  4258   3716     fts5StructureRelease(pStruct);
  4259   3717     sqlite3_free(aBuf);
  4260   3718   }
         3719  +
         3720  +static int fts5IndexReturn(Fts5Index *p){
         3721  +  int rc = p->rc;
         3722  +  p->rc = SQLITE_OK;
         3723  +  return rc;
         3724  +}
         3725  +
         3726  +/*
         3727  +** Run internal checks to ensure that the FTS index (a) is internally 
         3728  +** consistent and (b) contains entries for which the XOR of the checksums
         3729  +** as calculated by fts5IndexEntryCksum() is cksum.
         3730  +**
         3731  +** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
         3732  +** checksum does not match. Return SQLITE_OK if all checks pass without
         3733  +** error, or some other SQLite error code if another error (e.g. OOM)
         3734  +** occurs.
         3735  +*/
         3736  +int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
         3737  +  Fts5Config *pConfig = p->pConfig;
         3738  +  int iIdx;                       /* Used to iterate through indexes */
         3739  +  int rc;                         /* Return code */
         3740  +  u64 cksum2 = 0;                 /* Checksum based on contents of indexes */
         3741  +
         3742  +  /* Check that the checksum of the index matches the argument checksum */
         3743  +  for(iIdx=0; iIdx<=pConfig->nPrefix; iIdx++){
         3744  +    Fts5MultiSegIter *pIter;
         3745  +    Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
         3746  +    for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, -1, 0, &pIter);
         3747  +        fts5MultiIterEof(p, pIter)==0;
         3748  +        fts5MultiIterNext(p, pIter, 0, 0)
         3749  +    ){
         3750  +      Fts5PosIter sPos;           /* Used to iterate through position list */
         3751  +      int n;                      /* Size of term in bytes */
         3752  +      i64 iRowid = fts5MultiIterRowid(pIter);
         3753  +      char *z = (char*)fts5MultiIterTerm(pIter, &n);
         3754  +
         3755  +      for(fts5PosIterInit(p, pIter, &sPos);
         3756  +          fts5PosIterEof(p, &sPos)==0;
         3757  +          fts5PosIterNext(p, &sPos)
         3758  +      ){
         3759  +        cksum2 ^= fts5IndexEntryCksum(iRowid, sPos.iCol, sPos.iPos, z, n);
         3760  +#if 0
         3761  +        fprintf(stdout, "rowid=%d ", (int)iRowid);
         3762  +        fprintf(stdout, "term=%.*s ", n, z);
         3763  +        fprintf(stdout, "col=%d ", sPos.iCol);
         3764  +        fprintf(stdout, "off=%d\n", sPos.iPos);
         3765  +        fflush(stdout);
         3766  +#endif
         3767  +      }
         3768  +    }
         3769  +    fts5MultiIterFree(p, pIter);
         3770  +    fts5StructureRelease(pStruct);
         3771  +  }
         3772  +  rc = p->rc;
         3773  +  if( rc==SQLITE_OK && cksum!=cksum2 ) rc = FTS5_CORRUPT;
         3774  +
         3775  +  /* Check that the internal nodes of each segment match the leaves */
         3776  +  for(iIdx=0; rc==SQLITE_OK && iIdx<=pConfig->nPrefix; iIdx++){
         3777  +    Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
         3778  +    if( pStruct ){
         3779  +      int iLvl, iSeg;
         3780  +      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
         3781  +        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
         3782  +          Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
         3783  +          fts5IndexIntegrityCheckSegment(p, iIdx, pSeg);
         3784  +        }
         3785  +      }
         3786  +    }
         3787  +    fts5StructureRelease(pStruct);
         3788  +    rc = p->rc;
         3789  +  }
         3790  +
         3791  +  return rc;
         3792  +}
         3793  +
         3794  +
         3795  +/*
         3796  +** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain
         3797  +** to the document with rowid iRowid.
         3798  +*/
         3799  +int sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){
         3800  +  assert( p->rc==SQLITE_OK );
         3801  +  if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){
         3802  +    fts5IndexFlush(p);
         3803  +  }
         3804  +  p->iWriteRowid = iRowid;
         3805  +  return fts5IndexReturn(p);
         3806  +}
         3807  +
         3808  +/*
         3809  +** Commit data to disk.
         3810  +*/
         3811  +int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
         3812  +  assert( p->rc==SQLITE_OK );
         3813  +  fts5IndexFlush(p);
         3814  +  if( bCommit ) fts5CloseReader(p);
         3815  +  return fts5IndexReturn(p);
         3816  +}
         3817  +
         3818  +/*
         3819  +** Discard any data stored in the in-memory hash tables. Do not write it
         3820  +** to the database. Additionally, assume that the contents of the %_data
         3821  +** table may have changed on disk. So any in-memory caches of %_data 
         3822  +** records must be invalidated.
         3823  +*/
         3824  +int sqlite3Fts5IndexRollback(Fts5Index *p){
         3825  +  fts5CloseReader(p);
         3826  +  fts5IndexDiscardData(p);
         3827  +  assert( p->rc==SQLITE_OK );
         3828  +  return SQLITE_OK;
         3829  +}
         3830  +
         3831  +/*
         3832  +** Open a new Fts5Index handle. If the bCreate argument is true, create
         3833  +** and initialize the underlying %_data table.
         3834  +**
         3835  +** If successful, set *pp to point to the new object and return SQLITE_OK.
         3836  +** Otherwise, set *pp to NULL and return an SQLite error code.
         3837  +*/
         3838  +int sqlite3Fts5IndexOpen(
         3839  +  Fts5Config *pConfig, 
         3840  +  int bCreate, 
         3841  +  Fts5Index **pp,
         3842  +  char **pzErr
         3843  +){
         3844  +  int rc = SQLITE_OK;
         3845  +  Fts5Index *p;                   /* New object */
         3846  +
         3847  +  *pp = p = (Fts5Index*)sqlite3_malloc(sizeof(Fts5Index));
         3848  +  if( !p ) return SQLITE_NOMEM;
         3849  +
         3850  +  memset(p, 0, sizeof(Fts5Index));
         3851  +  p->pConfig = pConfig;
         3852  +  p->nCrisisMerge = FTS5_CRISIS_MERGE;
         3853  +  p->nWorkUnit = FTS5_WORK_UNIT;
         3854  +  p->nMaxPendingData = 1024*1024;
         3855  +  p->zDataTbl = sqlite3_mprintf("%s_data", pConfig->zName);
         3856  +  if( p->zDataTbl==0 ){
         3857  +    rc = SQLITE_NOMEM;
         3858  +  }else if( bCreate ){
         3859  +    int i;
         3860  +    Fts5Structure s;
         3861  +    rc = sqlite3Fts5CreateTable(
         3862  +        pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr
         3863  +    );
         3864  +    if( rc==SQLITE_OK ){
         3865  +      memset(&s, 0, sizeof(Fts5Structure));
         3866  +      for(i=0; i<pConfig->nPrefix+1; i++){
         3867  +        fts5StructureWrite(p, i, &s);
         3868  +      }
         3869  +      rc = p->rc;
         3870  +    }
         3871  +    if( rc==SQLITE_OK ){
         3872  +      rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0);
         3873  +    }
         3874  +  }
         3875  +
         3876  +  assert( p->rc==SQLITE_OK || rc!=SQLITE_OK );
         3877  +  if( rc ){
         3878  +    sqlite3Fts5IndexClose(p, 0);
         3879  +    *pp = 0;
         3880  +  }
         3881  +  return rc;
         3882  +}
         3883  +
         3884  +/*
         3885  +** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen().
         3886  +*/
         3887  +int sqlite3Fts5IndexClose(Fts5Index *p, int bDestroy){
         3888  +  int rc = SQLITE_OK;
         3889  +  if( p ){
         3890  +    if( bDestroy ){
         3891  +      rc = sqlite3Fts5DropTable(p->pConfig, "data");
         3892  +    }
         3893  +    assert( p->pReader==0 );
         3894  +    sqlite3_finalize(p->pWriter);
         3895  +    sqlite3_finalize(p->pDeleter);
         3896  +    if( p->apHash ){
         3897  +      int i;
         3898  +      for(i=0; i<=p->pConfig->nPrefix; i++){
         3899  +        sqlite3Fts5HashFree(p->apHash[i]);
         3900  +      }
         3901  +      sqlite3_free(p->apHash);
         3902  +    }
         3903  +    sqlite3_free(p->zDataTbl);
         3904  +    sqlite3_free(p);
         3905  +  }
         3906  +  return rc;
         3907  +}
         3908  +
         3909  +/*
         3910  +** Calculate and return a checksum that is the XOR of the index entry
         3911  +** checksum of all entries that would be generated by the token specified
         3912  +** by the final 5 arguments.
         3913  +*/
         3914  +u64 sqlite3Fts5IndexCksum(
         3915  +  Fts5Config *pConfig,            /* Configuration object */
         3916  +  i64 iRowid,                     /* Document term appears in */
         3917  +  int iCol,                       /* Column term appears in */
         3918  +  int iPos,                       /* Position term appears in */
         3919  +  const char *pTerm, int nTerm    /* Term at iPos */
         3920  +){
         3921  +  u64 ret = 0;                    /* Return value */
         3922  +  int iIdx;                       /* For iterating through indexes */
         3923  +
         3924  +  for(iIdx=0; iIdx<=pConfig->nPrefix; iIdx++){
         3925  +    int n = ((iIdx==pConfig->nPrefix) ? nTerm : pConfig->aPrefix[iIdx]);
         3926  +    if( n<=nTerm ){
         3927  +      ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, pTerm, n);
         3928  +    }
         3929  +  }
         3930  +
         3931  +  return ret;
         3932  +}
         3933  +
         3934  +/*
         3935  +** Insert or remove data to or from the index. Each time a document is 
         3936  +** added to or removed from the index, this function is called one or more
         3937  +** times.
         3938  +**
         3939  +** For an insert, it must be called once for each token in the new document.
         3940  +** If the operation is a delete, it must be called (at least) once for each
         3941  +** unique token in the document with an iCol value less than zero. The iPos
         3942  +** argument is ignored for a delete.
         3943  +*/
         3944  +int sqlite3Fts5IndexWrite(
         3945  +  Fts5Index *p,                   /* Index to write to */
         3946  +  int iCol,                       /* Column token appears in (-ve -> delete) */
         3947  +  int iPos,                       /* Position of token within column */
         3948  +  const char *pToken, int nToken  /* Token to add or remove to or from index */
         3949  +){
         3950  +  int i;                          /* Used to iterate through indexes */
         3951  +  Fts5Config *pConfig = p->pConfig;
         3952  +  assert( p->rc==SQLITE_OK );
         3953  +
         3954  +  /* Allocate hash tables if they have not already been allocated */
         3955  +  if( p->apHash==0 ){
         3956  +    int nHash = pConfig->nPrefix + 1;
         3957  +    p->apHash = (Fts5Hash**)fts5IdxMalloc(p, sizeof(Fts5Hash*) * nHash);
         3958  +    for(i=0; p->rc==SQLITE_OK && i<nHash; i++){
         3959  +      p->rc = sqlite3Fts5HashNew(&p->apHash[i], &p->nPendingData);
         3960  +    }
         3961  +  }
         3962  +
         3963  +  /* Add the new token to the main terms hash table. And to each of the
         3964  +  ** prefix hash tables that it is large enough for. */
         3965  +  fts5AddTermToHash(p, 0, iCol, iPos, pToken, nToken);
         3966  +  for(i=0; i<pConfig->nPrefix; i++){
         3967  +    if( nToken>=pConfig->aPrefix[i] ){
         3968  +      fts5AddTermToHash(p, i+1, iCol, iPos, pToken, pConfig->aPrefix[i]);
         3969  +    }
         3970  +  }
         3971  +
         3972  +  return fts5IndexReturn(p);
         3973  +}
  4261   3974   
  4262   3975   /*
  4263   3976   ** Open a new iterator to iterate though all docids that match the 
  4264   3977   ** specified token or token prefix.
  4265   3978   */
  4266         -Fts5IndexIter *sqlite3Fts5IndexQuery(
         3979  +int sqlite3Fts5IndexQuery(
  4267   3980     Fts5Index *p,                   /* FTS index to query */
  4268   3981     const char *pToken, int nToken, /* Token (or prefix) to query for */
  4269         -  int flags                       /* Mask of FTS5INDEX_QUERY_X flags */
         3982  +  int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
         3983  +  Fts5IndexIter **ppIter          /* OUT: New iterator object */
  4270   3984   ){
  4271   3985     Fts5IndexIter *pRet;
  4272   3986     int iIdx = 0;
  4273   3987   
  4274   3988     if( flags & FTS5INDEX_QUERY_PREFIX ){
  4275   3989       Fts5Config *pConfig = p->pConfig;
  4276   3990       for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){
................................................................................
  4277   3991         if( pConfig->aPrefix[iIdx-1]==nToken ) break;
  4278   3992       }
  4279   3993       if( iIdx>pConfig->nPrefix ){
  4280   3994         iIdx = -1;
  4281   3995       }
  4282   3996     }
  4283   3997   
  4284         -  pRet = (Fts5IndexIter*)sqlite3_malloc(sizeof(Fts5IndexIter));
         3998  +  pRet = (Fts5IndexIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5IndexIter));
  4285   3999     if( pRet ){
  4286   4000       memset(pRet, 0, sizeof(Fts5IndexIter));
  4287   4001   
  4288   4002       pRet->pIndex = p;
  4289   4003       if( iIdx>=0 ){
  4290   4004         pRet->pStruct = fts5StructureRead(p, iIdx);
  4291   4005         if( pRet->pStruct ){
................................................................................
  4299   4013       }
  4300   4014     }
  4301   4015   
  4302   4016     if( p->rc ){
  4303   4017       sqlite3Fts5IterClose(pRet);
  4304   4018       pRet = 0;
  4305   4019     }
  4306         -  return pRet;
         4020  +  *ppIter = pRet;
         4021  +  return fts5IndexReturn(p);
  4307   4022   }
  4308   4023   
  4309   4024   /*
  4310   4025   ** Return true if the iterator passed as the only argument is at EOF.
  4311   4026   */
  4312   4027   int sqlite3Fts5IterEof(Fts5IndexIter *pIter){
         4028  +  assert( pIter->pIndex->rc==SQLITE_OK );
  4313   4029     if( pIter->pDoclist ){ 
  4314   4030       return pIter->pDoclist->aPoslist==0; 
  4315   4031     }else{
  4316   4032       return fts5MultiIterEof(pIter->pIndex, pIter->pMulti);
  4317   4033     }
  4318   4034   }
  4319   4035   
  4320   4036   /*
  4321   4037   ** Move to the next matching rowid. 
  4322   4038   */
  4323         -void sqlite3Fts5IterNext(Fts5IndexIter *pIter){
         4039  +int sqlite3Fts5IterNext(Fts5IndexIter *pIter){
         4040  +  assert( pIter->pIndex->rc==SQLITE_OK );
  4324   4041     if( pIter->pDoclist ){
  4325   4042       fts5DoclistIterNext(pIter->pDoclist);
  4326   4043     }else{
  4327   4044       fts5BufferZero(&pIter->poslist);
  4328   4045       fts5MultiIterNext(pIter->pIndex, pIter->pMulti, 0, 0);
  4329   4046     }
         4047  +  return fts5IndexReturn(pIter->pIndex);
  4330   4048   }
  4331   4049   
  4332   4050   /*
  4333   4051   ** Move to the next matching rowid that occurs at or after iMatch. The
  4334   4052   ** definition of "at or after" depends on whether this iterator iterates
  4335   4053   ** in ascending or descending rowid order.
  4336   4054   */
  4337         -void sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){
         4055  +int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){
  4338   4056     if( pIter->pDoclist ){
  4339   4057       assert( 0 );
  4340   4058       /* fts5DoclistIterNextFrom(pIter->pDoclist, iMatch); */
  4341   4059     }else{
  4342   4060       fts5MultiIterNextFrom(pIter->pIndex, pIter->pMulti, iMatch);
  4343   4061     }
         4062  +  return fts5IndexReturn(pIter->pIndex);
  4344   4063   }
  4345   4064   
  4346   4065   /*
  4347   4066   ** Return the current rowid.
  4348   4067   */
  4349   4068   i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){
  4350   4069     if( pIter->pDoclist ){
................................................................................
  4359   4078   ** Return a pointer to a buffer containing a copy of the position list for
  4360   4079   ** the current entry. Output variable *pn is set to the size of the buffer 
  4361   4080   ** in bytes before returning.
  4362   4081   **
  4363   4082   ** The returned buffer does not include the 0x00 terminator byte stored on
  4364   4083   ** disk.
  4365   4084   */
  4366         -const u8 *sqlite3Fts5IterPoslist(Fts5IndexIter *pIter, int *pn){
         4085  +int sqlite3Fts5IterPoslist(Fts5IndexIter *pIter, const u8 **pp, int *pn){
         4086  +  assert( pIter->pIndex->rc==SQLITE_OK );
  4367   4087     if( pIter->pDoclist ){
  4368   4088       *pn = pIter->pDoclist->nPoslist;
  4369         -    return pIter->pDoclist->aPoslist;
         4089  +    *pp = pIter->pDoclist->aPoslist;
  4370   4090     }else{
  4371   4091       Fts5Index *p = pIter->pIndex;
  4372   4092       fts5BufferZero(&pIter->poslist);
  4373   4093       fts5MultiIterPoslist(p, pIter->pMulti, 0, &pIter->poslist);
  4374         -    assert( p->rc==SQLITE_OK );
  4375         -    if( p->rc ) return 0;
  4376   4094       *pn = pIter->poslist.n;
  4377         -    return pIter->poslist.p;
         4095  +    *pp = pIter->poslist.p;
  4378   4096     }
         4097  +  return fts5IndexReturn(pIter->pIndex);
  4379   4098   }
  4380   4099   
  4381   4100   /*
  4382   4101   ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery().
  4383   4102   */
  4384   4103   void sqlite3Fts5IterClose(Fts5IndexIter *pIter){
  4385   4104     if( pIter ){
................................................................................
  4398   4117   
  4399   4118   /*
  4400   4119   ** Read the "averages" record into the buffer supplied as the second 
  4401   4120   ** argument. Return SQLITE_OK if successful, or an SQLite error code
  4402   4121   ** if an error occurs.
  4403   4122   */
  4404   4123   int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf){
         4124  +  assert( p->rc==SQLITE_OK );
  4405   4125     fts5DataReadOrBuffer(p, pBuf, FTS5_AVERAGES_ROWID);
  4406         -  return p->rc;
         4126  +  return fts5IndexReturn(p);
  4407   4127   }
  4408   4128   
  4409   4129   /*
  4410   4130   ** Replace the current "averages" record with the contents of the buffer 
  4411   4131   ** supplied as the second argument.
  4412   4132   */
  4413   4133   int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){
         4134  +  assert( p->rc==SQLITE_OK );
  4414   4135     fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData);
  4415         -  return p->rc;
         4136  +  return fts5IndexReturn(p);
  4416   4137   }
  4417   4138   
  4418   4139   /*
  4419   4140   ** Return the total number of blocks this module has read from the %_data
  4420   4141   ** table since it was created.
  4421   4142   */
  4422   4143   int sqlite3Fts5IndexReads(Fts5Index *p){
  4423   4144     return p->nRead;
  4424   4145   }
  4425   4146   
  4426   4147   /*
  4427         -** Set the 32-bit cookie value at the start of all structure records to
  4428         -** the value passed as the second argument.
         4148  +** Set the 32-bit cookie value stored at the start of all structure 
         4149  +** records to the value passed as the second argument.
  4429   4150   **
  4430   4151   ** Return SQLITE_OK if successful, or an SQLite error code if an error
  4431   4152   ** occurs.
  4432   4153   */
  4433   4154   int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){
  4434   4155     int rc = SQLITE_OK;
  4435   4156     Fts5Config *pConfig = p->pConfig;
  4436   4157     u8 aCookie[4];
  4437   4158     int i;
  4438   4159   
         4160  +  assert( p->rc==SQLITE_OK );
  4439   4161     sqlite3Fts5Put32(aCookie, iNew);
  4440   4162     for(i=0; rc==SQLITE_OK && i<=pConfig->nPrefix; i++){
  4441   4163       sqlite3_blob *pBlob = 0;
  4442   4164       i64 iRowid = FTS5_STRUCTURE_ROWID(i);
  4443   4165       rc = sqlite3_blob_open(
  4444   4166           pConfig->db, pConfig->zDb, p->zDataTbl, "block", iRowid, 1, &pBlob
  4445   4167       );
................................................................................
  4447   4169         sqlite3_blob_write(pBlob, aCookie, 4, 0);
  4448   4170         rc = sqlite3_blob_close(pBlob);
  4449   4171       }
  4450   4172     }
  4451   4173   
  4452   4174     return rc;
  4453   4175   }
         4176  +
         4177  +/*************************************************************************
         4178  +**************************************************************************
         4179  +** Below this point is the implementation of the fts5_decode() scalar
         4180  +** function only.
         4181  +*/
         4182  +
         4183  +/*
         4184  +** Decode a segment-data rowid from the %_data table. This function is
         4185  +** the opposite of macro FTS5_SEGMENT_ROWID().
         4186  +*/
         4187  +static void fts5DecodeRowid(
         4188  +  i64 iRowid,                     /* Rowid from %_data table */
         4189  +  int *piIdx,                     /* OUT: Index */
         4190  +  int *piSegid,                   /* OUT: Segment id */
         4191  +  int *piHeight,                  /* OUT: Height */
         4192  +  int *piPgno                     /* OUT: Page number */
         4193  +){
         4194  +  *piPgno = (int)(iRowid & (((i64)1 << FTS5_DATA_PAGE_B) - 1));
         4195  +  iRowid >>= FTS5_DATA_PAGE_B;
         4196  +
         4197  +  *piHeight = (int)(iRowid & (((i64)1 << FTS5_DATA_HEIGHT_B) - 1));
         4198  +  iRowid >>= FTS5_DATA_HEIGHT_B;
         4199  +
         4200  +  *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1));
         4201  +  iRowid >>= FTS5_DATA_ID_B;
         4202  +
         4203  +  *piIdx = (int)(iRowid & (((i64)1 << FTS5_DATA_IDX_B) - 1));
         4204  +}
         4205  +
         4206  +static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
         4207  +  int iIdx,iSegid,iHeight,iPgno;  /* Rowid compenents */
         4208  +  fts5DecodeRowid(iKey, &iIdx, &iSegid, &iHeight, &iPgno);
         4209  +
         4210  +  if( iSegid==0 ){
         4211  +    if( iKey==FTS5_AVERAGES_ROWID ){
         4212  +      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(averages) ");
         4213  +    }else{
         4214  +      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
         4215  +          "{structure idx=%d}", (int)(iKey-10)
         4216  +      );
         4217  +    }
         4218  +  }
         4219  +  else if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){
         4220  +    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(dlidx idx=%d segid=%d pgno=%d)",
         4221  +        iIdx, iSegid, iPgno
         4222  +    );
         4223  +  }else{
         4224  +    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(idx=%d segid=%d h=%d pgno=%d)",
         4225  +        iIdx, iSegid, iHeight, iPgno
         4226  +    );
         4227  +  }
         4228  +}
         4229  +
         4230  +static void fts5DebugStructure(
         4231  +  int *pRc,                       /* IN/OUT: error code */
         4232  +  Fts5Buffer *pBuf,
         4233  +  Fts5Structure *p
         4234  +){
         4235  +  int iLvl, iSeg;                 /* Iterate through levels, segments */
         4236  +
         4237  +  for(iLvl=0; iLvl<p->nLevel; iLvl++){
         4238  +    Fts5StructureLevel *pLvl = &p->aLevel[iLvl];
         4239  +    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
         4240  +        " {lvl=%d nMerge=%d", iLvl, pLvl->nMerge
         4241  +    );
         4242  +    for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
         4243  +      Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
         4244  +      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
         4245  +          " {id=%d h=%d leaves=%d..%d}", pSeg->iSegid, pSeg->nHeight, 
         4246  +          pSeg->pgnoFirst, pSeg->pgnoLast
         4247  +      );
         4248  +    }
         4249  +    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
         4250  +  }
         4251  +}
         4252  +
         4253  +/*
         4254  +** This is part of the fts5_decode() debugging aid.
         4255  +**
         4256  +** Arguments pBlob/nBlob contain a serialized Fts5Structure object. This
         4257  +** function appends a human-readable representation of the same object
         4258  +** to the buffer passed as the second argument. 
         4259  +*/
         4260  +static void fts5DecodeStructure(
         4261  +  int *pRc,                       /* IN/OUT: error code */
         4262  +  Fts5Buffer *pBuf,
         4263  +  const u8 *pBlob, int nBlob
         4264  +){
         4265  +  int rc;                         /* Return code */
         4266  +  Fts5Structure *p = 0;           /* Decoded structure object */
         4267  +
         4268  +  rc = fts5StructureDecode(pBlob, nBlob, 0, &p);
         4269  +  if( rc!=SQLITE_OK ){
         4270  +    *pRc = rc;
         4271  +    return;
         4272  +  }
         4273  +
         4274  +  fts5DebugStructure(pRc, pBuf, p);
         4275  +  fts5StructureRelease(p);
         4276  +}
         4277  +
         4278  +/*
         4279  +** Buffer (a/n) is assumed to contain a list of serialized varints. Read
         4280  +** each varint and append its string representation to buffer pBuf. Return
         4281  +** after either the input buffer is exhausted or a 0 value is read.
         4282  +**
         4283  +** The return value is the number of bytes read from the input buffer.
         4284  +*/
         4285  +static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
         4286  +  int iOff = 0;
         4287  +  while( iOff<n ){
         4288  +    int iVal;
         4289  +    iOff += getVarint32(&a[iOff], iVal);
         4290  +    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %d", iVal);
         4291  +  }
         4292  +  return iOff;
         4293  +}
         4294  +
         4295  +/*
         4296  +** The start of buffer (a/n) contains the start of a doclist. The doclist
         4297  +** may or may not finish within the buffer. This function appends a text
         4298  +** representation of the part of the doclist that is present to buffer
         4299  +** pBuf. 
         4300  +**
         4301  +** The return value is the number of bytes read from the input buffer.
         4302  +*/
         4303  +static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
         4304  +  i64 iDocid;
         4305  +  int iOff = 0;
         4306  +
         4307  +  if( iOff<n ){
         4308  +    iOff += sqlite3GetVarint(&a[iOff], (u64*)&iDocid);
         4309  +    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
         4310  +  }
         4311  +  while( iOff<n ){
         4312  +    int nPos;
         4313  +    iOff += getVarint32(&a[iOff], nPos);
         4314  +    iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos));
         4315  +    if( iOff<n ){
         4316  +      i64 iDelta;
         4317  +      iOff += sqlite3GetVarint(&a[iOff], (u64*)&iDelta);
         4318  +      if( iDelta==0 ) return iOff;
         4319  +      iDocid -= iDelta;
         4320  +      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
         4321  +    }
         4322  +  }
         4323  +
         4324  +  return iOff;
         4325  +}
         4326  +
         4327  +/*
         4328  +** The implementation of user-defined scalar function fts5_decode().
         4329  +*/
         4330  +static void fts5DecodeFunction(
         4331  +  sqlite3_context *pCtx,          /* Function call context */
         4332  +  int nArg,                       /* Number of args (always 2) */
         4333  +  sqlite3_value **apVal           /* Function arguments */
         4334  +){
         4335  +  i64 iRowid;                     /* Rowid for record being decoded */
         4336  +  int iIdx,iSegid,iHeight,iPgno;  /* Rowid components */
         4337  +  const u8 *a; int n;             /* Record to decode */
         4338  +  Fts5Buffer s;                   /* Build up text to return here */
         4339  +  int rc = SQLITE_OK;             /* Return code */
         4340  +
         4341  +  assert( nArg==2 );
         4342  +  memset(&s, 0, sizeof(Fts5Buffer));
         4343  +  iRowid = sqlite3_value_int64(apVal[0]);
         4344  +  n = sqlite3_value_bytes(apVal[1]);
         4345  +  a = sqlite3_value_blob(apVal[1]);
         4346  +  fts5DecodeRowid(iRowid, &iIdx, &iSegid, &iHeight, &iPgno);
         4347  +
         4348  +  fts5DebugRowid(&rc, &s, iRowid);
         4349  +  if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){
         4350  +    int i = 0;
         4351  +    i64 iPrev;
         4352  +    if( n>0 ){
         4353  +      i = getVarint(&a[i], (u64*)&iPrev);
         4354  +      sqlite3Fts5BufferAppendPrintf(&rc, &s, " %lld", iPrev);
         4355  +    }
         4356  +    while( i<n ){
         4357  +      i64 iVal;
         4358  +      i += getVarint(&a[i], (u64*)&iVal);
         4359  +      if( iVal==0 ){
         4360  +        sqlite3Fts5BufferAppendPrintf(&rc, &s, " x");
         4361  +      }else{
         4362  +        iPrev = iPrev - iVal;
         4363  +        sqlite3Fts5BufferAppendPrintf(&rc, &s, " %lld", iPrev);
         4364  +      }
         4365  +    }
         4366  +
         4367  +  }else
         4368  +  if( iSegid==0 ){
         4369  +    if( iRowid==FTS5_AVERAGES_ROWID ){
         4370  +      /* todo */
         4371  +    }else{
         4372  +      fts5DecodeStructure(&rc, &s, a, n);
         4373  +    }
         4374  +  }else{
         4375  +
         4376  +    Fts5Buffer term;
         4377  +    memset(&term, 0, sizeof(Fts5Buffer));
         4378  +
         4379  +    if( iHeight==0 ){
         4380  +      int iTermOff = 0;
         4381  +      int iRowidOff = 0;
         4382  +      int iOff;
         4383  +      int nKeep = 0;
         4384  +
         4385  +      iRowidOff = fts5GetU16(&a[0]);
         4386  +      iTermOff = fts5GetU16(&a[2]);
         4387  +
         4388  +      if( iRowidOff ){
         4389  +        iOff = iRowidOff;
         4390  +      }else if( iTermOff ){
         4391  +        iOff = iTermOff;
         4392  +      }else{
         4393  +        iOff = n;
         4394  +      }
         4395  +      fts5DecodePoslist(&rc, &s, &a[4], iOff-4);
         4396  +
         4397  +
         4398  +      assert( iRowidOff==0 || iOff==iRowidOff );
         4399  +      if( iRowidOff ){
         4400  +        iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff);
         4401  +      }
         4402  +
         4403  +      assert( iTermOff==0 || iOff==iTermOff );
         4404  +      while( iOff<n ){
         4405  +        int nByte;
         4406  +        iOff += getVarint32(&a[iOff], nByte);
         4407  +        term.n= nKeep;
         4408  +        fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff]);
         4409  +        iOff += nByte;
         4410  +
         4411  +        sqlite3Fts5BufferAppendPrintf(
         4412  +            &rc, &s, " term=%.*s", term.n, (const char*)term.p
         4413  +        );
         4414  +        iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff);
         4415  +        if( iOff<n ){
         4416  +          iOff += getVarint32(&a[iOff], nKeep);
         4417  +        }
         4418  +      }
         4419  +      fts5BufferFree(&term);
         4420  +    }else{
         4421  +      Fts5NodeIter ss;
         4422  +      for(fts5NodeIterInit(a, n, &ss); ss.aData; fts5NodeIterNext(&rc, &ss)){
         4423  +        if( ss.term.n==0 ){
         4424  +          sqlite3Fts5BufferAppendPrintf(&rc, &s, " left=%d", ss.iChild);
         4425  +        }else{
         4426  +          sqlite3Fts5BufferAppendPrintf(&rc,&s, " \"%.*s\"", 
         4427  +              ss.term.n, ss.term.p
         4428  +          );
         4429  +        }
         4430  +        if( ss.nEmpty ){
         4431  +          sqlite3Fts5BufferAppendPrintf(&rc, &s, " empty=%d%s", ss.nEmpty,
         4432  +              ss.bDlidx ? "*" : ""
         4433  +          );
         4434  +        }
         4435  +      }
         4436  +      fts5NodeIterFree(&ss);
         4437  +    }
         4438  +  }
         4439  +  
         4440  +  if( rc==SQLITE_OK ){
         4441  +    sqlite3_result_text(pCtx, (const char*)s.p, s.n, SQLITE_TRANSIENT);
         4442  +  }else{
         4443  +    sqlite3_result_error_code(pCtx, rc);
         4444  +  }
         4445  +  fts5BufferFree(&s);
         4446  +}
         4447  +
         4448  +
         4449  +/*
         4450  +** This is called as part of registering the FTS5 module with database
         4451  +** connection db. It registers several user-defined scalar functions useful
         4452  +** with FTS5.
         4453  +**
         4454  +** If successful, SQLITE_OK is returned. If an error occurs, some other
         4455  +** SQLite error code is returned instead.
         4456  +*/
         4457  +int sqlite3Fts5IndexInit(sqlite3 *db){
         4458  +  int rc = sqlite3_create_function(
         4459  +      db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0
         4460  +  );
         4461  +  return rc;
         4462  +}
  4454   4463   

Changes to ext/fts5/fts5_storage.c.

   222    222   }
   223    223   
   224    224   /*
   225    225   ** Close a handle opened by an earlier call to sqlite3Fts5StorageOpen().
   226    226   */
   227    227   int sqlite3Fts5StorageClose(Fts5Storage *p, int bDestroy){
   228    228     int rc = SQLITE_OK;
   229         -  int i;
          229  +  if( p ){
          230  +    int i;
   230    231   
   231         -  /* Finalize all SQL statements */
   232         -  for(i=0; i<ArraySize(p->aStmt); i++){
   233         -    sqlite3_finalize(p->aStmt[i]);
          232  +    /* Finalize all SQL statements */
          233  +    for(i=0; i<ArraySize(p->aStmt); i++){
          234  +      sqlite3_finalize(p->aStmt[i]);
          235  +    }
          236  +
          237  +    /* If required, remove the shadow tables from the database */
          238  +    if( bDestroy ){
          239  +      rc = sqlite3Fts5DropTable(p->pConfig, "content");
          240  +      if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "docsize");
          241  +      if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "config");
          242  +    }
          243  +
          244  +    sqlite3_free(p);
   234    245     }
   235         -
   236         -  /* If required, remove the shadow tables from the database */
   237         -  if( bDestroy ){
   238         -    rc = sqlite3Fts5DropTable(p->pConfig, "content");
   239         -    if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "docsize");
   240         -    if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "config");
   241         -  }
   242         -
   243         -  sqlite3_free(p);
   244    246     return rc;
   245    247   }
   246    248   
   247    249   typedef struct Fts5InsertCtx Fts5InsertCtx;
   248    250   struct Fts5InsertCtx {
   249    251     Fts5Storage *pStorage;
   250    252     int iCol;
................................................................................
   261    263     int iStart,                     /* Start offset of token */
   262    264     int iEnd,                       /* End offset of token */
   263    265     int iPos                        /* Position offset of token */
   264    266   ){
   265    267     Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
   266    268     Fts5Index *pIdx = pCtx->pStorage->pIndex;
   267    269     pCtx->szCol = iPos+1;
   268         -  sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, iPos, pToken, nToken);
   269         -  return SQLITE_OK;
          270  +  return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, iPos, pToken, nToken);
   270    271   }
   271    272   
   272    273   /*
   273    274   ** If a row with rowid iDel is present in the %_content table, add the
   274    275   ** delete-markers to the FTS index necessary to delete it. Do not actually
   275    276   ** remove the %_content row at this time though.
   276    277   */
................................................................................
   284    285       int rc2;
   285    286       sqlite3_bind_int64(pSeek, 1, iDel);
   286    287       if( sqlite3_step(pSeek)==SQLITE_ROW ){
   287    288         int iCol;
   288    289         Fts5InsertCtx ctx;
   289    290         ctx.pStorage = p;
   290    291         ctx.iCol = -1;
   291         -      sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
   292         -      for(iCol=1; iCol<=pConfig->nCol; iCol++){
          292  +      rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
          293  +      for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
   293    294           rc = sqlite3Fts5Tokenize(pConfig, 
   294    295               (const char*)sqlite3_column_text(pSeek, iCol),
   295    296               sqlite3_column_bytes(pSeek, iCol),
   296    297               (void*)&ctx,
   297    298               fts5StorageInsertCallback
   298    299           );
   299    300           p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
................................................................................
   471    472     if( rc==SQLITE_OK ){
   472    473       sqlite3_step(pInsert);
   473    474       rc = sqlite3_reset(pInsert);
   474    475     }
   475    476     *piRowid = sqlite3_last_insert_rowid(pConfig->db);
   476    477   
   477    478     /* Add new entries to the FTS index */
   478         -  sqlite3Fts5IndexBeginWrite(p->pIndex, *piRowid);
   479         -  ctx.pStorage = p;
          479  +  if( rc==SQLITE_OK ){
          480  +    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, *piRowid);
          481  +    ctx.pStorage = p;
          482  +  }
   480    483     for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
   481    484       ctx.szCol = 0;
   482    485       rc = sqlite3Fts5Tokenize(pConfig, 
   483    486           (const char*)sqlite3_value_text(apVal[ctx.iCol+2]),
   484    487           sqlite3_value_bytes(apVal[ctx.iCol+2]),
   485    488           (void*)&ctx,
   486    489           fts5StorageInsertCallback

Added test/fts5fault1.test.

            1  +# 2014 June 17
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +source $testdir/malloc_common.tcl
           18  +set testprefix fts5fault1
           19  +
           20  +# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           21  +ifcapable !fts5 {
           22  +  finish_test
           23  +  return
           24  +}
           25  +
           26  +faultsim_save_and_close
           27  +do_faultsim_test 1 -prep {
           28  +  faultsim_restore_and_reopen
           29  +} -body {
           30  +  execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) }
           31  +} -test {
           32  +  faultsim_test_result {0 {}} 
           33  +}
           34  +
           35  +
           36  +
           37  +finish_test

Changes to test/permutations.test.

   221    221     fts3varint.test
   222    222     fts4growth.test fts4growth2.test
   223    223   }
   224    224   
   225    225   test_suite "fts5" -prefix "" -description {
   226    226     All FTS5 tests.
   227    227   } -files {
   228         -  fts5aa.test fts5ab.test fts5ac.test fts5ad.test fts5ae.test fts5ea.test
          228  +  fts5aa.test fts5ab.test fts5ac.test fts5ad.test fts5ae.test 
   229    229     fts5af.test fts5ag.test fts5ah.test fts5ai.test fts5aj.test
          230  +  fts5ak.test fts5al.test
          231  +  fts5ea.test
          232  +
          233  +  fts5fault1.test
   230    234   }
   231    235   
   232    236   test_suite "nofaultsim" -prefix "" -description {
   233    237     "Very" quick test suite. Runs in less than 5 minutes on a workstation. 
   234    238     This test suite is the same as the "quick" tests, except that some files
   235    239     that test malloc and IO errors are omitted.
   236    240   } -files [