/ Check-in [49c1e745]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Improve fts5 integrity-check so that it checks that DESC queries return the same as ASC. Change the poslist format slightly to make room for a delete-flag.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 49c1e74522a26e5dbe6f8305bc96487279b80dfb
User & Date: dan 2015-04-11 16:23:31
Context
2015-04-11
18:25
Have fts5 integrity check verify that prefix indexes contain the same values as returned by prefix queries on the main terms index. check-in: bdb8e82a user: dan tags: fts5
16:23
Improve fts5 integrity-check so that it checks that DESC queries return the same as ASC. Change the poslist format slightly to make room for a delete-flag. check-in: 49c1e745 user: dan tags: fts5
2015-03-21
15:45
Merge trunk changes with this branch. check-in: 14274391 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5Int.h.

   378    378   );
   379    379   
   380    380   /*
   381    381   ** Empty (but do not delete) a hash table.
   382    382   */
   383    383   void sqlite3Fts5HashClear(Fts5Hash*);
   384    384   
   385         -/*
   386         -** Iterate through the contents of the hash table.
   387         -*/
   388         -int sqlite3Fts5HashIterate(
   389         -  Fts5Hash*,
   390         -  void *pCtx,
   391         -  int (*xTerm)(void*, const char*, int),
   392         -  int (*xEntry)(void*, i64, const u8*, int),
   393         -  int (*xTermDone)(void*)
   394         -);
   395         -
   396    385   int sqlite3Fts5HashQuery(
   397    386     Fts5Hash*,                      /* Hash table to query */
   398    387     const char *pTerm, int nTerm,   /* Query term */
   399    388     const u8 **ppDoclist,           /* OUT: Pointer to doclist for pTerm */
   400    389     int *pnDoclist                  /* OUT: Size of doclist in bytes */
   401    390   );
   402    391   

Changes to ext/fts5/fts5_hash.c.

   163    163     pHash->nSlot = nNew;
   164    164     pHash->aSlot = apNew;
   165    165     return SQLITE_OK;
   166    166   }
   167    167   
   168    168   static void fts5HashAddPoslistSize(Fts5HashEntry *p){
   169    169     if( p->iSzPoslist ){
          170  +    /* WRITEPOSLISTSIZE */
   170    171       u8 *pPtr = (u8*)p;
   171         -    int nSz = p->nData - p->iSzPoslist - 1;
          172  +    int nSz = (p->nData - p->iSzPoslist - 1) * 2;
   172    173   
   173    174       if( nSz<=127 ){
   174    175         pPtr[p->iSzPoslist] = nSz;
   175    176       }else{
   176    177         int nByte = sqlite3Fts5GetVarintLen((u32)nSz);
   177         -      memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
          178  +      /* WRITEPOSLISTSIZE */
          179  +      memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz/2);
   178    180         sqlite3PutVarint(&pPtr[p->iSzPoslist], nSz);
   179    181         p->nData += (nByte-1);
   180    182       }
   181    183       p->iSzPoslist = 0;
   182    184     }
   183    185   }
   184    186   

Changes to ext/fts5/fts5_index.c.

    38     38   **     the doclist. This is used to speed up seek operations, and merges of
    39     39   **     large doclists with very small doclists.
    40     40   **
    41     41   **   * extra fields in the "structure record" record the state of ongoing
    42     42   **     incremental merge operations.
    43     43   **
    44     44   */
           45  +
    45     46   
    46     47   #define FTS5_OPT_WORK_UNIT  1000  /* Number of leaf pages per optimize step */
    47     48   #define FTS5_WORK_UNIT      64    /* Number of leaf pages in unit of work */
    48     49   
    49     50   #define FTS5_MIN_DLIDX_SIZE 4     /* Add dlidx if this many empty pages */
    50     51   
    51     52   /*
................................................................................
   117    118   **           varint:  rowid delta (always > 0)
   118    119   **           poslist: next poslist
   119    120   **         }
   120    121   **         0x00 byte
   121    122   **
   122    123   **     poslist format:
   123    124   **
   124         -**         varint: size of poslist in bytes. not including this field.
          125  +**         varint: size of poslist in bytes multiplied by 2, not including
          126  +**                 this field. Plus 1 if this entry carries the "delete" flag.
   125    127   **         collist: collist for column 0
   126    128   **         zero-or-more {
   127    129   **           0x01 byte
   128    130   **           varint: column number (I)
   129    131   **           collist: collist for column I
   130    132   **         }
   131    133   **
................................................................................
  1625   1627       pIter->iLeafOffset = fts5GetU16(&a[2]);
  1626   1628       fts5SegIterLoadTerm(p, pIter, 0);
  1627   1629     }
  1628   1630   }
  1629   1631   
  1630   1632   /*
  1631   1633   ** This function is only ever called on iterators created by calls to
  1632         -** Fts5IndexQuery() with the FTS5INDEX_QUERY_ASC flag set.
         1634  +** Fts5IndexQuery() with the FTS5INDEX_QUERY_DESC flag set.
  1633   1635   **
  1634   1636   ** When this function is called, iterator pIter points to the first rowid
  1635   1637   ** on the current leaf associated with the term being queried. This function
  1636   1638   ** advances it to point to the last such rowid and, if necessary, initializes
  1637   1639   ** the aRowidOffset[] and iRowidOffset variables.
  1638   1640   */
  1639   1641   static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){
................................................................................
  1642   1644     u8 *a = pIter->pLeaf->p;
  1643   1645     int iRowidOffset = 0;
  1644   1646   
  1645   1647     while( p->rc==SQLITE_OK && i<n ){
  1646   1648       i64 iDelta = 0;
  1647   1649       int nPos;
  1648   1650   
         1651  +    /* READPOSLISTSIZE */
  1649   1652       i += fts5GetVarint32(&a[i], nPos);
  1650         -    i += nPos;
         1653  +    i += nPos / 2;
  1651   1654       if( i>=n ) break;
  1652   1655       i += getVarint(&a[i], (u64*)&iDelta);
  1653   1656       if( iDelta==0 ) break;
  1654   1657       pIter->iRowid += iDelta;
  1655   1658   
  1656   1659       if( iRowidOffset>=pIter->nRowidOffset ){
  1657   1660         int nNew = pIter->nRowidOffset + 8;
................................................................................
  1761   1764           u8 *a = pIter->pLeaf->p;
  1762   1765           int iOff;
  1763   1766           int nPos;
  1764   1767           i64 iDelta;
  1765   1768           pIter->iRowidOffset--;
  1766   1769   
  1767   1770           pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset];
         1771  +        /* READPOSLISTSIZE */
  1768   1772           iOff += fts5GetVarint32(&a[iOff], nPos);
  1769         -        iOff += nPos;
         1773  +        iOff += (nPos / 2);
  1770   1774           getVarint(&a[iOff], (u64*)&iDelta);
  1771   1775           pIter->iRowid -= iDelta;
  1772   1776         }else{
  1773   1777           fts5SegIterReverseNewPage(p, pIter);
  1774   1778         }
  1775   1779       }else{
  1776   1780         Fts5Data *pLeaf = pIter->pLeaf;
................................................................................
  1781   1785         /* Search for the end of the position list within the current page. */
  1782   1786         u8 *a = pLeaf->p;
  1783   1787         int n = pLeaf->n;
  1784   1788   
  1785   1789         iOff = pIter->iLeafOffset;
  1786   1790         if( iOff<n ){
  1787   1791           int nPoslist;
         1792  +        /* READPOSLISTSIZE */
  1788   1793           iOff += fts5GetVarint32(&a[iOff], nPoslist);
  1789         -        iOff += nPoslist;
         1794  +        iOff += nPoslist / 2;
  1790   1795         }
  1791   1796   
  1792   1797         if( iOff<n ){
  1793   1798           /* The next entry is on the current page */
  1794   1799           u64 iDelta;
  1795   1800           iOff += sqlite3GetVarint(&a[iOff], &iDelta);
  1796   1801           pIter->iLeafOffset = iOff;
................................................................................
  1818   1823             pIter->pLeaf = 0;
  1819   1824           }else{
  1820   1825             pIter->pLeaf->p = (u8*)pList;
  1821   1826             pIter->pLeaf->n = nList;
  1822   1827             sqlite3Fts5BufferSet(&p->rc, &pIter->term, strlen(zTerm), (u8*)zTerm);
  1823   1828             pIter->iLeafOffset = getVarint(pList, (u64*)&pIter->iRowid);
  1824   1829             if( pIter->flags & FTS5_SEGITER_REVERSE ){
         1830  +            assert( 0 );
  1825   1831               fts5SegIterReverseInitPage(p, pIter);
  1826   1832             }
  1827   1833           }
  1828   1834         }else{
  1829   1835           iOff = 0;
  1830   1836           /* Next entry is not on the current page */
  1831   1837           while( iOff==0 ){
................................................................................
  1877   1883       pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, pgnoLast));
  1878   1884     }else{
  1879   1885       while( iOff<pLeaf->n ){
  1880   1886         int nPos;
  1881   1887         i64 iDelta;
  1882   1888   
  1883   1889         /* Position list size in bytes */
         1890  +      /* READPOSLISTSIZE */
  1884   1891         iOff += fts5GetVarint32(&pLeaf->p[iOff], nPos);
  1885         -      iOff += nPos;
         1892  +      iOff += (nPos / 2);
  1886   1893         if( iOff>=pLeaf->n ) break;
  1887   1894   
  1888   1895         /* Rowid delta. Or, if 0x00, the end of doclist marker. */
  1889   1896         nPos = getVarint(&pLeaf->p[iOff], (u64*)&iDelta);
  1890   1897         if( iDelta==0 ) break;
  1891   1898         iOff += nPos;
  1892   1899       }
................................................................................
  1960   1967     ** term. */
  1961   1968     if( pIter->iTermLeafPgno==pIter->iLeafPgno ){
  1962   1969       while( iOff<pLeaf->n ){
  1963   1970         i64 iDelta;
  1964   1971         int nPoslist;
  1965   1972   
  1966   1973         /* iOff is currently the offset of the size field of a position list. */
         1974  +      /* READPOSLISTSIZE */
  1967   1975         iOff += fts5GetVarint32(&pLeaf->p[iOff], nPoslist);
  1968         -      iOff += nPoslist;
         1976  +      iOff += nPoslist / 2;
  1969   1977   
  1970   1978         if( iOff<pLeaf->n ){
  1971   1979           iOff += getVarint(&pLeaf->p[iOff], (u64*)&iDelta);
  1972   1980           if( iDelta==0 ) return;
  1973   1981         }
  1974   1982       }
  1975   1983     }
................................................................................
  2652   2660       pIter->nRem = 1;
  2653   2661       fts5ChunkIterNext(p, pIter);
  2654   2662       if( p->rc ) return;
  2655   2663       iOff = 4;
  2656   2664       pLeaf = pIter->pLeaf;
  2657   2665     }
  2658   2666   
         2667  +  /* READPOSLISTSIZE */
  2659   2668     iOff += fts5GetVarint32(&pLeaf->p[iOff], pIter->nRem);
         2669  +  pIter->nRem = pIter->nRem / 2;
  2660   2670     pIter->n = MIN(pLeaf->n - iOff, pIter->nRem);
  2661   2671     pIter->p = pLeaf->p + iOff;
  2662   2672   
  2663   2673     if( pIter->n==0 ){
  2664   2674       fts5ChunkIterNext(p, pIter);
  2665   2675     }
  2666   2676   }
................................................................................
  3365   3375           bRequireDoclistTerm = 1;
  3366   3376         }
  3367   3377   
  3368   3378         /* Append the rowid to the output */
  3369   3379         fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter));
  3370   3380   
  3371   3381         /* Copy the position list from input to output */
  3372         -      fts5WriteAppendPoslistInt(p, &writer, sPos.nRem);
         3382  +      /* WRITEPOSLISTSIZE */
         3383  +      fts5WriteAppendPoslistInt(p, &writer, sPos.nRem * 2);
  3373   3384         for(/* noop */; !fts5ChunkIterEof(p, &sPos); fts5ChunkIterNext(p, &sPos)){
  3374   3385           fts5WriteAppendPoslistData(p, &writer, sPos.p, sPos.n);
  3375   3386         }
  3376   3387       }
  3377   3388   
  3378   3389       fts5ChunkIterRelease(&sPos);
  3379   3390     }
................................................................................
  3583   3594         const u8 *pDoclist;
  3584   3595         int nDoclist;
  3585   3596         int nSuffix;                /* Size of term suffix */
  3586   3597   
  3587   3598         sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
  3588   3599         nTerm = strlen(zTerm);
  3589   3600   
  3590         -      /* Decide if the term fits on the current leaf. If not, flush it
  3591         -      ** to disk.  */
         3601  +      /* Decide if the term will fit on the current leaf. If it will not, 
         3602  +      ** flush the leaf to disk here.  */
  3592   3603         if( (pBuf->n + nTerm + 2) > pgsz ){
  3593   3604           fts5WriteFlushLeaf(p, &writer);
  3594   3605           pBuf = &writer.aWriter[0].buf;
  3595   3606           if( (nTerm + 32) > pBuf->nSpace ){
  3596   3607             fts5BufferGrow(&p->rc, pBuf, nTerm + 32 - pBuf->n);
  3597   3608             if( p->rc ) break;
  3598   3609           }
................................................................................
  3629   3640           /* The entire doclist will not fit on this leaf. The following 
  3630   3641           ** loop iterates through the poslists that make up the current 
  3631   3642           ** doclist.  */
  3632   3643           while( iOff<nDoclist ){
  3633   3644             u32 nPos;
  3634   3645             int nCopy;
  3635   3646             iOff += getVarint(&pDoclist[iOff], (u64*)&iDelta);
         3647  +          /* READPOSLISTSIZE */
  3636   3648             nCopy = fts5GetVarint32(&pDoclist[iOff], nPos);
  3637         -          nCopy += nPos;
         3649  +          nCopy += (nPos / 2);
  3638   3650             iRowid += iDelta;
  3639   3651             
  3640   3652             if( bFirstDocid ){
  3641   3653               fts5PutU16(&pBuf->p[0], pBuf->n);   /* first docid on page */
  3642   3654               pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iRowid);
  3643   3655               bFirstDocid = 0;
  3644   3656             }else{
................................................................................
  4067   4079     if( p->rc==SQLITE_OK ){
  4068   4080       Fts5ChunkIter iter;
  4069   4081       Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
  4070   4082       assert( fts5MultiIterEof(p, pMulti)==0 );
  4071   4083       fts5ChunkIterInit(p, pSeg, &iter);
  4072   4084       if( fts5ChunkIterEof(p, &iter)==0 ){
  4073   4085         if( bSz ){
  4074         -        fts5BufferAppendVarint(&p->rc, pBuf, iter.nRem);
         4086  +        /* WRITEPOSLISTSIZE */
         4087  +        fts5BufferAppendVarint(&p->rc, pBuf, iter.nRem * 2);
  4075   4088         }
  4076   4089         while( fts5ChunkIterEof(p, &iter)==0 ){
  4077   4090           fts5BufferAppendBlob(&p->rc, pBuf, iter.n, iter.p);
  4078   4091           fts5ChunkIterNext(p, &iter);
  4079   4092         }
  4080   4093       }
  4081   4094       fts5ChunkIterRelease(&iter);
................................................................................
  4091   4104           pIter->iRowid -= iDelta;
  4092   4105         }else{
  4093   4106           pIter->iRowid += iDelta;
  4094   4107         }
  4095   4108       }else{
  4096   4109         pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid);
  4097   4110       }
         4111  +    /* READPOSLISTSIZE */
  4098   4112       pIter->i += fts5GetVarint32(&pIter->a[pIter->i], pIter->nPoslist);
         4113  +    pIter->nPoslist = pIter->nPoslist / 2;
  4099   4114       pIter->aPoslist = &pIter->a[pIter->i];
  4100   4115       pIter->i += pIter->nPoslist;
  4101   4116     }else{
  4102   4117       pIter->aPoslist = 0;
  4103   4118     }
  4104   4119   }
  4105   4120   
................................................................................
  4162   4177       fts5DoclistIterInit(p2, bDesc, &i2);
  4163   4178       while( i1.aPoslist!=0 || i2.aPoslist!=0 ){
  4164   4179         if( i2.aPoslist==0 || (i1.aPoslist && 
  4165   4180              ( (bDesc && i1.iRowid>i2.iRowid) || (!bDesc && i1.iRowid<i2.iRowid) )
  4166   4181         )){
  4167   4182           /* Copy entry from i1 */
  4168   4183           fts5MergeAppendDocid(&p->rc, bDesc, &out, &iLastRowid, i1.iRowid);
  4169         -        fts5BufferAppendVarint(&p->rc, &out, i1.nPoslist);
         4184  +        /* WRITEPOSLISTSIZE */
         4185  +        fts5BufferAppendVarint(&p->rc, &out, i1.nPoslist * 2);
  4170   4186           fts5BufferAppendBlob(&p->rc, &out, i1.nPoslist, i1.aPoslist);
  4171   4187           fts5DoclistIterNext(&i1);
  4172   4188         }
  4173   4189         else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){
  4174   4190           /* Copy entry from i2 */
  4175   4191           fts5MergeAppendDocid(&p->rc, bDesc, &out, &iLastRowid, i2.iRowid);
  4176         -        fts5BufferAppendVarint(&p->rc, &out, i2.nPoslist);
         4192  +        /* WRITEPOSLISTSIZE */
         4193  +        fts5BufferAppendVarint(&p->rc, &out, i2.nPoslist * 2);
  4177   4194           fts5BufferAppendBlob(&p->rc, &out, i2.nPoslist, i2.aPoslist);
  4178   4195           fts5DoclistIterNext(&i2);
  4179   4196         }
  4180   4197         else{
  4181   4198           Fts5PoslistReader r1;
  4182   4199           Fts5PoslistReader r2;
  4183   4200           Fts5PoslistWriter writer;
................................................................................
  4198   4215               iNew = r2.iPos;
  4199   4216               sqlite3Fts5PoslistReaderNext(&r2);
  4200   4217               if( r1.iPos==r2.iPos ) sqlite3Fts5PoslistReaderNext(&r1);
  4201   4218             }
  4202   4219             p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
  4203   4220           }
  4204   4221   
  4205         -        fts5BufferAppendVarint(&p->rc, &out, tmp.n);
         4222  +        /* WRITEPOSLISTSIZE */
         4223  +        fts5BufferAppendVarint(&p->rc, &out, tmp.n * 2);
  4206   4224           fts5BufferAppendBlob(&p->rc, &out, tmp.n, tmp.p);
  4207   4225           fts5DoclistIterNext(&i1);
  4208   4226           fts5DoclistIterNext(&i2);
  4209   4227         }
  4210   4228       }
  4211   4229   
  4212   4230       fts5BufferSet(&p->rc, p1, out.n, out.p);
................................................................................
  4293   4311         fts5DoclistIterInit(&doclist, bDesc, pIter->pDoclist);
  4294   4312       }
  4295   4313     }
  4296   4314   
  4297   4315     fts5StructureRelease(pStruct);
  4298   4316     sqlite3_free(aBuf);
  4299   4317   }
         4318  +
         4319  +static int fts5QueryCksum(
         4320  +  Fts5Index *p,
         4321  +  const char *z,
         4322  +  int n,
         4323  +  int flags,
         4324  +  u64 *pCksum
         4325  +){
         4326  +  u64 cksum = *pCksum;
         4327  +  Fts5IndexIter *pIdxIter = 0;
         4328  +  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, &pIdxIter);
         4329  +
         4330  +  while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
         4331  +    const u8 *pPos;
         4332  +    int nPos;
         4333  +    i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
         4334  +    rc = sqlite3Fts5IterPoslist(pIdxIter, &pPos, &nPos);
         4335  +    if( rc==SQLITE_OK ){
         4336  +      Fts5PoslistReader sReader;
         4337  +      for(sqlite3Fts5PoslistReaderInit(-1, pPos, nPos, &sReader);
         4338  +          sReader.bEof==0;
         4339  +          sqlite3Fts5PoslistReaderNext(&sReader)
         4340  +      ){
         4341  +        int iCol = FTS5_POS2COLUMN(sReader.iPos);
         4342  +        int iOff = FTS5_POS2OFFSET(sReader.iPos);
         4343  +        cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, z, n);
         4344  +      }
         4345  +      rc = sqlite3Fts5IterNext(pIdxIter);
         4346  +    }
         4347  +  }
         4348  +  sqlite3Fts5IterClose(pIdxIter);
         4349  +
         4350  +  *pCksum = cksum;
         4351  +  return rc;
         4352  +}
  4300   4353   
  4301   4354   /*
  4302   4355   ** Run internal checks to ensure that the FTS index (a) is internally 
  4303   4356   ** consistent and (b) contains entries for which the XOR of the checksums
  4304   4357   ** as calculated by fts5IndexEntryCksum() is cksum.
  4305   4358   **
  4306   4359   ** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
................................................................................
  4362   4415             fts5PosIterNext(p, &sPos)
  4363   4416         ){
  4364   4417           cksum2 ^= fts5IndexEntryCksum(iRowid, sPos.iCol, sPos.iPos, z, n);
  4365   4418         }
  4366   4419   
  4367   4420         /* If this is a new term, query for it. Update cksum3 with the results. */
  4368   4421         if( p->rc==SQLITE_OK && (term.n!=n || memcmp(term.p, z, n)) ){
  4369         -        Fts5IndexIter *pIdxIter = 0;
         4422  +        int rc;
  4370   4423           int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX);
  4371         -        int rc = sqlite3Fts5IndexQuery(p, z, n, flags, &pIdxIter);
  4372         -        while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
  4373         -          const u8 *pPos;
  4374         -          int nPos;
  4375         -          i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
  4376         -          rc = sqlite3Fts5IterPoslist(pIdxIter, &pPos, &nPos);
  4377         -          if( rc==SQLITE_OK ){
  4378         -            Fts5PoslistReader sReader;
  4379         -            for(sqlite3Fts5PoslistReaderInit(-1, pPos, nPos, &sReader);
  4380         -                sReader.bEof==0;
  4381         -                sqlite3Fts5PoslistReaderNext(&sReader)
  4382         -            ){
  4383         -              int iCol = FTS5_POS2COLUMN(sReader.iPos);
  4384         -              int iOff = FTS5_POS2OFFSET(sReader.iPos);
  4385         -              cksum3 ^= fts5IndexEntryCksum(rowid, iCol, iOff, z, n);
  4386         -            }
  4387         -            rc = sqlite3Fts5IterNext(pIdxIter);
  4388         -          }
         4424  +        u64 ck1 = 0;
         4425  +        u64 ck2 = 0;
         4426  +
         4427  +        /* Check that the results returned for ASC and DESC queries are
         4428  +        ** the same. If not, call this corruption.  */
         4429  +        rc = fts5QueryCksum(p, z, n, flags, &ck1);
         4430  +        if( rc==SQLITE_OK ){
         4431  +          rc = fts5QueryCksum(p, z, n, flags | FTS5INDEX_QUERY_DESC, &ck2);
  4389   4432           }
  4390         -        sqlite3Fts5IterClose(pIdxIter);
         4433  +        if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
         4434  +
         4435  +        cksum3 ^= ck1;
  4391   4436           fts5BufferSet(&rc, &term, n, (const u8*)z);
  4392   4437           p->rc = rc;
  4393   4438         }
  4394   4439       }
  4395   4440       fts5MultiIterFree(p, pIter);
  4396   4441       fts5StructureRelease(pStruct);
  4397   4442     }
................................................................................
  4769   4814   
  4770   4815   
  4771   4816   /*
  4772   4817   ** Return a pointer to a buffer containing a copy of the position list for
  4773   4818   ** the current entry. Output variable *pn is set to the size of the buffer 
  4774   4819   ** in bytes before returning.
  4775   4820   **
  4776         -** The returned buffer does not include the 0x00 terminator byte stored on
  4777         -** disk.
         4821  +** The returned position list does not include the "number of bytes" varint
         4822  +** field that starts the position list on disk.
  4778   4823   */
  4779   4824   int sqlite3Fts5IterPoslist(Fts5IndexIter *pIter, const u8 **pp, int *pn){
  4780   4825     assert( pIter->pIndex->rc==SQLITE_OK );
  4781   4826     if( pIter->pDoclist ){
  4782   4827       *pn = pIter->pDoclist->nPoslist;
  4783   4828       *pp = pIter->pDoclist->aPoslist;
  4784   4829     }else{
................................................................................
  5007   5052   
  5008   5053     if( iOff<n ){
  5009   5054       iOff += sqlite3GetVarint(&a[iOff], (u64*)&iDocid);
  5010   5055       sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
  5011   5056     }
  5012   5057     while( iOff<n ){
  5013   5058       int nPos;
         5059  +    /* READPOSLISTSIZE */
  5014   5060       iOff += fts5GetVarint32(&a[iOff], nPos);
  5015         -    iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos));
         5061  +    iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos / 2));
  5016   5062       if( iOff<n ){
  5017   5063         i64 iDelta;
  5018   5064         iOff += sqlite3GetVarint(&a[iOff], (u64*)&iDelta);
  5019   5065         if( iDelta==0 ) return iOff;
  5020   5066         iDocid += iDelta;
  5021   5067         sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
  5022   5068       }