/ Check-in [90b82d3e]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix fts5_index.c to use doclist-indexes when possible. Only some cases work so far.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 90b82d3ef613b2915e0e280dc1d2e5a2b617d59c
User & Date: dan 2014-08-04 20:07:40
Context
2014-08-05
19:00
Use doclist-indexes with "ORDER BY rowid ASC" fts5 queries as well. check-in: d028ba65 user: dan tags: fts5
2014-08-04
20:07
Fix fts5_index.c to use doclist-indexes when possible. Only some cases work so far. check-in: 90b82d3e user: dan tags: fts5
2014-08-02
20:49
Start changing things to use doclist indexes as required. code is not activated yet. check-in: b8864da9 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_index.c.

   262    262   #endif
   263    263   
   264    264   
   265    265   typedef struct Fts5BtreeIter Fts5BtreeIter;
   266    266   typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel;
   267    267   typedef struct Fts5ChunkIter Fts5ChunkIter;
   268    268   typedef struct Fts5Data Fts5Data;
          269  +typedef struct Fts5DlidxIter Fts5DlidxIter;
   269    270   typedef struct Fts5MultiSegIter Fts5MultiSegIter;
   270    271   typedef struct Fts5NodeIter Fts5NodeIter;
   271    272   typedef struct Fts5PageWriter Fts5PageWriter;
   272    273   typedef struct Fts5PendingDoclist Fts5PendingDoclist;
   273    274   typedef struct Fts5PendingPoslist Fts5PendingPoslist;
   274    275   typedef struct Fts5PosIter Fts5PosIter;
   275    276   typedef struct Fts5SegIter Fts5SegIter;
   276    277   typedef struct Fts5DoclistIter Fts5DoclistIter;
   277    278   typedef struct Fts5SegWriter Fts5SegWriter;
   278    279   typedef struct Fts5Structure Fts5Structure;
   279    280   typedef struct Fts5StructureLevel Fts5StructureLevel;
   280    281   typedef struct Fts5StructureSegment Fts5StructureSegment;
   281         -
   282    282   
   283    283   /*
   284    284   ** One object per %_data table.
   285    285   */
   286    286   struct Fts5Index {
   287    287     Fts5Config *pConfig;            /* Virtual table configuration */
   288    288     char *zDataTbl;                 /* Name of %_data table */
................................................................................
   317    317   
   318    318     /* Output variables. aPoslist==0 at EOF */
   319    319     i64 iRowid;
   320    320     u8 *aPoslist;
   321    321     int nPoslist;
   322    322   };
   323    323   
          324  +/*
          325  +** Each iterator used by external modules is an instance of this type.
          326  +*/
   324    327   struct Fts5IndexIter {
   325    328     Fts5Index *pIndex;
   326    329     Fts5Structure *pStruct;
   327    330     Fts5MultiSegIter *pMulti;
   328    331     Fts5DoclistIter *pDoclist;
   329    332     Fts5Buffer poslist;             /* Buffer containing current poslist */
   330    333   };
................................................................................
   485    488     int iTermLeafOffset;
   486    489   
   487    490     /* The following are only used if the FTS5_SEGITER_REVERSE flag is set. */
   488    491     int iRowidOffset;               /* Current entry in aRowidOffset[] */
   489    492     int nRowidOffset;               /* Allocated size of aRowidOffset[] array */
   490    493     int *aRowidOffset;              /* Array of offset to rowid fields */
   491    494   
          495  +  Fts5DlidxIter *pDlidx;          /* If there is a doclist-index */
          496  +
   492    497     /* Variables populated based on current entry. */
   493    498     Fts5Buffer term;                /* Current term */
   494    499     i64 iRowid;                     /* Current rowid */
   495    500   };
   496    501   
   497    502   #define FTS5_SEGITER_ONETERM 0x01
   498    503   #define FTS5_SEGITER_REVERSE 0x02
................................................................................
   535    540   
   536    541     /* Output variables */
   537    542     Fts5Buffer term;
   538    543     int nEmpty;
   539    544     int iChild;
   540    545     int bDlidx;
   541    546   };
          547  +
          548  +/*
          549  +** An instance of the following type is used to iterate through the contents
          550  +** of a doclist-index record.
          551  +**
          552  +** pData:
          553  +**   A reference to the dlidx record.
          554  +*/
          555  +struct Fts5DlidxIter {
          556  +  Fts5Data *pData;              /* Data for doclist index, if any */
          557  +  int iOff;                     /* Current offset into pDlidx */
          558  +  int bRowidValid;              /* iRowid is valid */
          559  +  int bEof;                     /* At EOF already */
          560  +
          561  +  /* Output variables */
          562  +  int iLeafPgno;                /* Page number of current leaf page */
          563  +  int bZero;                    /* True if current leaf has no rowids */
          564  +  i64 iRowid;                   /* If bZero==0, first rowid on leaf */
          565  +};
          566  +
   542    567   
   543    568   /*
   544    569   ** An Fts5BtreeIter object is used to iterate through all entries in the
   545    570   ** b-tree hierarchy belonging to a single fts5 segment. In this case the
   546    571   ** "b-tree hierarchy" is all b-tree nodes except leaves. Each entry in the
   547    572   ** b-tree hierarchy consists of the following:
   548    573   **
................................................................................
   572    597     /* Output variables */
   573    598     Fts5Buffer term;                /* Current term */
   574    599     int iLeaf;                      /* Leaf containing terms >= current term */
   575    600     int nEmpty;                     /* Number of "empty" leaves following iLeaf */
   576    601     int bEof;                       /* Set to true at EOF */
   577    602     int bDlidx;                     /* True if there exists a dlidx */
   578    603   };
          604  +
          605  +
          606  +/*
          607  +** Decode a segment-data rowid from the %_data table. This function is
          608  +** the opposite of macro FTS5_SEGMENT_ROWID().
          609  +*/
          610  +static void fts5DecodeRowid(
          611  +  i64 iRowid,                     /* Rowid from %_data table */
          612  +  int *piIdx,                     /* OUT: Index */
          613  +  int *piSegid,                   /* OUT: Segment id */
          614  +  int *piHeight,                  /* OUT: Height */
          615  +  int *piPgno                     /* OUT: Page number */
          616  +){
          617  +  *piPgno = (int)(iRowid & (((i64)1 << FTS5_DATA_PAGE_B) - 1));
          618  +  iRowid >>= FTS5_DATA_PAGE_B;
          619  +
          620  +  *piHeight = (int)(iRowid & (((i64)1 << FTS5_DATA_HEIGHT_B) - 1));
          621  +  iRowid >>= FTS5_DATA_HEIGHT_B;
          622  +
          623  +  *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1));
          624  +  iRowid >>= FTS5_DATA_ID_B;
          625  +
          626  +  *piIdx = (int)(iRowid & (((i64)1 << FTS5_DATA_IDX_B) - 1));
          627  +}
          628  +
          629  +static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
          630  +  int iIdx,iSegid,iHeight,iPgno;  /* Rowid compenents */
          631  +  fts5DecodeRowid(iKey, &iIdx, &iSegid, &iHeight, &iPgno);
          632  +
          633  +  if( iSegid==0 ){
          634  +    if( iKey==FTS5_AVERAGES_ROWID ){
          635  +      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(averages) ");
          636  +    }else{
          637  +      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
          638  +          "(structure idx=%d)", (int)(iKey-10)
          639  +      );
          640  +    }
          641  +  }
          642  +  else if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){
          643  +    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(dlidx idx=%d segid=%d pgno=%d)",
          644  +        iIdx, iSegid, iPgno
          645  +    );
          646  +  }else{
          647  +    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(idx=%d segid=%d h=%d pgno=%d)",
          648  +        iIdx, iSegid, iHeight, iPgno
          649  +    );
          650  +  }
          651  +}
          652  +
   579    653   
   580    654   static void fts5PutU16(u8 *aOut, u16 iVal){
   581    655     aOut[0] = (iVal>>8);
   582    656     aOut[1] = (iVal&0xFF);
   583    657   }
   584    658   
   585    659   static u16 fts5GetU16(const u8 *aIn){
................................................................................
   662    736   static Fts5Data *fts5DataReadOrBuffer(
   663    737     Fts5Index *p, 
   664    738     Fts5Buffer *pBuf, 
   665    739     i64 iRowid
   666    740   ){
   667    741     Fts5Data *pRet = 0;
   668    742     if( p->rc==SQLITE_OK ){
   669         -    int rc;
          743  +    int rc = SQLITE_OK;
          744  +
          745  +#if 0
          746  +Fts5Buffer buf = {0,0,0};
          747  +fts5DebugRowid(&rc, &buf, iRowid);
          748  +fprintf(stdout, "read: %s\n", buf.p);
          749  +fflush(stdout);
          750  +sqlite3_free(buf.p);
          751  +#endif
   670    752   
   671    753       /* If the blob handle is not yet open, open and seek it. Otherwise, use
   672    754       ** the blob_reopen() API to reseek the existing blob handle.  */
   673    755       if( p->pReader==0 ){
   674    756         Fts5Config *pConfig = p->pConfig;
   675    757         rc = sqlite3_blob_open(pConfig->db, 
   676    758             pConfig->zDb, p->zDataTbl, "block", iRowid, 0, &p->pReader
................................................................................
  1036   1118   
  1037   1119   /*
  1038   1120   ** Free any memory allocated by the iterator object.
  1039   1121   */
  1040   1122   static void fts5NodeIterFree(Fts5NodeIter *pIter){
  1041   1123     fts5BufferFree(&pIter->term);
  1042   1124   }
         1125  +
         1126  +/*
         1127  +** Return non-zero if EOF is reached.
         1128  +*/
         1129  +static int fts5DlidxIterNext(Fts5DlidxIter *pIter, int bRev){
         1130  +  if( bRev ){
         1131  +    i64 iVal;
         1132  +    int iOff = pIter->iOff;
         1133  +    int iLimit;
         1134  +    u8 *a = pIter->pData->p;
         1135  +
         1136  +    /* Currently iOff points to the first byte of a varint. This block 
         1137  +    ** decrements iOff until it points to the first byte of the previous 
         1138  +    ** varint. Taking care not to read any memory locations that occur
         1139  +    ** before the buffer in memory.  */
         1140  +    iLimit = (iOff>9 ? iOff-9 : 0);
         1141  +    for(iOff--; iOff>iLimit; iOff--){
         1142  +      if( (a[iOff-1] & 0x80)==0 ) break;
         1143  +    }
         1144  +    pIter->iOff = iOff;
         1145  +
         1146  +    if( iOff<=0 ){
         1147  +      pIter->bEof = 1;
         1148  +      return 1;
         1149  +    }
         1150  +
         1151  +    getVarint(&a[iOff], (u64*)&iVal);
         1152  +    if( iVal==0 ){
         1153  +      pIter->bZero = 1;
         1154  +    }else if( iOff==0 ){
         1155  +      pIter->iRowid = iVal;
         1156  +    }else{
         1157  +      pIter->iRowid += iVal;
         1158  +    }
         1159  +    pIter->iLeafPgno--;
         1160  +  }else{
         1161  +    i64 iVal;
         1162  +    if( pIter->iOff>=pIter->pData->n ){
         1163  +      pIter->bEof = 1;
         1164  +      return 1;
         1165  +    }
         1166  +    pIter->iOff += getVarint(&pIter->pData->p[pIter->iOff], (u64*)&iVal);
         1167  +    if( iVal==0 ){
         1168  +      pIter->bZero = 1;
         1169  +    }else{
         1170  +      pIter->bZero = 0;
         1171  +      if( pIter->bRowidValid ){
         1172  +        pIter->iRowid -= iVal;
         1173  +      }else{
         1174  +        pIter->bRowidValid = 1;
         1175  +        pIter->iRowid = iVal;
         1176  +      }
         1177  +    }
         1178  +    pIter->iLeafPgno++;
         1179  +  }
         1180  +  return 0;
         1181  +}
         1182  +
         1183  +static void fts5DlidxIterLast(Fts5DlidxIter *pIter){
         1184  +  while( 0==fts5DlidxIterNext(pIter, 0) );
         1185  +  assert( pIter->iOff==pIter->pData->n && pIter->bEof==1 );
         1186  +  pIter->bEof = 0;
         1187  +}
         1188  +
         1189  +static int fts5DlidxIterEof(Fts5Index *p, Fts5DlidxIter *pIter){
         1190  +  return (p->rc!=SQLITE_OK || pIter->bEof);
         1191  +}
         1192  +
         1193  +static void fts5DlidxIterInit(
         1194  +  Fts5Index *p,                   /* Fts5 Backend to iterate within */
         1195  +  int bRev,                       /* True for ORDER BY ASC */
         1196  +  int iIdx, int iSegid,           /* Segment iSegid within index iIdx */
         1197  +  int iLeafPgno,                  /* Leaf page number to load dlidx for */
         1198  +  Fts5DlidxIter **ppIter          /* OUT: Populated iterator */
         1199  +){
         1200  +  Fts5DlidxIter *pIter = *ppIter;
         1201  +  Fts5Data *pDlidx;
         1202  +
         1203  +  pDlidx = fts5DataRead(p, FTS5_DOCLIST_IDX_ROWID(iIdx, iSegid, iLeafPgno));
         1204  +  if( pDlidx==0 ) return;
         1205  +  if( pIter==0 ){
         1206  +    *ppIter = pIter = (Fts5DlidxIter*)fts5IdxMalloc(p, sizeof(Fts5DlidxIter));
         1207  +    if( pIter==0 ){ 
         1208  +      fts5DataRelease(pDlidx);
         1209  +      return;
         1210  +    }
         1211  +  }else{
         1212  +    memset(pIter, 0, sizeof(Fts5DlidxIter));
         1213  +  }
         1214  +
         1215  +  pIter->pData = pDlidx;
         1216  +
         1217  +  pIter->iLeafPgno = iLeafPgno;
         1218  +  if( bRev==0 ){
         1219  +    fts5DlidxIterNext(pIter, 0);
         1220  +  }else{
         1221  +    fts5DlidxIterLast(pIter);
         1222  +  }
         1223  +}
         1224  +
         1225  +/*
         1226  +** Free a doclist-index iterator object allocated by fts5DlidxIterInit().
         1227  +*/
         1228  +static void fts5DlidxIterFree(Fts5DlidxIter *pIter){
         1229  +  if( pIter ){
         1230  +    fts5DataRelease(pIter->pData);
         1231  +    sqlite3_free(pIter);
         1232  +  }
         1233  +}
  1043   1234   
  1044   1235   /*
  1045   1236   ** Load the next leaf page into the segment iterator.
  1046   1237   */
  1047   1238   static void fts5SegIterNextPage(
  1048   1239     Fts5Index *p,                   /* FTS5 backend object */
  1049   1240     Fts5SegIter *pIter              /* Iterator to advance to next page */
................................................................................
  1171   1362   
  1172   1363       pIter->aRowidOffset[iRowidOffset++] = pIter->iLeafOffset;
  1173   1364       pIter->iLeafOffset = i;
  1174   1365     }
  1175   1366     pIter->iRowidOffset = iRowidOffset;
  1176   1367   }
  1177   1368   
         1369  +/*
         1370  +**
         1371  +*/
         1372  +static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){
         1373  +  assert( pIter->flags & FTS5_SEGITER_REVERSE );
         1374  +  assert( pIter->flags & FTS5_SEGITER_ONETERM );
         1375  +
         1376  +  fts5DataRelease(pIter->pLeaf);
         1377  +  pIter->pLeaf = 0;
         1378  +  while( p->rc==SQLITE_OK && pIter->iLeafPgno>pIter->iTermLeafPgno ){
         1379  +    Fts5Data *pNew;
         1380  +    pIter->iLeafPgno--;
         1381  +    pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID(
         1382  +          pIter->iIdx, pIter->pSeg->iSegid, 0, pIter->iLeafPgno
         1383  +    ));
         1384  +    if( pNew ){
         1385  +      if( pIter->iLeafPgno==pIter->iTermLeafPgno ){
         1386  +        if( pIter->iTermLeafOffset<pNew->n ){
         1387  +          pIter->pLeaf = pNew;
         1388  +          pIter->iLeafOffset = pIter->iTermLeafOffset;
         1389  +        }
         1390  +      }else{
         1391  +        int iRowidOff, dummy;
         1392  +        fts5LeafHeader(pNew, &iRowidOff, &dummy);
         1393  +        if( iRowidOff ){
         1394  +          pIter->pLeaf = pNew;
         1395  +          pIter->iLeafOffset = iRowidOff;
         1396  +        }
         1397  +      }
         1398  +
         1399  +      if( pIter->pLeaf ){
         1400  +        u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset];
         1401  +        pIter->iLeafOffset += getVarint(a, (u64*)&pIter->iRowid);
         1402  +        break;
         1403  +      }else{
         1404  +        fts5DataRelease(pNew);
         1405  +      }
         1406  +    }
         1407  +  }
         1408  +
         1409  +  if( pIter->pLeaf ){
         1410  +    fts5SegIterReverseInitPage(p, pIter);
         1411  +  }
         1412  +}
  1178   1413   
  1179   1414   /*
  1180   1415   ** Advance iterator pIter to the next entry. 
  1181   1416   **
  1182   1417   ** If an error occurs, Fts5Index.rc is set to an appropriate error code. It 
  1183   1418   ** is not considered an error if the iterator reaches EOF. If an error has 
  1184   1419   ** already occurred when this function is called, it is a no-op.
................................................................................
  1198   1433   
  1199   1434           pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset];
  1200   1435           iOff += getVarint32(&a[iOff], nPos);
  1201   1436           iOff += nPos;
  1202   1437           getVarint(&a[iOff], (u64*)&iDelta);
  1203   1438           pIter->iRowid += iDelta;
  1204   1439         }else{
         1440  +        fts5SegIterReverseNewPage(p, pIter);
         1441  +#if 0
  1205   1442           fts5DataRelease(pIter->pLeaf);
  1206   1443           pIter->pLeaf = 0;
  1207   1444           while( p->rc==SQLITE_OK && pIter->iLeafPgno>pIter->iTermLeafPgno ){
  1208   1445             Fts5Data *pNew;
  1209   1446             pIter->iLeafPgno--;
  1210   1447             pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID(
  1211   1448                   pIter->iIdx, pIter->pSeg->iSegid, 0, pIter->iLeafPgno
................................................................................
  1234   1471               }
  1235   1472             }
  1236   1473           }
  1237   1474   
  1238   1475           if( pIter->pLeaf ){
  1239   1476             fts5SegIterReverseInitPage(p, pIter);
  1240   1477           }
         1478  +#endif
  1241   1479         }
  1242   1480       }else{
  1243   1481         Fts5Data *pLeaf = pIter->pLeaf;
  1244   1482         int iOff;
  1245   1483         int bNewTerm = 0;
  1246   1484         int nKeep = 0;
  1247   1485   
................................................................................
  1375   1613       iOff += getVarint(&pLast->p[iOff], (u64*)&pIter->iRowid);
  1376   1614       pIter->iLeafOffset = iOff;
  1377   1615     }
  1378   1616   
  1379   1617     fts5SegIterReverseInitPage(p, pIter);
  1380   1618     pIter->flags |= FTS5_SEGITER_REVERSE;
  1381   1619   }
         1620  +
         1621  +/*
         1622  +** Iterator pIter currently points to the first rowid of a doclist within
         1623  +** index iIdx. There is a doclist-index associated with the final term on
         1624  +** the current page. If the current term is the last term on the page, 
         1625  +** load the doclist-index from disk and initialize an iterator at 
         1626  +** (pIter->pDlidx).
         1627  +*/
         1628  +static void fts5SegIterLoadDlidx(Fts5Index *p, int iIdx, Fts5SegIter *pIter){
         1629  +  int iSegid = pIter->pSeg->iSegid;
         1630  +  int bRev = (pIter->flags & FTS5_SEGITER_REVERSE);
         1631  +  Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */
         1632  +  int iOff = pIter->iLeafOffset;  /* Byte offset within current leaf */
         1633  +
         1634  +  assert( pIter->flags & FTS5_SEGITER_ONETERM );
         1635  +  assert( pIter->pDlidx==0 );
         1636  +
         1637  +  /* Check if the current doclist ends on this page. If it does, return
         1638  +  ** early without loading the doclist-index (as it belongs to a different
         1639  +  ** term. */
         1640  +  while( iOff<pLeaf->n ){
         1641  +    i64 iDelta;
         1642  +    int nPoslist;
         1643  +
         1644  +    /* iOff is currently the offset of the size field of a position list. */
         1645  +    iOff += getVarint32(&pLeaf->p[iOff], nPoslist);
         1646  +    iOff += nPoslist;
         1647  +
         1648  +    if( iOff<pLeaf->n ){
         1649  +      iOff += getVarint(&pLeaf->p[iOff], (u64*)&iDelta);
         1650  +      if( iDelta==0 ) return;
         1651  +    }
         1652  +  }
         1653  +
         1654  +  fts5DlidxIterInit(p, bRev, iIdx, iSegid, pIter->iLeafPgno, &pIter->pDlidx);
         1655  +}
  1382   1656   
  1383   1657   /*
  1384   1658   ** Initialize the object pIter to point to term pTerm/nTerm within segment
  1385   1659   ** pSeg, index iIdx. If there is no such term in the index, the iterator
  1386   1660   ** is set to EOF.
  1387   1661   **
  1388   1662   ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If 
................................................................................
  1395   1669     int flags,                      /* Mask of FTS5INDEX_XXX flags */
  1396   1670     Fts5StructureSegment *pSeg,     /* Description of segment */
  1397   1671     Fts5SegIter *pIter              /* Object to populate */
  1398   1672   ){
  1399   1673     int iPg = 1;
  1400   1674     int h;
  1401   1675     int bGe = ((flags & FTS5INDEX_QUERY_PREFIX) && iIdx==0);
         1676  +  int bDlidx = 0;                 /* True if there is a doclist-index */
  1402   1677   
  1403   1678     assert( bGe==0 || (flags & FTS5INDEX_QUERY_ASC)==0 );
  1404   1679     assert( pTerm && nTerm );
  1405   1680     memset(pIter, 0, sizeof(*pIter));
  1406   1681     pIter->pSeg = pSeg;
  1407   1682     pIter->iIdx = iIdx;
  1408   1683   
................................................................................
  1414   1689       Fts5Data *pNode = fts5DataRead(p, iRowid);
  1415   1690       if( pNode==0 ) break;
  1416   1691   
  1417   1692       fts5NodeIterInit(pNode->p, pNode->n, &node);
  1418   1693       assert( node.term.n==0 );
  1419   1694   
  1420   1695       iPg = node.iChild;
         1696  +    bDlidx = node.bDlidx;
  1421   1697       for(fts5NodeIterNext(&p->rc, &node);
  1422   1698           node.aData && fts5BufferCompareBlob(&node.term, pTerm, nTerm)<=0;
  1423   1699           fts5NodeIterNext(&p->rc, &node)
  1424   1700       ){
  1425   1701         iPg = node.iChild;
         1702  +      bDlidx = node.bDlidx;
  1426   1703       }
  1427   1704       fts5NodeIterFree(&node);
  1428   1705       fts5DataRelease(pNode);
  1429   1706     }
  1430   1707   
  1431   1708     if( iPg<pSeg->pgnoFirst ){
  1432   1709       iPg = pSeg->pgnoFirst;
         1710  +    bDlidx = 0;
  1433   1711     }
  1434   1712   
  1435   1713     pIter->iLeafPgno = iPg - 1;
  1436   1714     fts5SegIterNextPage(p, pIter);
  1437   1715   
  1438   1716     if( pIter->pLeaf ){
  1439   1717       int res;
................................................................................
  1450   1728         fts5DataRelease(pIter->pLeaf);
  1451   1729         pIter->pLeaf = 0;
  1452   1730       }
  1453   1731     }
  1454   1732   
  1455   1733     if( bGe==0 ){
  1456   1734       pIter->flags |= FTS5_SEGITER_ONETERM;
  1457         -    if( pIter->pLeaf && (flags & FTS5INDEX_QUERY_ASC) ){
  1458         -      fts5SegIterReverse(p, iIdx, pIter);
         1735  +    if( pIter->pLeaf ){
         1736  +      if( bDlidx ){
         1737  +        fts5SegIterLoadDlidx(p, iIdx, pIter);
         1738  +      }
         1739  +      if( flags & FTS5INDEX_QUERY_ASC ){
         1740  +        fts5SegIterReverse(p, iIdx, pIter);
         1741  +      }
  1459   1742       }
  1460   1743     }
  1461   1744   }
  1462   1745   
  1463   1746   /*
  1464   1747   ** Zero the iterator passed as the only argument.
  1465   1748   */
  1466   1749   static void fts5SegIterClear(Fts5SegIter *pIter){
  1467   1750     fts5BufferFree(&pIter->term);
  1468   1751     fts5DataRelease(pIter->pLeaf);
         1752  +  fts5DlidxIterFree(pIter->pDlidx);
  1469   1753     sqlite3_free(pIter->aRowidOffset);
  1470   1754     memset(pIter, 0, sizeof(Fts5SegIter));
  1471   1755   }
  1472   1756   
  1473   1757   /*
  1474   1758   ** Do the comparison necessary to populate pIter->aFirst[iOut].
  1475   1759   **
................................................................................
  1546   1830       int iEq;
  1547   1831       if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){
  1548   1832         fts5SegIterNext(p, &pIter->aSeg[iEq]);
  1549   1833         i = pIter->nSeg + iEq;
  1550   1834       }
  1551   1835     }
  1552   1836   }
         1837  +
         1838  +/*
         1839  +** Move the seg-iter so that it points to the first rowid on page iLeafPgno.
         1840  +** It is an error if leaf iLeafPgno contains no rowid.
         1841  +*/
         1842  +static void fts5SegIterGotoPage(
         1843  +  Fts5Index *p,                   /* FTS5 backend object */
         1844  +  Fts5SegIter *pIter,             /* Iterator to advance */
         1845  +  int iLeafPgno
         1846  +){
         1847  +  assert( iLeafPgno>pIter->iLeafPgno );
         1848  +  if( p->rc==SQLITE_OK ){
         1849  +    pIter->iLeafPgno = iLeafPgno-1;
         1850  +    fts5SegIterNextPage(p, pIter);
         1851  +    assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno );
         1852  +  }
         1853  +
         1854  +  if( p->rc==SQLITE_OK ){
         1855  +    int iOff;
         1856  +    u8 *a = pIter->pLeaf->p;
         1857  +    int n = pIter->pLeaf->n;
         1858  +
         1859  +    iOff = fts5GetU16(&a[0]);
         1860  +    if( iOff<4 || iOff>=n ){
         1861  +      p->rc = FTS5_CORRUPT;
         1862  +    }else{
         1863  +      iOff += getVarint(&a[iOff], (u64*)&pIter->iRowid);
         1864  +      pIter->iLeafOffset = iOff;
         1865  +    }
         1866  +  }
         1867  +}
         1868  +
         1869  +/*
         1870  +** Advance the iterator passed as the second argument until it is at or 
         1871  +** past rowid iFrom. Regardless of the value of iFrom, the iterator is
         1872  +** always advanced at least once.
         1873  +*/
         1874  +static void fts5SegIterNextFrom(
         1875  +  Fts5Index *p,                   /* FTS5 backend object */
         1876  +  Fts5SegIter *pIter,             /* Iterator to advance */
         1877  +  i64 iMatch                      /* Advance iterator at least this far */
         1878  +){
         1879  +  int bRev = (pIter->flags & FTS5_SEGITER_REVERSE);
         1880  +  Fts5DlidxIter *pDlidx = pIter->pDlidx;
         1881  +  int iLeafPgno = pIter->iLeafPgno;
         1882  +
         1883  +  assert( pIter->flags & FTS5_SEGITER_ONETERM );
         1884  +  assert( pIter->pDlidx );
         1885  +  assert( pIter->pLeaf );
         1886  +
         1887  +
         1888  +  if( bRev==0 ){
         1889  +    while( fts5DlidxIterEof(p, pDlidx)==0 && iMatch<pDlidx->iRowid ){
         1890  +      if( pDlidx->bZero==0 ) iLeafPgno = pDlidx->iLeafPgno;
         1891  +      fts5DlidxIterNext(pDlidx, 0);
         1892  +    }
         1893  +    assert( iLeafPgno>=pIter->iLeafPgno || p->rc );
         1894  +    if( iLeafPgno>pIter->iLeafPgno ){
         1895  +      fts5SegIterGotoPage(p, pIter, iLeafPgno);
         1896  +    }
         1897  +  }else if( 0 ){
         1898  +    while( fts5DlidxIterEof(p, pDlidx)==0 && iMatch>pDlidx->iRowid ){
         1899  +      fts5DlidxIterNext(pDlidx, 0);
         1900  +      if( pDlidx->bZero==0 ) iLeafPgno = pDlidx->iLeafPgno;
         1901  +    }
         1902  +    assert( iLeafPgno<=pIter->iLeafPgno || p->rc );
         1903  +    if( iLeafPgno<pIter->iLeafPgno ){
         1904  +      fts5SegIterGotoPage(p, pIter, iLeafPgno);
         1905  +    }
         1906  +  }
         1907  +
         1908  +  while( 1 ){
         1909  +    fts5SegIterNext(p, pIter);
         1910  +    if( pIter->pLeaf==0 ) break;
         1911  +    if( bRev==0 && pIter->iRowid<=iMatch ) break;
         1912  +    if( bRev!=0 && pIter->iRowid>=iMatch ) break;
         1913  +  }
         1914  +}
  1553   1915   
  1554   1916   /*
  1555   1917   ** Move the iterator to the next entry. 
  1556   1918   **
  1557   1919   ** If an error occurs, an error code is left in Fts5Index.rc. It is not 
  1558   1920   ** considered an error if the iterator reaches EOF, or if it is already at 
  1559   1921   ** EOF when this function is called.
  1560   1922   */
  1561         -static void fts5MultiIterNext(Fts5Index *p, Fts5MultiSegIter *pIter){
         1923  +static void fts5MultiIterNext(
         1924  +  Fts5Index *p, 
         1925  +  Fts5MultiSegIter *pIter,
         1926  +  int bFrom,                      /* True if argument iFrom is valid */
         1927  +  i64 iFrom                       /* Advance at least as far as this */
         1928  +){
  1562   1929     if( p->rc==SQLITE_OK ){
  1563   1930       int iFirst = pIter->aFirst[1];
  1564         -    fts5SegIterNext(p, &pIter->aSeg[iFirst]);
         1931  +    Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
         1932  +    if( bFrom && pSeg->pDlidx ){
         1933  +      fts5SegIterNextFrom(p, pSeg, iFrom);
         1934  +    }else{
         1935  +      fts5SegIterNext(p, pSeg);
         1936  +    }
  1565   1937       fts5MultiIterAdvanced(p, pIter, iFirst, 1);
  1566   1938     }
  1567   1939   }
  1568   1940   
  1569   1941   /*
  1570   1942   ** Allocate a new Fts5MultiSegIter object.
  1571   1943   **
................................................................................
  1678   2050   static void fts5MultiIterNextFrom(
  1679   2051     Fts5Index *p, 
  1680   2052     Fts5MultiSegIter *pIter, 
  1681   2053     i64 iMatch
  1682   2054   ){
  1683   2055     while( 1 ){
  1684   2056       i64 iRowid;
  1685         -    fts5MultiIterNext(p, pIter);
         2057  +    fts5MultiIterNext(p, pIter, 1, iMatch);
  1686   2058       if( fts5MultiIterEof(p, pIter) ) break;
  1687   2059       iRowid = fts5MultiIterRowid(pIter);
  1688   2060       if( pIter->bRev==0 && iRowid<=iMatch ) break;
  1689   2061       if( pIter->bRev!=0 && iRowid>=iMatch ) break;
  1690   2062     }
  1691   2063   }
  1692   2064   
................................................................................
  2585   2957   #if 0
  2586   2958   fprintf(stdout, "merging %d segments from level %d!", nInput, iLvl);
  2587   2959   fflush(stdout);
  2588   2960   #endif
  2589   2961   
  2590   2962     for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, iLvl, nInput, &pIter);
  2591   2963         fts5MultiIterEof(p, pIter)==0;
  2592         -      fts5MultiIterNext(p, pIter)
         2964  +      fts5MultiIterNext(p, pIter, 0, 0)
  2593   2965     ){
  2594   2966       Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1] ];
  2595   2967       Fts5ChunkIter sPos;           /* Used to iterate through position list */
  2596   2968       int nTerm;
  2597   2969       const u8 *pTerm = fts5MultiIterTerm(pIter, &nTerm);
  2598   2970   
  2599   2971       if( nTerm!=term.n || memcmp(pTerm, term.p, nTerm) ){
................................................................................
  3001   3373         pLvl->pData = 0;
  3002   3374       }
  3003   3375     }
  3004   3376     sqlite3_free(pIter->aLvl);
  3005   3377     fts5BufferFree(&pIter->term);
  3006   3378   }
  3007   3379   
  3008         -typedef struct DoclistIdxIter DoclistIdxIter;
  3009         -struct DoclistIdxIter {
  3010         -  Fts5Data *pDlidx;             /* Data for doclist index, if any */
  3011         -  int iOff;                     /* Current offset into pDlidx */
  3012         -  int bRowidValid;              /* iRowid is valid */
  3013   3380   
  3014         -  int bZero;                    /* True if current leaf has no rowid */
  3015         -  i64 iRowid;                   /* If bZero==0, first rowid on leaf */
  3016         -};
  3017         -
  3018         -/*
  3019         -** Return non-zero if EOF is reached.
  3020         -*/
  3021         -static int fts5IndexDoclistIterNext(DoclistIdxIter *pIter){
  3022         -  i64 iVal;
  3023         -  if( pIter->iOff>=pIter->pDlidx->n ) return 1;
  3024         -  pIter->iOff += getVarint(&pIter->pDlidx->p[pIter->iOff], (u64*)&iVal);
  3025         -  if( iVal==0 ){
  3026         -    pIter->bZero = 1;
  3027         -  }else{
  3028         -    pIter->bZero = 0;
  3029         -    if( pIter->bRowidValid ){
  3030         -      pIter->iRowid -= iVal;
  3031         -    }else{
  3032         -      pIter->bRowidValid = 1;
  3033         -      pIter->iRowid = iVal;
  3034         -    }
  3035         -  }
  3036         -  return 0;
  3037         -}
  3038   3381   
  3039   3382   static void fts5IndexIntegrityCheckSegment(
  3040   3383     Fts5Index *p,                   /* FTS5 backend object */
  3041   3384     int iIdx,                       /* Index that pSeg is a part of */
  3042   3385     Fts5StructureSegment *pSeg      /* Segment to check internal consistency */
  3043   3386   ){
  3044   3387     Fts5BtreeIter iter;             /* Used to iterate through b-tree hierarchy */
................................................................................
  3048   3391         iter.bEof==0;
  3049   3392         fts5BtreeIterNext(&iter)
  3050   3393     ){
  3051   3394       i64 iRow;                     /* Rowid for this leaf */
  3052   3395       Fts5Data *pLeaf;              /* Data for this leaf */
  3053   3396       int iOff;                     /* Offset of first term on leaf */
  3054   3397       int i;                        /* Used to iterate through empty leaves */
  3055         -    DoclistIdxIter dliter;        /* For iterating through any doclist index */
  3056   3398   
  3057   3399       /* If the leaf in question has already been trimmed from the segment, 
  3058   3400       ** ignore this b-tree entry. Otherwise, load it into memory. */
  3059   3401       if( iter.iLeaf<pSeg->pgnoFirst ) continue;
  3060   3402       iRow = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, 0, iter.iLeaf);
  3061   3403       pLeaf = fts5DataRead(p, iRow);
  3062   3404       if( pLeaf==0 ) break;
................................................................................
  3075   3417         if( res<0 ){
  3076   3418           p->rc = FTS5_CORRUPT;
  3077   3419         }
  3078   3420       }
  3079   3421       fts5DataRelease(pLeaf);
  3080   3422       if( p->rc ) break;
  3081   3423   
  3082         -    memset(&dliter, 0, sizeof(DoclistIdxIter));
  3083         -    if( iter.bDlidx ){
  3084         -      i64 iDlidxRowid = FTS5_DOCLIST_IDX_ROWID(iIdx, pSeg->iSegid, iter.iLeaf);
  3085         -      dliter.pDlidx = fts5DataRead(p, iDlidxRowid);
  3086         -    }
  3087   3424   
  3088   3425       /* Now check that the iter.nEmpty leaves following the current leaf
  3089   3426       ** (a) exist and (b) contain no terms. */
  3090         -    for(i=1; i<=iter.nEmpty; i++){
         3427  +    for(i=1; p->rc==SQLITE_OK && i<=iter.nEmpty; i++){
  3091   3428         pLeaf = fts5DataRead(p, iRow+i);
  3092   3429         if( pLeaf && 0!=fts5GetU16(&pLeaf->p[2]) ){
  3093   3430           p->rc = FTS5_CORRUPT;
  3094   3431         }
  3095         -      if( pLeaf && dliter.pDlidx ){
  3096         -        if( fts5IndexDoclistIterNext(&dliter) ){
  3097         -          p->rc = FTS5_CORRUPT;
  3098         -        }else{
         3432  +      fts5DataRelease(pLeaf);
         3433  +    }
         3434  +
         3435  +    /* If there is a doclist-index, check that it looks right. */
         3436  +    if( iter.bDlidx ){
         3437  +      Fts5DlidxIter *pDlidx = 0;  /* For iterating through doclist index */
         3438  +      int nEntry = 0;
         3439  +      int iSegid = pSeg->iSegid;
         3440  +      int bRev = 0;
         3441  +
         3442  +      for(fts5DlidxIterInit(p, bRev, iIdx, iSegid, iter.iLeaf, &pDlidx);
         3443  +          fts5DlidxIterEof(p, pDlidx)==0;
         3444  +          fts5DlidxIterNext(pDlidx, bRev)
         3445  +      ){
         3446  +        i64 iKey = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, pDlidx->iLeafPgno);
         3447  +        pLeaf = fts5DataRead(p, iKey);
         3448  +        if( pLeaf ){
  3099   3449             int iRowidOff = fts5GetU16(&pLeaf->p[0]);
  3100         -          if( dliter.bZero ){
         3450  +          if( pDlidx->bZero ){
  3101   3451               if( iRowidOff!=0 ) p->rc = FTS5_CORRUPT;
  3102   3452             }else{
  3103   3453               i64 iRowid;
  3104   3454               getVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid);
  3105         -            if( iRowid!=dliter.iRowid ) p->rc = FTS5_CORRUPT;
  3106         -          }
  3107         -        }
  3108         -      }
  3109         -      fts5DataRelease(pLeaf);
  3110         -    }
  3111         -
  3112         -    /* There may (or may not be) a final entry in the doclist. The entry
  3113         -    ** is only present if the page following the nEmpty termless pages
  3114         -    ** (a) exists and (b) contains at least one rowid that is part of
  3115         -    ** the doclist.  */
  3116         -    if( dliter.pDlidx ){
  3117         -      if( (iter.iLeaf + iter.nEmpty)==pSeg->pgnoLast ){
  3118         -        /* The next page does not exist. So the iterator should be at EOF. */
  3119         -        if( fts5IndexDoclistIterNext(&dliter)==0 ) p->rc = FTS5_CORRUPT;
  3120         -      }else{
  3121         -        Fts5Data *pLeaf = fts5DataRead(p, iRow+i);
  3122         -        if( pLeaf ){
  3123         -          int iRowidOff = fts5GetU16(&pLeaf->p[0]);
  3124         -          if( iRowidOff==0 ){
  3125         -            if( fts5IndexDoclistIterNext(&dliter)==0 ) p->rc = FTS5_CORRUPT;
  3126         -          }else{
  3127         -            if( fts5IndexDoclistIterNext(&dliter) ){
  3128         -              p->rc = FTS5_CORRUPT;
  3129         -            }else{
  3130         -              i64 iRowid;
  3131         -              getVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid);
  3132         -              if( iRowid!=dliter.iRowid ) p->rc = FTS5_CORRUPT;
  3133         -            }
         3455  +            if( iRowid!=pDlidx->iRowid ) p->rc = FTS5_CORRUPT;
  3134   3456             }
  3135   3457             fts5DataRelease(pLeaf);
  3136   3458           }
         3459  +        nEntry++;
  3137   3460         }
         3461  +
         3462  +      /* Check that the doclist-index was the right length */
         3463  +      if( p->rc==SQLITE_OK && nEntry!=iter.nEmpty && nEntry!=iter.nEmpty+1 ){
         3464  +        p->rc = FTS5_CORRUPT;
         3465  +      }
         3466  +      fts5DlidxIterFree(pDlidx);
  3138   3467       }
  3139         -
  3140         -    fts5DataRelease(dliter.pDlidx);
  3141   3468     }
  3142   3469   
  3143   3470     if( p->rc==SQLITE_OK && iter.iLeaf!=pSeg->pgnoLast ){
  3144   3471       p->rc = FTS5_CORRUPT;
  3145   3472     }
  3146   3473   
  3147   3474     fts5BtreeIterFree(&iter);
................................................................................
  3165   3492   
  3166   3493     /* Check that the checksum of the index matches the argument checksum */
  3167   3494     for(iIdx=0; iIdx<=pConfig->nPrefix; iIdx++){
  3168   3495       Fts5MultiSegIter *pIter;
  3169   3496       Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
  3170   3497       for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, -1, 0, &pIter);
  3171   3498           fts5MultiIterEof(p, pIter)==0;
  3172         -        fts5MultiIterNext(p, pIter)
         3499  +        fts5MultiIterNext(p, pIter, 0, 0)
  3173   3500       ){
  3174   3501         Fts5PosIter sPos;           /* Used to iterate through position list */
  3175   3502         int n;                      /* Size of term in bytes */
  3176   3503         i64 iRowid = fts5MultiIterRowid(pIter);
  3177   3504         char *z = (char*)fts5MultiIterTerm(pIter, &n);
  3178   3505   
  3179   3506         for(fts5PosIterInit(p, pIter, &sPos);
................................................................................
  3246   3573       }
  3247   3574       sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
  3248   3575     }
  3249   3576   
  3250   3577     fts5StructureRelease(p);
  3251   3578   }
  3252   3579   
  3253         -/*
  3254         -** Decode a segment-data rowid from the %_data table. This function is
  3255         -** the opposite of macro FTS5_SEGMENT_ROWID().
  3256         -*/
  3257         -static void fts5DecodeRowid(
  3258         -  i64 iRowid,                     /* Rowid from %_data table */
  3259         -  int *piIdx,                     /* OUT: Index */
  3260         -  int *piSegid,                   /* OUT: Segment id */
  3261         -  int *piHeight,                  /* OUT: Height */
  3262         -  int *piPgno                     /* OUT: Page number */
  3263         -){
  3264         -  *piPgno = (int)(iRowid & (((i64)1 << FTS5_DATA_PAGE_B) - 1));
  3265         -  iRowid >>= FTS5_DATA_PAGE_B;
  3266         -
  3267         -  *piHeight = (int)(iRowid & (((i64)1 << FTS5_DATA_HEIGHT_B) - 1));
  3268         -  iRowid >>= FTS5_DATA_HEIGHT_B;
  3269         -
  3270         -  *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1));
  3271         -  iRowid >>= FTS5_DATA_ID_B;
  3272         -
  3273         -  *piIdx = (int)(iRowid & (((i64)1 << FTS5_DATA_IDX_B) - 1));
  3274         -}
  3275         -
  3276   3580   /*
  3277   3581   ** Buffer (a/n) is assumed to contain a list of serialized varints. Read
  3278   3582   ** each varint and append its string representation to buffer pBuf. Return
  3279   3583   ** after either the input buffer is exhausted or a 0 value is read.
  3280   3584   **
  3281   3585   ** The return value is the number of bytes read from the input buffer.
  3282   3586   */
................................................................................
  3327   3631   */
  3328   3632   static void fts5DecodeFunction(
  3329   3633     sqlite3_context *pCtx,          /* Function call context */
  3330   3634     int nArg,                       /* Number of args (always 2) */
  3331   3635     sqlite3_value **apVal           /* Function arguments */
  3332   3636   ){
  3333   3637     i64 iRowid;                     /* Rowid for record being decoded */
  3334         -  int iIdx,iSegid,iHeight,iPgno;  /* Rowid compenents */
         3638  +  int iIdx,iSegid,iHeight,iPgno;  /* Rowid components */
  3335   3639     const u8 *a; int n;             /* Record to decode */
  3336   3640     Fts5Buffer s;                   /* Build up text to return here */
  3337   3641     int rc = SQLITE_OK;             /* Return code */
  3338   3642   
  3339   3643     assert( nArg==2 );
  3340   3644     memset(&s, 0, sizeof(Fts5Buffer));
  3341   3645     iRowid = sqlite3_value_int64(apVal[0]);
  3342   3646     n = sqlite3_value_bytes(apVal[1]);
  3343   3647     a = sqlite3_value_blob(apVal[1]);
  3344   3648     fts5DecodeRowid(iRowid, &iIdx, &iSegid, &iHeight, &iPgno);
  3345   3649   
         3650  +  fts5DebugRowid(&rc, &s, iRowid);
  3346   3651     if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){
  3347   3652       int i = 0;
  3348   3653       i64 iPrev;
  3349         -    sqlite3Fts5BufferAppendPrintf(&rc, &s, "(dlidx idx=%d segid=%d pgno=%d)",
  3350         -        iIdx, iSegid, iPgno
  3351         -    );
  3352   3654       if( n>0 ){
  3353   3655         i = getVarint(&a[i], (u64*)&iPrev);
  3354   3656         sqlite3Fts5BufferAppendPrintf(&rc, &s, " %lld", iPrev);
  3355   3657       }
  3356   3658       while( i<n ){
  3357   3659         i64 iVal;
  3358   3660         i += getVarint(&a[i], (u64*)&iVal);
................................................................................
  3363   3665           sqlite3Fts5BufferAppendPrintf(&rc, &s, " %lld", iPrev);
  3364   3666         }
  3365   3667       }
  3366   3668   
  3367   3669     }else
  3368   3670     if( iSegid==0 ){
  3369   3671       if( iRowid==FTS5_AVERAGES_ROWID ){
  3370         -      sqlite3Fts5BufferAppendPrintf(&rc, &s, "{averages} ");
         3672  +      /* todo */
  3371   3673       }else{
  3372         -      sqlite3Fts5BufferAppendPrintf(&rc, &s, 
  3373         -          "{structure idx=%d}", (int)(iRowid-10)
  3374         -      );
  3375   3674         fts5DecodeStructure(&rc, &s, a, n);
  3376   3675       }
  3377   3676     }else{
  3378   3677   
  3379   3678       Fts5Buffer term;
  3380   3679       memset(&term, 0, sizeof(Fts5Buffer));
  3381         -    sqlite3Fts5BufferAppendPrintf(&rc, &s, "(idx=%d segid=%d h=%d pgno=%d) ",
  3382         -        iIdx, iSegid, iHeight, iPgno
  3383         -    );
  3384   3680   
  3385   3681       if( iHeight==0 ){
  3386   3682         int iTermOff = 0;
  3387   3683         int iRowidOff = 0;
  3388   3684         int iOff;
  3389   3685         int nKeep = 0;
  3390   3686   
................................................................................
  3662   3958       i64 iLastRowid;
  3663   3959       Fts5MultiSegIter *p1 = 0;     /* Iterator used to gather data from index */
  3664   3960       Fts5Buffer doclist;
  3665   3961   
  3666   3962       memset(&doclist, 0, sizeof(doclist));
  3667   3963       for(fts5MultiIterNew(p, pStruct, 0, 1, pToken, nToken, -1, 0, &p1);
  3668   3964           fts5MultiIterEof(p, p1)==0;
  3669         -        fts5MultiIterNext(p, p1)
         3965  +        fts5MultiIterNext(p, p1, 0, 0)
  3670   3966       ){
  3671   3967         i64 iRowid = fts5MultiIterRowid(p1);
  3672   3968         int nTerm;
  3673   3969         const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm);
  3674   3970         assert( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
  3675   3971         if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break;
  3676   3972   
................................................................................
  3781   4077   ** Move to the next matching rowid. 
  3782   4078   */
  3783   4079   void sqlite3Fts5IterNext(Fts5IndexIter *pIter){
  3784   4080     if( pIter->pDoclist ){
  3785   4081       fts5DoclistIterNext(pIter->pDoclist);
  3786   4082     }else{
  3787   4083       fts5BufferZero(&pIter->poslist);
  3788         -    fts5MultiIterNext(pIter->pIndex, pIter->pMulti);
         4084  +    fts5MultiIterNext(pIter->pIndex, pIter->pMulti, 0, 0);
  3789   4085     }
  3790   4086   }
  3791   4087   
  3792   4088   /*
  3793   4089   ** Move to the next matching rowid that occurs at or after iMatch. The
  3794   4090   ** definition of "at or after" depends on whether this iterator iterates
  3795   4091   ** in ascending or descending rowid order.

Changes to test/fts5aa.test.

    46     46   }
    47     47   do_execsql_test 2.1 {
    48     48     INSERT INTO t1 VALUES('a b c', 'd e f');
    49     49   }
    50     50   do_execsql_test 2.2 {
    51     51     SELECT fts5_decode(id, block) FROM t1_data WHERE id==10
    52     52   } {
    53         -  {{structure idx=0} {lvl=0 nMerge=0 {id=27723 h=1 leaves=1..1}}}
           53  +  {(structure idx=0) {lvl=0 nMerge=0 {id=27723 h=1 leaves=1..1}}}
    54     54   }
    55     55   do_execsql_test 2.3 {
    56     56     INSERT INTO t1(t1) VALUES('integrity-check');
    57     57   }
    58     58   
    59     59   #-------------------------------------------------------------------------
    60     60   #
................................................................................
   178    178         set y [doc]
   179    179         set z [doc]
   180    180         set rowid [expr int(rand() * 100)]
   181    181         execsql { REPLACE INTO t1(rowid,x,y,z) VALUES($rowid, $x, $y, $z) }
   182    182       }
   183    183       execsql { INSERT INTO t1(t1) VALUES('integrity-check'); }
   184    184     } {}
   185         -  if {[set_test_counter errors]} exit
   186    185   }
   187    186   
   188    187   #-------------------------------------------------------------------------
   189    188   #
   190    189   reset_db
   191    190   do_execsql_test 8.0 {
   192    191     CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3");

Changes to test/fts5ah.test.

    51     51   
    52     52   proc reads {} {
    53     53     db one {SELECT t1 FROM t1 WHERE t1 MATCH '*reads'}
    54     54   }
    55     55   
    56     56   do_test 1.4 {
    57     57     set nRead [reads]
    58         -  db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x' }
    59         -  set a [expr [reads] - $nRead]
    60         -} {}
           58  +  execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'x' }
           59  +  set nReadX [expr [reads] - $nRead]
           60  +  expr $nReadX>1000
           61  +} {1}
           62  +
           63  +foreach {tn q res} "
           64  +  1 { SELECT rowid FROM t1 WHERE t1 MATCH 'w + x'   }  [list $W]
           65  +  2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x + w'   }  [list $W]
           66  +  3 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w' }  [list $W]
           67  +  4 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x' }  [list $Y]
           68  +" {
           69  +
           70  +  do_test 1.5.$tn.1 {
           71  +    set nRead [reads]
           72  +    execsql $q
           73  +    set n [expr [reads] - $nRead]
           74  +    expr {$n < ($nReadX / 10)}
           75  +  } {1}
           76  +
           77  +  do_test 1.5.$tn.2 {
           78  +    set nRead [reads]
           79  +    execsql "$q ORDER BY rowid ASC"
           80  +    set n [expr [reads] - $nRead]
           81  +    expr {$n < ($nReadX / 10)}
           82  +  } {1}
    61     83   
    62         -do_test 1.5 {
    63         -  set nRead [reads]
    64         -  db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x + w' }
    65         -  set a [expr [reads] - $nRead]
    66         -} {}
           84  +  do_execsql_test 1.5.$tn.3 $q [lsort -int -decr $res]
           85  +  do_execsql_test 1.5.$tn.4 "$q ORDER BY rowid ASC" [lsort -int -incr $res]
           86  +}
    67     87   
           88  +#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}
    68     89   
    69     90   finish_test
    70     91