/ Check-in [898618cc]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Reduce the number of calls to malloc() made by fts5.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 898618ccf61d3ad166d9fc742e132d518338b5c3
User & Date: dan 2015-07-09 19:02:19
Context
2015-07-09
20:46
Improve the performance of docid merges in fts5. check-in: b2de77a0 user: dan tags: trunk
19:02
Reduce the number of calls to malloc() made by fts5. check-in: 898618cc user: dan tags: trunk
2015-07-08
17:59
Fix two problems that could cause fts3 auxiliary functions to occasionally misbehave if used with match expressions that contain both OR and NEAR. check-in: 372c1db2 user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5Int.h.

32
33
34
35
36
37
38

39
40
41
42
43
44
45
#define ArraySize(x) (sizeof(x) / sizeof(x[0]))

#define testcase(x)
#define ALWAYS(x) 1
#define NEVER(x) 0

#define MIN(x,y) (((x) < (y)) ? (x) : (y))


#endif


/*
** Maximum number of prefix indexes on single FTS5 table. This must be
** less than 32. If it is set to anything large than that, an #error







>







32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
#define ArraySize(x) (sizeof(x) / sizeof(x[0]))

#define testcase(x)
#define ALWAYS(x) 1
#define NEVER(x) 0

#define MIN(x,y) (((x) < (y)) ? (x) : (y))
#define MAX(x,y) (((x) > (y)) ? (x) : (y))

#endif


/*
** Maximum number of prefix indexes on single FTS5 table. This must be
** less than 32. If it is set to anything large than that, an #error

Changes to ext/fts5/fts5_index.c.

600
601
602
603
604
605
606















607
608
609
610
611
612
613
...
720
721
722
723
724
725
726
727
728
729
730


731
732
733
734
735
736


737
738
739
740
741
742
743
....
1015
1016
1017
1018
1019
1020
1021
1022
1023

1024
1025
1026


1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
....
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
....
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
....
2090
2091
2092
2093
2094
2095
2096

2097
2098
2099
2100
2101
2102
2103
....
2226
2227
2228
2229
2230
2231
2232


2233

2234
2235
2236
2237
2238
2239
2240
....
2283
2284
2285
2286
2287
2288
2289

2290
2291
2292
2293
2294
2295
2296
2297
2298
2299



2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
....
2336
2337
2338
2339
2340
2341
2342














2343
2344
2345
2346
2347
2348
2349
....
2791
2792
2793
2794
2795
2796
2797

2798
2799
2800
2801
2802
2803
2804
....
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
....
2862
2863
2864
2865
2866
2867
2868

2869
2870
2871
2872
2873
2874
2875
....
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
  Fts5Buffer term;                /* Current term */
  int iLeaf;                      /* Leaf containing terms >= current term */
  int nEmpty;                     /* Number of "empty" leaves following iLeaf */
  int bEof;                       /* Set to true at EOF */
  int bDlidx;                     /* True if there exists a dlidx */
};

















static void fts5PutU16(u8 *aOut, u16 iVal){
  aOut[0] = (iVal>>8);
  aOut[1] = (iVal&0xFF);
}

static u16 fts5GetU16(const u8 *aIn){
................................................................................
    ** backing store corruption.  */
    if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT;

    if( rc==SQLITE_OK ){
      u8 *aOut;                   /* Read blob data into this buffer */
      int nByte = sqlite3_blob_bytes(p->pReader);
      if( pBuf ){
        fts5BufferZero(pBuf);
        fts5BufferGrow(&rc, pBuf, nByte);
        aOut = pBuf->p;
        pBuf->n = nByte;


      }else{
        int nSpace = nByte + FTS5_DATA_ZERO_PADDING;
        pRet = (Fts5Data*)sqlite3Fts5MallocZero(&rc, nSpace+sizeof(Fts5Data));
        if( pRet ){
          pRet->n = nByte;
          aOut = pRet->p = (u8*)&pRet[1];


        }
      }

      if( rc==SQLITE_OK ){
        rc = sqlite3_blob_read(p->pReader, aOut, nByte, 0);
      }
      if( rc!=SQLITE_OK ){
................................................................................
** If an error occurs, NULL is returned and an error code left in the
** Fts5Index handle. If an error has already occurred when this function
** is called, it is a no-op.
*/
static Fts5Structure *fts5StructureRead(Fts5Index *p){
  Fts5Config *pConfig = p->pConfig;
  Fts5Structure *pRet = 0;        /* Object to return */
  Fts5Data *pData;                /* %_data entry containing structure record */
  int iCookie;                    /* Configuration cookie */


  pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
  if( !pData ) return 0;


  p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet);

  if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
    p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
  }

  fts5DataRelease(pData);
  if( p->rc!=SQLITE_OK ){
    fts5StructureRelease(pRet);
    pRet = 0;
  }
  return pRet;
}

................................................................................
  }

  pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno);
}

#ifdef SQLITE_DEBUG
static void fts5AssertNodeSeekOk(
  Fts5Data *pNode,
  const u8 *pTerm, int nTerm,     /* Term to search for */
  int iExpectPg,
  int bExpectDlidx
){
  int bDlidx;
  int iPg;
  int rc = SQLITE_OK;
................................................................................
** It returns the associated page number. Or, if (pTerm/nTerm) is smaller
** than all terms within the node, the leftmost child page number. 
**
** Before returning, (*pbDlidx) is set to true if the last term on the
** returned child page number has a doclist-index. Or left as is otherwise.
*/
static int fts5NodeSeek(
  Fts5Data *pNode,                /* Node to search */
  const u8 *pTerm, int nTerm,     /* Term to search for */
  int *pbDlidx                    /* OUT: True if dlidx flag is set */
){
  int iPg;
  u8 *pPtr = pNode->p;
  u8 *pEnd = &pPtr[pNode->n];
  int nMatch = 0;                 /* Number of bytes of pTerm already matched */
................................................................................
    /* If there is a "no terms" record at pPtr, read it now. Store the
    ** number of termless pages in nEmpty. If it indicates a doclist-index, 
    ** set (*pbDlidx) to true.*/
    if( *pPtr<2 ){
      *pbDlidx = (*pPtr==0x01);
      pPtr++;
      pPtr += fts5GetVarint32(pPtr, nEmpty);

    }

    /* Read the next "term" pointer. Set nKeep to the number of bytes to
    ** keep from the previous term, and nNew to the number of bytes of
    ** new data that will be appended to it. */
    nKeep = (int)*pPtr++;
    nNew = (int)*pPtr++;
................................................................................

      /* Skip past docid delta */
      fts5IndexSkipVarint(a, iOff);

      /* Skip past position list */
      fts5IndexGetVarint32(a, iOff, nPos);
      iOff += (nPos >> 1);


      if( iOff>=n ) goto search_failed;


      /* If this is the end of the doclist, break out of the loop */
      if( a[iOff]==0x00 ){
        iOff++;
        break;
      }
    };
................................................................................
** pSeg. If there is no such term in the index, the iterator is set to EOF.
**
** If an error occurs, Fts5Index.rc is set to an appropriate error code. If 
** an error has already occurred when this function is called, it is a no-op.
*/
static void fts5SegIterSeekInit(
  Fts5Index *p,                   /* FTS5 backend */

  const u8 *pTerm, int nTerm,     /* Term to seek to */
  int flags,                      /* Mask of FTS5INDEX_XXX flags */
  Fts5StructureSegment *pSeg,     /* Description of segment */
  Fts5SegIter *pIter              /* Object to populate */
){
  int iPg = 1;
  int h;
  int bGe = (flags & FTS5INDEX_QUERY_SCAN);
  int bDlidx = 0;                 /* True if there is a doclist-index */
  Fts5Data *pLeaf;




  assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 );
  assert( pTerm && nTerm );
  memset(pIter, 0, sizeof(*pIter));
  pIter->pSeg = pSeg;

  /* This block sets stack variable iPg to the leaf page number that may
  ** contain term (pTerm/nTerm), if it is present in the segment. */
  for(h=pSeg->nHeight-1; h>0; h--){
    i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, h, iPg);
    Fts5Data *pNode = fts5DataRead(p, iRowid);
    if( pNode==0 ) break;

    iPg = fts5NodeSeek(pNode, pTerm, nTerm, &bDlidx);
    fts5DataRelease(pNode);
  }

  if( iPg<pSeg->pgnoFirst ){
    iPg = pSeg->pgnoFirst;
    bDlidx = 0;
  }

................................................................................
        fts5SegIterLoadDlidx(p, pIter);
      }
      if( flags & FTS5INDEX_QUERY_DESC ){
        fts5SegIterReverse(p, pIter);
      }
    }
  }














}

/*
** Initialize the object pIter to point to term pTerm/nTerm within the
** in-memory hash table. If there is no such term in the hash-table, the 
** iterator is set to EOF.
**
................................................................................
  int iLevel,                     /* Level to iterate (-1 for all) */
  int nSegment,                   /* Number of segments to merge (iLevel>=0) */
  Fts5IndexIter **ppOut           /* New object */
){
  int nSeg = 0;                   /* Number of segment-iters in use */
  int iIter = 0;                  /* */
  int iSeg;                       /* Used to iterate through segments */

  Fts5StructureLevel *pLvl;
  Fts5IndexIter *pNew;

  assert( (pTerm==0 && nTerm==0) || iLevel<0 );

  /* Allocate space for the new multi-seg-iterator. */
  if( p->rc==SQLITE_OK ){
................................................................................
    for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){
      for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){
        Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
        Fts5SegIter *pIter = &pNew->aSeg[iIter++];
        if( pTerm==0 ){
          fts5SegIterInit(p, pSeg, pIter);
        }else{
          fts5SegIterSeekInit(p, pTerm, nTerm, flags, pSeg, pIter);
        }
      }
    }
  }else{
    pLvl = &pStruct->aLevel[iLevel];
    for(iSeg=nSeg-1; iSeg>=0; iSeg--){
      fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]);
................................................................................
    if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){
      fts5MultiIterNext(p, pNew, 0, 0);
    }
  }else{
    fts5MultiIterFree(p, pNew);
    *ppOut = 0;
  }

}

/*
** Create an Fts5IndexIter that iterates through the doclist provided
** as the second argument.
*/
static void fts5MultiIterNew2(
................................................................................
  Fts5Structure *p
){
  int iLvl, iSeg;                 /* Iterate through levels, segments */

  for(iLvl=0; iLvl<p->nLevel; iLvl++){
    Fts5StructureLevel *pLvl = &p->aLevel[iLvl];
    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
        " {lvl=%d nMerge=%d", iLvl, pLvl->nMerge
    );
    for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
      Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
          " {id=%d h=%d leaves=%d..%d}", pSeg->iSegid, pSeg->nHeight, 
          pSeg->pgnoFirst, pSeg->pgnoLast
      );







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|
|

|
>
>


|



>
>







 







<

>

|
|
>
>
|





|







 







|







 







|







 







>







 







>
>
|
>







 







>










>
>
>










|
|
<
|
<







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>







 







|







 







>







 







|







600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
...
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
....
1034
1035
1036
1037
1038
1039
1040

1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
....
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
....
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
....
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
....
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
....
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340

2341

2342
2343
2344
2345
2346
2347
2348
....
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
....
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
....
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
....
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
....
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
  Fts5Buffer term;                /* Current term */
  int iLeaf;                      /* Leaf containing terms >= current term */
  int nEmpty;                     /* Number of "empty" leaves following iLeaf */
  int bEof;                       /* Set to true at EOF */
  int bDlidx;                     /* True if there exists a dlidx */
};


/*
** The first argument passed to this macro is a pointer to an Fts5Buffer
** object.
*/
#define fts5BufferSize(pBuf,n) {                \
  if( pBuf->nSpace<n ) {                        \
    u8 *pNew = sqlite3_realloc(pBuf->p, n);     \
    if( pNew==0 ){                              \
      sqlite3_free(pBuf->p);                    \
    }                                           \
    pBuf->nSpace = n;                           \
    pBuf->p = pNew;                             \
  }                                             \
}

static void fts5PutU16(u8 *aOut, u16 iVal){
  aOut[0] = (iVal>>8);
  aOut[1] = (iVal&0xFF);
}

static u16 fts5GetU16(const u8 *aIn){
................................................................................
    ** backing store corruption.  */
    if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT;

    if( rc==SQLITE_OK ){
      u8 *aOut;                   /* Read blob data into this buffer */
      int nByte = sqlite3_blob_bytes(p->pReader);
      if( pBuf ){
        fts5BufferSize(pBuf, MAX(nByte, p->pConfig->pgsz) + 20);
        pBuf->n = nByte;
        aOut = pBuf->p;
        if( aOut==0 ){
          rc = SQLITE_NOMEM;
        }
      }else{
        int nSpace = nByte + FTS5_DATA_ZERO_PADDING;
        pRet = (Fts5Data*)sqlite3_malloc(nSpace+sizeof(Fts5Data));
        if( pRet ){
          pRet->n = nByte;
          aOut = pRet->p = (u8*)&pRet[1];
        }else{
          rc = SQLITE_NOMEM;
        }
      }

      if( rc==SQLITE_OK ){
        rc = sqlite3_blob_read(p->pReader, aOut, nByte, 0);
      }
      if( rc!=SQLITE_OK ){
................................................................................
** If an error occurs, NULL is returned and an error code left in the
** Fts5Index handle. If an error has already occurred when this function
** is called, it is a no-op.
*/
static Fts5Structure *fts5StructureRead(Fts5Index *p){
  Fts5Config *pConfig = p->pConfig;
  Fts5Structure *pRet = 0;        /* Object to return */

  int iCookie;                    /* Configuration cookie */
  Fts5Buffer buf = {0, 0, 0};

  fts5DataBuffer(p, &buf, FTS5_STRUCTURE_ROWID);
  if( buf.p==0 ) return 0;
  assert( buf.nSpace>=(buf.n + FTS5_DATA_ZERO_PADDING) );
  memset(&buf.p[buf.n], 0, FTS5_DATA_ZERO_PADDING);
  p->rc = fts5StructureDecode(buf.p, buf.n, &iCookie, &pRet);

  if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
    p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
  }

  fts5BufferFree(&buf);
  if( p->rc!=SQLITE_OK ){
    fts5StructureRelease(pRet);
    pRet = 0;
  }
  return pRet;
}

................................................................................
  }

  pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno);
}

#ifdef SQLITE_DEBUG
static void fts5AssertNodeSeekOk(
  Fts5Buffer *pNode,
  const u8 *pTerm, int nTerm,     /* Term to search for */
  int iExpectPg,
  int bExpectDlidx
){
  int bDlidx;
  int iPg;
  int rc = SQLITE_OK;
................................................................................
** It returns the associated page number. Or, if (pTerm/nTerm) is smaller
** than all terms within the node, the leftmost child page number. 
**
** Before returning, (*pbDlidx) is set to true if the last term on the
** returned child page number has a doclist-index. Or left as is otherwise.
*/
static int fts5NodeSeek(
  Fts5Buffer *pNode,              /* Node to search */
  const u8 *pTerm, int nTerm,     /* Term to search for */
  int *pbDlidx                    /* OUT: True if dlidx flag is set */
){
  int iPg;
  u8 *pPtr = pNode->p;
  u8 *pEnd = &pPtr[pNode->n];
  int nMatch = 0;                 /* Number of bytes of pTerm already matched */
................................................................................
    /* If there is a "no terms" record at pPtr, read it now. Store the
    ** number of termless pages in nEmpty. If it indicates a doclist-index, 
    ** set (*pbDlidx) to true.*/
    if( *pPtr<2 ){
      *pbDlidx = (*pPtr==0x01);
      pPtr++;
      pPtr += fts5GetVarint32(pPtr, nEmpty);
      if( pPtr>=pEnd ) break;
    }

    /* Read the next "term" pointer. Set nKeep to the number of bytes to
    ** keep from the previous term, and nNew to the number of bytes of
    ** new data that will be appended to it. */
    nKeep = (int)*pPtr++;
    nNew = (int)*pPtr++;
................................................................................

      /* Skip past docid delta */
      fts5IndexSkipVarint(a, iOff);

      /* Skip past position list */
      fts5IndexGetVarint32(a, iOff, nPos);
      iOff += (nPos >> 1);
      if( iOff>=(n-1) ){
        iOff = n;
        goto search_failed;
      }

      /* If this is the end of the doclist, break out of the loop */
      if( a[iOff]==0x00 ){
        iOff++;
        break;
      }
    };
................................................................................
** pSeg. If there is no such term in the index, the iterator is set to EOF.
**
** If an error occurs, Fts5Index.rc is set to an appropriate error code. If 
** an error has already occurred when this function is called, it is a no-op.
*/
static void fts5SegIterSeekInit(
  Fts5Index *p,                   /* FTS5 backend */
  Fts5Buffer *pBuf,               /* Buffer to use for loading pages */
  const u8 *pTerm, int nTerm,     /* Term to seek to */
  int flags,                      /* Mask of FTS5INDEX_XXX flags */
  Fts5StructureSegment *pSeg,     /* Description of segment */
  Fts5SegIter *pIter              /* Object to populate */
){
  int iPg = 1;
  int h;
  int bGe = (flags & FTS5INDEX_QUERY_SCAN);
  int bDlidx = 0;                 /* True if there is a doclist-index */
  Fts5Data *pLeaf;

  static int nCall = 0;
  nCall++;

  assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 );
  assert( pTerm && nTerm );
  memset(pIter, 0, sizeof(*pIter));
  pIter->pSeg = pSeg;

  /* This block sets stack variable iPg to the leaf page number that may
  ** contain term (pTerm/nTerm), if it is present in the segment. */
  for(h=pSeg->nHeight-1; h>0; h--){
    i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, h, iPg);
    fts5DataBuffer(p, pBuf, iRowid);
    if( p->rc ) break;

    iPg = fts5NodeSeek(pBuf, pTerm, nTerm, &bDlidx);

  }

  if( iPg<pSeg->pgnoFirst ){
    iPg = pSeg->pgnoFirst;
    bDlidx = 0;
  }

................................................................................
        fts5SegIterLoadDlidx(p, pIter);
      }
      if( flags & FTS5INDEX_QUERY_DESC ){
        fts5SegIterReverse(p, pIter);
      }
    }
  }

  /* Either:
  **
  **   1) an error has occurred, or
  **   2) the iterator points to EOF, or
  **   3) the iterator points to an entry with term (pTerm/nTerm), or
  **   4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points
  **      to an entry with a term greater than or equal to (pTerm/nTerm).
  */
  assert( p->rc!=SQLITE_OK                                          /* 1 */
   || pIter->pLeaf==0                                               /* 2 */
   || fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)==0          /* 3 */
   || (bGe && fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)>0)  /* 4 */
  );
}

/*
** Initialize the object pIter to point to term pTerm/nTerm within the
** in-memory hash table. If there is no such term in the hash-table, the 
** iterator is set to EOF.
**
................................................................................
  int iLevel,                     /* Level to iterate (-1 for all) */
  int nSegment,                   /* Number of segments to merge (iLevel>=0) */
  Fts5IndexIter **ppOut           /* New object */
){
  int nSeg = 0;                   /* Number of segment-iters in use */
  int iIter = 0;                  /* */
  int iSeg;                       /* Used to iterate through segments */
  Fts5Buffer buf = {0,0,0};       /* Buffer used by fts5SegIterSeekInit() */
  Fts5StructureLevel *pLvl;
  Fts5IndexIter *pNew;

  assert( (pTerm==0 && nTerm==0) || iLevel<0 );

  /* Allocate space for the new multi-seg-iterator. */
  if( p->rc==SQLITE_OK ){
................................................................................
    for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){
      for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){
        Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
        Fts5SegIter *pIter = &pNew->aSeg[iIter++];
        if( pTerm==0 ){
          fts5SegIterInit(p, pSeg, pIter);
        }else{
          fts5SegIterSeekInit(p, &buf, pTerm, nTerm, flags, pSeg, pIter);
        }
      }
    }
  }else{
    pLvl = &pStruct->aLevel[iLevel];
    for(iSeg=nSeg-1; iSeg>=0; iSeg--){
      fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]);
................................................................................
    if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){
      fts5MultiIterNext(p, pNew, 0, 0);
    }
  }else{
    fts5MultiIterFree(p, pNew);
    *ppOut = 0;
  }
  fts5BufferFree(&buf);
}

/*
** Create an Fts5IndexIter that iterates through the doclist provided
** as the second argument.
*/
static void fts5MultiIterNew2(
................................................................................
  Fts5Structure *p
){
  int iLvl, iSeg;                 /* Iterate through levels, segments */

  for(iLvl=0; iLvl<p->nLevel; iLvl++){
    Fts5StructureLevel *pLvl = &p->aLevel[iLvl];
    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
        " {lvl=%d nMerge=%d nSeg=%d", iLvl, pLvl->nMerge, pLvl->nSeg
    );
    for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
      Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
          " {id=%d h=%d leaves=%d..%d}", pSeg->iSegid, pSeg->nHeight, 
          pSeg->pgnoFirst, pSeg->pgnoLast
      );

Changes to ext/fts5/test/fts5_common.tcl.

120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
  }
}

proc fts5_level_segs {tbl} {
  set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
  set ret [list]
  foreach L [lrange [db one $sql] 1 end] {
    lappend ret [expr [llength $L] - 2]
  }
  set ret
} 

proc fts5_level_segids {tbl} {
  set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
  set ret [list]
  foreach L [lrange [db one $sql] 1 end] {
    set lvl [list]
    foreach S [lrange $L 2 end] {
      regexp {id=([1234567890]*)} $S -> segid
      lappend lvl $segid
    }
    lappend ret $lvl
  }
  set ret
}







|









|







120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
  }
}

proc fts5_level_segs {tbl} {
  set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
  set ret [list]
  foreach L [lrange [db one $sql] 1 end] {
    lappend ret [expr [llength $L] - 3]
  }
  set ret
} 

proc fts5_level_segids {tbl} {
  set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
  set ret [list]
  foreach L [lrange [db one $sql] 1 end] {
    set lvl [list]
    foreach S [lrange $L 3 end] {
      regexp {id=([1234567890]*)} $S -> segid
      lappend lvl $segid
    }
    lappend ret $lvl
  }
  set ret
}

Changes to ext/fts5/test/fts5aa.test.

45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
}
do_execsql_test 2.1 {
  INSERT INTO t1 VALUES('a b c', 'd e f');
}
do_test 2.2 {
  execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 }
} {/{\(structure\) {lvl=0 nMerge=0 {id=[0123456789]* h=1 leaves=1..1}}}/}

foreach w {a b c d e f} {
  do_execsql_test 2.3.$w.asc {
    SELECT rowid FROM t1 WHERE t1 MATCH $w;
  } {1}
  do_execsql_test 2.3.$w.desc {
    SELECT rowid FROM t1 WHERE t1 MATCH $w ORDER BY rowid DESC;







|







45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
}
do_execsql_test 2.1 {
  INSERT INTO t1 VALUES('a b c', 'd e f');
}
do_test 2.2 {
  execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 }
} {/{\(structure\) {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* h=1 leaves=1..1}}}/}

foreach w {a b c d e f} {
  do_execsql_test 2.3.$w.asc {
    SELECT rowid FROM t1 WHERE t1 MATCH $w;
  } {1}
  do_execsql_test 2.3.$w.desc {
    SELECT rowid FROM t1 WHERE t1 MATCH $w ORDER BY rowid DESC;

Changes to ext/fts5/test/fts5rowid.test.

67
68
69
70
71
72
73
74

75
76
77
78
79
80
81
82
83
84
85


86
87
88
89
90
91
92
93
94
95
96

set res [db one {SELECT count(*) FROM x1_data}]
do_execsql_test 2.3 {
  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
} $res
do_execsql_test 2.4 {
  UPDATE x1_data SET block = X'';
  SELECT count(fts5_decode(rowid, block)) FROM x1_data;

} $res

do_execsql_test 2.5 {
  INSERT INTO x1(x1, rank) VALUES('pgsz', 1024);
  INSERT INTO x1(x1) VALUES('rebuild');
}

set res [db one {SELECT count(*) FROM x1_data}]
do_execsql_test 2.6 {
  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
} $res


do_execsql_test 2.7 {
  UPDATE x1_data SET block = X'';
  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
} $res

#-------------------------------------------------------------------------
# Tests with very large tokens.
#
set strlist [list \
  "[string repeat x 400]"                       \
  "[string repeat x 300][string repeat w 100]"  \







|
>











>
>
|
|
|
|







67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

set res [db one {SELECT count(*) FROM x1_data}]
do_execsql_test 2.3 {
  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
} $res
do_execsql_test 2.4 {
  UPDATE x1_data SET block = X'';
  -- SELECT count(fts5_decode(rowid, block)) FROM x1_data;
  SELECT count(*) FROM x1_data;
} $res

do_execsql_test 2.5 {
  INSERT INTO x1(x1, rank) VALUES('pgsz', 1024);
  INSERT INTO x1(x1) VALUES('rebuild');
}

set res [db one {SELECT count(*) FROM x1_data}]
do_execsql_test 2.6 {
  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
} $res

# This is really a corruption test...
#do_execsql_test 2.7 {
#  UPDATE x1_data SET block = X'';
#  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
#} $res

#-------------------------------------------------------------------------
# Tests with very large tokens.
#
set strlist [list \
  "[string repeat x 400]"                       \
  "[string repeat x 300][string repeat w 100]"  \

Changes to ext/fts5/tool/showfts5.tcl.

1
2




3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19





20
21
22
23
24
25



26
27
28
29
30
31
32






proc usage {} {
  puts stderr "usage: $::argv0 database table"
  puts stderr ""
  exit 1
}

set o(vtab)       fts5
set o(tok)        ""
set o(limit)      0
set o(automerge)  -1
set o(crisismerge)  -1

if {[llength $argv]!=2} usage

set database [lindex $argv 0]
set tbl [lindex $argv 1]






sqlite3 db $database
catch { load_static_extension db fts5 }

db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id=10" {
  foreach lvl [lrange $d 1 end] {
    puts $lvl



  }
}







>
>
>
>





<
<
<
<
<
<
<

<



>
>
>
>
>





|
>
>
>







1
2
3
4
5
6
7
8
9
10
11







12

13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36



#-------------------------------------------------------------------------
# Process command line arguments.
#
proc usage {} {
  puts stderr "usage: $::argv0 database table"
  puts stderr ""
  exit 1
}







if {[llength $argv]!=2} usage

set database [lindex $argv 0]
set tbl [lindex $argv 1]



#-------------------------------------------------------------------------
# Start of main program.
#
sqlite3 db $database
catch { load_static_extension db fts5 }

db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id=10" {
  foreach lvl [lrange $d 1 end] {
    puts [lrange $lvl 0 2]
    foreach seg [lrange $lvl 3 end] {
      puts "        $seg"
    }
  }
}