/ Check-in [aa7ba302]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:When possible, have FTS use 32-bit operations to read varints from the database.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: aa7ba302ed13aedde89b5bcbe9119799c0da8a42
User & Date: dan 2013-11-12 17:46:44
Context
2013-11-12
18:37
Break out the structure and macro definitions of where.c into a separate header file whereInt.h for easier editing and debugging. check-in: c4446712 user: drh tags: trunk
17:46
When possible, have FTS use 32-bit operations to read varints from the database. check-in: aa7ba302 user: dan tags: trunk
15:33
A better (simpler) fix to the count(*) problem addressed in the previous check-in. check-in: 0f924c6e user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

326
327
328
329
330
331
332




333
334
335
336
337
338
339
340
341
342
343
344
345
346
347










348
349
350
351
352
353
354
355

356










357
358
359
360
361
362
363
364
365
....
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
....
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
....
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
....
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
....
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
....
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
....
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
    vu >>= 7;
  }while( vu!=0 );
  q[-1] &= 0x7f;  /* turn off high bit in final byte */
  assert( q - (unsigned char *)p <= FTS3_VARINT_MAX );
  return (int) (q - (unsigned char *)p);
}





/* 
** Read a 64-bit variable-length integer from memory starting at p[0].
** Return the number of bytes read, or 0 on error.
** The value is stored in *v.
*/
int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){
  const unsigned char *q = (const unsigned char *) p;
  sqlite_uint64 x = 0, y = 1;
  while( (*q&0x80)==0x80 && q-(unsigned char *)p<FTS3_VARINT_MAX ){
    x += y * (*q++ & 0x7f);
    y <<= 7;
  }
  x += y * (*q++);
  *v = (sqlite_int64) x;
  return (int) (q - (unsigned char *)p);










}

/*
** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a
** 32-bit integer before it is returned.
*/
int sqlite3Fts3GetVarint32(const char *p, int *pi){
 sqlite_int64 i;

 int ret = sqlite3Fts3GetVarint(p, &i);










 *pi = (int) i;
 return ret;
}

/*
** Return the number of bytes required to encode v as a varint
*/
int sqlite3Fts3VarintLen(sqlite3_uint64 v){
  int i = 0;
................................................................................
    int nSuffix;                  /* Size of term suffix */
    int nPrefix = 0;              /* Size of term prefix */
    int nBuffer;                  /* Total term size */
  
    /* Load the next term on the node into zBuffer. Use realloc() to expand
    ** the size of zBuffer if required.  */
    if( !isFirstTerm ){
      zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
    }
    isFirstTerm = 0;
    zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
    
    if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){
      rc = FTS_CORRUPT_VTAB;
      goto finish_scan;
    }
    if( nPrefix+nSuffix>nAlloc ){
      char *zNew;
................................................................................
  sqlite3_int64 *piLeaf2          /* Selected leaf node */
){
  int rc;                         /* Return code */
  int iHeight;                    /* Height of this node in tree */

  assert( piLeaf || piLeaf2 );

  sqlite3Fts3GetVarint32(zNode, &iHeight);
  rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2);
  assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) );

  if( rc==SQLITE_OK && iHeight>1 ){
    char *zBlob = 0;              /* Blob read from %_segments table */
    int nBlob;                    /* Size of zBlob in bytes */

................................................................................
  char *p1 = *pp1;
  char *p2 = *pp2;

  while( *p1 || *p2 ){
    int iCol1;         /* The current column index in pp1 */
    int iCol2;         /* The current column index in pp2 */

    if( *p1==POS_COLUMN ) sqlite3Fts3GetVarint32(&p1[1], &iCol1);
    else if( *p1==POS_END ) iCol1 = POSITION_LIST_END;
    else iCol1 = 0;

    if( *p2==POS_COLUMN ) sqlite3Fts3GetVarint32(&p2[1], &iCol2);
    else if( *p2==POS_END ) iCol2 = POSITION_LIST_END;
    else iCol2 = 0;

    if( iCol1==iCol2 ){
      sqlite3_int64 i1 = 0;       /* Last position from pp1 */
      sqlite3_int64 i2 = 0;       /* Last position from pp2 */
      sqlite3_int64 iPrev = 0;
................................................................................

  /* Never set both isSaveLeft and isExact for the same invocation. */
  assert( isSaveLeft==0 || isExact==0 );

  assert( p!=0 && *p1!=0 && *p2!=0 );
  if( *p1==POS_COLUMN ){ 
    p1++;
    p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
  }
  if( *p2==POS_COLUMN ){ 
    p2++;
    p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
  }

  while( 1 ){
    if( iCol1==iCol2 ){
      char *pSave = p;
      sqlite3_int64 iPrev = 0;
      sqlite3_int64 iPos1 = 0;
................................................................................

      fts3ColumnlistCopy(0, &p1);
      fts3ColumnlistCopy(0, &p2);
      assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 );
      if( 0==*p1 || 0==*p2 ) break;

      p1++;
      p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
      p2++;
      p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
    }

    /* Advance pointer p1 or p2 (whichever corresponds to the smaller of
    ** iCol1 and iCol2) so that it points to either the 0x00 that marks the
    ** end of the position list, or the 0x01 that precedes the next 
    ** column-number in the position list. 
    */
    else if( iCol1<iCol2 ){
      fts3ColumnlistCopy(0, &p1);
      if( 0==*p1 ) break;
      p1++;
      p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
    }else{
      fts3ColumnlistCopy(0, &p2);
      if( 0==*p2 ) break;
      p2++;
      p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
    }
  }

  fts3PoslistCopy(0, &p2);
  fts3PoslistCopy(0, &p1);
  *pp1 = p1;
  *pp2 = p2;
................................................................................
        /* aMI[iCol*3 + 1] = Number of occurrences
        ** aMI[iCol*3 + 2] = Number of rows containing at least one instance
        */
        pExpr->aMI[iCol*3 + 1] += iCnt;
        pExpr->aMI[iCol*3 + 2] += (iCnt>0);
        if( *p==0x00 ) break;
        p++;
        p += sqlite3Fts3GetVarint32(p, &iCol);
      }
    }

    fts3EvalUpdateCounts(pExpr->pLeft);
    fts3EvalUpdateCounts(pExpr->pRight);
  }
}
................................................................................

    if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0;
  }
  if( pIter==0 ) return SQLITE_OK;

  if( *pIter==0x01 ){
    pIter++;
    pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
  }else{
    iThis = 0;
  }
  while( iThis<iCol ){
    fts3ColumnlistCopy(0, &pIter);
    if( *pIter==0x00 ) return 0;
    pIter++;
    pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
  }

  *ppOut = ((iCol==iThis)?pIter:0);
  return SQLITE_OK;
}

/*







>
>
>
>






|
|
|
|
|
|
|
|
|
>
>
>
>
>
>
>
>
>
>







|
>
|
>
>
>
>
>
>
>
>
>
>
|
|







 







|


|







 







|







 







|



|







 







|



|







 







|

|











|




|







 







|







 







|







|







326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
....
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
....
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
....
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
....
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
....
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
....
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
....
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
    vu >>= 7;
  }while( vu!=0 );
  q[-1] &= 0x7f;  /* turn off high bit in final byte */
  assert( q - (unsigned char *)p <= FTS3_VARINT_MAX );
  return (int) (q - (unsigned char *)p);
}

#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \
  v = (v & mask1) | ( (*ptr++) << shift );                    \
  if( (v & mask2)==0 ){ var = v; return ret; }

/* 
** Read a 64-bit variable-length integer from memory starting at p[0].
** Return the number of bytes read, or 0 on error.
** The value is stored in *v.
*/
int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){
  const char *pStart = p;
  u32 a;
  u64 b;
  int ret;
  int shift;

  GETVARINT_STEP(a, p, 0,  0x00,     0x80, *v, 1);
  GETVARINT_STEP(a, p, 7,  0x7F,     0x4000, *v, 2);
  GETVARINT_STEP(a, p, 14, 0x3FFF,   0x200000, *v, 3);
  GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *v, 4);
  b = (a & 0x0FFFFFFF );

  for(shift=28; shift<=63; shift+=7){
    u64 c = *p++;
    b += (c&0x7F) << shift;
    if( (c & 0x80)==0 ) break;
  }
  *v = b;
  return p - pStart;
}

/*
** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a
** 32-bit integer before it is returned.
*/
int sqlite3Fts3GetVarint32(const char *p, int *pi){
  u32 a;

#ifndef fts3GetVarint32
  GETVARINT_STEP(a, p, 0,  0x00,     0x80, *pi, 1);
#else
  a = (*p++);
  assert( a & 0x80 );
#endif

  GETVARINT_STEP(a, p, 7,  0x7F,     0x4000, *pi, 2);
  GETVARINT_STEP(a, p, 14, 0x3FFF,   0x200000, *pi, 3);
  GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4);
  a = (a & 0x0FFFFFFF );
  *pi = (int)(a | ((u32)(*p & 0x0F) << 28));
  return 5;
}

/*
** Return the number of bytes required to encode v as a varint
*/
int sqlite3Fts3VarintLen(sqlite3_uint64 v){
  int i = 0;
................................................................................
    int nSuffix;                  /* Size of term suffix */
    int nPrefix = 0;              /* Size of term prefix */
    int nBuffer;                  /* Total term size */
  
    /* Load the next term on the node into zBuffer. Use realloc() to expand
    ** the size of zBuffer if required.  */
    if( !isFirstTerm ){
      zCsr += fts3GetVarint32(zCsr, &nPrefix);
    }
    isFirstTerm = 0;
    zCsr += fts3GetVarint32(zCsr, &nSuffix);
    
    if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){
      rc = FTS_CORRUPT_VTAB;
      goto finish_scan;
    }
    if( nPrefix+nSuffix>nAlloc ){
      char *zNew;
................................................................................
  sqlite3_int64 *piLeaf2          /* Selected leaf node */
){
  int rc;                         /* Return code */
  int iHeight;                    /* Height of this node in tree */

  assert( piLeaf || piLeaf2 );

  fts3GetVarint32(zNode, &iHeight);
  rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2);
  assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) );

  if( rc==SQLITE_OK && iHeight>1 ){
    char *zBlob = 0;              /* Blob read from %_segments table */
    int nBlob;                    /* Size of zBlob in bytes */

................................................................................
  char *p1 = *pp1;
  char *p2 = *pp2;

  while( *p1 || *p2 ){
    int iCol1;         /* The current column index in pp1 */
    int iCol2;         /* The current column index in pp2 */

    if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1);
    else if( *p1==POS_END ) iCol1 = POSITION_LIST_END;
    else iCol1 = 0;

    if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2);
    else if( *p2==POS_END ) iCol2 = POSITION_LIST_END;
    else iCol2 = 0;

    if( iCol1==iCol2 ){
      sqlite3_int64 i1 = 0;       /* Last position from pp1 */
      sqlite3_int64 i2 = 0;       /* Last position from pp2 */
      sqlite3_int64 iPrev = 0;
................................................................................

  /* Never set both isSaveLeft and isExact for the same invocation. */
  assert( isSaveLeft==0 || isExact==0 );

  assert( p!=0 && *p1!=0 && *p2!=0 );
  if( *p1==POS_COLUMN ){ 
    p1++;
    p1 += fts3GetVarint32(p1, &iCol1);
  }
  if( *p2==POS_COLUMN ){ 
    p2++;
    p2 += fts3GetVarint32(p2, &iCol2);
  }

  while( 1 ){
    if( iCol1==iCol2 ){
      char *pSave = p;
      sqlite3_int64 iPrev = 0;
      sqlite3_int64 iPos1 = 0;
................................................................................

      fts3ColumnlistCopy(0, &p1);
      fts3ColumnlistCopy(0, &p2);
      assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 );
      if( 0==*p1 || 0==*p2 ) break;

      p1++;
      p1 += fts3GetVarint32(p1, &iCol1);
      p2++;
      p2 += fts3GetVarint32(p2, &iCol2);
    }

    /* Advance pointer p1 or p2 (whichever corresponds to the smaller of
    ** iCol1 and iCol2) so that it points to either the 0x00 that marks the
    ** end of the position list, or the 0x01 that precedes the next 
    ** column-number in the position list. 
    */
    else if( iCol1<iCol2 ){
      fts3ColumnlistCopy(0, &p1);
      if( 0==*p1 ) break;
      p1++;
      p1 += fts3GetVarint32(p1, &iCol1);
    }else{
      fts3ColumnlistCopy(0, &p2);
      if( 0==*p2 ) break;
      p2++;
      p2 += fts3GetVarint32(p2, &iCol2);
    }
  }

  fts3PoslistCopy(0, &p2);
  fts3PoslistCopy(0, &p1);
  *pp1 = p1;
  *pp2 = p2;
................................................................................
        /* aMI[iCol*3 + 1] = Number of occurrences
        ** aMI[iCol*3 + 2] = Number of rows containing at least one instance
        */
        pExpr->aMI[iCol*3 + 1] += iCnt;
        pExpr->aMI[iCol*3 + 2] += (iCnt>0);
        if( *p==0x00 ) break;
        p++;
        p += fts3GetVarint32(p, &iCol);
      }
    }

    fts3EvalUpdateCounts(pExpr->pLeft);
    fts3EvalUpdateCounts(pExpr->pRight);
  }
}
................................................................................

    if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0;
  }
  if( pIter==0 ) return SQLITE_OK;

  if( *pIter==0x01 ){
    pIter++;
    pIter += fts3GetVarint32(pIter, &iThis);
  }else{
    iThis = 0;
  }
  while( iThis<iCol ){
    fts3ColumnlistCopy(0, &pIter);
    if( *pIter==0x00 ) return 0;
    pIter++;
    pIter += fts3GetVarint32(pIter, &iThis);
  }

  *ppOut = ((iCol==iThis)?pIter:0);
  return SQLITE_OK;
}

/*

Changes to ext/fts3/fts3Int.h.

519
520
521
522
523
524
525




526
527
528
529
530
531
532
  char *zTerm;                    /* Pointer to term buffer */
  int nTerm;                      /* Size of zTerm in bytes */
  char *aDoclist;                 /* Pointer to doclist buffer */
  int nDoclist;                   /* Size of aDoclist[] in bytes */
};

int sqlite3Fts3Incrmerge(Fts3Table*,int,int);





/* fts3.c */
int sqlite3Fts3PutVarint(char *, sqlite3_int64);
int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
int sqlite3Fts3GetVarint32(const char *, int *);
int sqlite3Fts3VarintLen(sqlite3_uint64);
void sqlite3Fts3Dequote(char *);







>
>
>
>







519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
  char *zTerm;                    /* Pointer to term buffer */
  int nTerm;                      /* Size of zTerm in bytes */
  char *aDoclist;                 /* Pointer to doclist buffer */
  int nDoclist;                   /* Size of aDoclist[] in bytes */
};

int sqlite3Fts3Incrmerge(Fts3Table*,int,int);

#define fts3GetVarint32(p, piVal) (                                           \
  (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \
)

/* fts3.c */
int sqlite3Fts3PutVarint(char *, sqlite3_int64);
int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
int sqlite3Fts3GetVarint32(const char *, int *);
int sqlite3Fts3VarintLen(sqlite3_uint64);
void sqlite3Fts3Dequote(char *);

Changes to ext/fts3/fts3_snippet.c.

124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
** When this function is called, *pp points to the start of an element of
** the list. *piPos contains the value of the previous entry in the list.
** After it returns, *piPos contains the value of the next element of the
** list and *pp is advanced to the following varint.
*/
static void fts3GetDeltaPosition(char **pp, int *piPos){
  int iVal;
  *pp += sqlite3Fts3GetVarint32(*pp, &iVal);
  *piPos += (iVal-2);
}

/*
** Helper function for fts3ExprIterate() (see below).
*/
static int fts3ExprIterate2(







|







124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
** When this function is called, *pp points to the start of an element of
** the list. *piPos contains the value of the previous entry in the list.
** After it returns, *piPos contains the value of the next element of the
** list and *pp is advanced to the following varint.
*/
static void fts3GetDeltaPosition(char **pp, int *piPos){
  int iVal;
  *pp += fts3GetVarint32(*pp, &iVal);
  *piPos += (iVal-2);
}

/*
** Helper function for fts3ExprIterate() (see below).
*/
static int fts3ExprIterate2(

Changes to ext/fts3/fts3_test.c.

512
513
514
515
516
517
518













































519
520
521
522
523
524
525
526
527
528
529
530
531




532
533
534
535
  Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(
    (const unsigned char *)&pPtr, sizeof(sqlite3_tokenizer_module *)
  ));
#endif
  UNUSED_PARAMETER(clientData);
  return TCL_OK;
}














































/* 
** End of tokenizer code.
**************************************************************************/ 

int Sqlitetestfts3_Init(Tcl_Interp *interp){
  Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);
  Tcl_CreateObjCommand(interp, 
      "fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0
  );
  Tcl_CreateObjCommand(
      interp, "fts3_test_tokenizer", fts3_test_tokenizer_cmd, 0, 0
  );




  return TCL_OK;
}
#endif                  /* SQLITE_ENABLE_FTS3 || SQLITE_ENABLE_FTS4 */
#endif                  /* ifdef SQLITE_TEST */







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>













>
>
>
>




512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
  Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(
    (const unsigned char *)&pPtr, sizeof(sqlite3_tokenizer_module *)
  ));
#endif
  UNUSED_PARAMETER(clientData);
  return TCL_OK;
}

static int fts3_test_varint_cmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
#ifdef SQLITE_ENABLE_FTS3
  char aBuf[24];
  int rc;
  Tcl_WideInt w, w2;
  int nByte, nByte2;

  if( objc!=2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "INTEGER");
    return TCL_ERROR;
  }

  rc = Tcl_GetWideIntFromObj(interp, objv[1], &w);
  if( rc!=TCL_OK ) return rc;

  nByte = sqlite3Fts3PutVarint(aBuf, w);
  nByte2 = sqlite3Fts3GetVarint(aBuf, &w2);
  if( w!=w2 || nByte!=nByte2 ){
    char *zErr = sqlite3_mprintf("error testing %lld", w);
    Tcl_ResetResult(interp);
    Tcl_AppendResult(interp, zErr, 0);
    return TCL_ERROR;
  }

  if( w<=2147483647 && w>=0 ){
    int i;
    nByte2 = fts3GetVarint32(aBuf, &i);
    if( (int)w!=i || nByte!=nByte2 ){
      char *zErr = sqlite3_mprintf("error testing %lld (32-bit)", w);
      Tcl_ResetResult(interp);
      Tcl_AppendResult(interp, zErr, 0);
      return TCL_ERROR;
    }
  }

#endif
  UNUSED_PARAMETER(clientData);
  return TCL_OK;
}

/* 
** End of tokenizer code.
**************************************************************************/ 

int Sqlitetestfts3_Init(Tcl_Interp *interp){
  Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);
  Tcl_CreateObjCommand(interp, 
      "fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0
  );
  Tcl_CreateObjCommand(
      interp, "fts3_test_tokenizer", fts3_test_tokenizer_cmd, 0, 0
  );

  Tcl_CreateObjCommand(
      interp, "fts3_test_varint", fts3_test_varint_cmd, 0, 0
  );
  return TCL_OK;
}
#endif                  /* SQLITE_ENABLE_FTS3 || SQLITE_ENABLE_FTS4 */
#endif                  /* ifdef SQLITE_TEST */

Changes to ext/fts3/fts3_write.c.

1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
....
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
....
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
....
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
....
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
  assert( !fts3SegReaderIsPending(pReader) );

  rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
  if( rc!=SQLITE_OK ) return rc;
  
  /* Because of the FTS3_NODE_PADDING bytes of padding, the following is 
  ** safe (no risk of overread) even if the node data is corrupted. */
  pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
  pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
  if( nPrefix<0 || nSuffix<=0 
   || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 
  ){
    return FTS_CORRUPT_VTAB;
  }

  if( nPrefix+nSuffix>pReader->nTermAlloc ){
................................................................................

  rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX);
  if( rc!=SQLITE_OK ) return rc;

  memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
  pReader->nTerm = nPrefix+nSuffix;
  pNext += nSuffix;
  pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
  pReader->aDoclist = pNext;
  pReader->pOffsetList = 0;

  /* Check that the doclist does not appear to extend past the end of the
  ** b-tree node. And that the final byte of the doclist is 0x00. If either 
  ** of these statements is untrue, then the data structure is corrupt.
  */
................................................................................

    nList -= (int)(p - pList);
    pList = p;
    if( nList==0 ){
      break;
    }
    p = &pList[1];
    p += sqlite3Fts3GetVarint32(p, &iCurrent);
  }

  if( bZero && &pList[nList]!=pEnd ){
    memset(&pList[nList], 0, pEnd - &pList[nList]);
  }
  *ppList = pList;
  *pnList = nList;
................................................................................
  assert( p->aNode );
  if( p->iChild && bFirst==0 ) p->iChild++;
  if( p->iOff>=p->nNode ){
    /* EOF */
    p->aNode = 0;
  }else{
    if( bFirst==0 ){
      p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nPrefix);
    }
    p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);

    blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
    if( rc==SQLITE_OK ){
      memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
      p->term.n = nPrefix+nSuffix;
      p->iOff += nSuffix;
      if( p->iChild==0 ){
        p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
        p->aDoclist = &p->aNode[p->iOff];
        p->iOff += p->nDoclist;
      }
    }
  }

  assert( p->iOff<=p->nNode );
................................................................................

  i = pHint->n-2;
  while( i>0 && (pHint->a[i-1] & 0x80) ) i--;
  while( i>0 && (pHint->a[i-1] & 0x80) ) i--;

  pHint->n = i;
  i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel);
  i += sqlite3Fts3GetVarint32(&pHint->a[i], pnInput);
  if( i!=nHint ) return SQLITE_CORRUPT_VTAB;

  return SQLITE_OK;
}


/*







|
|







 







|







 







|







 







|

|







|







 







|







1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
....
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
....
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
....
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
....
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
  assert( !fts3SegReaderIsPending(pReader) );

  rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
  if( rc!=SQLITE_OK ) return rc;
  
  /* Because of the FTS3_NODE_PADDING bytes of padding, the following is 
  ** safe (no risk of overread) even if the node data is corrupted. */
  pNext += fts3GetVarint32(pNext, &nPrefix);
  pNext += fts3GetVarint32(pNext, &nSuffix);
  if( nPrefix<0 || nSuffix<=0 
   || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 
  ){
    return FTS_CORRUPT_VTAB;
  }

  if( nPrefix+nSuffix>pReader->nTermAlloc ){
................................................................................

  rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX);
  if( rc!=SQLITE_OK ) return rc;

  memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
  pReader->nTerm = nPrefix+nSuffix;
  pNext += nSuffix;
  pNext += fts3GetVarint32(pNext, &pReader->nDoclist);
  pReader->aDoclist = pNext;
  pReader->pOffsetList = 0;

  /* Check that the doclist does not appear to extend past the end of the
  ** b-tree node. And that the final byte of the doclist is 0x00. If either 
  ** of these statements is untrue, then the data structure is corrupt.
  */
................................................................................

    nList -= (int)(p - pList);
    pList = p;
    if( nList==0 ){
      break;
    }
    p = &pList[1];
    p += fts3GetVarint32(p, &iCurrent);
  }

  if( bZero && &pList[nList]!=pEnd ){
    memset(&pList[nList], 0, pEnd - &pList[nList]);
  }
  *ppList = pList;
  *pnList = nList;
................................................................................
  assert( p->aNode );
  if( p->iChild && bFirst==0 ) p->iChild++;
  if( p->iOff>=p->nNode ){
    /* EOF */
    p->aNode = 0;
  }else{
    if( bFirst==0 ){
      p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix);
    }
    p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);

    blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
    if( rc==SQLITE_OK ){
      memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
      p->term.n = nPrefix+nSuffix;
      p->iOff += nSuffix;
      if( p->iChild==0 ){
        p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
        p->aDoclist = &p->aNode[p->iOff];
        p->iOff += p->nDoclist;
      }
    }
  }

  assert( p->iOff<=p->nNode );
................................................................................

  i = pHint->n-2;
  while( i>0 && (pHint->a[i-1] & 0x80) ) i--;
  while( i>0 && (pHint->a[i-1] & 0x80) ) i--;

  pHint->n = i;
  i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel);
  i += fts3GetVarint32(&pHint->a[i], pnInput);
  if( i!=nHint ) return SQLITE_CORRUPT_VTAB;

  return SQLITE_OK;
}


/*

Added test/fts3varint.test.













































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
# 2007 November 23
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file runs all tests.
#
# $Id: fts3.test,v 1.2 2008/07/23 18:17:32 drh Exp $

set testdir [file dirname $argv0]
source $testdir/permutations.test
set testprefix fts3varint

ifcapable !fts3 {
  finish_test
  return
}

proc test_list {list} {
  foreach n $list { fts3_test_varint $n }
}

proc do_fts3_varint_test {tn list} {
  uplevel [list do_test $tn [list test_list $list] {}]
}

do_fts3_varint_test 1.0 {
  1 10 100 1000 10000 100000 1000000 
  10000000 100000000 1000000000 10000000000
  100000000000 1000000000000 10000000000000
}

do_fts3_varint_test 1.1 {
  -1 -10 -100 -1000 -10000 -100000 -1000000 
  -10000000 -100000000 -1000000000 -10000000000
  -100000000000 -1000000000000 -10000000000000
}

do_fts3_varint_test 2.0 { 0 1 2 }
do_fts3_varint_test 2.1 { 1 2 3 }
do_fts3_varint_test 2.2 { 3 4 5 }
do_fts3_varint_test 2.3 { 7 8 9 }
do_fts3_varint_test 2.4 { 15 16 17 }
do_fts3_varint_test 2.5 { 31 32 33 }
do_fts3_varint_test 2.6 { 63 64 65 }
do_fts3_varint_test 2.7 { 127 128 129 }
do_fts3_varint_test 2.8 { 255 256 257 }
do_fts3_varint_test 2.9 { 511 512 513 }
do_fts3_varint_test 2.10 { 1023 1024 1025 }
do_fts3_varint_test 2.11 { 2047 2048 2049 }
do_fts3_varint_test 2.12 { 4095 4096 4097 }
do_fts3_varint_test 2.13 { 8191 8192 8193 }
do_fts3_varint_test 2.14 { 16383 16384 16385 }
do_fts3_varint_test 2.15 { 32767 32768 32769 }
do_fts3_varint_test 2.16 { 65535 65536 65537 }
do_fts3_varint_test 2.17 { 131071 131072 131073 }
do_fts3_varint_test 2.18 { 262143 262144 262145 }
do_fts3_varint_test 2.19 { 524287 524288 524289 }
do_fts3_varint_test 2.20 { 1048575 1048576 1048577 }
do_fts3_varint_test 2.21 { 2097151 2097152 2097153 }
do_fts3_varint_test 2.22 { 4194303 4194304 4194305 }
do_fts3_varint_test 2.23 { 8388607 8388608 8388609 }
do_fts3_varint_test 2.24 { 16777215 16777216 16777217 }
do_fts3_varint_test 2.25 { 33554431 33554432 33554433 }
do_fts3_varint_test 2.26 { 67108863 67108864 67108865 }
do_fts3_varint_test 2.27 { 134217727 134217728 134217729 }
do_fts3_varint_test 2.28 { 268435455 268435456 268435457 }
do_fts3_varint_test 2.29 { 536870911 536870912 536870913 }
do_fts3_varint_test 2.30 { 1073741823 1073741824 1073741825 }
do_fts3_varint_test 2.31 { 2147483647 2147483648 2147483649 }
do_fts3_varint_test 2.32 { 4294967295 4294967296 4294967297 }
do_fts3_varint_test 2.33 { 8589934591 8589934592 8589934593 }
do_fts3_varint_test 2.34 { 17179869183 17179869184 17179869185 }
do_fts3_varint_test 2.35 { 34359738367 34359738368 34359738369 }
do_fts3_varint_test 2.36 { 68719476735 68719476736 68719476737 }
do_fts3_varint_test 2.37 { 137438953471 137438953472 137438953473 }
do_fts3_varint_test 2.38 { 274877906943 274877906944 274877906945 }
do_fts3_varint_test 2.39 { 549755813887 549755813888 549755813889 }
do_fts3_varint_test 2.40 { 1099511627775 1099511627776 1099511627777 }
do_fts3_varint_test 2.41 { 2199023255551 2199023255552 2199023255553 }
do_fts3_varint_test 2.42 { 4398046511103 4398046511104 4398046511105 }
do_fts3_varint_test 2.43 { 8796093022207 8796093022208 8796093022209 }
do_fts3_varint_test 2.44 { 17592186044415 17592186044416 17592186044417 }
do_fts3_varint_test 2.45 { 35184372088831 35184372088832 35184372088833 }
do_fts3_varint_test 2.46 { 70368744177663 70368744177664 70368744177665 }
do_fts3_varint_test 2.47 { 140737488355327 140737488355328 140737488355329 }
do_fts3_varint_test 2.48 { 281474976710655 281474976710656 281474976710657 }
do_fts3_varint_test 2.49 { 562949953421311 562949953421312 562949953421313 }
do_fts3_varint_test 2.50 { 1125899906842623 1125899906842624 1125899906842625 }
do_fts3_varint_test 2.51 { 2251799813685247 2251799813685248 2251799813685249 }
do_fts3_varint_test 2.52 { 4503599627370495 4503599627370496 4503599627370497 }
do_fts3_varint_test 2.53 { 9007199254740991 9007199254740992 9007199254740993 }
do_fts3_varint_test 2.54 { 
    18014398509481983 18014398509481984 18014398509481985 }
do_fts3_varint_test 2.55 { 
    36028797018963967 36028797018963968 36028797018963969 }
do_fts3_varint_test 2.56 { 
    72057594037927935 72057594037927936 72057594037927937 }
do_fts3_varint_test 2.57 { 
    144115188075855871 144115188075855872 144115188075855873 }
do_fts3_varint_test 2.58 { 
    288230376151711743 288230376151711744 288230376151711745 }
do_fts3_varint_test 2.59 { 
    576460752303423487 576460752303423488 576460752303423489 }
do_fts3_varint_test 2.60 { 
    1152921504606846975 1152921504606846976 1152921504606846977 }
do_fts3_varint_test 2.61 { 
    2305843009213693951 2305843009213693952 2305843009213693953 }
do_fts3_varint_test 2.62 { 
    4611686018427387903 4611686018427387904 4611686018427387905 }
do_fts3_varint_test 2.63 { 
    9223372036854775807 9223372036854775808 9223372036854775809 }

do_fts3_varint_test 3.0 { 18446744073709551615 -18446744073709551615 }

finish_test

Changes to test/permutations.test.

191
192
193
194
195
196
197

198
199
200
201
202
203
204
  fts3sort.test
  fts3fault.test fts3malloc.test fts3matchinfo.test
  fts3aux1.test fts3comp1.test fts3auto.test
  fts4aa.test fts4content.test
  fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
  fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test
  fts4check.test fts4unicode.test fts4noti.test

}

test_suite "nofaultsim" -prefix "" -description {
  "Very" quick test suite. Runs in less than 5 minutes on a workstation. 
  This test suite is the same as the "quick" tests, except that some files
  that test malloc and IO errors are omitted.
} -files [







>







191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
  fts3sort.test
  fts3fault.test fts3malloc.test fts3matchinfo.test
  fts3aux1.test fts3comp1.test fts3auto.test
  fts4aa.test fts4content.test
  fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
  fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test
  fts4check.test fts4unicode.test fts4noti.test
  fts3varint.test
}

test_suite "nofaultsim" -prefix "" -description {
  "Very" quick test suite. Runs in less than 5 minutes on a workstation. 
  This test suite is the same as the "quick" tests, except that some files
  that test malloc and IO errors are omitted.
} -files [