/ Check-in [35b1b8d4]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge the 3.8.12 beta changes from trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | sessions
Files: files | file ages | folders
SHA1: 35b1b8d4b97715030700e37b292bb4f1bb3f44d6
User & Date: drh 2015-10-08 14:55:30
Context
2015-10-09
15:29
Merge the version 3.9.0 changes and the incorporation of JSON1 and FTS5 into the amalgamation from trunk. check-in: c1d96fb6 user: drh tags: sessions
2015-10-08
14:55
Merge the 3.8.12 beta changes from trunk. check-in: 35b1b8d4 user: drh tags: sessions
02:44
Remove two unused lines of code - discovered by scan-build. check-in: 77b707b7 user: drh tags: trunk
2015-10-01
16:35
Changes to the sesssions module ensure that tables appear within changesets and patchsets in the same order that they were attached to the sessions object. check-in: 76958902 user: dan tags: sessions
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3_expr.c.

789
790
791
792
793
794
795

796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912



913




914


















915
916
917
918
919
920
921
  Fts3Expr *pFree = 0;            /* List of free nodes. Linked by pParent. */
  int eType = pRoot->eType;       /* Type of node in this tree */

  if( nMaxDepth==0 ){
    rc = SQLITE_ERROR;
  }


  if( rc==SQLITE_OK && (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){
    Fts3Expr **apLeaf;
    apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth);
    if( 0==apLeaf ){
      rc = SQLITE_NOMEM;
    }else{
      memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth);
    }

    if( rc==SQLITE_OK ){
      int i;
      Fts3Expr *p;

      /* Set $p to point to the left-most leaf in the tree of eType nodes. */
      for(p=pRoot; p->eType==eType; p=p->pLeft){
        assert( p->pParent==0 || p->pParent->pLeft==p );
        assert( p->pLeft && p->pRight );
      }

      /* This loop runs once for each leaf in the tree of eType nodes. */
      while( 1 ){
        int iLvl;
        Fts3Expr *pParent = p->pParent;     /* Current parent of p */

        assert( pParent==0 || pParent->pLeft==p );
        p->pParent = 0;
        if( pParent ){
          pParent->pLeft = 0;
        }else{
          pRoot = 0;
        }
        rc = fts3ExprBalance(&p, nMaxDepth-1);
        if( rc!=SQLITE_OK ) break;

        for(iLvl=0; p && iLvl<nMaxDepth; iLvl++){
          if( apLeaf[iLvl]==0 ){
            apLeaf[iLvl] = p;
            p = 0;
          }else{
            assert( pFree );
            pFree->pLeft = apLeaf[iLvl];
            pFree->pRight = p;
            pFree->pLeft->pParent = pFree;
            pFree->pRight->pParent = pFree;

            p = pFree;
            pFree = pFree->pParent;
            p->pParent = 0;
            apLeaf[iLvl] = 0;
          }
        }
        if( p ){
          sqlite3Fts3ExprFree(p);
          rc = SQLITE_TOOBIG;
          break;
        }

        /* If that was the last leaf node, break out of the loop */
        if( pParent==0 ) break;

        /* Set $p to point to the next leaf in the tree of eType nodes */
        for(p=pParent->pRight; p->eType==eType; p=p->pLeft);

        /* Remove pParent from the original tree. */
        assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent );
        pParent->pRight->pParent = pParent->pParent;
        if( pParent->pParent ){
          pParent->pParent->pLeft = pParent->pRight;
        }else{
          assert( pParent==pRoot );
          pRoot = pParent->pRight;
        }

        /* Link pParent into the free node list. It will be used as an
        ** internal node of the new tree.  */
        pParent->pParent = pFree;
        pFree = pParent;
      }

      if( rc==SQLITE_OK ){
        p = 0;
        for(i=0; i<nMaxDepth; i++){
          if( apLeaf[i] ){
            if( p==0 ){
              p = apLeaf[i];
              p->pParent = 0;
            }else{
              assert( pFree!=0 );
              pFree->pRight = p;
              pFree->pLeft = apLeaf[i];
              pFree->pLeft->pParent = pFree;
              pFree->pRight->pParent = pFree;

              p = pFree;
              pFree = pFree->pParent;
              p->pParent = 0;
            }
          }
        }
        pRoot = p;
      }else{
        /* An error occurred. Delete the contents of the apLeaf[] array 
        ** and pFree list. Everything else is cleaned up by the call to
        ** sqlite3Fts3ExprFree(pRoot) below.  */
        Fts3Expr *pDel;
        for(i=0; i<nMaxDepth; i++){
          sqlite3Fts3ExprFree(apLeaf[i]);
        }
        while( (pDel=pFree)!=0 ){
          pFree = pDel->pParent;
          sqlite3_free(pDel);
        }
      }

      assert( pFree==0 );
      sqlite3_free( apLeaf );
    }



  }























  if( rc!=SQLITE_OK ){
    sqlite3Fts3ExprFree(pRoot);
    pRoot = 0;
  }
  *pp = pRoot;
  return rc;
}







>
|
|
|
|
|
|
|
|

|
|
|

|
|
|
|
|

|
|
|
|

|
|
|
|
|
|
|
|
|

|
|
|
|
|
|
|
|
|
|

|
|
|
|
|
|
|
|
|
|
|

|
|

|
|

|
|
|
|
|
|
|
|
|

|
|
|
|
|

|
|
|
|
|
|
|
|
|
|
|
|
|

|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|

|
|
|
>
>
>
|
>
>
>
>

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
  Fts3Expr *pFree = 0;            /* List of free nodes. Linked by pParent. */
  int eType = pRoot->eType;       /* Type of node in this tree */

  if( nMaxDepth==0 ){
    rc = SQLITE_ERROR;
  }

  if( rc==SQLITE_OK ){
    if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){
      Fts3Expr **apLeaf;
      apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth);
      if( 0==apLeaf ){
        rc = SQLITE_NOMEM;
      }else{
        memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth);
      }

      if( rc==SQLITE_OK ){
        int i;
        Fts3Expr *p;

        /* Set $p to point to the left-most leaf in the tree of eType nodes. */
        for(p=pRoot; p->eType==eType; p=p->pLeft){
          assert( p->pParent==0 || p->pParent->pLeft==p );
          assert( p->pLeft && p->pRight );
        }

        /* This loop runs once for each leaf in the tree of eType nodes. */
        while( 1 ){
          int iLvl;
          Fts3Expr *pParent = p->pParent;     /* Current parent of p */

          assert( pParent==0 || pParent->pLeft==p );
          p->pParent = 0;
          if( pParent ){
            pParent->pLeft = 0;
          }else{
            pRoot = 0;
          }
          rc = fts3ExprBalance(&p, nMaxDepth-1);
          if( rc!=SQLITE_OK ) break;

          for(iLvl=0; p && iLvl<nMaxDepth; iLvl++){
            if( apLeaf[iLvl]==0 ){
              apLeaf[iLvl] = p;
              p = 0;
            }else{
              assert( pFree );
              pFree->pLeft = apLeaf[iLvl];
              pFree->pRight = p;
              pFree->pLeft->pParent = pFree;
              pFree->pRight->pParent = pFree;

              p = pFree;
              pFree = pFree->pParent;
              p->pParent = 0;
              apLeaf[iLvl] = 0;
            }
          }
          if( p ){
            sqlite3Fts3ExprFree(p);
            rc = SQLITE_TOOBIG;
            break;
          }

          /* If that was the last leaf node, break out of the loop */
          if( pParent==0 ) break;

          /* Set $p to point to the next leaf in the tree of eType nodes */
          for(p=pParent->pRight; p->eType==eType; p=p->pLeft);

          /* Remove pParent from the original tree. */
          assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent );
          pParent->pRight->pParent = pParent->pParent;
          if( pParent->pParent ){
            pParent->pParent->pLeft = pParent->pRight;
          }else{
            assert( pParent==pRoot );
            pRoot = pParent->pRight;
          }

          /* Link pParent into the free node list. It will be used as an
          ** internal node of the new tree.  */
          pParent->pParent = pFree;
          pFree = pParent;
        }

        if( rc==SQLITE_OK ){
          p = 0;
          for(i=0; i<nMaxDepth; i++){
            if( apLeaf[i] ){
              if( p==0 ){
                p = apLeaf[i];
                p->pParent = 0;
              }else{
                assert( pFree!=0 );
                pFree->pRight = p;
                pFree->pLeft = apLeaf[i];
                pFree->pLeft->pParent = pFree;
                pFree->pRight->pParent = pFree;

                p = pFree;
                pFree = pFree->pParent;
                p->pParent = 0;
              }
            }
          }
          pRoot = p;
        }else{
          /* An error occurred. Delete the contents of the apLeaf[] array 
          ** and pFree list. Everything else is cleaned up by the call to
          ** sqlite3Fts3ExprFree(pRoot) below.  */
          Fts3Expr *pDel;
          for(i=0; i<nMaxDepth; i++){
            sqlite3Fts3ExprFree(apLeaf[i]);
          }
          while( (pDel=pFree)!=0 ){
            pFree = pDel->pParent;
            sqlite3_free(pDel);
          }
        }

        assert( pFree==0 );
        sqlite3_free( apLeaf );
      }
    }else if( eType==FTSQUERY_NOT ){
      Fts3Expr *pLeft = pRoot->pLeft;
      Fts3Expr *pRight = pRoot->pRight;

      pRoot->pLeft = 0;
      pRoot->pRight = 0;
      pLeft->pParent = 0;
      pRight->pParent = 0;

      rc = fts3ExprBalance(&pLeft, nMaxDepth-1);
      if( rc==SQLITE_OK ){
        rc = fts3ExprBalance(&pRight, nMaxDepth-1);
      }

      if( rc!=SQLITE_OK ){
        sqlite3Fts3ExprFree(pRight);
        sqlite3Fts3ExprFree(pLeft);
      }else{
        assert( pLeft && pRight );
        pRoot->pLeft = pLeft;
        pLeft->pParent = pRoot;
        pRoot->pRight = pRight;
        pRight->pParent = pRoot;
      }
    }
  }
  
  if( rc!=SQLITE_OK ){
    sqlite3Fts3ExprFree(pRoot);
    pRoot = 0;
  }
  *pp = pRoot;
  return rc;
}

Changes to ext/fts3/fts3_write.c.

1318
1319
1320
1321
1322
1323
1324
1325
1326
1327

1328

1329
1330



1331
1332
1333
1334
1335
1336
1337
1338
1339
....
1565
1566
1567
1568
1569
1570
1571

1572
1573

1574
1575
1576
1577

1578
1579
1580
1581
1582
1583
1584
    pNext = &pReader->aDoclist[pReader->nDoclist];
  }

  if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){

    if( fts3SegReaderIsPending(pReader) ){
      Fts3HashElem *pElem = *(pReader->ppNextElem);
      if( pElem==0 ){
        pReader->aNode = 0;
      }else{

        PendingList *pList = (PendingList *)fts3HashData(pElem);

        pReader->zTerm = (char *)fts3HashKey(pElem);
        pReader->nTerm = fts3HashKeysize(pElem);



        pReader->nNode = pReader->nDoclist = pList->nData + 1;
        pReader->aNode = pReader->aDoclist = pList->aData;
        pReader->ppNextElem++;
        assert( pReader->aNode );
      }
      return SQLITE_OK;
    }

    fts3SegReaderSetEof(pReader);
................................................................................
}

/*
** Free all allocations associated with the iterator passed as the 
** second argument.
*/
void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){

  if( pReader && !fts3SegReaderIsPending(pReader) ){
    sqlite3_free(pReader->zTerm);

    if( !fts3SegReaderIsRootOnly(pReader) ){
      sqlite3_free(pReader->aNode);
      sqlite3_blob_close(pReader->pBlob);
    }

  }
  sqlite3_free(pReader);
}

/*
** Allocate a new SegReader object.
*/







|
|
|
>

>


>
>
>
|
|







 







>
|
|
>


<

>







1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
....
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582

1583
1584
1585
1586
1587
1588
1589
1590
1591
    pNext = &pReader->aDoclist[pReader->nDoclist];
  }

  if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){

    if( fts3SegReaderIsPending(pReader) ){
      Fts3HashElem *pElem = *(pReader->ppNextElem);
      sqlite3_free(pReader->aNode);
      pReader->aNode = 0;
      if( pElem ){
        char *aCopy;
        PendingList *pList = (PendingList *)fts3HashData(pElem);
        int nCopy = pList->nData+1;
        pReader->zTerm = (char *)fts3HashKey(pElem);
        pReader->nTerm = fts3HashKeysize(pElem);
        aCopy = (char*)sqlite3_malloc(nCopy);
        if( !aCopy ) return SQLITE_NOMEM;
        memcpy(aCopy, pList->aData, nCopy);
        pReader->nNode = pReader->nDoclist = nCopy;
        pReader->aNode = pReader->aDoclist = aCopy;
        pReader->ppNextElem++;
        assert( pReader->aNode );
      }
      return SQLITE_OK;
    }

    fts3SegReaderSetEof(pReader);
................................................................................
}

/*
** Free all allocations associated with the iterator passed as the 
** second argument.
*/
void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
  if( pReader ){
    if( !fts3SegReaderIsPending(pReader) ){
      sqlite3_free(pReader->zTerm);
    }
    if( !fts3SegReaderIsRootOnly(pReader) ){
      sqlite3_free(pReader->aNode);

    }
    sqlite3_blob_close(pReader->pBlob);
  }
  sqlite3_free(pReader);
}

/*
** Allocate a new SegReader object.
*/

Changes to ext/fts5/fts5Int.h.

77
78
79
80
81
82
83














84
85
86
87
88
89
90
...
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
...
317
318
319
320
321
322
323

324
325
326
327
328
329
330
331
...
366
367
368
369
370
371
372

373
374
375
376
377
378
379
...
429
430
431
432
433
434
435









436
437
438
439
440
441
442
...
522
523
524
525
526
527
528
529

530
531
532
533
534
535
536
...
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
...
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
extern int sqlite3_fts5_may_be_corrupt;
# define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
#else
# define assert_nc(x) assert(x)
#endif

typedef struct Fts5Global Fts5Global;















/**************************************************************************
** Interface to code in fts5_config.c. fts5_config.c contains contains code
** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
*/

typedef struct Fts5Config Fts5Config;
................................................................................
** Create/destroy an Fts5Index object.
*/
int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
int sqlite3Fts5IndexClose(Fts5Index *p);

/*
** for(
**   pIter = sqlite3Fts5IndexQuery(p, "token", 5, 0);
**   0==sqlite3Fts5IterEof(pIter);
**   sqlite3Fts5IterNext(pIter)
** ){
**   i64 iRowid = sqlite3Fts5IterRowid(pIter);
** }
*/

................................................................................
** Open a new iterator to iterate though all rowids that match the 
** specified token or token prefix.
*/
int sqlite3Fts5IndexQuery(
  Fts5Index *p,                   /* FTS index to query */
  const char *pToken, int nToken, /* Token (or prefix) to query for */
  int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */

  Fts5IndexIter **ppIter
);

/*
** The various operations on open token or token prefix iterators opened
** using sqlite3Fts5IndexQuery().
*/
int sqlite3Fts5IterEof(Fts5IndexIter*);
................................................................................

/*
** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
** document iDocid.
*/
int sqlite3Fts5IndexBeginWrite(
  Fts5Index *p,                   /* Index to write to */

  i64 iDocid                      /* Docid to add or remove data from */
);

/*
** Flush any data stored in the in-memory hash tables to the database.
** If the bCommit flag is true, also close any open blob handles.
*/
................................................................................
int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
int sqlite3Fts5GetVarintLen(u32 iVal);
u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
int sqlite3Fts5PutVarint(unsigned char *p, u64 v);

#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
#define fts5GetVarint    sqlite3Fts5GetVarint










/*
** End of interface to code in fts5_varint.c.
**************************************************************************/


/**************************************************************************
................................................................................
int sqlite3Fts5StorageClose(Fts5Storage *p);
int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);

int sqlite3Fts5DropAll(Fts5Config*);
int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);

int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
int sqlite3Fts5StorageInsert(Fts5Storage *p, sqlite3_value **apVal, int, i64*);


int sqlite3Fts5StorageIntegrity(Fts5Storage *p);

int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);

int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
................................................................................
*/
typedef struct Fts5Expr Fts5Expr;
typedef struct Fts5ExprNode Fts5ExprNode;
typedef struct Fts5Parse Fts5Parse;
typedef struct Fts5Token Fts5Token;
typedef struct Fts5ExprPhrase Fts5ExprPhrase;
typedef struct Fts5ExprNearset Fts5ExprNearset;
typedef struct Fts5ExprColset Fts5ExprColset;

struct Fts5Token {
  const char *p;                  /* Token text (not NULL terminated) */
  int n;                          /* Size of buffer p in bytes */
};

/* Parse a MATCH expression. */
................................................................................

Fts5ExprNearset *sqlite3Fts5ParseNearset(
  Fts5Parse*, 
  Fts5ExprNearset*,
  Fts5ExprPhrase* 
);

Fts5ExprColset *sqlite3Fts5ParseColset(
  Fts5Parse*, 
  Fts5ExprColset*, 
  Fts5Token *
);

void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);

void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5ExprColset*);
void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);

/*
** End of interface to code in fts5_expr.c.
**************************************************************************/








>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|







 







>
|







 







>







 







>
>
>
>
>
>
>
>
>







 







|
>







 







<







 







|

|








|







77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
...
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
...
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
...
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
...
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
...
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
...
587
588
589
590
591
592
593

594
595
596
597
598
599
600
...
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
extern int sqlite3_fts5_may_be_corrupt;
# define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
#else
# define assert_nc(x) assert(x)
#endif

typedef struct Fts5Global Fts5Global;
typedef struct Fts5Colset Fts5Colset;

/* If a NEAR() clump or phrase may only match a specific set of columns, 
** then an object of the following type is used to record the set of columns.
** Each entry in the aiCol[] array is a column that may be matched.
**
** This object is used by fts5_expr.c and fts5_index.c.
*/
struct Fts5Colset {
  int nCol;
  int aiCol[1];
};



/**************************************************************************
** Interface to code in fts5_config.c. fts5_config.c contains contains code
** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
*/

typedef struct Fts5Config Fts5Config;
................................................................................
** Create/destroy an Fts5Index object.
*/
int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
int sqlite3Fts5IndexClose(Fts5Index *p);

/*
** for(
**   sqlite3Fts5IndexQuery(p, "token", 5, 0, 0, &pIter);
**   0==sqlite3Fts5IterEof(pIter);
**   sqlite3Fts5IterNext(pIter)
** ){
**   i64 iRowid = sqlite3Fts5IterRowid(pIter);
** }
*/

................................................................................
** Open a new iterator to iterate though all rowids that match the 
** specified token or token prefix.
*/
int sqlite3Fts5IndexQuery(
  Fts5Index *p,                   /* FTS index to query */
  const char *pToken, int nToken, /* Token (or prefix) to query for */
  int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
  Fts5Colset *pColset,            /* Match these columns only */
  Fts5IndexIter **ppIter          /* OUT: New iterator object */
);

/*
** The various operations on open token or token prefix iterators opened
** using sqlite3Fts5IndexQuery().
*/
int sqlite3Fts5IterEof(Fts5IndexIter*);
................................................................................

/*
** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
** document iDocid.
*/
int sqlite3Fts5IndexBeginWrite(
  Fts5Index *p,                   /* Index to write to */
  int bDelete,                    /* True if current operation is a delete */
  i64 iDocid                      /* Docid to add or remove data from */
);

/*
** Flush any data stored in the in-memory hash tables to the database.
** If the bCommit flag is true, also close any open blob handles.
*/
................................................................................
int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
int sqlite3Fts5GetVarintLen(u32 iVal);
u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
int sqlite3Fts5PutVarint(unsigned char *p, u64 v);

#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
#define fts5GetVarint    sqlite3Fts5GetVarint

#define fts5FastGetVarint32(a, iOff, nVal) {      \
  nVal = (a)[iOff++];                             \
  if( nVal & 0x80 ){                              \
    iOff--;                                       \
    iOff += fts5GetVarint32(&(a)[iOff], nVal);    \
  }                                               \
}


/*
** End of interface to code in fts5_varint.c.
**************************************************************************/


/**************************************************************************
................................................................................
int sqlite3Fts5StorageClose(Fts5Storage *p);
int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);

int sqlite3Fts5DropAll(Fts5Config*);
int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);

int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);

int sqlite3Fts5StorageIntegrity(Fts5Storage *p);

int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);

int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
................................................................................
*/
typedef struct Fts5Expr Fts5Expr;
typedef struct Fts5ExprNode Fts5ExprNode;
typedef struct Fts5Parse Fts5Parse;
typedef struct Fts5Token Fts5Token;
typedef struct Fts5ExprPhrase Fts5ExprPhrase;
typedef struct Fts5ExprNearset Fts5ExprNearset;


struct Fts5Token {
  const char *p;                  /* Token text (not NULL terminated) */
  int n;                          /* Size of buffer p in bytes */
};

/* Parse a MATCH expression. */
................................................................................

Fts5ExprNearset *sqlite3Fts5ParseNearset(
  Fts5Parse*, 
  Fts5ExprNearset*,
  Fts5ExprPhrase* 
);

Fts5Colset *sqlite3Fts5ParseColset(
  Fts5Parse*, 
  Fts5Colset*, 
  Fts5Token *
);

void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);

void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);

/*
** End of interface to code in fts5_expr.c.
**************************************************************************/

Changes to ext/fts5/fts5_buffer.c.

181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
...
229
230
231
232
233
234
235

236
237
238

239
240
241

242

243
244
245
246
247
248
249
...
286
287
288
289
290
291
292

293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
  if( i>=n ){
    /* EOF */
    *piOff = -1;
    return 1;  
  }else{
    i64 iOff = *piOff;
    int iVal;
    i += fts5GetVarint32(&a[i], iVal);
    if( iVal==1 ){
      i += fts5GetVarint32(&a[i], iVal);
      iOff = ((i64)iVal) << 32;
      i += fts5GetVarint32(&a[i], iVal);
    }
    *piOff = iOff + (iVal-2);
    *pi = i;
    return 0;
  }
}

................................................................................
int sqlite3Fts5PoslistWriterAppend(
  Fts5Buffer *pBuf, 
  Fts5PoslistWriter *pWriter,
  i64 iPos
){
  static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
  int rc = SQLITE_OK;

  if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
    fts5BufferAppendVarint(&rc, pBuf, 1);
    fts5BufferAppendVarint(&rc, pBuf, (iPos >> 32));

    pWriter->iPrev = (iPos & colmask);
  }
  fts5BufferAppendVarint(&rc, pBuf, (iPos - pWriter->iPrev) + 2);

  pWriter->iPrev = iPos;

  return rc;
}

void *sqlite3Fts5MallocZero(int *pRc, int nByte){
  void *pRet = 0;
  if( *pRc==SQLITE_OK ){
    pRet = sqlite3_malloc(nByte);
................................................................................
** Return true if character 't' may be part of an FTS5 bareword, or false
** otherwise. Characters that may be part of barewords:
**
**   * All non-ASCII characters,
**   * The 52 upper and lower case ASCII characters, and
**   * The 10 integer ASCII characters.
**   * The underscore character "_" (0x5F).

*/
int sqlite3Fts5IsBareword(char t){
  u8 aBareword[128] = {
    0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0,   /* 0x00 .. 0x0F */
    0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0,   /* 0x10 .. 0x1F */
    0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0,   /* 0x20 .. 0x2F */
    1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 0, 0, 0, 0, 0, 0,   /* 0x30 .. 0x3F */
    0, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1,   /* 0x40 .. 0x4F */
    1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 0, 0, 0, 0, 1,   /* 0x50 .. 0x5F */
    0, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1,   /* 0x60 .. 0x6F */
    1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 0, 0, 0, 0, 0    /* 0x70 .. 0x7F */
  };

  return (t & 0x80) || aBareword[(int)t];
}









|

|

|







 







>
|
<
|
>
|
|
<
>
|
>







 







>




|












181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
...
229
230
231
232
233
234
235
236
237

238
239
240
241

242
243
244
245
246
247
248
249
250
251
...
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
  if( i>=n ){
    /* EOF */
    *piOff = -1;
    return 1;  
  }else{
    i64 iOff = *piOff;
    int iVal;
    fts5FastGetVarint32(a, i, iVal);
    if( iVal==1 ){
      fts5FastGetVarint32(a, i, iVal);
      iOff = ((i64)iVal) << 32;
      fts5FastGetVarint32(a, i, iVal);
    }
    *piOff = iOff + (iVal-2);
    *pi = i;
    return 0;
  }
}

................................................................................
int sqlite3Fts5PoslistWriterAppend(
  Fts5Buffer *pBuf, 
  Fts5PoslistWriter *pWriter,
  i64 iPos
){
  static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
  int rc = SQLITE_OK;
  if( 0==sqlite3Fts5BufferGrow(&rc, pBuf, 5+5+5) ){
    if( (iPos & colmask) != (pWriter->iPrev & colmask) ){

      pBuf->p[pBuf->n++] = 1;
      pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
      pWriter->iPrev = (iPos & colmask);
    }

    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2);
    pWriter->iPrev = iPos;
  }
  return rc;
}

void *sqlite3Fts5MallocZero(int *pRc, int nByte){
  void *pRet = 0;
  if( *pRc==SQLITE_OK ){
    pRet = sqlite3_malloc(nByte);
................................................................................
** Return true if character 't' may be part of an FTS5 bareword, or false
** otherwise. Characters that may be part of barewords:
**
**   * All non-ASCII characters,
**   * The 52 upper and lower case ASCII characters, and
**   * The 10 integer ASCII characters.
**   * The underscore character "_" (0x5F).
**   * The unicode "subsitute" character (0x1A).
*/
int sqlite3Fts5IsBareword(char t){
  u8 aBareword[128] = {
    0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0,   /* 0x00 .. 0x0F */
    0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 1, 0, 0, 0, 0, 0,   /* 0x10 .. 0x1F */
    0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0,   /* 0x20 .. 0x2F */
    1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 0, 0, 0, 0, 0, 0,   /* 0x30 .. 0x3F */
    0, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1,   /* 0x40 .. 0x4F */
    1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 0, 0, 0, 0, 1,   /* 0x50 .. 0x5F */
    0, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1,   /* 0x60 .. 0x6F */
    1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 0, 0, 0, 0, 0    /* 0x70 .. 0x7F */
  };

  return (t & 0x80) || aBareword[(int)t];
}


Changes to ext/fts5/fts5_expr.c.

85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
...
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
...
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
...
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
...
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
...
998
999
1000
1001
1002
1003
1004

1005
1006
1007
1008
1009
1010
1011
....
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
....
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
....
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
struct Fts5ExprPhrase {
  Fts5ExprNode *pNode;            /* FTS5_STRING node this phrase is part of */
  Fts5Buffer poslist;             /* Current position list */
  int nTerm;                      /* Number of entries in aTerm[] */
  Fts5ExprTerm aTerm[1];          /* Terms that make up this phrase */
};

/*
** If a NEAR() clump may only match a specific set of columns, then
** Fts5ExprNearset.pColset points to an object of the following type.
** Each entry in the aiCol[] array
*/
struct Fts5ExprColset {
  int nCol;
  int aiCol[1];
};

/*
** One or more phrases that must appear within a certain token distance of
** each other within each matching document.
*/
struct Fts5ExprNearset {
  int nNear;                      /* NEAR parameter */
  Fts5ExprColset *pColset;        /* Columns to search (NULL -> all columns) */
  int nPhrase;                    /* Number of entries in aPhrase[] array */
  Fts5ExprPhrase *apPhrase[1];    /* Array of phrase pointers */
};


/*
** Parse context.
................................................................................
  if( p ){
    sqlite3Fts5ParseNodeFree(p->pRoot);
    sqlite3_free(p->apExprPhrase);
    sqlite3_free(p);
  }
}

static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){
  int i;
  for(i=0; i<pColset->nCol; i++){
    if( pColset->aiCol[i]==iCol ) return 1;
  }
  return 0;
}

................................................................................
**
** SQLITE_OK is returned if an error occurs, or an SQLite error code 
** otherwise. It is not considered an error code if the current rowid is 
** not a match.
*/
static int fts5ExprPhraseIsMatch(
  Fts5ExprNode *pNode,            /* Node pPhrase belongs to */
  Fts5ExprColset *pColset,        /* Restrict matches to these columns */
  Fts5ExprPhrase *pPhrase,        /* Phrase object to initialize */
  int *pbMatch                    /* OUT: Set to true if really a match */
){
  Fts5PoslistWriter writer = {0};
  Fts5PoslistReader aStatic[4];
  Fts5PoslistReader *aIter = aStatic;
  int i;
................................................................................
  while( p<pEnd && ((prev & 0x80) || *p!=0x01) ){
    prev = *p++;
  }
  return p - (*pa);
}

static int fts5ExprExtractColset (
  Fts5ExprColset *pColset,        /* Colset to filter on */
  const u8 *pPos, int nPos,       /* Position list */
  Fts5Buffer *pBuf                /* Output buffer */
){
  int rc = SQLITE_OK;
  int i;

  fts5BufferZero(pBuf);
................................................................................
  ** of a single term only, grab pointers into the poslist managed by the
  ** fts5_index.c iterator object. This is much faster than synthesizing 
  ** a new poslist the way we have to for more complicated phrase or NEAR
  ** expressions.  */
  Fts5ExprNearset *pNear = pNode->pNear;
  Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
  Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
  Fts5ExprColset *pColset = pNear->pColset;
  const u8 *pPos;
  int nPos;
  int rc;

  assert( pNode->eType==FTS5_TERM );
  assert( pNear->nPhrase==1 && pPhrase->nTerm==1 );
  assert( pPhrase->aTerm[0].pSynonym==0 );
................................................................................
          sqlite3Fts5IterClose(p->pIter);
          p->pIter = 0;
        }
        rc = sqlite3Fts5IndexQuery(
            pExpr->pIndex, p->zTerm, strlen(p->zTerm),
            (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
            (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),

            &p->pIter
        );
        assert( rc==SQLITE_OK || p->pIter==0 );
        if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){
          bEof = 0;
        }
      }
................................................................................
    nNear = FTS5_DEFAULT_NEARDIST;
  }
  pNear->nNear = nNear;
}

/*
** The second argument passed to this function may be NULL, or it may be
** an existing Fts5ExprColset object. This function returns a pointer to
** a new colset object containing the contents of (p) with new value column
** number iCol appended. 
**
** If an OOM error occurs, store an error code in pParse and return NULL.
** The old colset object (if any) is not freed in this case.
*/
static Fts5ExprColset *fts5ParseColset(
  Fts5Parse *pParse,              /* Store SQLITE_NOMEM here if required */
  Fts5ExprColset *p,              /* Existing colset object */
  int iCol                        /* New column to add to colset object */
){
  int nCol = p ? p->nCol : 0;     /* Num. columns already in colset object */
  Fts5ExprColset *pNew;           /* New colset object to return */

  assert( pParse->rc==SQLITE_OK );
  assert( iCol>=0 && iCol<pParse->pConfig->nCol );

  pNew = sqlite3_realloc(p, sizeof(Fts5ExprColset) + sizeof(int)*nCol);
  if( pNew==0 ){
    pParse->rc = SQLITE_NOMEM;
  }else{
    int *aiCol = pNew->aiCol;
    int i, j;
    for(i=0; i<nCol; i++){
      if( aiCol[i]==iCol ) return pNew;
................................................................................
    for(i=1; i<pNew->nCol; i++) assert( pNew->aiCol[i]>pNew->aiCol[i-1] );
#endif
  }

  return pNew;
}

Fts5ExprColset *sqlite3Fts5ParseColset(
  Fts5Parse *pParse,              /* Store SQLITE_NOMEM here if required */
  Fts5ExprColset *pColset,        /* Existing colset object */
  Fts5Token *p
){
  Fts5ExprColset *pRet = 0;
  int iCol;
  char *z;                        /* Dequoted copy of token p */

  z = sqlite3Fts5Strndup(&pParse->rc, p->p, p->n);
  if( pParse->rc==SQLITE_OK ){
    Fts5Config *pConfig = pParse->pConfig;
    sqlite3Fts5Dequote(z);
................................................................................

  return pRet;
}

void sqlite3Fts5ParseSetColset(
  Fts5Parse *pParse, 
  Fts5ExprNearset *pNear, 
  Fts5ExprColset *pColset 
){
  if( pNear ){
    pNear->pColset = pColset;
  }else{
    sqlite3_free(pColset);
  }
}







<
<
<
<
<
<
<
<
<
<






|







 







|







 







|







 







|







 







|







 







>







 







|






|

|



|




|







 







|

|


|







 







|







85
86
87
88
89
90
91










92
93
94
95
96
97
98
99
100
101
102
103
104
105
...
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
...
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
...
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
...
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
...
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
....
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
....
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
....
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
struct Fts5ExprPhrase {
  Fts5ExprNode *pNode;            /* FTS5_STRING node this phrase is part of */
  Fts5Buffer poslist;             /* Current position list */
  int nTerm;                      /* Number of entries in aTerm[] */
  Fts5ExprTerm aTerm[1];          /* Terms that make up this phrase */
};











/*
** One or more phrases that must appear within a certain token distance of
** each other within each matching document.
*/
struct Fts5ExprNearset {
  int nNear;                      /* NEAR parameter */
  Fts5Colset *pColset;            /* Columns to search (NULL -> all columns) */
  int nPhrase;                    /* Number of entries in aPhrase[] array */
  Fts5ExprPhrase *apPhrase[1];    /* Array of phrase pointers */
};


/*
** Parse context.
................................................................................
  if( p ){
    sqlite3Fts5ParseNodeFree(p->pRoot);
    sqlite3_free(p->apExprPhrase);
    sqlite3_free(p);
  }
}

static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){
  int i;
  for(i=0; i<pColset->nCol; i++){
    if( pColset->aiCol[i]==iCol ) return 1;
  }
  return 0;
}

................................................................................
**
** SQLITE_OK is returned if an error occurs, or an SQLite error code 
** otherwise. It is not considered an error code if the current rowid is 
** not a match.
*/
static int fts5ExprPhraseIsMatch(
  Fts5ExprNode *pNode,            /* Node pPhrase belongs to */
  Fts5Colset *pColset,            /* Restrict matches to these columns */
  Fts5ExprPhrase *pPhrase,        /* Phrase object to initialize */
  int *pbMatch                    /* OUT: Set to true if really a match */
){
  Fts5PoslistWriter writer = {0};
  Fts5PoslistReader aStatic[4];
  Fts5PoslistReader *aIter = aStatic;
  int i;
................................................................................
  while( p<pEnd && ((prev & 0x80) || *p!=0x01) ){
    prev = *p++;
  }
  return p - (*pa);
}

static int fts5ExprExtractColset (
  Fts5Colset *pColset,            /* Colset to filter on */
  const u8 *pPos, int nPos,       /* Position list */
  Fts5Buffer *pBuf                /* Output buffer */
){
  int rc = SQLITE_OK;
  int i;

  fts5BufferZero(pBuf);
................................................................................
  ** of a single term only, grab pointers into the poslist managed by the
  ** fts5_index.c iterator object. This is much faster than synthesizing 
  ** a new poslist the way we have to for more complicated phrase or NEAR
  ** expressions.  */
  Fts5ExprNearset *pNear = pNode->pNear;
  Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
  Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
  Fts5Colset *pColset = pNear->pColset;
  const u8 *pPos;
  int nPos;
  int rc;

  assert( pNode->eType==FTS5_TERM );
  assert( pNear->nPhrase==1 && pPhrase->nTerm==1 );
  assert( pPhrase->aTerm[0].pSynonym==0 );
................................................................................
          sqlite3Fts5IterClose(p->pIter);
          p->pIter = 0;
        }
        rc = sqlite3Fts5IndexQuery(
            pExpr->pIndex, p->zTerm, strlen(p->zTerm),
            (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
            (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
            pNear->pColset,
            &p->pIter
        );
        assert( rc==SQLITE_OK || p->pIter==0 );
        if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){
          bEof = 0;
        }
      }
................................................................................
    nNear = FTS5_DEFAULT_NEARDIST;
  }
  pNear->nNear = nNear;
}

/*
** The second argument passed to this function may be NULL, or it may be
** an existing Fts5Colset object. This function returns a pointer to
** a new colset object containing the contents of (p) with new value column
** number iCol appended. 
**
** If an OOM error occurs, store an error code in pParse and return NULL.
** The old colset object (if any) is not freed in this case.
*/
static Fts5Colset *fts5ParseColset(
  Fts5Parse *pParse,              /* Store SQLITE_NOMEM here if required */
  Fts5Colset *p,                  /* Existing colset object */
  int iCol                        /* New column to add to colset object */
){
  int nCol = p ? p->nCol : 0;     /* Num. columns already in colset object */
  Fts5Colset *pNew;               /* New colset object to return */

  assert( pParse->rc==SQLITE_OK );
  assert( iCol>=0 && iCol<pParse->pConfig->nCol );

  pNew = sqlite3_realloc(p, sizeof(Fts5Colset) + sizeof(int)*nCol);
  if( pNew==0 ){
    pParse->rc = SQLITE_NOMEM;
  }else{
    int *aiCol = pNew->aiCol;
    int i, j;
    for(i=0; i<nCol; i++){
      if( aiCol[i]==iCol ) return pNew;
................................................................................
    for(i=1; i<pNew->nCol; i++) assert( pNew->aiCol[i]>pNew->aiCol[i-1] );
#endif
  }

  return pNew;
}

Fts5Colset *sqlite3Fts5ParseColset(
  Fts5Parse *pParse,              /* Store SQLITE_NOMEM here if required */
  Fts5Colset *pColset,            /* Existing colset object */
  Fts5Token *p
){
  Fts5Colset *pRet = 0;
  int iCol;
  char *z;                        /* Dequoted copy of token p */

  z = sqlite3Fts5Strndup(&pParse->rc, p->p, p->n);
  if( pParse->rc==SQLITE_OK ){
    Fts5Config *pConfig = pParse->pConfig;
    sqlite3Fts5Dequote(z);
................................................................................

  return pRet;
}

void sqlite3Fts5ParseSetColset(
  Fts5Parse *pParse, 
  Fts5ExprNearset *pNear, 
  Fts5Colset *pColset 
){
  if( pNear ){
    pNear->pColset = pColset;
  }else{
    sqlite3_free(pColset);
  }
}

Changes to ext/fts5/fts5_index.c.

287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
...
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317

318
319
320
321
322
323
324
....
1776
1777
1778
1779
1780
1781
1782

1783
1784
1785
1786
1787
1788
1789
....
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
....
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
....
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
....
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711





3712
3713
3714
3715
3716
3717
3718
....
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946





































































3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958

3959
3960

3961








3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978



3979
3980
3981
3982
3983
3984



3985
3986
3987










3988
3989






3990
3991
3992


3993



3994

3995
3996
3997
3998
3999
4000
4001
4002
4003




4004
4005




4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019

4020
4021



4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035







4036
4037
4038
4039
4040
4041
4042
....
4052
4053
4054
4055
4056
4057
4058

4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081



4082

4083
4084
4085
4086
4087
4088

4089


4090
4091
4092

4093
4094
4095
4096
4097
4098



4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
....
4121
4122
4123
4124
4125
4126
4127

4128
4129
4130
4131
4132
4133
4134
4135
....
4160
4161
4162
4163
4164
4165
4166

4167
4168
4169




4170





4171
4172
4173

4174

4175
4176
4177
4178
4179
4180
4181
....
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206


4207

4208
4209

4210

4211
4212
4213
4214
4215
4216
4217
....
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
....
4363
4364
4365
4366
4367
4368
4369

4370
4371
4372
4373
4374
4375
4376
....
4389
4390
4391
4392
4393
4394
4395

4396
4397
4398
4399
4400
4401
4402
....
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
....
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554

4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
....
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
....
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
....
5238
5239
5240
5241
5242
5243
5244























5245
5246
5247
5248
5249
5250
5251
....
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
  ** Variables related to the accumulation of tokens and doclists within the
  ** in-memory hash tables before they are flushed to disk.
  */
  Fts5Hash *pHash;                /* Hash table for in-memory data */
  int nMaxPendingData;            /* Max pending data before flush to disk */
  int nPendingData;               /* Current bytes of pending data */
  i64 iWriteRowid;                /* Rowid for current doc being written */
  Fts5Buffer scratch;

  /* Error state. */
  int rc;                         /* Current error code */

  /* State used by the fts5DataXXX() functions. */
  sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
  sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
................................................................................
  sqlite3_stmt *pIdxWriter;       /* "INSERT ... %_idx VALUES(?,?,?,?)" */
  sqlite3_stmt *pIdxDeleter;      /* "DELETE FROM %_idx WHERE segid=? */
  sqlite3_stmt *pIdxSelect;
  int nRead;                      /* Total number of blocks read */
};

struct Fts5DoclistIter {
  u8 *a;
  int n;
  int i;

  /* Output variables. aPoslist==0 at EOF */
  i64 iRowid;
  u8 *aPoslist;
  int nPoslist;

};

/*
** The contents of the "structure" record for each index are represented
** using an Fts5Structure record in memory. Which uses instances of the 
** other Fts5StructureXXX types as components.
*/
................................................................................
        }else{
          pIter->pLeaf->p = (u8*)pList;
          pIter->pLeaf->nn = nList;
          pIter->pLeaf->szLeaf = nList;
          pIter->iEndofDoclist = nList+1;
          sqlite3Fts5BufferSet(&p->rc, &pIter->term, strlen(zTerm), (u8*)zTerm);
          pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid);

        }
      }else{
        iOff = 0;
        /* Next entry is not on the current page */
        while( iOff==0 ){
          fts5SegIterNextPage(p, pIter);
          pLeaf = pIter->pLeaf;
................................................................................
  ){
    return;
  }

  pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno);
}

#define fts5IndexGetVarint32(a, iOff, nVal) {     \
  nVal = (a)[iOff++];                             \
  if( nVal & 0x80 ){                              \
    iOff--;                                       \
    iOff += fts5GetVarint32(&(a)[iOff], nVal);    \
  }                                               \
}

#define fts5IndexSkipVarint(a, iOff) {            \
  int iEnd = iOff+9;                              \
  while( (a[iOff++] & 0x80) && iOff<iEnd );       \
}

/*
** The iterator object passed as the second argument currently contains
................................................................................
  iPgidx = szLeaf;
  iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff);
  iOff = iTermOff;

  while( 1 ){

    /* Figure out how many new bytes are in this term */
    fts5IndexGetVarint32(a, iOff, nNew);
    if( nKeep<nMatch ){
      goto search_failed;
    }

    assert( nKeep>=nMatch );
    if( nKeep==nMatch ){
      int nCmp;
................................................................................
    }

    iPgidx += fts5GetVarint32(&a[iPgidx], nKeep);
    iTermOff += nKeep;
    iOff = iTermOff;

    /* Read the nKeep field of the next term. */
    fts5IndexGetVarint32(a, iOff, nKeep);
  }

 search_failed:
  if( bGe==0 ){
    fts5DataRelease(pIter->pLeaf);
    pIter->pLeaf = 0;
    return;
................................................................................
      if( (ret + i) > nMax ) break;
      ret += i;
    }
  }
  return ret;
}

#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \
  assert( pBuf->nSpace>=(pBuf->n+nBlob) );             \
  memcpy(&pBuf->p[pBuf->n], pBlob, nBlob);             \
  pBuf->n += nBlob;                                    \





}

/*
** Flush the contents of in-memory hash table iHash to a new level-0 
** segment on disk. Also update the corresponding structure record.
**
** If an error occurs, set the Fts5Index.rc error code. If an error has 
................................................................................
  fts5StructureRelease(pStruct);

  return fts5IndexReturn(p);
}

static void fts5PoslistCallback(
  Fts5Index *p, 
  void *pCtx, 
  const u8 *pChunk, int nChunk
){
  assert_nc( nChunk>=0 );
  if( nChunk>0 ){
    fts5BufferAppendBlob(&p->rc, (Fts5Buffer*)pCtx, nChunk, pChunk);





































































  }
}

/*
** Iterator pIter currently points to a valid entry (not EOF). This
** function appends the position list data for the current entry to
** buffer pBuf. It does not make a copy of the position-list size
** field.
*/
static void fts5SegiterPoslist(
  Fts5Index *p,
  Fts5SegIter *pSeg,

  Fts5Buffer *pBuf
){

  fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback);








}

/*
** Iterator pMulti currently points to a valid entry (not EOF). This
** function appends a copy of the position-list of the entry pMulti 
** currently points to to buffer pBuf.
**
** If an error occurs, an error code is left in p->rc. It is assumed
** no error has already occurred when this function is called.
*/
static void fts5MultiIterPoslist(
  Fts5Index *p,
  Fts5IndexIter *pMulti,
  int bSz,                        /* Append a size field before the data */
  Fts5Buffer *pBuf
){
  if( p->rc==SQLITE_OK ){



    Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
    assert( fts5MultiIterEof(p, pMulti)==0 );

    if( bSz ){
      /* WRITEPOSLISTSIZE */
      fts5BufferAppendVarint(&p->rc, pBuf, pSeg->nPos*2);



    }
    fts5SegiterPoslist(p, pSeg, pBuf);
  }










}







static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
  if( pIter->i<pIter->n ){
    int bDummy;


    if( pIter->i ){



      i64 iDelta;

      pIter->i += fts5GetVarint(&pIter->a[pIter->i], (u64*)&iDelta);
      pIter->iRowid += iDelta;
    }else{
      pIter->i += fts5GetVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid);
    }
    pIter->i += fts5GetPoslistSize(
        &pIter->a[pIter->i], &pIter->nPoslist, &bDummy
    );
    pIter->aPoslist = &pIter->a[pIter->i];




    pIter->i += pIter->nPoslist;
  }else{




    pIter->aPoslist = 0;
  }
}

static void fts5DoclistIterInit(
  Fts5Buffer *pBuf, 
  Fts5DoclistIter *pIter
){
  memset(pIter, 0, sizeof(*pIter));
  pIter->a = pBuf->p;
  pIter->n = pBuf->n;
  fts5DoclistIterNext(pIter);
}


/*
** Append a doclist to buffer pBuf.



*/
static void fts5MergeAppendDocid(
  int *pRc,                       /* IN/OUT: Error code */
  Fts5Buffer *pBuf,               /* Buffer to write to */
  i64 *piLastRowid,               /* IN/OUT: Previous rowid written (if any) */
  i64 iRowid                      /* Rowid to append */
){
  if( pBuf->n==0 ){
    fts5BufferAppendVarint(pRc, pBuf, iRowid);
  }else{
    fts5BufferAppendVarint(pRc, pBuf, iRowid - *piLastRowid);
  }
  *piLastRowid = iRowid;
}








/*
** Buffers p1 and p2 contain doclists. This function merges the content
** of the two doclists together and sets buffer p1 to the result before
** returning.
**
** If an error occurs, an error code is left in p->rc. If an error has
................................................................................
    Fts5DoclistIter i1;
    Fts5DoclistIter i2;
    Fts5Buffer out;
    Fts5Buffer tmp;
    memset(&out, 0, sizeof(out));
    memset(&tmp, 0, sizeof(tmp));


    fts5DoclistIterInit(p1, &i1);
    fts5DoclistIterInit(p2, &i2);
    while( p->rc==SQLITE_OK && (i1.aPoslist!=0 || i2.aPoslist!=0) ){
      if( i2.aPoslist==0 || (i1.aPoslist && i1.iRowid<i2.iRowid) ){
        /* Copy entry from i1 */
        fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i1.iRowid);
        /* WRITEPOSLISTSIZE */
        fts5BufferAppendVarint(&p->rc, &out, i1.nPoslist * 2);
        fts5BufferAppendBlob(&p->rc, &out, i1.nPoslist, i1.aPoslist);
        fts5DoclistIterNext(&i1);
      }
      else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){
        /* Copy entry from i2 */
        fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid);
        /* WRITEPOSLISTSIZE */
        fts5BufferAppendVarint(&p->rc, &out, i2.nPoslist * 2);
        fts5BufferAppendBlob(&p->rc, &out, i2.nPoslist, i2.aPoslist);
        fts5DoclistIterNext(&i2);
      }
      else{
        Fts5PoslistReader r1;
        Fts5PoslistReader r2;
        Fts5PoslistWriter writer;





        memset(&writer, 0, sizeof(writer));

        /* Merge the two position lists. */ 
        fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid);
        fts5BufferZero(&tmp);
        sqlite3Fts5PoslistReaderInit(-1, i1.aPoslist, i1.nPoslist, &r1);

        sqlite3Fts5PoslistReaderInit(-1, i2.aPoslist, i2.nPoslist, &r2);


        while( p->rc==SQLITE_OK && (r1.bEof==0 || r2.bEof==0) ){
          i64 iNew;
          if( r2.bEof || (r1.bEof==0 && r1.iPos<r2.iPos) ){

            iNew = r1.iPos;
            sqlite3Fts5PoslistReaderNext(&r1);
          }else{
            iNew = r2.iPos;
            sqlite3Fts5PoslistReaderNext(&r2);
            if( r1.iPos==r2.iPos ) sqlite3Fts5PoslistReaderNext(&r1);



          }
          p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
        }

        /* WRITEPOSLISTSIZE */
        fts5BufferAppendVarint(&p->rc, &out, tmp.n * 2);
        fts5BufferAppendBlob(&p->rc, &out, tmp.n, tmp.p);
        fts5DoclistIterNext(&i1);
        fts5DoclistIterNext(&i2);
      }
    }

    fts5BufferSet(&p->rc, p1, out.n, out.p);
    fts5BufferFree(&tmp);
................................................................................
}

static void fts5SetupPrefixIter(
  Fts5Index *p,                   /* Index to read from */
  int bDesc,                      /* True for "ORDER BY rowid DESC" */
  const u8 *pToken,               /* Buffer containing prefix to match */
  int nToken,                     /* Size of buffer pToken in bytes */

  Fts5IndexIter **ppIter       /* OUT: New iterator */
){
  Fts5Structure *pStruct;
  Fts5Buffer *aBuf;
  const int nBuf = 32;

  aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf);
  pStruct = fts5StructureRead(p);
................................................................................
            fts5BufferSwap(&doclist, &aBuf[i]);
            fts5BufferZero(&doclist);
          }else{
            fts5MergePrefixLists(p, &doclist, &aBuf[i]);
            fts5BufferZero(&aBuf[i]);
          }
        }

      }

      fts5MergeAppendDocid(&p->rc, &doclist, &iLastRowid, iRowid);




      fts5MultiIterPoslist(p, p1, 1, &doclist);





    }

    for(i=0; i<nBuf; i++){

      fts5MergePrefixLists(p, &doclist, &aBuf[i]);

      fts5BufferFree(&aBuf[i]);
    }
    fts5MultiIterFree(p, p1);

    pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n);
    if( pData ){
      pData->p = (u8*)&pData[1];
................................................................................
}


/*
** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain
** to the document with rowid iRowid.
*/
int sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){
  assert( p->rc==SQLITE_OK );

  /* Allocate the hash table if it has not already been allocated */
  if( p->pHash==0 ){
    p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData);
  }

  /* Flush the hash table to disk if required */


  if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){

    fts5IndexFlush(p);
  }

  p->iWriteRowid = iRowid;

  return fts5IndexReturn(p);
}

/*
** Commit data to disk.
*/
int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
................................................................................
    assert( p->pReader==0 );
    sqlite3_finalize(p->pWriter);
    sqlite3_finalize(p->pDeleter);
    sqlite3_finalize(p->pIdxWriter);
    sqlite3_finalize(p->pIdxDeleter);
    sqlite3_finalize(p->pIdxSelect);
    sqlite3Fts5HashFree(p->pHash);
    sqlite3Fts5BufferFree(&p->scratch);
    sqlite3_free(p->zDataTbl);
    sqlite3_free(p);
  }
  return rc;
}

/*
................................................................................
  const char *pToken, int nToken  /* Token to add or remove to or from index */
){
  int i;                          /* Used to iterate through indexes */
  int rc = SQLITE_OK;             /* Return code */
  Fts5Config *pConfig = p->pConfig;

  assert( p->rc==SQLITE_OK );


  /* Add the entry to the main terms index. */
  rc = sqlite3Fts5HashWrite(
      p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken
  );

  for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){
................................................................................
** Open a new iterator to iterate though all rowid that match the 
** specified token or token prefix.
*/
int sqlite3Fts5IndexQuery(
  Fts5Index *p,                   /* FTS index to query */
  const char *pToken, int nToken, /* Token (or prefix) to query for */
  int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */

  Fts5IndexIter **ppIter          /* OUT: New iterator object */
){
  Fts5Config *pConfig = p->pConfig;
  Fts5IndexIter *pRet = 0;
  int iIdx = 0;
  Fts5Buffer buf = {0, 0, 0};

................................................................................
      if( pStruct ){
        fts5MultiIterNew(p, pStruct, 1, flags, buf.p, nToken+1, -1, 0, &pRet);
        fts5StructureRelease(pStruct);
      }
    }else{
      int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0;
      buf.p[0] = FTS5_MAIN_PREFIX;
      fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, &pRet);
    }

    if( p->rc ){
      sqlite3Fts5IterClose(pRet);
      pRet = 0;
      fts5CloseReader(p);
    }
................................................................................
  assert( pIter->pIndex->rc==SQLITE_OK );
  *piRowid = pSeg->iRowid;
  *pn = pSeg->nPos;
  if( pSeg->iLeafOffset+pSeg->nPos <= pSeg->pLeaf->szLeaf ){
    *pp = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  }else{
    fts5BufferZero(&pIter->poslist);
    fts5SegiterPoslist(pIter->pIndex, pSeg, &pIter->poslist);
    *pp = pIter->poslist.p;
  }
  return fts5IndexReturn(pIter->pIndex);
}

/*
** This function is similar to sqlite3Fts5IterPoslist(), except that it
** copies the position list into the buffer supplied as the second 
** argument.
*/
int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){
  Fts5Index *p = pIter->pIndex;


  assert( p->rc==SQLITE_OK );
  fts5BufferZero(pBuf);
  fts5MultiIterPoslist(p, pIter, 0, pBuf);
  return fts5IndexReturn(p);
}

/*
** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery().
*/
void sqlite3Fts5IterClose(Fts5IndexIter *pIter){
................................................................................
  const char *z,                  /* Index key to query for */
  int n,                          /* Size of index key in bytes */
  int flags,                      /* Flags for Fts5IndexQuery */
  u64 *pCksum                     /* IN/OUT: Checksum value */
){
  u64 cksum = *pCksum;
  Fts5IndexIter *pIdxIter = 0;
  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, &pIdxIter);

  while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
    i64 dummy;
    const u8 *pPos;
    int nPos;
    i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
    rc = sqlite3Fts5IterPoslist(pIdxIter, &pPos, &nPos, &dummy);
................................................................................
    i64 iRowid = fts5MultiIterRowid(pIter);
    char *z = (char*)fts5MultiIterTerm(pIter, &n);

    /* If this is a new term, query for it. Update cksum3 with the results. */
    fts5TestTerm(p, &term, z, n, cksum2, &cksum3);

    poslist.n = 0;
    fts5MultiIterPoslist(p, pIter, 0, &poslist);
    while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
      int iCol = FTS5_POS2COLUMN(iPos);
      int iTokOff = FTS5_POS2OFFSET(iPos);
      cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
    }
  }
  fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3);
................................................................................
    *pRc = rc;
    return;
  }

  fts5DebugStructure(pRc, pBuf, p);
  fts5StructureRelease(p);
}
























/*
** Buffer (a/n) is assumed to contain a list of serialized varints. Read
** each varint and append its string representation to buffer pBuf. Return
** after either the input buffer is exhausted or a 0 value is read.
**
** The return value is the number of bytes read from the input buffer.
................................................................................
    for(fts5DlidxLvlNext(&lvl); lvl.bEof==0; fts5DlidxLvlNext(&lvl)){
      sqlite3Fts5BufferAppendPrintf(&rc, &s, 
          " %d(%lld)", lvl.iLeafPgno, lvl.iRowid
      );
    }
  }else if( iSegid==0 ){
    if( iRowid==FTS5_AVERAGES_ROWID ){
      /* todo */
    }else{
      fts5DecodeStructure(&rc, &s, a, n);
    }
  }else{
    Fts5Buffer term;              /* Current term read from page */
    int szLeaf;                   /* Offset of pgidx in a[] */
    int iPgidxOff;







|







 







|
<
<





>







 







>







 







<
<
<
<
<
<
<
<







 







|







 







|







 







|
|
|
|
>
>
>
>
>







 







|




|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>












>


>
|
>
>
>
>
>
>
>
>










|


|



>
>
>



<
|
<
>
>
>
|
|
|
>
>
>
>
>
>
>
>
>
>
|
|
>
>
>
>
>
>

<
<
>
>
|
>
>
>
|
>
|
|
<
<
|
<
<
<
<
>
>
>
>
|
|
>
>
>
>
|








|
|



>


>
>
>


<




|
<
<
|
<


>
>
>
>
>
>
>







 







>





|
|
<
<




|
|
<
<



|
|
|
>
>
>

>



|

<
>
|
>
>
|

<
>
|
|

|
|
<
>
>
>





|
|







 







>
|







 







>


<
>
>
>
>
|
>
>
>
>
>



>
|
>







 







|








>
>
|
>


>

>







 







<







 







>







 







>







 







|







 







|












<
>


|







 







|







 







|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|







287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
...
303
304
305
306
307
308
309
310


311
312
313
314
315
316
317
318
319
320
321
322
323
....
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
....
1936
1937
1938
1939
1940
1941
1942








1943
1944
1945
1946
1947
1948
1949
....
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
....
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
....
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
....
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060

4061

4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086


4087
4088
4089
4090
4091
4092
4093
4094
4095
4096


4097




4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129

4130
4131
4132
4133
4134


4135

4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
....
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175


4176
4177
4178
4179
4180
4181


4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197

4198
4199
4200
4201
4202
4203

4204
4205
4206
4207
4208
4209

4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
....
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
....
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284

4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
....
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
....
4433
4434
4435
4436
4437
4438
4439

4440
4441
4442
4443
4444
4445
4446
....
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
....
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
....
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
....
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685

4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
....
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
....
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
....
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
....
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
  ** Variables related to the accumulation of tokens and doclists within the
  ** in-memory hash tables before they are flushed to disk.
  */
  Fts5Hash *pHash;                /* Hash table for in-memory data */
  int nMaxPendingData;            /* Max pending data before flush to disk */
  int nPendingData;               /* Current bytes of pending data */
  i64 iWriteRowid;                /* Rowid for current doc being written */
  int bDelete;                    /* Current write is a delete */

  /* Error state. */
  int rc;                         /* Current error code */

  /* State used by the fts5DataXXX() functions. */
  sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
  sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
................................................................................
  sqlite3_stmt *pIdxWriter;       /* "INSERT ... %_idx VALUES(?,?,?,?)" */
  sqlite3_stmt *pIdxDeleter;      /* "DELETE FROM %_idx WHERE segid=? */
  sqlite3_stmt *pIdxSelect;
  int nRead;                      /* Total number of blocks read */
};

struct Fts5DoclistIter {
  u8 *aEof;                       /* Pointer to 1 byte past end of doclist */



  /* Output variables. aPoslist==0 at EOF */
  i64 iRowid;
  u8 *aPoslist;
  int nPoslist;
  int nSize;
};

/*
** The contents of the "structure" record for each index are represented
** using an Fts5Structure record in memory. Which uses instances of the 
** other Fts5StructureXXX types as components.
*/
................................................................................
        }else{
          pIter->pLeaf->p = (u8*)pList;
          pIter->pLeaf->nn = nList;
          pIter->pLeaf->szLeaf = nList;
          pIter->iEndofDoclist = nList+1;
          sqlite3Fts5BufferSet(&p->rc, &pIter->term, strlen(zTerm), (u8*)zTerm);
          pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid);
          if( pbNewTerm ) *pbNewTerm = 1;
        }
      }else{
        iOff = 0;
        /* Next entry is not on the current page */
        while( iOff==0 ){
          fts5SegIterNextPage(p, pIter);
          pLeaf = pIter->pLeaf;
................................................................................
  ){
    return;
  }

  pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno);
}









#define fts5IndexSkipVarint(a, iOff) {            \
  int iEnd = iOff+9;                              \
  while( (a[iOff++] & 0x80) && iOff<iEnd );       \
}

/*
** The iterator object passed as the second argument currently contains
................................................................................
  iPgidx = szLeaf;
  iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff);
  iOff = iTermOff;

  while( 1 ){

    /* Figure out how many new bytes are in this term */
    fts5FastGetVarint32(a, iOff, nNew);
    if( nKeep<nMatch ){
      goto search_failed;
    }

    assert( nKeep>=nMatch );
    if( nKeep==nMatch ){
      int nCmp;
................................................................................
    }

    iPgidx += fts5GetVarint32(&a[iPgidx], nKeep);
    iTermOff += nKeep;
    iOff = iTermOff;

    /* Read the nKeep field of the next term. */
    fts5FastGetVarint32(a, iOff, nKeep);
  }

 search_failed:
  if( bGe==0 ){
    fts5DataRelease(pIter->pLeaf);
    pIter->pLeaf = 0;
    return;
................................................................................
      if( (ret + i) > nMax ) break;
      ret += i;
    }
  }
  return ret;
}

#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) {     \
  assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) );             \
  memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob);             \
  (pBuf)->n += nBlob;                                      \
}

#define fts5BufferSafeAppendVarint(pBuf, iVal) {                \
  (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal));  \
  assert( (pBuf)->nSpace>=(pBuf)->n );                          \
}

/*
** Flush the contents of in-memory hash table iHash to a new level-0 
** segment on disk. Also update the corresponding structure record.
**
** If an error occurs, set the Fts5Index.rc error code. If an error has 
................................................................................
  fts5StructureRelease(pStruct);

  return fts5IndexReturn(p);
}

static void fts5PoslistCallback(
  Fts5Index *p, 
  void *pContext, 
  const u8 *pChunk, int nChunk
){
  assert_nc( nChunk>=0 );
  if( nChunk>0 ){
    fts5BufferAppendBlob(&p->rc, (Fts5Buffer*)pContext, nChunk, pChunk);
  }
}

typedef struct PoslistCallbackCtx PoslistCallbackCtx;
struct PoslistCallbackCtx {
  Fts5Buffer *pBuf;               /* Append to this buffer */
  Fts5Colset *pColset;            /* Restrict matches to this column */
  int eState;                     /* See above */
};

/*
** TODO: Make this more efficient!
*/
static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){
  int i;
  for(i=0; i<pColset->nCol; i++){
    if( pColset->aiCol[i]==iCol ) return 1;
  }
  return 0;
}

static void fts5PoslistFilterCallback(
  Fts5Index *p, 
  void *pContext, 
  const u8 *pChunk, int nChunk
){
  PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext;
  assert_nc( nChunk>=0 );
  if( nChunk>0 ){
    /* Search through to find the first varint with value 1. This is the
    ** start of the next columns hits. */
    int i = 0;
    int iStart = 0;

    if( pCtx->eState==2 ){
      int iCol;
      fts5FastGetVarint32(pChunk, i, iCol);
      if( fts5IndexColsetTest(pCtx->pColset, iCol) ){
        pCtx->eState = 1;
        fts5BufferAppendVarint(&p->rc, pCtx->pBuf, 1);
      }else{
        pCtx->eState = 0;
      }
    }

    do {
      while( i<nChunk && pChunk[i]!=0x01 ){
        while( pChunk[i] & 0x80 ) i++;
        i++;
      }
      if( pCtx->eState ){
        fts5BufferAppendBlob(&p->rc, pCtx->pBuf, i-iStart, &pChunk[iStart]);
      }
      if( i<nChunk ){
        int iCol;
        iStart = i;
        i++;
        if( i>=nChunk ){
          pCtx->eState = 2;
        }else{
          fts5FastGetVarint32(pChunk, i, iCol);
          pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol);
          if( pCtx->eState ){
            fts5BufferAppendBlob(&p->rc, pCtx->pBuf, i-iStart, &pChunk[iStart]);
            iStart = i;
          }
        }
      }
    }while( i<nChunk );
  }
}

/*
** Iterator pIter currently points to a valid entry (not EOF). This
** function appends the position list data for the current entry to
** buffer pBuf. It does not make a copy of the position-list size
** field.
*/
static void fts5SegiterPoslist(
  Fts5Index *p,
  Fts5SegIter *pSeg,
  Fts5Colset *pColset,
  Fts5Buffer *pBuf
){
  if( pColset==0 ){
    fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback);
  }else{
    PoslistCallbackCtx sCtx;
    sCtx.pBuf = pBuf;
    sCtx.pColset = pColset;
    sCtx.eState = pColset ? fts5IndexColsetTest(pColset, 0) : 1;
    assert( sCtx.eState==0 || sCtx.eState==1 );
    fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
  }
}

/*
** Iterator pMulti currently points to a valid entry (not EOF). This
** function appends a copy of the position-list of the entry pMulti 
** currently points to to buffer pBuf.
**
** If an error occurs, an error code is left in p->rc. It is assumed
** no error has already occurred when this function is called.
*/
static int fts5MultiIterPoslist(
  Fts5Index *p,
  Fts5IndexIter *pMulti,
  Fts5Colset *pColset,
  Fts5Buffer *pBuf
){
  if( p->rc==SQLITE_OK ){
    int iSz;
    int iData;

    Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
    assert( fts5MultiIterEof(p, pMulti)==0 );


    /* WRITEPOSLISTSIZE */

    iSz = pBuf->n;
    fts5BufferSafeAppendVarint(pBuf, pSeg->nPos*2);
    iData = pBuf->n;

    fts5SegiterPoslist(p, pSeg, pColset, pBuf);

    if( pColset ){
      int nActual = pBuf->n - iData;
      if( nActual!=pSeg->nPos ){
        /* WRITEPOSLISTSIZE */
        if( nActual==0 ){
          return 1;
        }else{
          int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2));
          while( iSz<(iData-nReq) ){ pBuf->p[iSz++] = 0x80; }
          sqlite3Fts5PutVarint(&pBuf->p[iSz], nActual*2);
        }
      }
    }
  }

  return 0;
}

static void fts5DoclistIterNext(Fts5DoclistIter *pIter){


  u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist;

  assert( pIter->aPoslist );
  if( p>=pIter->aEof ){
    pIter->aPoslist = 0;
  }else{
    i64 iDelta;

    p += fts5GetVarint(p, (u64*)&iDelta);
    pIter->iRowid += iDelta;







    /* Read position list size */
    if( p[0] & 0x80 ){
      int nPos;
      pIter->nSize = fts5GetVarint32(p, nPos);
      pIter->nPoslist = (nPos>>1);
    }else{
      pIter->nPoslist = ((int)(p[0])) >> 1;
      pIter->nSize = 1;
    }

    pIter->aPoslist = p;
  }
}

static void fts5DoclistIterInit(
  Fts5Buffer *pBuf, 
  Fts5DoclistIter *pIter
){
  memset(pIter, 0, sizeof(*pIter));
  pIter->aPoslist = pBuf->p;
  pIter->aEof = &pBuf->p[pBuf->n];
  fts5DoclistIterNext(pIter);
}

#if 0
/*
** Append a doclist to buffer pBuf.
**
** This function assumes that space within the buffer has already been
** allocated.
*/
static void fts5MergeAppendDocid(

  Fts5Buffer *pBuf,               /* Buffer to write to */
  i64 *piLastRowid,               /* IN/OUT: Previous rowid written (if any) */
  i64 iRowid                      /* Rowid to append */
){
  assert( pBuf->n!=0 || (*piLastRowid)==0 );


  fts5BufferSafeAppendVarint(pBuf, iRowid - *piLastRowid);

  *piLastRowid = iRowid;
}
#endif

#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) {       \
  assert( (pBuf)->n!=0 || (iLastRowid)==0 );                   \
  fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \
  (iLastRowid) = (iRowid);                                     \
}

/*
** Buffers p1 and p2 contain doclists. This function merges the content
** of the two doclists together and sets buffer p1 to the result before
** returning.
**
** If an error occurs, an error code is left in p->rc. If an error has
................................................................................
    Fts5DoclistIter i1;
    Fts5DoclistIter i2;
    Fts5Buffer out;
    Fts5Buffer tmp;
    memset(&out, 0, sizeof(out));
    memset(&tmp, 0, sizeof(tmp));

    sqlite3Fts5BufferGrow(&p->rc, &out, p1->n + p2->n);
    fts5DoclistIterInit(p1, &i1);
    fts5DoclistIterInit(p2, &i2);
    while( p->rc==SQLITE_OK && (i1.aPoslist!=0 || i2.aPoslist!=0) ){
      if( i2.aPoslist==0 || (i1.aPoslist && i1.iRowid<i2.iRowid) ){
        /* Copy entry from i1 */
        fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
        fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist+i1.nSize);


        fts5DoclistIterNext(&i1);
      }
      else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){
        /* Copy entry from i2 */
        fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
        fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist+i2.nSize);


        fts5DoclistIterNext(&i2);
      }
      else{
        i64 iPos1 = 0;
        i64 iPos2 = 0;
        int iOff1 = 0;
        int iOff2 = 0;
        u8 *a1 = &i1.aPoslist[i1.nSize];
        u8 *a2 = &i2.aPoslist[i2.nSize];

        Fts5PoslistWriter writer;
        memset(&writer, 0, sizeof(writer));

        /* Merge the two position lists. */ 
        fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
        fts5BufferZero(&tmp);


        sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
        sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);

        while( p->rc==SQLITE_OK && (iPos1>=0 || iPos2>=0) ){
          i64 iNew;

          if( iPos2<0 || (iPos1>=0 && iPos1<iPos2) ){
            iNew = iPos1;
            sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
          }else{
            iNew = iPos2;
            sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);

            if( iPos1==iPos2 ){
              sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1,&iPos1);
            }
          }
          p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
        }

        /* WRITEPOSLISTSIZE */
        fts5BufferSafeAppendVarint(&out, tmp.n * 2);
        fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n);
        fts5DoclistIterNext(&i1);
        fts5DoclistIterNext(&i2);
      }
    }

    fts5BufferSet(&p->rc, p1, out.n, out.p);
    fts5BufferFree(&tmp);
................................................................................
}

static void fts5SetupPrefixIter(
  Fts5Index *p,                   /* Index to read from */
  int bDesc,                      /* True for "ORDER BY rowid DESC" */
  const u8 *pToken,               /* Buffer containing prefix to match */
  int nToken,                     /* Size of buffer pToken in bytes */
  Fts5Colset *pColset,            /* Restrict matches to these columns */
  Fts5IndexIter **ppIter          /* OUT: New iterator */
){
  Fts5Structure *pStruct;
  Fts5Buffer *aBuf;
  const int nBuf = 32;

  aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf);
  pStruct = fts5StructureRead(p);
................................................................................
            fts5BufferSwap(&doclist, &aBuf[i]);
            fts5BufferZero(&doclist);
          }else{
            fts5MergePrefixLists(p, &doclist, &aBuf[i]);
            fts5BufferZero(&aBuf[i]);
          }
        }
        iLastRowid = 0;
      }


      if( 0==sqlite3Fts5BufferGrow(&p->rc, &doclist, 9) ){
        int iSave = doclist.n;
        assert( doclist.n!=0 || iLastRowid==0 );
        fts5BufferSafeAppendVarint(&doclist, iRowid - iLastRowid);
        if( fts5MultiIterPoslist(p, p1, pColset, &doclist) ){
          doclist.n = iSave;
        }else{
          iLastRowid = iRowid;
        }
      }
    }

    for(i=0; i<nBuf; i++){
      if( p->rc==SQLITE_OK ){
        fts5MergePrefixLists(p, &doclist, &aBuf[i]);
      }
      fts5BufferFree(&aBuf[i]);
    }
    fts5MultiIterFree(p, p1);

    pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n);
    if( pData ){
      pData->p = (u8*)&pData[1];
................................................................................
}


/*
** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain
** to the document with rowid iRowid.
*/
int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){
  assert( p->rc==SQLITE_OK );

  /* Allocate the hash table if it has not already been allocated */
  if( p->pHash==0 ){
    p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData);
  }

  /* Flush the hash table to disk if required */
  if( iRowid<p->iWriteRowid 
   || (iRowid==p->iWriteRowid && p->bDelete==0)
   || (p->nPendingData > p->nMaxPendingData) 
  ){
    fts5IndexFlush(p);
  }

  p->iWriteRowid = iRowid;
  p->bDelete = bDelete;
  return fts5IndexReturn(p);
}

/*
** Commit data to disk.
*/
int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
................................................................................
    assert( p->pReader==0 );
    sqlite3_finalize(p->pWriter);
    sqlite3_finalize(p->pDeleter);
    sqlite3_finalize(p->pIdxWriter);
    sqlite3_finalize(p->pIdxDeleter);
    sqlite3_finalize(p->pIdxSelect);
    sqlite3Fts5HashFree(p->pHash);

    sqlite3_free(p->zDataTbl);
    sqlite3_free(p);
  }
  return rc;
}

/*
................................................................................
  const char *pToken, int nToken  /* Token to add or remove to or from index */
){
  int i;                          /* Used to iterate through indexes */
  int rc = SQLITE_OK;             /* Return code */
  Fts5Config *pConfig = p->pConfig;

  assert( p->rc==SQLITE_OK );
  assert( (iCol<0)==p->bDelete );

  /* Add the entry to the main terms index. */
  rc = sqlite3Fts5HashWrite(
      p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken
  );

  for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){
................................................................................
** Open a new iterator to iterate though all rowid that match the 
** specified token or token prefix.
*/
int sqlite3Fts5IndexQuery(
  Fts5Index *p,                   /* FTS index to query */
  const char *pToken, int nToken, /* Token (or prefix) to query for */
  int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
  Fts5Colset *pColset,            /* Match these columns only */
  Fts5IndexIter **ppIter          /* OUT: New iterator object */
){
  Fts5Config *pConfig = p->pConfig;
  Fts5IndexIter *pRet = 0;
  int iIdx = 0;
  Fts5Buffer buf = {0, 0, 0};

................................................................................
      if( pStruct ){
        fts5MultiIterNew(p, pStruct, 1, flags, buf.p, nToken+1, -1, 0, &pRet);
        fts5StructureRelease(pStruct);
      }
    }else{
      int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0;
      buf.p[0] = FTS5_MAIN_PREFIX;
      fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet);
    }

    if( p->rc ){
      sqlite3Fts5IterClose(pRet);
      pRet = 0;
      fts5CloseReader(p);
    }
................................................................................
  assert( pIter->pIndex->rc==SQLITE_OK );
  *piRowid = pSeg->iRowid;
  *pn = pSeg->nPos;
  if( pSeg->iLeafOffset+pSeg->nPos <= pSeg->pLeaf->szLeaf ){
    *pp = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  }else{
    fts5BufferZero(&pIter->poslist);
    fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist);
    *pp = pIter->poslist.p;
  }
  return fts5IndexReturn(pIter->pIndex);
}

/*
** This function is similar to sqlite3Fts5IterPoslist(), except that it
** copies the position list into the buffer supplied as the second 
** argument.
*/
int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){
  Fts5Index *p = pIter->pIndex;

  Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  assert( p->rc==SQLITE_OK );
  fts5BufferZero(pBuf);
  fts5SegiterPoslist(p, pSeg, 0, pBuf);
  return fts5IndexReturn(p);
}

/*
** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery().
*/
void sqlite3Fts5IterClose(Fts5IndexIter *pIter){
................................................................................
  const char *z,                  /* Index key to query for */
  int n,                          /* Size of index key in bytes */
  int flags,                      /* Flags for Fts5IndexQuery */
  u64 *pCksum                     /* IN/OUT: Checksum value */
){
  u64 cksum = *pCksum;
  Fts5IndexIter *pIdxIter = 0;
  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter);

  while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
    i64 dummy;
    const u8 *pPos;
    int nPos;
    i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
    rc = sqlite3Fts5IterPoslist(pIdxIter, &pPos, &nPos, &dummy);
................................................................................
    i64 iRowid = fts5MultiIterRowid(pIter);
    char *z = (char*)fts5MultiIterTerm(pIter, &n);

    /* If this is a new term, query for it. Update cksum3 with the results. */
    fts5TestTerm(p, &term, z, n, cksum2, &cksum3);

    poslist.n = 0;
    fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst] , 0, &poslist);
    while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
      int iCol = FTS5_POS2COLUMN(iPos);
      int iTokOff = FTS5_POS2OFFSET(iPos);
      cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
    }
  }
  fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3);
................................................................................
    *pRc = rc;
    return;
  }

  fts5DebugStructure(pRc, pBuf, p);
  fts5StructureRelease(p);
}

/*
** This is part of the fts5_decode() debugging aid.
**
** Arguments pBlob/nBlob contain an "averages" record. This function 
** appends a human-readable representation of record to the buffer passed 
** as the second argument. 
*/
static void fts5DecodeAverages(
  int *pRc,                       /* IN/OUT: error code */
  Fts5Buffer *pBuf,
  const u8 *pBlob, int nBlob
){
  int i = 0;
  const char *zSpace = "";

  while( i<nBlob ){
    u64 iVal;
    i += sqlite3Fts5GetVarint(&pBlob[i], &iVal);
    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "%s%d", zSpace, (int)iVal);
    zSpace = " ";
  }
}

/*
** Buffer (a/n) is assumed to contain a list of serialized varints. Read
** each varint and append its string representation to buffer pBuf. Return
** after either the input buffer is exhausted or a 0 value is read.
**
** The return value is the number of bytes read from the input buffer.
................................................................................
    for(fts5DlidxLvlNext(&lvl); lvl.bEof==0; fts5DlidxLvlNext(&lvl)){
      sqlite3Fts5BufferAppendPrintf(&rc, &s, 
          " %d(%lld)", lvl.iLeafPgno, lvl.iRowid
      );
    }
  }else if( iSegid==0 ){
    if( iRowid==FTS5_AVERAGES_ROWID ){
      fts5DecodeAverages(&rc, &s, a, n);
    }else{
      fts5DecodeStructure(&rc, &s, a, n);
    }
  }else{
    Fts5Buffer term;              /* Current term read from page */
    int szLeaf;                   /* Offset of pgidx in a[] */
    int iPgidxOff;

Changes to ext/fts5/fts5_main.c.

435
436
437
438
439
440
441













442
443
444
445
446
447
448
...
488
489
490
491
492
493
494

495

496
497
498
499
500
501
502
503
...
542
543
544
545
546
547
548

549
550
551
552
553
554
555
....
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
....
1347
1348
1349
1350
1351
1352
1353
















1354
1355
1356
1357
1358









1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374






1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
























1388
1389
1390
1391
1392

1393
1394


1395
1396
1397
1398
1399
1400
1401
1402




1403
1404
1405
1406

1407
1408


1409
1410
1411
1412
1413



1414



1415
1416







1417
1418














1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
*/
#define FTS5_PLAN_MATCH          1       /* (<tbl> MATCH ?) */
#define FTS5_PLAN_SOURCE         2       /* A source cursor for SORTED_MATCH */
#define FTS5_PLAN_SPECIAL        3       /* An internal query */
#define FTS5_PLAN_SORTED_MATCH   4       /* (<tbl> MATCH ? ORDER BY rank) */
#define FTS5_PLAN_SCAN           5       /* No usable constraint */
#define FTS5_PLAN_ROWID          6       /* (rowid = ?) */














/*
** Implementation of the xBestIndex method for FTS5 tables. Within the 
** WHERE constraint, it searches for the following:
**
**   1. A MATCH constraint against the special column.
**   2. A MATCH constraint against the "rank" column.
................................................................................
  struct Constraint {
    int op;                       /* Mask against sqlite3_index_constraint.op */
    int fts5op;                   /* FTS5 mask for idxFlags */
    int iCol;                     /* 0==rowid, 1==tbl, 2==rank */
    int omit;                     /* True to omit this if found */
    int iConsIndex;               /* Index in pInfo->aConstraint[] */
  } aConstraint[] = {

    {SQLITE_INDEX_CONSTRAINT_MATCH, FTS5_BI_MATCH,    1, 1, -1},

    {SQLITE_INDEX_CONSTRAINT_MATCH, FTS5_BI_RANK,     2, 1, -1},
    {SQLITE_INDEX_CONSTRAINT_EQ,    FTS5_BI_ROWID_EQ, 0, 0, -1},
    {SQLITE_INDEX_CONSTRAINT_LT|SQLITE_INDEX_CONSTRAINT_LE, 
                                    FTS5_BI_ROWID_LE, 0, 0, -1},
    {SQLITE_INDEX_CONSTRAINT_GT|SQLITE_INDEX_CONSTRAINT_GE, 
                                    FTS5_BI_ROWID_GE, 0, 0, -1},
  };

................................................................................
    }
  }

  /* Calculate the estimated cost based on the flags set in idxFlags. */
  bHasMatch = BitFlagTest(idxFlags, FTS5_BI_MATCH);
  if( BitFlagTest(idxFlags, FTS5_BI_ROWID_EQ) ){
    pInfo->estimatedCost = bHasMatch ? 100.0 : 10.0;

  }else if( BitFlagAllTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){
    pInfo->estimatedCost = bHasMatch ? 500.0 : 250000.0;
  }else if( BitFlagTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){
    pInfo->estimatedCost = bHasMatch ? 750.0 : 750000.0;
  }else{
    pInfo->estimatedCost = bHasMatch ? 1000.0 : 1000000.0;
  }
................................................................................
**
** The commands implemented by this function are documented in the "Special
** INSERT Directives" section of the documentation. It should be updated if
** more commands are added to this function.
*/
static int fts5SpecialInsert(
  Fts5Table *pTab,                /* Fts5 table object */
  sqlite3_value *pCmd,            /* Value inserted into special column */
  sqlite3_value *pVal             /* Value inserted into rank column */
){
  Fts5Config *pConfig = pTab->pConfig;
  const char *z = (const char*)sqlite3_value_text(pCmd);
  int rc = SQLITE_OK;
  int bError = 0;

  if( 0==sqlite3_stricmp("delete-all", z) ){
    if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
      fts5SetVtabError(pTab, 
          "'delete-all' may only be used with a "
          "contentless or external content fts5 table"
      );
      rc = SQLITE_ERROR;
    }else{
      rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage);
    }
  }else if( 0==sqlite3_stricmp("rebuild", z) ){
    if( pConfig->eContent==FTS5_CONTENT_NONE ){
      fts5SetVtabError(pTab, 
          "'rebuild' may not be used with a contentless fts5 table"
      );
      rc = SQLITE_ERROR;
    }else{
      rc = sqlite3Fts5StorageRebuild(pTab->pStorage);
    }
  }else if( 0==sqlite3_stricmp("optimize", z) ){
    rc = sqlite3Fts5StorageOptimize(pTab->pStorage);
  }else if( 0==sqlite3_stricmp("merge", z) ){
    int nMerge = sqlite3_value_int(pVal);
    rc = sqlite3Fts5StorageMerge(pTab->pStorage, nMerge);
  }else if( 0==sqlite3_stricmp("integrity-check", z) ){
    rc = sqlite3Fts5StorageIntegrity(pTab->pStorage);
#ifdef SQLITE_DEBUG
  }else if( 0==sqlite3_stricmp("prefix-index", z) ){
    pConfig->bPrefixIndex = sqlite3_value_int(pVal);
#endif
  }else{
    rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex);
    if( rc==SQLITE_OK ){
      rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, z, pVal, &bError);
    }
    if( rc==SQLITE_OK ){
      if( bError ){
        rc = SQLITE_ERROR;
      }else{
        rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, z, pVal, 0);
      }
    }
  }
  return rc;
}

static int fts5SpecialDelete(
................................................................................
  int eType1 = sqlite3_value_type(apVal[1]);
  if( eType1==SQLITE_INTEGER ){
    sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]);
    rc = sqlite3Fts5StorageSpecialDelete(pTab->pStorage, iDel, &apVal[2]);
  }
  return rc;
}

















/* 
** This function is the implementation of the xUpdate callback used by 
** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
** inserted, updated or deleted.









*/
static int fts5UpdateMethod(
  sqlite3_vtab *pVtab,            /* Virtual table handle */
  int nArg,                       /* Size of argument array */
  sqlite3_value **apVal,          /* Array of arguments */
  sqlite_int64 *pRowid            /* OUT: The affected (or effected) rowid */
){
  Fts5Table *pTab = (Fts5Table*)pVtab;
  Fts5Config *pConfig = pTab->pConfig;
  int eType0;                     /* value_type() of apVal[0] */
  int eConflict;                  /* ON CONFLICT for this DML */
  int rc = SQLITE_OK;             /* Return code */

  /* A transaction must be open when this is called. */
  assert( pTab->ts.eState==1 );







  assert( pTab->pConfig->pzErrmsg==0 );
  pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg;

  /* A delete specifies a single argument - the rowid of the row to remove.
  ** Update and insert operations pass:
  **
  **   1. The "old" rowid, or NULL.
  **   2. The "new" rowid.
  **   3. Values for each of the nCol matchable columns.
  **   4. Values for the two hidden columns (<tablename> and "rank").
  */

  eType0 = sqlite3_value_type(apVal[0]);
























  eConflict = sqlite3_vtab_on_conflict(pConfig->db);

  assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
  assert( pVtab->zErrMsg==0 );
  assert( (nArg==1 && eType0==SQLITE_INTEGER) || nArg==(2+pConfig->nCol+2) );


  fts5TripCursors(pTab);


  if( eType0==SQLITE_INTEGER ){
    if( fts5IsContentless(pTab) ){
      pTab->base.zErrMsg = sqlite3_mprintf(
          "cannot %s contentless fts5 table: %s", 
          (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName
      );
      rc = SQLITE_ERROR;
    }else{




      i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
    }
  }else{

    sqlite3_value *pCmd = apVal[2 + pConfig->nCol];
    assert( nArg>1 );


    if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){
      const char *z = (const char*)sqlite3_value_text(pCmd);
      if( pConfig->eContent!=FTS5_CONTENT_NORMAL 
       && 0==sqlite3_stricmp("delete", z) 
      ){



        rc = fts5SpecialDelete(pTab, apVal, pRowid);



      }else{
        rc = fts5SpecialInsert(pTab, pCmd, apVal[2 + pConfig->nCol + 1]);







      }
      goto update_method_out;














    }
  }


  if( rc==SQLITE_OK && nArg>1 ){
    rc = sqlite3Fts5StorageInsert(pTab->pStorage, apVal, eConflict, pRowid);
  }

 update_method_out:
  pTab->pConfig->pzErrmsg = 0;
  return rc;
}

/*
** Implementation of xSync() method. 
*/







>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
|
>
|







 







>







 







|



<



|









|








|

|


|


|





|





|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>





>
>
>
>
>
>
>
>
>










<





>
>
>
>
>
>



|
|
<
<
<
<
<
<


>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|

|
<
<
>

<
>
>
|
<





<
>
>
>
>



<
>
|
|
>
>
|
<
<
<

>
>
>
|
>
>
>
|
<
>
>
>
>
>
>
>
|
<
>
>
>
>
>
>
>
>
>
>
>
>
>
>



<
<
<
<
<
<







435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
...
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
...
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
....
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306

1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
....
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408

1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424






1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453


1454
1455

1456
1457
1458

1459
1460
1461
1462
1463

1464
1465
1466
1467
1468
1469
1470

1471
1472
1473
1474
1475
1476



1477
1478
1479
1480
1481
1482
1483
1484
1485

1486
1487
1488
1489
1490
1491
1492
1493

1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510






1511
1512
1513
1514
1515
1516
1517
*/
#define FTS5_PLAN_MATCH          1       /* (<tbl> MATCH ?) */
#define FTS5_PLAN_SOURCE         2       /* A source cursor for SORTED_MATCH */
#define FTS5_PLAN_SPECIAL        3       /* An internal query */
#define FTS5_PLAN_SORTED_MATCH   4       /* (<tbl> MATCH ? ORDER BY rank) */
#define FTS5_PLAN_SCAN           5       /* No usable constraint */
#define FTS5_PLAN_ROWID          6       /* (rowid = ?) */

/*
** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this
** extension is currently being used by a version of SQLite too old to
** support index-info flags. In that case this function is a no-op.
*/
static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){
#if SQLITE_VERSION_NUMBER>=3008012
  if( sqlite3_libversion_number()>=3008012 ){
    pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE;
  }
#endif
}

/*
** Implementation of the xBestIndex method for FTS5 tables. Within the 
** WHERE constraint, it searches for the following:
**
**   1. A MATCH constraint against the special column.
**   2. A MATCH constraint against the "rank" column.
................................................................................
  struct Constraint {
    int op;                       /* Mask against sqlite3_index_constraint.op */
    int fts5op;                   /* FTS5 mask for idxFlags */
    int iCol;                     /* 0==rowid, 1==tbl, 2==rank */
    int omit;                     /* True to omit this if found */
    int iConsIndex;               /* Index in pInfo->aConstraint[] */
  } aConstraint[] = {
    {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, 
                                    FTS5_BI_MATCH,    1, 1, -1},
    {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, 
                                    FTS5_BI_RANK,     2, 1, -1},
    {SQLITE_INDEX_CONSTRAINT_EQ,    FTS5_BI_ROWID_EQ, 0, 0, -1},
    {SQLITE_INDEX_CONSTRAINT_LT|SQLITE_INDEX_CONSTRAINT_LE, 
                                    FTS5_BI_ROWID_LE, 0, 0, -1},
    {SQLITE_INDEX_CONSTRAINT_GT|SQLITE_INDEX_CONSTRAINT_GE, 
                                    FTS5_BI_ROWID_GE, 0, 0, -1},
  };

................................................................................
    }
  }

  /* Calculate the estimated cost based on the flags set in idxFlags. */
  bHasMatch = BitFlagTest(idxFlags, FTS5_BI_MATCH);
  if( BitFlagTest(idxFlags, FTS5_BI_ROWID_EQ) ){
    pInfo->estimatedCost = bHasMatch ? 100.0 : 10.0;
    if( bHasMatch==0 ) fts5SetUniqueFlag(pInfo);
  }else if( BitFlagAllTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){
    pInfo->estimatedCost = bHasMatch ? 500.0 : 250000.0;
  }else if( BitFlagTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){
    pInfo->estimatedCost = bHasMatch ? 750.0 : 750000.0;
  }else{
    pInfo->estimatedCost = bHasMatch ? 1000.0 : 1000000.0;
  }
................................................................................
**
** The commands implemented by this function are documented in the "Special
** INSERT Directives" section of the documentation. It should be updated if
** more commands are added to this function.
*/
static int fts5SpecialInsert(
  Fts5Table *pTab,                /* Fts5 table object */
  const char *zCmd,               /* Text inserted into table-name column */
  sqlite3_value *pVal             /* Value inserted into rank column */
){
  Fts5Config *pConfig = pTab->pConfig;

  int rc = SQLITE_OK;
  int bError = 0;

  if( 0==sqlite3_stricmp("delete-all", zCmd) ){
    if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
      fts5SetVtabError(pTab, 
          "'delete-all' may only be used with a "
          "contentless or external content fts5 table"
      );
      rc = SQLITE_ERROR;
    }else{
      rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage);
    }
  }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){
    if( pConfig->eContent==FTS5_CONTENT_NONE ){
      fts5SetVtabError(pTab, 
          "'rebuild' may not be used with a contentless fts5 table"
      );
      rc = SQLITE_ERROR;
    }else{
      rc = sqlite3Fts5StorageRebuild(pTab->pStorage);
    }
  }else if( 0==sqlite3_stricmp("optimize", zCmd) ){
    rc = sqlite3Fts5StorageOptimize(pTab->pStorage);
  }else if( 0==sqlite3_stricmp("merge", zCmd) ){
    int nMerge = sqlite3_value_int(pVal);
    rc = sqlite3Fts5StorageMerge(pTab->pStorage, nMerge);
  }else if( 0==sqlite3_stricmp("integrity-check", zCmd) ){
    rc = sqlite3Fts5StorageIntegrity(pTab->pStorage);
#ifdef SQLITE_DEBUG
  }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){
    pConfig->bPrefixIndex = sqlite3_value_int(pVal);
#endif
  }else{
    rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex);
    if( rc==SQLITE_OK ){
      rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, zCmd, pVal, &bError);
    }
    if( rc==SQLITE_OK ){
      if( bError ){
        rc = SQLITE_ERROR;
      }else{
        rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, zCmd, pVal, 0);
      }
    }
  }
  return rc;
}

static int fts5SpecialDelete(
................................................................................
  int eType1 = sqlite3_value_type(apVal[1]);
  if( eType1==SQLITE_INTEGER ){
    sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]);
    rc = sqlite3Fts5StorageSpecialDelete(pTab->pStorage, iDel, &apVal[2]);
  }
  return rc;
}

static void fts5StorageInsert(
  int *pRc, 
  Fts5Table *pTab, 
  sqlite3_value **apVal, 
  i64 *piRowid
){
  int rc = *pRc;
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, piRowid);
  }
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *piRowid);
  }
  *pRc = rc;
}

/* 
** This function is the implementation of the xUpdate callback used by 
** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
** inserted, updated or deleted.
**
** A delete specifies a single argument - the rowid of the row to remove.
** 
** Update and insert operations pass:
**
**   1. The "old" rowid, or NULL.
**   2. The "new" rowid.
**   3. Values for each of the nCol matchable columns.
**   4. Values for the two hidden columns (<tablename> and "rank").
*/
static int fts5UpdateMethod(
  sqlite3_vtab *pVtab,            /* Virtual table handle */
  int nArg,                       /* Size of argument array */
  sqlite3_value **apVal,          /* Array of arguments */
  sqlite_int64 *pRowid            /* OUT: The affected (or effected) rowid */
){
  Fts5Table *pTab = (Fts5Table*)pVtab;
  Fts5Config *pConfig = pTab->pConfig;
  int eType0;                     /* value_type() of apVal[0] */

  int rc = SQLITE_OK;             /* Return code */

  /* A transaction must be open when this is called. */
  assert( pTab->ts.eState==1 );

  assert( pVtab->zErrMsg==0 );
  assert( nArg==1 || nArg==(2+pConfig->nCol+2) );
  assert( nArg==1 
      || sqlite3_value_type(apVal[1])==SQLITE_INTEGER 
      || sqlite3_value_type(apVal[1])==SQLITE_NULL 
  );
  assert( pTab->pConfig->pzErrmsg==0 );
  pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg;

  /* Put any active cursors into REQUIRE_SEEK state. */
  fts5TripCursors(pTab);







  eType0 = sqlite3_value_type(apVal[0]);
  if( eType0==SQLITE_NULL 
   && sqlite3_value_type(apVal[2+pConfig->nCol])!=SQLITE_NULL 
  ){
    /* A "special" INSERT op. These are handled separately. */
    const char *z = (const char*)sqlite3_value_text(apVal[2+pConfig->nCol]);
    if( pConfig->eContent!=FTS5_CONTENT_NORMAL 
      && 0==sqlite3_stricmp("delete", z) 
    ){
      rc = fts5SpecialDelete(pTab, apVal, pRowid);
    }else{
      rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]);
    }
  }else{
    /* A regular INSERT, UPDATE or DELETE statement. The trick here is that
    ** any conflict on the rowid value must be detected before any 
    ** modifications are made to the database file. There are 4 cases:
    **
    **   1) DELETE
    **   2) UPDATE (rowid not modified)
    **   3) UPDATE (rowid modified)
    **   4) INSERT
    **
    ** Cases 3 and 4 may violate the rowid constraint.
    */
    int eConflict = sqlite3_vtab_on_conflict(pConfig->db);

    assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );


    assert( nArg!=1 || eType0==SQLITE_INTEGER );


    /* Filter out attempts to run UPDATE or DELETE on contentless tables.
    ** This is not suported.  */
    if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){

      pTab->base.zErrMsg = sqlite3_mprintf(
          "cannot %s contentless fts5 table: %s", 
          (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName
      );
      rc = SQLITE_ERROR;

    }

    /* Case 1: DELETE */
    else if( nArg==1 ){
      i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
    }


    /* Case 2: INSERT */
    else if( eType0!=SQLITE_INTEGER ){     
      /* If this is a REPLACE, first remove the current entry (if any) */
      if( eConflict==SQLITE_REPLACE 
       && sqlite3_value_type(apVal[1])==SQLITE_INTEGER 



      ){
        i64 iNew = sqlite3_value_int64(apVal[1]);  /* Rowid to delete */
        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew);
      }
      fts5StorageInsert(&rc, pTab, apVal, pRowid);
    }

    /* Case 2: UPDATE */
    else{

      i64 iOld = sqlite3_value_int64(apVal[0]);  /* Old rowid */
      i64 iNew = sqlite3_value_int64(apVal[1]);  /* New rowid */
      if( iOld!=iNew ){
        if( eConflict==SQLITE_REPLACE ){
          rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
          if( rc==SQLITE_OK ){
            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew);
          }

          fts5StorageInsert(&rc, pTab, apVal, pRowid);
        }else{
          rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid);
          if( rc==SQLITE_OK ){
            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
          }
          if( rc==SQLITE_OK ){
            rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *pRowid);
          }
        }
      }else{
        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
        fts5StorageInsert(&rc, pTab, apVal, pRowid);
      }
    }
  }







  pTab->pConfig->pzErrmsg = 0;
  return rc;
}

/*
** Implementation of xSync() method. 
*/

Changes to ext/fts5/fts5_storage.c.

388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
...
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
...
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
...
701
702
703
704
705
706
707
708
709













































710
711
712
713
714

715
716
717
718
719
720
721
722
723
724

725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
...
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
    int rc2;
    sqlite3_bind_int64(pSeek, 1, iDel);
    if( sqlite3_step(pSeek)==SQLITE_ROW ){
      int iCol;
      Fts5InsertCtx ctx;
      ctx.pStorage = p;
      ctx.iCol = -1;
      rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
      for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
        if( pConfig->abUnindexed[iCol-1] ) continue;
        ctx.szCol = 0;
        rc = sqlite3Fts5Tokenize(pConfig, 
            FTS5_TOKENIZE_DOCUMENT,
            (const char*)sqlite3_column_text(pSeek, iCol),
            sqlite3_column_bytes(pSeek, iCol),
................................................................................
  /* Delete the index records */
  if( rc==SQLITE_OK ){
    int iCol;
    Fts5InsertCtx ctx;
    ctx.pStorage = p;
    ctx.iCol = -1;

    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
    for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){
      if( pConfig->abUnindexed[iCol] ) continue;
      ctx.szCol = 0;
      rc = sqlite3Fts5Tokenize(pConfig, 
        FTS5_TOKENIZE_DOCUMENT,
        (const char*)sqlite3_value_text(apVal[iCol]),
        sqlite3_value_bytes(apVal[iCol]),
................................................................................
    rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
  }

  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){
    i64 iRowid = sqlite3_column_int64(pScan, 0);

    sqlite3Fts5BufferZero(&buf);
    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iRowid);
    for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
      ctx.szCol = 0;
      if( pConfig->abUnindexed[ctx.iCol]==0 ){
        rc = sqlite3Fts5Tokenize(pConfig, 
            FTS5_TOKENIZE_DOCUMENT,
            (const char*)sqlite3_column_text(pScan, ctx.iCol+1),
            sqlite3_column_bytes(pScan, ctx.iCol+1),
................................................................................
      *piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
    }
  }
  return rc;
}

/*
** Insert a new row into the FTS table.
*/













































int sqlite3Fts5StorageInsert(
  Fts5Storage *p,                 /* Storage module to write to */
  sqlite3_value **apVal,          /* Array of values passed to xUpdate() */
  int eConflict,                  /* on conflict clause */
  i64 *piRowid                    /* OUT: rowid of new record */

){
  Fts5Config *pConfig = p->pConfig;
  int rc = SQLITE_OK;             /* Return code */
  sqlite3_stmt *pInsert = 0;      /* Statement used to write %_content table */
  int eStmt = 0;                  /* Type of statement used on %_content */
  int i;                          /* Counter variable */
  Fts5InsertCtx ctx;              /* Tokenization callback context object */
  Fts5Buffer buf;                 /* Buffer used to build up %_docsize blob */

  memset(&buf, 0, sizeof(Fts5Buffer));

  rc = fts5StorageLoadTotals(p, 1);

  /* Insert the new row into the %_content table. */
  if( rc==SQLITE_OK ){
    if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
      if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
        *piRowid = sqlite3_value_int64(apVal[1]);
      }else{
        rc = fts5StorageNewRowid(p, piRowid);
      }
    }else{
      if( eConflict==SQLITE_REPLACE ){
        eStmt = FTS5_STMT_REPLACE_CONTENT;
        rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
      }else{
        eStmt = FTS5_STMT_INSERT_CONTENT;
      }
      if( rc==SQLITE_OK ){
        rc = fts5StorageGetStmt(p, eStmt, &pInsert, 0);
      }
      for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
        rc = sqlite3_bind_value(pInsert, i, apVal[i]);
      }
      if( rc==SQLITE_OK ){
        sqlite3_step(pInsert);
        rc = sqlite3_reset(pInsert);
      }
      *piRowid = sqlite3_last_insert_rowid(pConfig->db);
    }
  }

  /* Add new entries to the FTS index */
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, *piRowid);
    ctx.pStorage = p;
  }
  for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
    ctx.szCol = 0;
    if( pConfig->abUnindexed[ctx.iCol]==0 ){
      rc = sqlite3Fts5Tokenize(pConfig, 
          FTS5_TOKENIZE_DOCUMENT,
          (const char*)sqlite3_value_text(apVal[ctx.iCol+2]),
................................................................................
    sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
    p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
  }
  p->nTotalRow++;

  /* Write the %_docsize record */
  if( rc==SQLITE_OK ){
    rc = fts5StorageInsertDocsize(p, *piRowid, &buf);
  }
  sqlite3_free(buf.p);

  /* Write the averages record */
  if( rc==SQLITE_OK ){
    rc = fts5StorageSaveTotals(p);
  }







|







 







|







 







|







 







|

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
|
|
<
<
>



<
<
<




>


<

<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|
<







 







|







388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
...
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
...
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
...
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757


758
759
760
761



762
763
764
765
766
767
768

769





























770

771
772
773
774
775
776
777
...
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
    int rc2;
    sqlite3_bind_int64(pSeek, 1, iDel);
    if( sqlite3_step(pSeek)==SQLITE_ROW ){
      int iCol;
      Fts5InsertCtx ctx;
      ctx.pStorage = p;
      ctx.iCol = -1;
      rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
      for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
        if( pConfig->abUnindexed[iCol-1] ) continue;
        ctx.szCol = 0;
        rc = sqlite3Fts5Tokenize(pConfig, 
            FTS5_TOKENIZE_DOCUMENT,
            (const char*)sqlite3_column_text(pSeek, iCol),
            sqlite3_column_bytes(pSeek, iCol),
................................................................................
  /* Delete the index records */
  if( rc==SQLITE_OK ){
    int iCol;
    Fts5InsertCtx ctx;
    ctx.pStorage = p;
    ctx.iCol = -1;

    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
    for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){
      if( pConfig->abUnindexed[iCol] ) continue;
      ctx.szCol = 0;
      rc = sqlite3Fts5Tokenize(pConfig, 
        FTS5_TOKENIZE_DOCUMENT,
        (const char*)sqlite3_value_text(apVal[iCol]),
        sqlite3_value_bytes(apVal[iCol]),
................................................................................
    rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
  }

  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){
    i64 iRowid = sqlite3_column_int64(pScan, 0);

    sqlite3Fts5BufferZero(&buf);
    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
    for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
      ctx.szCol = 0;
      if( pConfig->abUnindexed[ctx.iCol]==0 ){
        rc = sqlite3Fts5Tokenize(pConfig, 
            FTS5_TOKENIZE_DOCUMENT,
            (const char*)sqlite3_column_text(pScan, ctx.iCol+1),
            sqlite3_column_bytes(pScan, ctx.iCol+1),
................................................................................
      *piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
    }
  }
  return rc;
}

/*
** Insert a new row into the FTS content table.
*/
int sqlite3Fts5StorageContentInsert(
  Fts5Storage *p, 
  sqlite3_value **apVal, 
  i64 *piRowid
){
  Fts5Config *pConfig = p->pConfig;
  int rc = SQLITE_OK;

  /* Insert the new row into the %_content table. */
  if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
    if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
      *piRowid = sqlite3_value_int64(apVal[1]);
    }else{
      rc = fts5StorageNewRowid(p, piRowid);
    }
  }else{
    sqlite3_stmt *pInsert = 0;    /* Statement to write %_content table */
    int i;                        /* Counter variable */
#if 0
    if( eConflict==SQLITE_REPLACE ){
      eStmt = FTS5_STMT_REPLACE_CONTENT;
      rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
    }else{
      eStmt = FTS5_STMT_INSERT_CONTENT;
    }
#endif
    if( rc==SQLITE_OK ){
      rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
    }
    for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
      rc = sqlite3_bind_value(pInsert, i, apVal[i]);
    }
    if( rc==SQLITE_OK ){
      sqlite3_step(pInsert);
      rc = sqlite3_reset(pInsert);
    }
    *piRowid = sqlite3_last_insert_rowid(pConfig->db);
  }

  return rc;
}

/*
** Insert new entries into the FTS index and %_docsize table.
*/
int sqlite3Fts5StorageIndexInsert(
  Fts5Storage *p, 
  sqlite3_value **apVal, 


  i64 iRowid
){
  Fts5Config *pConfig = p->pConfig;
  int rc = SQLITE_OK;             /* Return code */



  Fts5InsertCtx ctx;              /* Tokenization callback context object */
  Fts5Buffer buf;                 /* Buffer used to build up %_docsize blob */

  memset(&buf, 0, sizeof(Fts5Buffer));
  ctx.pStorage = p;
  rc = fts5StorageLoadTotals(p, 1);


  if( rc==SQLITE_OK ){





























    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);

  }
  for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
    ctx.szCol = 0;
    if( pConfig->abUnindexed[ctx.iCol]==0 ){
      rc = sqlite3Fts5Tokenize(pConfig, 
          FTS5_TOKENIZE_DOCUMENT,
          (const char*)sqlite3_value_text(apVal[ctx.iCol+2]),
................................................................................
    sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
    p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
  }
  p->nTotalRow++;

  /* Write the %_docsize record */
  if( rc==SQLITE_OK ){
    rc = fts5StorageInsertDocsize(p, iRowid, &buf);
  }
  sqlite3_free(buf.p);

  /* Write the averages record */
  if( rc==SQLITE_OK ){
    rc = fts5StorageSaveTotals(p);
  }

Changes to ext/fts5/fts5_vocab.c.

398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
  sqlite3_value **apVal           /* Arguments for the indexing scheme */
){
  Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
  int rc;
  const int flags = FTS5INDEX_QUERY_SCAN;

  fts5VocabResetCursor(pCsr);
  rc = sqlite3Fts5IndexQuery(pCsr->pIndex, 0, 0, flags, &pCsr->pIter);
  if( rc==SQLITE_OK ){
    rc = fts5VocabNextMethod(pCursor);
  }

  return rc;
}








|







398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
  sqlite3_value **apVal           /* Arguments for the indexing scheme */
){
  Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
  int rc;
  const int flags = FTS5INDEX_QUERY_SCAN;

  fts5VocabResetCursor(pCsr);
  rc = sqlite3Fts5IndexQuery(pCsr->pIndex, 0, 0, flags, 0, &pCsr->pIter);
  if( rc==SQLITE_OK ){
    rc = fts5VocabNextMethod(pCursor);
  }

  return rc;
}

Changes to ext/fts5/fts5parse.y.

97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
}
cnearset(A) ::= colset(X) COLON nearset(Y). { 
  sqlite3Fts5ParseSetColset(pParse, Y, X);
  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
}

%type colset {Fts5ExprColset*}
%destructor colset { sqlite3_free($$); }
%type colsetlist {Fts5ExprColset*}
%destructor colsetlist { sqlite3_free($$); }

colset(A) ::= LCP colsetlist(X) RCP. { A = X; }
colset(A) ::= STRING(X). {
  A = sqlite3Fts5ParseColset(pParse, 0, &X);
}








|

|







97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
}
cnearset(A) ::= colset(X) COLON nearset(Y). { 
  sqlite3Fts5ParseSetColset(pParse, Y, X);
  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
}

%type colset {Fts5Colset*}
%destructor colset { sqlite3_free($$); }
%type colsetlist {Fts5Colset*}
%destructor colsetlist { sqlite3_free($$); }

colset(A) ::= LCP colsetlist(X) RCP. { A = X; }
colset(A) ::= STRING(X). {
  A = sqlite3Fts5ParseColset(pParse, 0, &X);
}

Changes to ext/fts5/test/fts5al.test.

247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262

263
264
265
266
267
268
269












270
271
272
273
274
275
276
277
278
279
280
281
  INSERT INTO t3 VALUES('a one');
  INSERT INTO t3 VALUES('a two');
  INSERT INTO t3 VALUES('a three');
  INSERT INTO t3 VALUES('a four');
  INSERT INTO t3 VALUES('a five');
  INSERT INTO t3(t3, rank) VALUES('rank', 'bm25()');
}
breakpoint

do_execsql_test 4.3.2 {
  SELECT * FROM t3
  WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(4)' 
  ORDER BY rank ASC
} {
  {a four} {a one} {a five} {a two} {a three}
}

do_execsql_test 4.3.3 {
  SELECT *, rank FROM t3
  WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(3)' 
  ORDER BY rank ASC
} {
  {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 
}













do_catchsql_test 4.4.3 {
  SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'xyz(3)' 
} {1 {no such function: xyz}}
do_catchsql_test 4.4.4 {
  SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
} {1 {parse error in rank function: }}



finish_test








<








>







>
>
>
>
>
>
>
>
>
>
>
>












247
248
249
250
251
252
253

254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
  INSERT INTO t3 VALUES('a one');
  INSERT INTO t3 VALUES('a two');
  INSERT INTO t3 VALUES('a three');
  INSERT INTO t3 VALUES('a four');
  INSERT INTO t3 VALUES('a five');
  INSERT INTO t3(t3, rank) VALUES('rank', 'bm25()');
}


do_execsql_test 4.3.2 {
  SELECT * FROM t3
  WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(4)' 
  ORDER BY rank ASC
} {
  {a four} {a one} {a five} {a two} {a three}
}

do_execsql_test 4.3.3 {
  SELECT *, rank FROM t3
  WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(3)' 
  ORDER BY rank ASC
} {
  {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 
}

do_execsql_test 4.3.4 {
  SELECT * FROM t3('a', 'rowidmod(4)') ORDER BY rank ASC;
} {
  {a four} {a one} {a five} {a two} {a three}
}

do_execsql_test 4.3.5 {
  SELECT *, rank FROM t3('a', 'rowidmod(3)') ORDER BY rank ASC
} {
  {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 
}

do_catchsql_test 4.4.3 {
  SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'xyz(3)' 
} {1 {no such function: xyz}}
do_catchsql_test 4.4.4 {
  SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
} {1 {parse error in rank function: }}



finish_test

Added ext/fts5/test/fts5onepass.test.











































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
# 2015 Sep 27
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5onepass

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE ft USING fts5(content);
  INSERT INTO ft(rowid, content) VALUES(1, '1 2 3');
  INSERT INTO ft(rowid, content) VALUES(2, '4 5 6');
  INSERT INTO ft(rowid, content) VALUES(3, '7 8 9');
}

#-------------------------------------------------------------------------
# Check that UPDATE and DELETE statements that feature "WHERE rowid=?" or 
# or "WHERE rowid=?" clauses do not use statement journals. But that other
# DELETE and UPDATE statements do.
#
# Note: "MATCH ? AND rowid=?" does use a statement journal.
#
foreach {tn sql uses} {
  1.1 { DELETE FROM ft } 1
  1.2 { DELETE FROM ft WHERE rowid=? } 0
  1.3 { DELETE FROM ft WHERE rowid=? } 0
  1.4 { DELETE FROM ft WHERE ft MATCH '1' } 1
  1.5 { DELETE FROM ft WHERE ft MATCH '1' AND rowid=? } 1
  1.6 { DELETE FROM ft WHERE ft MATCH '1' AND rowid=? } 1

  2.1 { UPDATE ft SET content='a b c' } 1
  2.2 { UPDATE ft SET content='a b c' WHERE rowid=? } 0
  2.3 { UPDATE ft SET content='a b c' WHERE rowid=? } 0
  2.4 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' } 1
  2.5 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' AND rowid=? } 1
  2.6 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' AND rowid=? } 1
} {
  do_test 1.$tn { sql_uses_stmt db $sql } $uses
}

#-------------------------------------------------------------------------
# Check that putting a "DELETE/UPDATE ... WHERE rowid=?" statement in a
# trigger program does not prevent the VM from using a statement 
# transaction. Even if the calling statement cannot hit a constraint.
#
do_execsql_test 2.0 {
  CREATE TABLE t1(x);

  CREATE TRIGGER t1_ai AFTER INSERT ON t1 BEGIN
    DELETE FROM ft WHERE rowid=new.x;
  END;

  CREATE TRIGGER t1_ad AFTER DELETE ON t1 BEGIN
    UPDATE ft SET content = 'a b c' WHERE rowid=old.x;
  END;

  CREATE TRIGGER t1_bu BEFORE UPDATE ON t1 BEGIN
    DELETE FROM ft WHERE rowid=old.x;
  END;
}

foreach {tn sql uses} {
  1 { INSERT INTO t1 VALUES(1)      } 1
  2 { DELETE FROM t1 WHERE x=4      } 1
  3 { UPDATE t1 SET x=10 WHERE x=11 } 1
} {
  do_test 2.$tn { sql_uses_stmt db $sql } $uses
}

#-------------------------------------------------------------------------
# Test that an "UPDATE ... WHERE rowid=?" works and does not corrupt the
# index when it strikes a constraint. Both inside and outside a 
# transaction.
#
foreach {tn tcl1 tcl2}  {
  1 {} {}

  2 {
    execsql BEGIN
  } {
    if {[sqlite3_get_autocommit db]==1} { error "transaction rolled back!" }
    execsql COMMIT
  }
} {

  do_execsql_test 3.$tn.0 {
    DROP TABLE IF EXISTS ft2;
    CREATE VIRTUAL TABLE ft2 USING fts5(content);
    INSERT INTO ft2(rowid, content) VALUES(1, 'a b c');
    INSERT INTO ft2(rowid, content) VALUES(2, 'a b d');
    INSERT INTO ft2(rowid, content) VALUES(3, 'a b e');
  }

  eval $tcl1
  foreach {tn2 sql content} {
    1 { UPDATE ft2 SET rowid=2 WHERE rowid=1 }
      { 1 {a b c} 2 {a b d} 3 {a b e} }

    2 { 
      INSERT INTO ft2(rowid, content) VALUES(4, 'a b f');
      UPDATE ft2 SET rowid=5 WHERE rowid=4;
      UPDATE ft2 SET rowid=3 WHERE rowid=5;
    } { 1 {a b c} 2 {a b d} 3 {a b e} 5 {a b f} }

    3 {
      UPDATE ft2 SET rowid=3 WHERE rowid=4;           -- matches 0 rows
      UPDATE ft2 SET rowid=2 WHERE rowid=3;
    } { 1 {a b c} 2 {a b d} 3 {a b e} 5 {a b f} }

    4 {
      INSERT INTO ft2(rowid, content) VALUES(4, 'a b g');
      UPDATE ft2 SET rowid=-1 WHERE rowid=4;
      UPDATE ft2 SET rowid=3 WHERE rowid=-1;
    } {-1 {a b g} 1 {a b c} 2 {a b d} 3 {a b e} 5 {a b f} }

    5 {
      DELETE FROM ft2 WHERE rowid=451;
      DELETE FROM ft2 WHERE rowid=-1;
      UPDATE ft2 SET rowid = 2 WHERE rowid = 1;
    } {1 {a b c} 2 {a b d} 3 {a b e} 5 {a b f} }
  } {
    do_catchsql_test 3.$tn.$tn2.a $sql {1 {constraint failed}}
    do_execsql_test  3.$tn.$tn2.b { SELECT rowid, content FROM ft2 } $content

    do_execsql_test  3.$tn.$tn2.c { 
      INSERT INTO ft2(ft2) VALUES('integrity-check');
    }
  }
  eval $tcl2
}

#-------------------------------------------------------------------------
# Check that DELETE and UPDATE operations can be done without flushing
# the in-memory hash table to disk.
#
reset_db
do_execsql_test 4.1.1 {
  CREATE VIRTUAL TABLE ttt USING fts5(x);
  BEGIN;
    INSERT INTO ttt(rowid, x) VALUES(1, 'a b c');
    INSERT INTO ttt(rowid, x) VALUES(2, 'a b c');
    INSERT INTO ttt(rowid, x) VALUES(3, 'a b c');
  COMMIT
}
do_test 4.1.2 { fts5_level_segs ttt } {1}

do_execsql_test 4.2.1 {
  BEGIN;
    DELETE FROM ttt WHERE rowid=1;
    DELETE FROM ttt WHERE rowid=3;
    INSERT INTO ttt(rowid, x) VALUES(4, 'd e f');
    INSERT INTO ttt(rowid, x) VALUES(5, 'd e f');
  COMMIT;
} {}
do_test 4.2.2 { fts5_level_segs ttt } {2}


do_execsql_test 4.3.1 {
  BEGIN;
    UPDATE ttt SET x = 'd e f' WHERE rowid = 2;
    UPDATE ttt SET x = 'A B C' WHERE rowid = 4;
    INSERT INTO ttt(rowid, x) VALUES(6, 'd e f');
  COMMIT;
} {}
do_test 4.2.2 { fts5_level_segs ttt } {3}

finish_test

Added ext/fts5/test/fts5phrase.test.















































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
# 2014 Jan 08
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focused on phrase queries.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5phrase

# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t3 USING fts5(a, b, c);
  INSERT INTO t3 VALUES('d e a', 'd i j j f', 'i j i e b f h'); -- 1
  INSERT INTO t3 VALUES('g a e', 'f g i g a', 'h d g i g h c'); -- 2
  INSERT INTO t3 VALUES('e a d', 'e i h a f', 'c e h i f b i'); -- 3
  INSERT INTO t3 VALUES('a g c', 'd j d j c', 'c d f j i g j'); -- 4
  INSERT INTO t3 VALUES('b c b', 'j g c d f', 'j c j d g f b'); -- 5
  INSERT INTO t3 VALUES('j a d', 'e b i h h', 'c c f g d i d'); -- 6
  INSERT INTO t3 VALUES('a d f', 'h g i i i', 'e a g c i f b'); -- 7
  INSERT INTO t3 VALUES('g f d', 'f c g b j', 'b b h h h j j'); -- 8
  INSERT INTO t3 VALUES('f h g', 'c j f g j', 'd h d f e b h'); -- 9
  INSERT INTO t3 VALUES('f h d', 'c i a d b', 'g b j b a d e'); -- 10
  INSERT INTO t3 VALUES('j h h', 'j i h a g', 'd e i e a g j'); -- 11
  INSERT INTO t3 VALUES('a b e', 'h g a g c', 'h c a a d e g'); -- 12
  INSERT INTO t3 VALUES('a j g', 'i h i f i', 'a g h j g i b'); -- 13
  INSERT INTO t3 VALUES('j h e', 'f e d i e', 'i d c f e d c'); -- 14
  INSERT INTO t3 VALUES('d j d', 'd b i a c', 'g d h i d b e'); -- 15
  INSERT INTO t3 VALUES('h j e', 'e b b c f', 'j a f g h d j'); -- 16
  INSERT INTO t3 VALUES('c b j', 'c a b a i', 'h f i d a d c'); -- 17
  INSERT INTO t3 VALUES('e e d', 'i d f c c', 'g i d a f e a'); -- 18
  INSERT INTO t3 VALUES('e i g', 'e a b i h', 'i f d d a d f'); -- 19
  INSERT INTO t3 VALUES('h g f', 'b h h j d', 'i f d e g j a'); -- 20
  INSERT INTO t3 VALUES('e h f', 'j c b c f', 'j a j g h a c'); -- 21
  INSERT INTO t3 VALUES('d c h', 'b g i c e', 'i i c d e h i'); -- 22
  INSERT INTO t3 VALUES('a h i', 'a g d f f', 'e f i i b b h'); -- 23
  INSERT INTO t3 VALUES('d d g', 'c c b c g', 'g c h e b c e'); -- 24
  INSERT INTO t3 VALUES('a b b', 'b f a d i', 'd a h a b c i'); -- 25
  INSERT INTO t3 VALUES('a f d', 'a j e a h', 'j i h j a i f'); -- 26
  INSERT INTO t3 VALUES('d j d', 'h a d i a', 'h h f j h g a'); -- 27
  INSERT INTO t3 VALUES('g a e', 'd g f a g', 'i d b c g g j'); -- 28
  INSERT INTO t3 VALUES('j e h', 'g h j h g', 'd a e j a a h'); -- 29
  INSERT INTO t3 VALUES('e j e', 'g e j g c', 'f c e b e e a'); -- 30
  INSERT INTO t3 VALUES('h f f', 'i j g e c', 'j j f c a i j'); -- 31
  INSERT INTO t3 VALUES('a g c', 'c g d b i', 'g h c b a a f'); -- 32
  INSERT INTO t3 VALUES('c h i', 'j d h e e', 'a h i d c c j'); -- 33
  INSERT INTO t3 VALUES('d a c', 'e d d b j', 'c e b b h i h'); -- 34
  INSERT INTO t3 VALUES('d f h', 'c a f c c', 'j b b c c j f'); -- 35
  INSERT INTO t3 VALUES('b g h', 'g c c c f', 'c g c f h e e'); -- 36
  INSERT INTO t3 VALUES('f e a', 'b h f j h', 'j g h f d g f'); -- 37
  INSERT INTO t3 VALUES('h f a', 'a e i j g', 'f d a f d f c'); -- 38
  INSERT INTO t3 VALUES('f i c', 'f i i i i', 'e c f d h j f'); -- 39
  INSERT INTO t3 VALUES('h h d', 'd i e d i', 'd f e i a h a'); -- 40
  INSERT INTO t3 VALUES('f g c', 'd a f c h', 'b b g j c e g'); -- 41
  INSERT INTO t3 VALUES('h i h', 'h d j d e', 'e d b b i e g'); -- 42
  INSERT INTO t3 VALUES('b h i', 'j e i d a', 'j j h e e c a'); -- 43
  INSERT INTO t3 VALUES('g i g', 'f c c f d', 'a c i c a d a'); -- 44
  INSERT INTO t3 VALUES('c c f', 'a b j d b', 'c a e g f e c'); -- 45
  INSERT INTO t3 VALUES('d h j', 'g c b j d', 'e a h f h j g'); -- 46
  INSERT INTO t3 VALUES('a a d', 'j e j a i', 'i d c f f f b'); -- 47
  INSERT INTO t3 VALUES('b g j', 'e c i h f', 'd d h b g a d'); -- 48
  INSERT INTO t3 VALUES('c i a', 'a c c c c', 'e h i e h i e'); -- 49
  INSERT INTO t3 VALUES('f f c', 'f f b i i', 'f f a j e c i'); -- 50
}

proc pmatch {col expr} {
  return [expr {[string first $expr $col]>=0}]
}
db func pmatch pmatch

foreach {tn cols tokens} {
  1 a         "c c"
  2 b         "c c"
  3 c         "c c"
  4 {a b c}   "c c"
  5 {a b c}   "b h"
  6 {a b}     "b h"
  7 {a c}     "b h"
  8 {c a}     "b h"
  9 {c}       "i e"
  10 {b}      "i e"
  11 {a}      "i e"
} {
  set fts   "{$cols}:[join $tokens +]"
  set where [list]
  foreach c $cols { lappend where "pmatch($c, '$tokens')" }
  set where [join $where " OR "]

  set res [db eval "SELECT rowid FROM t3 WHERE $where"]
  do_execsql_test "1.$tn.$fts->([llength $res] rows)" { 
    SELECT rowid FROM t3($fts) 
  } $res
}

do_execsql_test 2.0 {
  SELECT rowid,
    highlight(t3, 0, '*', '*'),
    highlight(t3, 1, '*', '*'),
    highlight(t3, 2, '*', '*')
  FROM t3('a:f+f')
} {
  31 {h *f f*} {i j g e c} {j j f c a i j} 
  50 {*f f* c} {f f b i i} {f f a j e c i}
}

finish_test

Changes to ext/fts5/test/fts5prefix.test.

58
59
60
61
62
63
64


65



































































































































66
67

foreach {tn q res} {
  1 "SELECT rowid FROM t1 WHERE t1 MATCH '\xCA\xCB*'" 1
  2 "SELECT rowid FROM t1 WHERE t1 MATCH '\u1234\u5678*'" 2
} {
  do_execsql_test 2.3.$tn $q $res
}







































































































































finish_test









>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


>
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
foreach {tn q res} {
  1 "SELECT rowid FROM t1 WHERE t1 MATCH '\xCA\xCB*'" 1
  2 "SELECT rowid FROM t1 WHERE t1 MATCH '\u1234\u5678*'" 2
} {
  do_execsql_test 2.3.$tn $q $res
}

#-------------------------------------------------------------------------
# Check that prefix queries with:
#
#   * a column filter, and
#   * no prefix index.
#
# work Ok.
#
do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE t3 USING fts5(a, b, c);
  INSERT INTO t3(t3, rank) VALUES('pgsz', 32);
  BEGIN;
    INSERT INTO t3 VALUES('acb ccc bba', 'cca bba bca', 'bbc ccc bca'); -- 1
    INSERT INTO t3 VALUES('cbb cac cab', 'abb aac bba', 'aab ccc cac'); -- 2
    INSERT INTO t3 VALUES('aac bcb aac', 'acb bcb caa', 'aca bab bca'); -- 3
    INSERT INTO t3 VALUES('aab ccb ccc', 'aca cba cca', 'aca aac cbb'); -- 4
    INSERT INTO t3 VALUES('bac aab bab', 'ccb bac cba', 'acb aba abb'); -- 5
    INSERT INTO t3 VALUES('bab abc ccb', 'acb cba abb', 'cbb aaa cab'); -- 6
    INSERT INTO t3 VALUES('cbb bbc baa', 'aab aca baa', 'bcc cca aca'); -- 7
    INSERT INTO t3 VALUES('abc bba abb', 'cac abc cba', 'acc aac cac'); -- 8
    INSERT INTO t3 VALUES('bbc bbc cab', 'bcb ccb cba', 'bcc cac acb'); -- 9
  COMMIT;
}

foreach {tn match res} {
  1 "a : c*" {1 2 4 6 7 9}
  2 "b : c*" {1 3 4 5 6 8 9}
  3 "c : c*" {1 2 4 6 7 8 9}
  4 "a : b*" {1 3 5 6 7 8 9}
  5 "b : b*" {1 2 3 5 7 9}
  6 "c : b*" {1 3 7 9}
  7 "a : a*" {1 3 4 5 6 8}
  8 "b : a*" {2 3 4 6 7 8}
  9 "c : a*" {2 3 4 5 6 7 8 9}
} {
  do_execsql_test 3.1.$tn {
    SELECT rowid FROM t3($match)
  } $res
}

do_test 3.2 {
  expr srand(0)
  execsql { DELETE FROM t3 }
  for {set i 0} {$i < 1000} {incr i} {
    set a [fts5_rnddoc 3]
    set b [fts5_rnddoc 8]
    set c [fts5_rnddoc 20]
    execsql { INSERT INTO t3 VALUES($a, $b, $c) }
  }
  execsql { INSERT INTO t3(t3) VALUES('integrity-check') }
} {}

proc gmatch {col pattern} {
  expr {[lsearch -glob $col $pattern]>=0}
}
db func gmatch gmatch

proc ghl {col pattern} {
  foreach t $col {
    if {[string match $pattern $t]} {
      lappend res "*$t*"
    } else {
      lappend res $t
    }
  }
  set res
}
db func ghl ghl

set COLS(a) 0
set COLS(b) 1
set COLS(c) 2

for {set x 0} {$x<2} {incr x} {
  foreach {tn pattern} {
    1  {xa*}
    2  {xb*}
    3  {xc*}
    4  {xd*}
    5  {xe*}
    6  {xf*}
    7  {xg*}
    8  {xh*}
    9  {xi*}
    10 {xj*}
  } {
    foreach col {a b c} {

      # Check that the list of returned rowids is correct.
      #
      set res [db eval "SELECT rowid FROM t3 WHERE gmatch($col, '$pattern')"]
      set query "$col : $pattern"
      do_execsql_test 3.3.$x.$tn.$col.rowid {
        SELECT rowid FROM t3($query);
      } $res

      # Check that the highlight() function works.
      #
      set res [db eval \
        "SELECT ghl($col, '$pattern') FROM t3 WHERE gmatch($col, '$pattern')"
      ]
      set idx $COLS($col)
      do_execsql_test 3.3.$x.$tn.$col.highlight {
        SELECT highlight(t3, $idx, '*', '*') FROM t3($query);
      } $res
    }

    foreach colset {{a b} {b c} {c a} {a c} {b a}} {
      # Check that the list of returned rowids is correct.
      #
      foreach {col1 col2} $colset {}
      set expr "gmatch($col1, '$pattern') OR gmatch($col2, '$pattern')"
      set res [db eval "SELECT rowid FROM t3 WHERE $expr"]
      set query "{$colset} : $pattern"
      do_execsql_test 3.3.$x.$tn.{$colset}.rowid {
        SELECT rowid FROM t3($query);
      } $res

      set resq    "SELECT ghl($col1, '$pattern'), ghl($col2, '$pattern')"
      append resq " FROM t3 WHERE $expr"
      set res [db eval $resq]
      set idx1 $COLS($col1)
      set idx2 $COLS($col2)
      do_execsql_test 3.3.$x.$tn.{$colset}.highlight {
        SELECT highlight(t3, $idx1, '*', '*'), highlight(t3, $idx2, '*', '*')
          FROM t3($query)
      } $res
    }
  }
  execsql { INSERT INTO t3(t3) VALUES('optimize') }
  execsql { INSERT INTO t3(t3) VALUES('integrity-check') }
}


finish_test


Changes to ext/fts5/test/fts5simple.test.

15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
...
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
...
179
180
181
182
183
184
185
186

187



































































































188
189

# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

if 1 {
#-------------------------------------------------------------------------
#
set doc "x x [string repeat {y } 50]z z"
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  BEGIN;
................................................................................
  COMMIT;
} {}

do_execsql_test 5.4 {
  SELECT rowid FROM tt WHERE tt MATCH 'a*';
} {1 2}

}

do_execsql_test 5.5 {
  DELETE FROM tt;
  BEGIN;
    INSERT INTO tt VALUES('aa');
    INSERT INTO tt VALUES('ab');
    INSERT INTO tt VALUES('aa');
    INSERT INTO tt VALUES('ab');
................................................................................

do_catchsql_test 6.2 { 
  SELECT * FROM xyz WHERE xyz MATCH '' 
} {1 {fts5: syntax error near ""}}
do_catchsql_test 6.3 { 
  SELECT * FROM xyz WHERE xyz MATCH NULL 
} {1 {fts5: syntax error near ""}}






































































































finish_test








<







 







<
<







 








>

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


15
16
17
18
19
20
21

22
23
24
25
26
27
28
...
132
133
134
135
136
137
138


139
140
141
142
143
144
145
...
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286

# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}


#-------------------------------------------------------------------------
#
set doc "x x [string repeat {y } 50]z z"
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  BEGIN;
................................................................................
  COMMIT;
} {}

do_execsql_test 5.4 {
  SELECT rowid FROM tt WHERE tt MATCH 'a*';
} {1 2}



do_execsql_test 5.5 {
  DELETE FROM tt;
  BEGIN;
    INSERT INTO tt VALUES('aa');
    INSERT INTO tt VALUES('ab');
    INSERT INTO tt VALUES('aa');
    INSERT INTO tt VALUES('ab');
................................................................................

do_catchsql_test 6.2 { 
  SELECT * FROM xyz WHERE xyz MATCH '' 
} {1 {fts5: syntax error near ""}}
do_catchsql_test 6.3 { 
  SELECT * FROM xyz WHERE xyz MATCH NULL 
} {1 {fts5: syntax error near ""}}

#-------------------------------------------------------------------------

do_execsql_test 7.1 {
  CREATE VIRTUAL TABLE ft2 USING fts5(content);
  INSERT INTO ft2(rowid, content) VALUES(1, 'a b c');
  INSERT INTO ft2(rowid, content) VALUES(2, 'a b d');
} 

do_catchsql_test 7.2 {
  BEGIN;
    UPDATE ft2 SET rowid=2 WHERE rowid=1;
} {1 {constraint failed}} 

do_execsql_test 7.3 {
  COMMIT;
  INSERT INTO ft2(ft2) VALUES('integrity-check');
} {}

do_execsql_test 7.4 {
  SELECT * FROM ft2;
} {{a b c} {a b d}}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 8.1 {
  CREATE VIRTUAL TABLE ft2 USING fts5(content);
  INSERT INTO ft2(rowid, content) VALUES(1, 'a b');
}

do_execsql_test 8.2 {
  BEGIN;
    INSERT INTO ft2(rowid, content) VALUES(4, 'a x');
}

do_execsql_test 8.3 {
  INSERT INTO ft2(ft2) VALUES('integrity-check');
}

#-------------------------------------------------------------------------
# Check that the "table function" syntax works.
#
reset_db
do_execsql_test 9.1 {
  CREATE VIRTUAL TABLE ft2 USING fts5(content);
  INSERT INTO ft2(rowid, content) VALUES(1, 'a b');
  INSERT INTO ft2(rowid, content) VALUES(2, 'a b c d');
  INSERT INTO ft2(rowid, content) VALUES(3, 'c d e f');
}

do_execsql_test 9.2 {
  SELECT rowid FROM ft2('a');
} {1 2}

do_execsql_test 9.3 {
  SELECT rowid FROM ft2('b AND c');
} {2}

#-------------------------------------------------------------------------
#
do_execsql_test 10.0 {
  CREATE VIRTUAL TABLE t3 USING fts5(a, b, c);
  INSERT INTO t3 VALUES('bac aab bab', 'c bac c', 'acb aba abb'); -- 1
  INSERT INTO t3 VALUES('bab abc c', 'acb c abb', 'c aaa c');     -- 2
}

do_execsql_test 10.1 {
  SELECT rowid FROM t3('c: c*');
} {2}

#-------------------------------------------------------------------------
# Test that character 0x1A is allowed in fts5 barewords.
#
do_test 11.0 {
  execsql "CREATE VIRTUAL TABLE t4 USING fts5(x, tokenize=\"ascii tokenchars '\x1A'\")"
  execsql "
    INSERT INTO t4 VALUES('a b c \x1A');
    INSERT INTO t4 VALUES('a b c d\x1A');
    INSERT INTO t4 VALUES('a b c \x1Ad');
    INSERT INTO t4 VALUES('a b c d');
  "
} {}

do_test 11.1 {
  execsql "SELECT rowid FROM t4('\x1A')"
} {1}
do_test 11.2 {
  execsql "SELECT rowid FROM t4('\x1A*')"
} {1 3}
do_test 11.3 {
  execsql "SELECT rowid FROM t4('d\x1A')"
} {2}

do_test 11.4 {
  catchsql "SELECT rowid FROM t4('d\x1B')"
} {/fts5: syntax error/}
do_test 11.5 {
  catchsql "SELECT rowid FROM t4('d\x19')"
} {/fts5: syntax error/}


finish_test

Added ext/fts5/tool/fts5txt2db.tcl.















































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135


proc usage {} {
  puts stderr "$::argv0 ?OPTIONS? DATABASE FILE1..."
  puts stderr ""
  puts stderr "Options are"
  puts stderr "  -fts5"
  puts stderr "  -fts4"
  puts stderr "  -colsize <list of column sizes>"
  puts stderr {
This script is designed to create fts4/5 tables with more than one column.
The -colsize option should be set to a Tcl list of integer values, one for
each column in the table. Each value is the number of tokens that will be
inserted into the column value for each row. For example, setting the -colsize
option to "5 10" creates an FTS table with 2 columns, with roughly 5 and 10
tokens per row in each, respectively.

Each "FILE" argument should be a text file. The contents of these text files is
split on whitespace characters to form a list of tokens. The first N1 tokens
are used for the first column of the first row, where N1 is the first element
of the -colsize list. The next N2 are used for the second column of the first
row, and so on. Rows are added to the table until the entire list of tokens
is exhausted.
}
  exit -1
}

set O(aColsize)       [list 10 10 10]
set O(tblname)        t1
set O(fts)            fts5


set options_with_values {-colsize}

for {set i 0} {$i < [llength $argv]} {incr i} {
  set opt [lindex $argv $i]
  if {[string range $opt 0 0]!="-"} break

  if {[lsearch $options_with_values $opt]>=0} {
    incr i
    if {$i==[llength $argv]} usage
    set val [lindex $argv $i]
  }

  switch -- $opt {
    -colsize {
      set O(aColSize) $val
    }

    -fts4 {
      set O(fts) fts4
    }

    -fts5 {
      set O(fts) fts5
    }
  }
}

if {$i > [llength $argv]-2} usage
set O(db) [lindex $argv $i]
set O(files) [lrange $argv [expr $i+1] end]

foreach {k v} [lrange $argv 0 end-2] {
  switch -- $k {
    -colsize {
      set O(aColSize) $v
    }

    -colsize {
      set O(aColSize) $v
    }
  }

}

sqlite3 db $O(db)
load_static_extension db fts5


# Create the FTS table in the db. Return a list of the table columns.
#
proc create_table {} {
  global O
  set cols [list a b c d e f g h i j k l m n o p q r s t u v w x y z]

  set nCol [llength $O(aColsize)]
  set cols [lrange $cols 0 [expr $nCol-1]]

  set sql    "CREATE VIRTUAL TABLE IF NOT EXISTS $O(tblname) USING $O(fts) ("
  append sql [join $cols ,]
  append sql ");"

  db eval $sql
  return $cols
}

# Return a list of tokens from the named file.
#
proc readfile {file} {
  set fd [open $file]
  set data [read $fd]
  close $fd
  split $data
}


# Load all the data into a big list of tokens.
#
set tokens [list]
foreach f $O(files) {
  set tokens [concat $tokens [readfile $f]]
}

set N [llength $tokens]
set i 0
set cols [create_table]
set sql "INSERT INTO $O(tblname) VALUES(\$[lindex $cols 0]"
foreach c [lrange $cols 1 end] {
  append sql ", \$$c"
}
append sql ")"

db eval BEGIN
  while {$i < $N} {
    foreach c $cols s $O(aColsize) {
      set $c [lrange $tokens $i [expr $i+$s-1]]
      incr i $s
    }
    db eval $sql
  }
db eval COMMIT



Changes to ext/rtree/rtree1.test.

30
31
32
33
34
35
36





37
38
39
40
41
42
43
...
530
531
532
533
534
535
536
537


















































538
#   rtree-5.*: Test DELETE
#   rtree-6.*: Test UPDATE
#   rtree-7.*: Test renaming an r-tree table.
#   rtree-8.*: Test constrained scans of r-tree data.
#
#   rtree-12.*: Test that on-conflict clauses are supported.
#   rtree-13.*: Test that bug [d2889096e7bdeac6d] has been fixed.





#

ifcapable !rtree {
  finish_test
  return
}

................................................................................
  WITH r(x) AS (
    SELECT 1 UNION ALL
    SELECT 2 UNION ALL
    SELECT 3
  )
  SELECT * FROM r CROSS JOIN t9 WHERE id=x;
} {1 1 0.0 0.0 2 2 0.0 0.0}



















































finish_test







>
>
>
>
>







 








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
...
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
#   rtree-5.*: Test DELETE
#   rtree-6.*: Test UPDATE
#   rtree-7.*: Test renaming an r-tree table.
#   rtree-8.*: Test constrained scans of r-tree data.
#
#   rtree-12.*: Test that on-conflict clauses are supported.
#   rtree-13.*: Test that bug [d2889096e7bdeac6d] has been fixed.
#   rtree-14.*: Test if a non-integer is inserted into the PK column of an
#               r-tree table, it is converted to an integer before being
#               inserted. Also that if a non-numeric is inserted into one
#               of the min/max dimension columns, it is converted to the
#               required type before being inserted.
#

ifcapable !rtree {
  finish_test
  return
}

................................................................................
  WITH r(x) AS (
    SELECT 1 UNION ALL
    SELECT 2 UNION ALL
    SELECT 3
  )
  SELECT * FROM r CROSS JOIN t9 WHERE id=x;
} {1 1 0.0 0.0 2 2 0.0 0.0}

#-------------------------------------------------------------------------
# Test if a non-integer is inserted into the PK column of an r-tree
# table, it is converted to an integer before being inserted. Also
# that if a non-numeric is inserted into one of the min/max dimension
# columns, it is converted to the required type before being inserted.
#
do_execsql_test 14.1 {
  CREATE VIRTUAL TABLE t10 USING rtree(ii, x1, x2);
}

do_execsql_test 14.2 {
  INSERT INTO t10 VALUES(NULL,   1, 2);
  INSERT INTO t10 VALUES(NULL,   2, 3);
  INSERT INTO t10 VALUES('4xxx', 3, 4);
  INSERT INTO t10 VALUES(5.0,    4, 5);
  INSERT INTO t10 VALUES(6.4,    5, 6);
}
do_execsql_test 14.3 {
  SELECT * FROM t10;
} {
  1 1.0 2.0   2 2.0 3.0   4 3.0 4.0   5 4.0 5.0   6 5.0 6.0
}

do_execsql_test 14.4 {
  DELETE FROM t10;
  INSERT INTO t10 VALUES(1, 'one', 'two');
  INSERT INTO t10 VALUES(2, '52xyz', '81...');
}
do_execsql_test 14.5 {
  SELECT * FROM t10;
} {
  1 0.0 0.0
  2 52.0 81.0
}

do_execsql_test 14.4 {
  DROP TABLE t10;
  CREATE VIRTUAL TABLE t10 USING rtree_i32(ii, x1, x2);
  INSERT INTO t10 VALUES(1, 'one', 'two');
  INSERT INTO t10 VALUES(2, '52xyz', '81...');
  INSERT INTO t10 VALUES(3, 42.3, 49.9);
}
do_execsql_test 14.5 {
  SELECT * FROM t10;
} {
  1 0 0
  2 52 81
  3 42 49
}

finish_test

Changes to src/mutex_w32.c.

83
84
85
86
87
88
89
90



91
92
93
94
95
96
97
** compiled without mutexes (SQLITE_THREADSAFE=0).
*/
void sqlite3MemoryBarrier(void){
#if defined(SQLITE_MEMORY_BARRIER)
  SQLITE_MEMORY_BARRIER;
#elif defined(__GNUC__)
  __sync_synchronize();
#else



  MemoryBarrier();
#endif
}

/*
** Initialize and deinitialize the mutex subsystem.
*/







|
>
>
>







83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
** compiled without mutexes (SQLITE_THREADSAFE=0).
*/
void sqlite3MemoryBarrier(void){
#if defined(SQLITE_MEMORY_BARRIER)
  SQLITE_MEMORY_BARRIER;
#elif defined(__GNUC__)
  __sync_synchronize();
#elif !defined(SQLITE_DISABLE_INTRINSIC) && \
      defined(_MSC_VER) && _MSC_VER>=1300
  _ReadWriteBarrier();
#elif defined(MemoryBarrier)
  MemoryBarrier();
#endif
}

/*
** Initialize and deinitialize the mutex subsystem.
*/

Changes to src/pragma.c.

1357
1358
1359
1360
1361
1362
1363
1364

1365
1366
1367
1368
1369
1370
1371
1372
....
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575

    /* Code that appears at the end of the integrity check.  If no error
    ** messages have been generated, output OK.  Otherwise output the
    ** error message
    */
    static const int iLn = VDBE_OFFSET_LINENO(2);
    static const VdbeOpList endCode[] = {
      { OP_IfNeg,       1, 0,        0},    /* 0 */

      { OP_String8,     0, 3,        0},    /* 1 */
      { OP_ResultRow,   3, 1,        0},
    };

    int isQuick = (sqlite3Tolower(zLeft[0])=='q');

    /* If the PRAGMA command was of the form "PRAGMA <db>.integrity_check",
    ** then iDb is set to the index of the database identified by <db>.
................................................................................
          sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7);
          sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1);
        }
#endif /* SQLITE_OMIT_BTREECOUNT */
      } 
    }
    addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn);
    sqlite3VdbeChangeP3(v, addr, -mxErr);
    sqlite3VdbeJumpHere(v, addr);
    sqlite3VdbeChangeP4(v, addr+1, "ok", P4_STATIC);
  }
  break;
#endif /* SQLITE_OMIT_INTEGRITY_CHECK */

#ifndef SQLITE_OMIT_UTF16
  /*
  **   PRAGMA encoding







|
>
|







 







|
|
|







1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
....
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576

    /* Code that appears at the end of the integrity check.  If no error
    ** messages have been generated, output OK.  Otherwise output the
    ** error message
    */
    static const int iLn = VDBE_OFFSET_LINENO(2);
    static const VdbeOpList endCode[] = {
      { OP_AddImm,      1, 0,        0},    /* 0 */
      { OP_If,          1, 0,        0},    /* 1 */
      { OP_String8,     0, 3,        0},    /* 2 */
      { OP_ResultRow,   3, 1,        0},
    };

    int isQuick = (sqlite3Tolower(zLeft[0])=='q');

    /* If the PRAGMA command was of the form "PRAGMA <db>.integrity_check",
    ** then iDb is set to the index of the database identified by <db>.
................................................................................
          sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7);
          sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1);
        }
#endif /* SQLITE_OMIT_BTREECOUNT */
      } 
    }
    addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn);
    sqlite3VdbeChangeP2(v, addr, -mxErr);
    sqlite3VdbeJumpHere(v, addr+1);
    sqlite3VdbeChangeP4(v, addr+2, "ok", P4_STATIC);
  }
  break;
#endif /* SQLITE_OMIT_INTEGRITY_CHECK */

#ifndef SQLITE_OMIT_UTF16
  /*
  **   PRAGMA encoding

Changes to src/select.c.

575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
...
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
....
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
....
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860

1861
1862
1863
1864
1865
1866
1867
....
2269
2270
2271
2272
2273
2274
2275





2276
2277
2278
2279
2280
2281
2282
    int addr;
    int iLimit;
    if( pSelect->iOffset ){
      iLimit = pSelect->iOffset+1;
    }else{
      iLimit = pSelect->iLimit;
    }
    addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, -1); VdbeCoverage(v);
    sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor);
    sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor);
    sqlite3VdbeJumpHere(v, addr);
  }
}

/*
................................................................................
*/
static void codeOffset(
  Vdbe *v,          /* Generate code into this VM */
  int iOffset,      /* Register holding the offset counter */
  int iContinue     /* Jump here to skip the current record */
){
  if( iOffset>0 ){
    int addr;
    addr = sqlite3VdbeAddOp3(v, OP_IfNeg, iOffset, 0, -1); VdbeCoverage(v);
    sqlite3VdbeGoto(v, iContinue);
    VdbeComment((v, "skip OFFSET records"));
    sqlite3VdbeJumpHere(v, addr);
  }
}

/*
** Add code that will check to make sure the N registers starting at iMem
** form a distinct entry.  iTab is a sorting index that holds previously
** seen combinations of the N values.  A new entry is made in iTab
................................................................................
** the reuse of the same limit and offset registers across multiple
** SELECT statements.
*/
static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){
  Vdbe *v = 0;
  int iLimit = 0;
  int iOffset;
  int addr1, n;
  if( p->iLimit ) return;

  /* 
  ** "LIMIT -1" always shows all rows.  There is some
  ** controversy about what the correct behavior should be.
  ** The current implementation interprets "LIMIT 0" to mean
  ** no rows.
................................................................................
    }
    if( p->pOffset ){
      p->iOffset = iOffset = ++pParse->nMem;
      pParse->nMem++;   /* Allocate an extra register for limit+offset */
      sqlite3ExprCode(pParse, p->pOffset, iOffset);
      sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v);
      VdbeComment((v, "OFFSET counter"));
      addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iOffset); VdbeCoverage(v);
      sqlite3VdbeAddOp2(v, OP_Integer, 0, iOffset);
      sqlite3VdbeJumpHere(v, addr1);
      sqlite3VdbeAddOp3(v, OP_Add, iLimit, iOffset, iOffset+1);
      VdbeComment((v, "LIMIT+OFFSET"));
      addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iLimit); VdbeCoverage(v);
      sqlite3VdbeAddOp2(v, OP_Integer, -1, iOffset+1);
      sqlite3VdbeJumpHere(v, addr1);

    }
  }
}

#ifndef SQLITE_OMIT_COMPOUND_SELECT
/*
** Return the appropriate collating sequence for the iCol-th column of
................................................................................
      }
      p->pPrior = 0;
      p->iLimit = pPrior->iLimit;
      p->iOffset = pPrior->iOffset;
      if( p->iLimit ){
        addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v);
        VdbeComment((v, "Jump ahead if LIMIT reached"));





      }
      explainSetInteger(iSub2, pParse->iNextSelectId);
      rc = sqlite3Select(pParse, p, &dest);
      testcase( rc!=SQLITE_OK );
      pDelete = p->pPrior;
      p->pPrior = pPrior;
      p->nSelectRow += pPrior->nSelectRow;







|







 







<
|
<
|
<







 







|







 







|
<
<


<
<
<
>







 







>
>
>
>
>







575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
...
591
592
593
594
595
596
597

598

599

600
601
602
603
604
605
606
....
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
....
1843
1844
1845
1846
1847
1848
1849
1850


1851
1852



1853
1854
1855
1856
1857
1858
1859
1860
....
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
    int addr;
    int iLimit;
    if( pSelect->iOffset ){
      iLimit = pSelect->iOffset+1;
    }else{
      iLimit = pSelect->iLimit;
    }
    addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v);
    sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor);
    sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor);
    sqlite3VdbeJumpHere(v, addr);
  }
}

/*
................................................................................
*/
static void codeOffset(
  Vdbe *v,          /* Generate code into this VM */
  int iOffset,      /* Register holding the offset counter */
  int iContinue     /* Jump here to skip the current record */
){
  if( iOffset>0 ){

    sqlite3VdbeAddOp3(v, OP_IfPos, iOffset, iContinue, 1); VdbeCoverage(v);

    VdbeComment((v, "OFFSET"));

  }
}

/*
** Add code that will check to make sure the N registers starting at iMem
** form a distinct entry.  iTab is a sorting index that holds previously
** seen combinations of the N values.  A new entry is made in iTab
................................................................................
** the reuse of the same limit and offset registers across multiple
** SELECT statements.
*/
static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){
  Vdbe *v = 0;
  int iLimit = 0;
  int iOffset;
  int n;
  if( p->iLimit ) return;

  /* 
  ** "LIMIT -1" always shows all rows.  There is some
  ** controversy about what the correct behavior should be.
  ** The current implementation interprets "LIMIT 0" to mean
  ** no rows.
................................................................................
    }
    if( p->pOffset ){
      p->iOffset = iOffset = ++pParse->nMem;
      pParse->nMem++;   /* Allocate an extra register for limit+offset */
      sqlite3ExprCode(pParse, p->pOffset, iOffset);
      sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v);
      VdbeComment((v, "OFFSET counter"));
      sqlite3VdbeAddOp3(v, OP_SetIfNotPos, iOffset, iOffset, 0);


      sqlite3VdbeAddOp3(v, OP_Add, iLimit, iOffset, iOffset+1);
      VdbeComment((v, "LIMIT+OFFSET"));



      sqlite3VdbeAddOp3(v, OP_SetIfNotPos, iLimit, iOffset+1, -1);
    }
  }
}

#ifndef SQLITE_OMIT_COMPOUND_SELECT
/*
** Return the appropriate collating sequence for the iCol-th column of
................................................................................
      }
      p->pPrior = 0;
      p->iLimit = pPrior->iLimit;
      p->iOffset = pPrior->iOffset;
      if( p->iLimit ){
        addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v);
        VdbeComment((v, "Jump ahead if LIMIT reached"));
        if( p->iOffset ){
          sqlite3VdbeAddOp3(v, OP_SetIfNotPos, p->iOffset, p->iOffset, 0);
          sqlite3VdbeAddOp3(v, OP_Add, p->iLimit, p->iOffset, p->iOffset+1);
          sqlite3VdbeAddOp3(v, OP_SetIfNotPos, p->iLimit, p->iOffset+1, -1);
        }
      }
      explainSetInteger(iSub2, pParse->iNextSelectId);
      rc = sqlite3Select(pParse, p, &dest);
      testcase( rc!=SQLITE_OK );
      pDelete = p->pPrior;
      p->pPrior = pPrior;
      p->nSelectRow += pPrior->nSelectRow;

Changes to src/sqliteInt.h.

192
193
194
195
196
197
198

199
200
201
202
203
204
205
*/
#if !defined(SQLITE_DISABLE_INTRINSIC)
#  if defined(_MSC_VER) && _MSC_VER>=1300
#    if !defined(_WIN32_WCE)
#      include <intrin.h>
#      pragma intrinsic(_byteswap_ushort)
#      pragma intrinsic(_byteswap_ulong)

#    else
#      include <cmnintrin.h>
#    endif
#  endif
#endif

/*







>







192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
*/
#if !defined(SQLITE_DISABLE_INTRINSIC)
#  if defined(_MSC_VER) && _MSC_VER>=1300
#    if !defined(_WIN32_WCE)
#      include <intrin.h>
#      pragma intrinsic(_byteswap_ushort)
#      pragma intrinsic(_byteswap_ulong)
#      pragma intrinsic(_ReadWriteBarrier)
#    else
#      include <cmnintrin.h>
#    endif
#  endif
#endif

/*

Changes to src/vdbe.c.

5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749



5750
5751
5752
5753
5754
5755
5756
5757

5758
5759

5760
5761
5762
5763
5764



5765
5766
5767
5768
5769
5770
5771

5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
  if( pIn1->u.i<pIn2->u.i){
    pIn1->u.i = pIn2->u.i;
  }
  break;
}
#endif /* SQLITE_OMIT_AUTOINCREMENT */

/* Opcode: IfPos P1 P2 * * *
** Synopsis: if r[P1]>0 goto P2
**
** Register P1 must contain an integer.
** If the value of register P1 is 1 or greater, jump to P2 and
** add the literal value P3 to register P1.
**
** If the initial value of register P1 is less than 1, then the
** value is unchanged and control passes through to the next instruction.
*/
case OP_IfPos: {        /* jump, in1 */
  pIn1 = &aMem[pOp->p1];
  assert( pIn1->flags&MEM_Int );
  VdbeBranchTaken( pIn1->u.i>0, 2);
  if( pIn1->u.i>0 ) goto jump_to_p2;



  break;
}

/* Opcode: IfNeg P1 P2 P3 * *
** Synopsis: r[P1]+=P3, if r[P1]<0 goto P2
**
** Register P1 must contain an integer.  Add literal P3 to the value in
** register P1 then if the value of register P1 is less than zero, jump to P2. 

*/
case OP_IfNeg: {        /* jump, in1 */

  pIn1 = &aMem[pOp->p1];
  assert( pIn1->flags&MEM_Int );
  pIn1->u.i += pOp->p3;
  VdbeBranchTaken(pIn1->u.i<0, 2);
  if( pIn1->u.i<0 ) goto jump_to_p2;



  break;
}

/* Opcode: IfNotZero P1 P2 P3 * *
** Synopsis: if r[P1]!=0 then r[P1]+=P3, goto P2
**
** Register P1 must contain an integer.  If the content of register P1 is

** initially nonzero, then add P3 to P1 and jump to P2.  If register P1 is
** initially zero, leave it unchanged and fall through.
*/
case OP_IfNotZero: {        /* jump, in1 */
  pIn1 = &aMem[pOp->p1];
  assert( pIn1->flags&MEM_Int );
  VdbeBranchTaken(pIn1->u.i<0, 2);
  if( pIn1->u.i ){
     pIn1->u.i += pOp->p3;
     goto jump_to_p2;
  }
  break;
}

/* Opcode: DecrJumpZero P1 P2 * * *
** Synopsis: if (--r[P1])==0 goto P2







|
|


|
|








|
>
>
>



|
|

|
|
>

<
>


<
<
|
>
>
>




|


>
|
|






|







5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762

5763
5764
5765


5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
  if( pIn1->u.i<pIn2->u.i){
    pIn1->u.i = pIn2->u.i;
  }
  break;
}
#endif /* SQLITE_OMIT_AUTOINCREMENT */

/* Opcode: IfPos P1 P2 P3 * *
** Synopsis: if r[P1]>0 then r[P1]-=P3, goto P2
**
** Register P1 must contain an integer.
** If the value of register P1 is 1 or greater, subtract P3 from the
** value in P1 and jump to P2.
**
** If the initial value of register P1 is less than 1, then the
** value is unchanged and control passes through to the next instruction.
*/
case OP_IfPos: {        /* jump, in1 */
  pIn1 = &aMem[pOp->p1];
  assert( pIn1->flags&MEM_Int );
  VdbeBranchTaken( pIn1->u.i>0, 2);
  if( pIn1->u.i>0 ){
    pIn1->u.i -= pOp->p3;
    goto jump_to_p2;
  }
  break;
}

/* Opcode: SetIfNotPos P1 P2 P3 * *
** Synopsis: if r[P1]<=0 then r[P2]=P3
**
** Register P1 must contain an integer.
** If the value of register P1 is not positive (if it is less than 1) then
** set the value of register P2 to be the integer P3.
*/

case OP_SetIfNotPos: {        /* in1, in2 */
  pIn1 = &aMem[pOp->p1];
  assert( pIn1->flags&MEM_Int );


  if( pIn1->u.i<=0 ){
    pOut = out2Prerelease(p, pOp);
    pOut->u.i = pOp->p3;
  }
  break;
}

/* Opcode: IfNotZero P1 P2 P3 * *
** Synopsis: if r[P1]!=0 then r[P1]-=P3, goto P2
**
** Register P1 must contain an integer.  If the content of register P1 is
** initially nonzero, then subtract P3 from the value in register P1 and
** jump to P2.  If register P1 is initially zero, leave it unchanged
** and fall through.
*/
case OP_IfNotZero: {        /* jump, in1 */
  pIn1 = &aMem[pOp->p1];
  assert( pIn1->flags&MEM_Int );
  VdbeBranchTaken(pIn1->u.i<0, 2);
  if( pIn1->u.i ){
     pIn1->u.i -= pOp->p3;
     goto jump_to_p2;
  }
  break;
}

/* Opcode: DecrJumpZero P1 P2 * * *
** Synopsis: if (--r[P1])==0 goto P2

Changes to src/vtab.c.

1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155

/*
** Erase the eponymous virtual table instance associated with
** virtual table module pMod, if it exists.
*/
void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){
  Table *pTab = pMod->pEpoTab;
  if( (pTab = pMod->pEpoTab)!=0 ){
    sqlite3DeleteColumnNames(db, pTab);
    sqlite3VtabClear(db, pTab);
    sqlite3DbFree(db, pTab);
    pMod->pEpoTab = 0;
  }
}








|







1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155

/*
** Erase the eponymous virtual table instance associated with
** virtual table module pMod, if it exists.
*/
void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){
  Table *pTab = pMod->pEpoTab;
  if( pTab!=0 ){
    sqlite3DeleteColumnNames(db, pTab);
    sqlite3VtabClear(db, pTab);
    sqlite3DbFree(db, pTab);
    pMod->pEpoTab = 0;
  }
}

Changes to src/whereexpr.c.

946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
          eExtraOp = WO_EQUIV;
        }
      }else{
        pDup = pExpr;
        pNew = pTerm;
      }
      exprCommute(pParse, pDup);
      pLeft = sqlite3ExprSkipCollate(pDup->pLeft);
      pNew->leftCursor = iCur;
      pNew->u.leftColumn = iColumn;
      testcase( (prereqLeft | extraRight) != prereqLeft );
      pNew->prereqRight = prereqLeft | extraRight;
      pNew->prereqAll = prereqAll;
      pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask;
    }







<







946
947
948
949
950
951
952

953
954
955
956
957
958
959
          eExtraOp = WO_EQUIV;
        }
      }else{
        pDup = pExpr;
        pNew = pTerm;
      }
      exprCommute(pParse, pDup);

      pNew->leftCursor = iCur;
      pNew->u.leftColumn = iColumn;
      testcase( (prereqLeft | extraRight) != prereqLeft );
      pNew->prereqRight = prereqLeft | extraRight;
      pNew->prereqAll = prereqAll;
      pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask;
    }

Changes to test/fts3expr3.test.

117
118
119
120
121
122
123


124
125
126
127
128
129
130
...
197
198
199
200
201
202
203
204






























205
206
proc balanced_andor_tree {nEntry} {
  set tree [balanced_exprtree_structure $nEntry]
  set node "{[balanced_and_tree $nEntry]}"
  regsub -all AND $node OR node
  regsub -all xxx $tree $node tree
  return $tree
}



# Test that queries like "1 AND 2 AND 3 AND 4..." are transformed to 
# balanced trees by FTS.
#
for {set i 1} {$i < 100} {incr i} {
  do_test 1.$i {
    test_fts3expr2 [random_and_query $i]
................................................................................
set query [random_andor_query 12]
set result [balanced_andor_tree 12]
do_faultsim_test fts3expr3-fault-1 -faults oom-* -body {
  test_fts3expr2 $::query
} -test {
  faultsim_test_result [list 0 $::result]
}































set sqlite_fts3_enable_parentheses 0
finish_test







>
>







 








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
...
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
proc balanced_andor_tree {nEntry} {
  set tree [balanced_exprtree_structure $nEntry]
  set node "{[balanced_and_tree $nEntry]}"
  regsub -all AND $node OR node
  regsub -all xxx $tree $node tree
  return $tree
}

if 1 {

# Test that queries like "1 AND 2 AND 3 AND 4..." are transformed to 
# balanced trees by FTS.
#
for {set i 1} {$i < 100} {incr i} {
  do_test 1.$i {
    test_fts3expr2 [random_and_query $i]
................................................................................
set query [random_andor_query 12]
set result [balanced_andor_tree 12]
do_faultsim_test fts3expr3-fault-1 -faults oom-* -body {
  test_fts3expr2 $::query
} -test {
  faultsim_test_result [list 0 $::result]
}

}

#-------------------------------------------------------------------

foreach {tn expr res} {
  1 {1 OR 2 OR 3 OR 4}           {OR {OR {P 1} {P 2}} {OR {P 3} {P 4}}} 
  2 {1 OR (2 AND 3 AND 4 AND 5)} 
    {OR {P 1} {AND {AND {P 2} {P 3}} {AND {P 4} {P 5}}}}
  3 {(2 AND 3 AND 4 AND 5) OR 1} 
    {OR {AND {AND {P 2} {P 3}} {AND {P 4} {P 5}}} {P 1}}

  4 {1 AND (2 OR 3 OR 4 OR 5)} 
    {AND {P 1} {OR {OR {P 2} {P 3}} {OR {P 4} {P 5}}}}
  5 {(2 OR 3 OR 4 OR 5) AND 1} 
    {AND {OR {OR {P 2} {P 3}} {OR {P 4} {P 5}}} {P 1}}

  6 {(2 OR 3 OR 4 OR 5) NOT 1} 
    {NOT {OR {OR {P 2} {P 3}} {OR {P 4} {P 5}}} {P 1}}

  7 {1 NOT (2 OR 3 OR 4 OR 5)} 
    {NOT {P 1} {OR {OR {P 2} {P 3}} {OR {P 4} {P 5}}}}

  8 {(1 OR 2 OR 3 OR 4) NOT (5 AND 6 AND 7 AND 8)}
    {NOT {OR {OR {P 1} {P 2}} {OR {P 3} {P 4}}} {AND {AND {P 5} {P 6}} {AND {P 7} {P 8}}}}
} {
  do_test 5.1.$tn {
    test_fts3expr2 $expr
  } $res
}

set sqlite_fts3_enable_parentheses 0
finish_test

Added test/offset1.test.



































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
# 2015-10-06
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# 
# This file implements test cases for the [b65cb2c8d91f6685841d7d1e13b6]
# bug:  Correct handling of LIMIT and OFFSET on a UNION ALL query where
# the right-hand SELECT contains an ORDER BY in a subquery.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

ifcapable !compound {
  finish_test
  return
}

do_execsql_test offset1-1.1 {
  CREATE TABLE t1(a,b);
  INSERT INTO t1 VALUES(1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e');
  CREATE TABLE t2(x,y);
  INSERT INTO t2 VALUES(8,'y'),(9,'z'),(6,'w'),(7,'x');
  SELECT count(*) FROM t1, t2;
} {20}

do_execsql_test offset1-1.2.0 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 3 OFFSET 0;
} {1 a 2 b 3 c}
do_execsql_test offset1-1.2.1 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 3 OFFSET 1;
} {2 b 3 c 4 d}
do_execsql_test offset1-1.2.2 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 3 OFFSET 2;
} {3 c 4 d 5 e}
do_execsql_test offset1-1.2.3 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 3 OFFSET 3;
} {4 d 5 e 6 w}
do_execsql_test offset1-1.2.4 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 3 OFFSET 4;
} {5 e 6 w 7 x}
do_execsql_test offset1-1.2.5 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 3 OFFSET 5;
} {6 w 7 x 8 y}
do_execsql_test offset1-1.2.6 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 3 OFFSET 6;
} {7 x 8 y 9 z}
do_execsql_test offset1-1.2.7 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 3 OFFSET 7;
} {8 y 9 z}
do_execsql_test offset1-1.2.8 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 3 OFFSET 8;
} {9 z}
do_execsql_test offset1-1.2.9 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 3 OFFSET 9;
} {}

do_execsql_test offset1-1.3.0 {
  SELECT * FROM t1 LIMIT 0;
} {}

do_execsql_test offset1-1.4.0 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 0 OFFSET 1;
} {}
do_execsql_test offset1-1.4.1 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 1 OFFSET 1;
} {2 b}
do_execsql_test offset1-1.4.2 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 2 OFFSET 1;
} {2 b 3 c}
do_execsql_test offset1-1.4.3 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 3 OFFSET 1;
} {2 b 3 c 4 d}
do_execsql_test offset1-1.4.4 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 4 OFFSET 1;
} {2 b 3 c 4 d 5 e}
do_execsql_test offset1-1.4.5 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 5 OFFSET 1;
} {2 b 3 c 4 d 5 e 6 w}
do_execsql_test offset1-1.4.6 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 6 OFFSET 1;
} {2 b 3 c 4 d 5 e 6 w 7 x}
do_execsql_test offset1-1.4.7 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 7 OFFSET 1;
} {2 b 3 c 4 d 5 e 6 w 7 x 8 y}
do_execsql_test offset1-1.4.8 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 8 OFFSET 1;
} {2 b 3 c 4 d 5 e 6 w 7 x 8 y 9 z}
do_execsql_test offset1-1.4.9 {
  SELECT a, b FROM t1
  UNION ALL
  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
  LIMIT 9 OFFSET 1;
} {2 b 3 c 4 d 5 e 6 w 7 x 8 y 9 z}



finish_test

Changes to test/spellfix2.test.

72
73
74
75
76
77
78
79

80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101

102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
  WHERE word MATCH 'amstedam*' AND distance <= 100 AND top=20;
} {
  32 20
}

do_execsql_test 1.6 {
  SELECT word, distance, matchlen FROM demo 
  WHERE word MATCH 'amstedam*' AND distance <= 100;

} {
  amsterdam         100 9        amsterdamh        100 9
  amsterdamm        100 9        amsterdamn        100 9
  amsterdama        100 9        amsterdame        100 9
  amsterdami        100 9        amsterdamo        100 9
  amsterdamu        100 9        amsterdamy        100 9
  amsterdammetje    100 9        amsterdamania     100 9
  amsterdamb        100 9        amsterdamf        100 9
  amsterdamp        100 9        amsterdamv        100 9
  amsterdamw        100 9        amsterdamweg      100 9
  amsterdamc        100 9        amsterdamg        100 9
  amsterdamj        100 9        amsterdamk        100 9
  amsterdamq        100 9        amsterdams        100 9
  amsterdamx        100 9        amsterdamz        100 9
  amsterdamsestraat 100 9        amsterdamd        100 9
  amsterdamt        100 9        amsterdaml        100 9
  amsterdamlaan     100 9        amsterdamr        100 9
}

do_execsql_test 1.7 {
  SELECT word, distance, matchlen FROM demo 
  WHERE word MATCH 'amstedam*' AND distance <= 100 AND top=20;

} {
  amsterdam         100 9        amsterdamh        100 9
  amsterdamm        100 9        amsterdamn        100 9
  amsterdama        100 9        amsterdame        100 9
  amsterdami        100 9        amsterdamo        100 9
  amsterdamu        100 9        amsterdamy        100 9
  amsterdammetje    100 9        amsterdamania     100 9
  amsterdamb        100 9        amsterdamf        100 9
  amsterdamp        100 9        amsterdamv        100 9
  amsterdamw        100 9        amsterdamweg      100 9
  amsterdamc        100 9        amsterdamg        100 9
}


finish_test







|
>

|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|




|
>

|
|
|
|
|
|
|
|
|
|




72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
  WHERE word MATCH 'amstedam*' AND distance <= 100 AND top=20;
} {
  32 20
}

do_execsql_test 1.6 {
  SELECT word, distance, matchlen FROM demo 
  WHERE word MATCH 'amstedam*' AND distance <= 100
  ORDER BY distance, word;
} {
  amsterdam         100 9        amsterdama        100 9
  amsterdamania     100 9        amsterdamb        100 9
  amsterdamc        100 9        amsterdamd        100 9
  amsterdame        100 9        amsterdamf        100 9
  amsterdamg        100 9        amsterdamh        100 9
  amsterdami        100 9        amsterdamj        100 9
  amsterdamk        100 9        amsterdaml        100 9
  amsterdamlaan     100 9        amsterdamm        100 9
  amsterdammetje    100 9        amsterdamn        100 9
  amsterdamo        100 9        amsterdamp        100 9
  amsterdamq        100 9        amsterdamr        100 9
  amsterdams        100 9        amsterdamsestraat 100 9
  amsterdamt        100 9        amsterdamu        100 9
  amsterdamv        100 9        amsterdamw        100 9
  amsterdamweg      100 9        amsterdamx        100 9
  amsterdamy        100 9        amsterdamz        100 9
}

do_execsql_test 1.7 {
  SELECT word, distance, matchlen FROM demo 
  WHERE word MATCH 'amstedam*' AND distance <= 100 AND top=20
  ORDER BY distance, word;
} {
  amsterdam         100 9        amsterdama        100 9
  amsterdamania     100 9        amsterdamb        100 9
  amsterdamc        100 9        amsterdame        100 9
  amsterdamf        100 9        amsterdamg        100 9
  amsterdamh        100 9        amsterdami        100 9
  amsterdamm        100 9        amsterdammetje    100 9
  amsterdamn        100 9        amsterdamo        100 9
  amsterdamp        100 9        amsterdamu        100 9
  amsterdamv        100 9        amsterdamw        100 9
  amsterdamweg      100 9        amsterdamy        100 9
}


finish_test

Changes to test/uri.test.

59
60
61
62
63
64
65








66
67
68
69
70
71
72
    #
    # NOTE: Due to limits on legal characters for file names imposed by
    #       Windows, we must skip the final two tests here (i.e. the
    #       question mark is illegal in a file name on Windows).
    #
    if {$tn>14} break









    #
    # NOTE: On Windows, we need to account for the fact that the current
    #       directory does not start with a forward slash.
    #
    set uri  [string map [list PWD/ /[test_pwd /]] $uri]
  } else {
    set uri  [string map [list PWD/ [test_pwd /]] $uri]







>
>
>
>
>
>
>
>







59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
    #
    # NOTE: Due to limits on legal characters for file names imposed by
    #       Windows, we must skip the final two tests here (i.e. the
    #       question mark is illegal in a file name on Windows).
    #
    if {$tn>14} break

    #
    # NOTE: When running on Tcl 8.6 (or higher?) on Windows, a colon within
    #       the file name no longer tries to access an alternate data stream
    #       (ADS) named "test.db" for the "http" file, causing some spurious
    #       failures of this test.
    #
    if {$tn==12 && $::tcl_version>=8.6} continue

    #
    # NOTE: On Windows, we need to account for the fact that the current
    #       directory does not start with a forward slash.
    #
    set uri  [string map [list PWD/ /[test_pwd /]] $uri]
  } else {
    set uri  [string map [list PWD/ [test_pwd /]] $uri]

Deleted tool/diffdb.c.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
/*
** A utility for printing the differences between two SQLite database files.
*/
#include <stdio.h>
#include <ctype.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdlib.h>


#define PAGESIZE 1024
static int db1 = -1;
static int db2 = -1;

int main(int argc, char **argv){
  int iPg;
  unsigned char a1[PAGESIZE], a2[PAGESIZE];
  if( argc!=3 ){
    fprintf(stderr,"Usage: %s FILENAME FILENAME\n", argv[0]);
    exit(1);
  }
  db1 = open(argv[1], O_RDONLY);
  if( db1<0 ){
    fprintf(stderr,"%s: can't open %s\n", argv[0], argv[1]);
    exit(1);
  }
  db2 = open(argv[2], O_RDONLY);
  if( db2<0 ){
    fprintf(stderr,"%s: can't open %s\n", argv[0], argv[2]);
    exit(1);
  }
  iPg = 1;
  while( read(db1, a1, PAGESIZE)==PAGESIZE && read(db2,a2,PAGESIZE)==PAGESIZE ){
    if( memcmp(a1,a2,PAGESIZE) ){
      printf("Page %d\n", iPg);
    }
    iPg++;
  }
  printf("%d pages checked\n", iPg-1);
  close(db1);
  close(db2);
}
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
























































































Deleted tool/opcodeDoc.awk.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#
# Extract opcode documentation for sqliteVdbe.c and generate HTML
#
BEGIN {
  print "<html><body bgcolor=white>"
  print "<h1>SQLite Virtual Database Engine Opcodes</h1>"
  print "<table>"
}
/ Opcode: /,/\*\// {
  if( $2=="Opcode:" ){
    printf "<tr><td>%s&nbsp;%s&nbsp;%s&nbsp;%s</td>\n<td>\n", $3, $4, $5, $6
  }else if( $1=="*/" ){
    printf "</td></tr>\n"
  }else if( NF>1 ){
    sub(/^ *\*\* /,"")
    gsub(/</,"&lt;")
    gsub(/&/,"&amp;")
    print
  }
}
END {
  print "</table></body></html>"
}
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<














































Deleted tool/space_used.tcl.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
# Run this TCL script using "testfixture" in order get a report that shows
# how much disk space is used by a particular data to actually store data
# versus how much space is unused.
#

# Get the name of the database to analyze
#
if {[llength $argv]!=1} {
  puts stderr "Usage: $argv0 database-name"
  exit 1
}
set file_to_analyze [lindex $argv 0]

# Open the database
#
sqlite db [lindex $argv 0]
set DB [btree_open [lindex $argv 0]]

# Output the schema for the generated report
#
puts \
{BEGIN;
CREATE TABLE space_used(
   name clob,        -- Name of a table or index in the database file
   is_index boolean, -- TRUE if it is an index, false for a table
   payload int,      -- Total amount of data stored in this table or index
   pri_pages int,    -- Number of primary pages used
   ovfl_pages int,   -- Number of overflow pages used
   pri_unused int,   -- Number of unused bytes on primary pages
   ovfl_unused int   -- Number of unused bytes on overflow pages
);}

# This query will be used to find the root page number for every index and
# table in the database.
#
set sql {
  SELECT name, type, rootpage FROM sqlite_master
  UNION ALL
  SELECT 'sqlite_master', 'table', 2
  ORDER BY 1
}

# Initialize variables used for summary statistics.
#
set total_size 0
set total_primary 0
set total_overflow 0
set total_unused_primary 0
set total_unused_ovfl 0

# Analyze every table in the database, one at a time.
#
foreach {name type rootpage} [db eval $sql] {
  set cursor [btree_cursor $DB $rootpage 0]
  set go [btree_first $cursor]
  set size 0
  catch {unset pg_used}
  set unused_ovfl 0
  set n_overflow 0
  while {$go==0} {
    set payload [btree_payload_size $cursor]
    incr size $payload
    set stat [btree_cursor_dump $cursor]
    set pgno [lindex $stat 0]
    set freebytes [lindex $stat 4]
    set pg_used($pgno) $freebytes
    if {$payload>238} {
      set n [expr {($payload-238+1019)/1020}]
      incr n_overflow $n
      incr unused_ovfl [expr {$n*1020+238-$payload}]
    }
    set go [btree_next $cursor]
  }
  btree_close_cursor $cursor
  set n_primary [llength [array names pg_used]]
  set unused_primary 0
  foreach x [array names pg_used] {incr unused_primary $pg_used($x)}
  regsub -all ' $name '' name
  puts -nonewline "INSERT INTO space_used VALUES('$name'"
  puts -nonewline ",[expr {$type=="index"}]"
  puts ",$size,$n_primary,$n_overflow,$unused_primary,$unused_ovfl);"
  incr total_size $size
  incr total_primary $n_primary
  incr total_overflow $n_overflow
  incr total_unused_primary $unused_primary
  incr total_unused_ovfl $unused_ovfl
}

# Output summary statistics:
#
puts "-- Total payload size: $total_size"
puts "-- Total pages used: $total_primary primary and $total_overflow overflow"
set file_pgcnt [expr {[file size [lindex $argv 0]]/1024}]
puts -nonewline "-- Total unused bytes on primary pages: $total_unused_primary"
if {$total_primary>0} {
  set upp [expr {$total_unused_primary/$total_primary}]
  puts " (avg $upp bytes/page)"
} else {
  puts ""
}
puts -nonewline "-- Total unused bytes on overflow pages: $total_unused_ovfl"
if {$total_overflow>0} {
  set upp [expr {$total_unused_ovfl/$total_overflow}]
  puts " (avg $upp bytes/page)"
} else {
  puts ""
}
set n_free [expr {$file_pgcnt-$total_primary-$total_overflow}]
if {$n_free>0} {incr n_free -1}
puts "-- Total pages on freelist: $n_free"
puts "COMMIT;"
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<