/ Check-in [b8864da9]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Start changing things to use doclist indexes as required. code is not activated yet.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: b8864da95db2c0e611116304d607e35a86c9247d
User & Date: dan 2014-08-02 20:49:36
Context
2014-08-04
20:07
Fix fts5_index.c to use doclist-indexes when possible. Only some cases work so far. check-in: 90b82d3e user: dan tags: fts5
2014-08-02
20:49
Start changing things to use doclist indexes as required. code is not activated yet. check-in: b8864da9 user: dan tags: fts5
2014-08-01
20:13
Add a special case to the integrity-check code to check that the final integer in a doclist index is as expected. check-in: c9893415 user: dan tags: fts5
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

80
81
82
83
84
85
86






87
88
89
90
91
92
93
94
95
96

97
98
99
100
101
102
103
...
249
250
251
252
253
254
255

256
257
258
259
260
261
262
...
391
392
393
394
395
396
397

398
399
400
401
402
403
404
...
452
453
454
455
456
457
458





459
460
461
462
463
464
465
...
531
532
533
534
535
536
537




































538
539
540
541
542
543
544
...
555
556
557
558
559
560
561







562
563
564
565
566
567
568
569
570
571
572
573
574








575
576
577
578
579
580
581
582
583

584





585
586
587
588
589
590
591
...
625
626
627
628
629
630
631




632
633
634
635
636
637
638
...
645
646
647
648
649
650
651






652



653
654
655
656
657
658
659
....
1084
1085
1086
1087
1088
1089
1090






1091
1092
1093
1094
1095
1096
1097
  int nIdx;                       /* Number of entries in aIdx[] */
  int aIdx[0];                    /* Offsets into aPoslist for current row */
};


/*
** Virtual-table cursor object.






*/
struct Fts5Cursor {
  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
  int idxNum;                     /* idxNum passed to xFilter() */
  sqlite3_stmt *pStmt;            /* Statement used to read %_content */
  Fts5Expr *pExpr;                /* Expression for MATCH queries */
  Fts5Sorter *pSorter;            /* Sorter for "ORDER BY rank" queries */
  int csrflags;                   /* Mask of cursor flags (see below) */
  Fts5Cursor *pNext;              /* Next cursor in Fts5Cursor.pCsr list */
  Fts5Auxiliary *pRank;           /* Rank callback (or NULL) */


  /* Variables used by auxiliary functions */
  i64 iCsrId;                     /* Cursor id */
  Fts5Auxiliary *pAux;            /* Currently executing extension function */
  Fts5Auxdata *pAuxdata;          /* First in linked list of saved aux-data */
  int *aColumnSize;               /* Values for xColumnSize() */
};
................................................................................
** The three query plans xBestIndex may choose between.
*/
#define FTS5_PLAN_SCAN           1       /* No usable constraint */
#define FTS5_PLAN_MATCH          2       /* (<tbl> MATCH ?) */
#define FTS5_PLAN_SORTED_MATCH   3       /* (<tbl> MATCH ? ORDER BY rank) */
#define FTS5_PLAN_ROWID          4       /* (rowid = ?) */
#define FTS5_PLAN_SOURCE         5       /* A source cursor for SORTED_MATCH */


#define FTS5_PLAN(idxNum) ((idxNum) & 0x7)

#define FTS5_ORDER_DESC   8       /* ORDER BY rowid DESC */
#define FTS5_ORDER_ASC   16       /* ORDER BY rowid ASC */

/*
................................................................................
    sqlite3_free(pData);
  }

  /* Remove the cursor from the Fts5Global.pCsr list */
  for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
  *pp = pCsr->pNext;


  sqlite3_free(pCsr);
  return SQLITE_OK;
}

static int fts5SorterNext(Fts5Cursor *pCsr){
  Fts5Sorter *pSorter = pCsr->pSorter;
  int rc;
................................................................................
    case FTS5_PLAN_SOURCE:
      rc = sqlite3Fts5ExprNext(pCsr->pExpr);
      if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
        CsrFlagSet(pCsr, FTS5CSR_EOF);
      }
      CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
      break;






    case FTS5_PLAN_SORTED_MATCH: {
      rc = fts5SorterNext(pCsr);
      break;
    }

    default:
................................................................................
  rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, bAsc);
  if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
    CsrFlagSet(pCsr, FTS5CSR_EOF);
  }
  CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
  return rc;
}





































/*
** This is the xFilter interface for the virtual table.  See
** the virtual table xFilter method documentation for additional
** information.
*/
static int fts5FilterMethod(
................................................................................

  assert( pCsr->pStmt==0 );
  assert( pCsr->pExpr==0 );
  assert( pCsr->csrflags==0 );
  assert( pCsr->pRank==0 );

  if( pTab->pSortCsr ){







    pCsr->idxNum = FTS5_PLAN_SOURCE;
    pCsr->pRank = pTab->pSortCsr->pRank;
    pCsr->pExpr = pTab->pSortCsr->pExpr;
    rc = fts5CursorFirst(pTab, pCsr, bAsc);
  }else{
    int ePlan = FTS5_PLAN(idxNum);
    int eStmt = fts5StmtType(idxNum);
    pCsr->idxNum = idxNum;
    rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt);
    if( rc==SQLITE_OK ){
      if( ePlan==FTS5_PLAN_MATCH || ePlan==FTS5_PLAN_SORTED_MATCH ){
        char **pzErr = &pTab->base.zErrMsg;
        const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);








        pCsr->pRank = pTab->pGlobal->pAux;
        rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr);
        if( rc==SQLITE_OK ){
          if( ePlan==FTS5_PLAN_MATCH ){
            rc = fts5CursorFirst(pTab, pCsr, bAsc);
          }else{
            rc = fts5CursorFirstSorted(pTab, pCsr, bAsc);
          }
        }

      }else{





        if( ePlan==FTS5_PLAN_ROWID ){
          sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
        }
        rc = fts5NextMethod(pCursor);
      }
    }
  }
................................................................................
*/
static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int ePlan = FTS5_PLAN(pCsr->idxNum);
  
  assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
  switch( ePlan ){




    case FTS5_PLAN_SOURCE:
    case FTS5_PLAN_MATCH:
    case FTS5_PLAN_SORTED_MATCH:
      *pRowid = fts5CursorRowid(pCsr);
      break;

    default:
................................................................................

/*
** If the cursor requires seeking (bSeekRequired flag is set), seek it.
** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise.
*/
static int fts5SeekCursor(Fts5Cursor *pCsr){
  int rc = SQLITE_OK;






  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){



    assert( pCsr->pExpr );
    sqlite3_reset(pCsr->pStmt);
    sqlite3_bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr));
    rc = sqlite3_step(pCsr->pStmt);
    if( rc==SQLITE_ROW ){
      rc = SQLITE_OK;
      CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT);
................................................................................
  int iCol                        /* Index of column to read value from */
){
  Fts5Config *pConfig = ((Fts5Table*)(pCursor->pVtab))->pConfig;
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int rc = SQLITE_OK;
  
  assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );







  if( iCol==pConfig->nCol ){
    if( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE ){
      fts5PoslistBlob(pCtx, pCsr);
    }else{
      /* User is requesting the value of the special column with the same name
      ** as the table. Return the cursor integer id number. This value is only







>
>
>
>
>
>










>







 







>







 







>







 







>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>






<

<
<
|
<
|
>
>
>
>
>
>
>
>









>
|
>
>
>
>
>







 







>
>
>
>







 







>
>
>
>
>
>
|
>
>
>







 







>
>
>
>
>
>







80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
...
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
...
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
...
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
...
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
...
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624

625


626

627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
...
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
...
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
....
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
  int nIdx;                       /* Number of entries in aIdx[] */
  int aIdx[0];                    /* Offsets into aPoslist for current row */
};


/*
** Virtual-table cursor object.
**
** zSpecial:
**   If this is a 'special' query (refer to function fts5SpecialMatch()), 
**   then this variable points to a nul-terminated buffer containing the
**   result to return through the table-name column. It is nul-terminated
**   and should eventually be freed using sqlite3_free().
*/
struct Fts5Cursor {
  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
  int idxNum;                     /* idxNum passed to xFilter() */
  sqlite3_stmt *pStmt;            /* Statement used to read %_content */
  Fts5Expr *pExpr;                /* Expression for MATCH queries */
  Fts5Sorter *pSorter;            /* Sorter for "ORDER BY rank" queries */
  int csrflags;                   /* Mask of cursor flags (see below) */
  Fts5Cursor *pNext;              /* Next cursor in Fts5Cursor.pCsr list */
  Fts5Auxiliary *pRank;           /* Rank callback (or NULL) */
  char *zSpecial;                 /* Result of special query */

  /* Variables used by auxiliary functions */
  i64 iCsrId;                     /* Cursor id */
  Fts5Auxiliary *pAux;            /* Currently executing extension function */
  Fts5Auxdata *pAuxdata;          /* First in linked list of saved aux-data */
  int *aColumnSize;               /* Values for xColumnSize() */
};
................................................................................
** The three query plans xBestIndex may choose between.
*/
#define FTS5_PLAN_SCAN           1       /* No usable constraint */
#define FTS5_PLAN_MATCH          2       /* (<tbl> MATCH ?) */
#define FTS5_PLAN_SORTED_MATCH   3       /* (<tbl> MATCH ? ORDER BY rank) */
#define FTS5_PLAN_ROWID          4       /* (rowid = ?) */
#define FTS5_PLAN_SOURCE         5       /* A source cursor for SORTED_MATCH */
#define FTS5_PLAN_SPECIAL        6       /* An internal query */

#define FTS5_PLAN(idxNum) ((idxNum) & 0x7)

#define FTS5_ORDER_DESC   8       /* ORDER BY rowid DESC */
#define FTS5_ORDER_ASC   16       /* ORDER BY rowid ASC */

/*
................................................................................
    sqlite3_free(pData);
  }

  /* Remove the cursor from the Fts5Global.pCsr list */
  for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
  *pp = pCsr->pNext;

  sqlite3_free(pCsr->zSpecial);
  sqlite3_free(pCsr);
  return SQLITE_OK;
}

static int fts5SorterNext(Fts5Cursor *pCsr){
  Fts5Sorter *pSorter = pCsr->pSorter;
  int rc;
................................................................................
    case FTS5_PLAN_SOURCE:
      rc = sqlite3Fts5ExprNext(pCsr->pExpr);
      if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
        CsrFlagSet(pCsr, FTS5CSR_EOF);
      }
      CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
      break;

    case FTS5_PLAN_SPECIAL: {
      CsrFlagSet(pCsr, FTS5CSR_EOF);
      break;
    }

    case FTS5_PLAN_SORTED_MATCH: {
      rc = fts5SorterNext(pCsr);
      break;
    }

    default:
................................................................................
  rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, bAsc);
  if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
    CsrFlagSet(pCsr, FTS5CSR_EOF);
  }
  CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
  return rc;
}

/*
** Process a "special" query. A special query is identified as one with a
** MATCH expression that begins with a '*' character. The remainder of
** the text passed to the MATCH operator are used as  the special query
** parameters.
*/
static int fts5SpecialMatch(
  Fts5Table *pTab, 
  Fts5Cursor *pCsr, 
  const char *zQuery
){
  int rc = SQLITE_OK;             /* Return code */
  const char *z = zQuery;         /* Special query text */
  int n;                          /* Number of bytes in text at z */

  while( z[0]==' ' ) z++;
  for(n=0; z[n] && z[n]!=' '; n++);

  assert( pTab->base.zErrMsg==0 );
  assert( pCsr->zSpecial==0 );

  if( 0==sqlite3_strnicmp("reads", z, n) ){
    pCsr->zSpecial = sqlite3_mprintf("%d", sqlite3Fts5IndexReads(pTab->pIndex));
    pCsr->idxNum = FTS5_PLAN_SPECIAL;
    if( pCsr->zSpecial==0 ) rc = SQLITE_NOMEM;
  }
  else{
    /* An unrecognized directive. Return an error message. */
    pTab->base.zErrMsg = sqlite3_mprintf("unknown special query: %.*s", n, z);
    rc = SQLITE_ERROR;
  }

  return rc;
}


/*
** This is the xFilter interface for the virtual table.  See
** the virtual table xFilter method documentation for additional
** information.
*/
static int fts5FilterMethod(
................................................................................

  assert( pCsr->pStmt==0 );
  assert( pCsr->pExpr==0 );
  assert( pCsr->csrflags==0 );
  assert( pCsr->pRank==0 );

  if( pTab->pSortCsr ){
    /* If pSortCsr is non-NULL, then this call is being made as part of 
    ** processing for a "... MATCH <expr> ORDER BY rank" query (ePlan is
    ** set to FTS5_PLAN_SORTED_MATCH). pSortCsr is the cursor that will
    ** return results to the user for this query. The current cursor 
    ** (pCursor) is used to execute the query issued by function 
    ** fts5CursorFirstSorted() above.  */
    assert( FTS5_PLAN(idxNum)==FTS5_PLAN_SCAN );
    pCsr->idxNum = FTS5_PLAN_SOURCE;
    pCsr->pRank = pTab->pSortCsr->pRank;
    pCsr->pExpr = pTab->pSortCsr->pExpr;
    rc = fts5CursorFirst(pTab, pCsr, bAsc);
  }else{
    int ePlan = FTS5_PLAN(idxNum);

    pCsr->idxNum = idxNum;


    if( ePlan==FTS5_PLAN_MATCH || ePlan==FTS5_PLAN_SORTED_MATCH ){

      const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);

      if( zExpr[0]=='*' ){
        /* The user has issued a query of the form "MATCH '*...'". This
        ** indicates that the MATCH expression is not a full text query,
        ** but a request for an internal parameter.  */
        rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]);
      }else{
        char **pzErr = &pTab->base.zErrMsg;
        pCsr->pRank = pTab->pGlobal->pAux;
        rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr);
        if( rc==SQLITE_OK ){
          if( ePlan==FTS5_PLAN_MATCH ){
            rc = fts5CursorFirst(pTab, pCsr, bAsc);
          }else{
            rc = fts5CursorFirstSorted(pTab, pCsr, bAsc);
          }
        }
      }
    }else{
      /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup
      ** by rowid (ePlan==FTS5_PLAN_ROWID).  */
      int eStmt = fts5StmtType(idxNum);
      rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt);
      if( rc==SQLITE_OK ){
        if( ePlan==FTS5_PLAN_ROWID ){
          sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
        }
        rc = fts5NextMethod(pCursor);
      }
    }
  }
................................................................................
*/
static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int ePlan = FTS5_PLAN(pCsr->idxNum);
  
  assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
  switch( ePlan ){
    case FTS5_PLAN_SPECIAL:
      *pRowid = 0;
      break;

    case FTS5_PLAN_SOURCE:
    case FTS5_PLAN_MATCH:
    case FTS5_PLAN_SORTED_MATCH:
      *pRowid = fts5CursorRowid(pCsr);
      break;

    default:
................................................................................

/*
** If the cursor requires seeking (bSeekRequired flag is set), seek it.
** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise.
*/
static int fts5SeekCursor(Fts5Cursor *pCsr){
  int rc = SQLITE_OK;

  /* If the cursor does not yet have a statement handle, obtain one now. */ 
  if( pCsr->pStmt==0 ){
    Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
    int eStmt = fts5StmtType(pCsr->idxNum);
    rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt);
    assert( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) );
  }

  if( rc==SQLITE_OK && CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){
    assert( pCsr->pExpr );
    sqlite3_reset(pCsr->pStmt);
    sqlite3_bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr));
    rc = sqlite3_step(pCsr->pStmt);
    if( rc==SQLITE_ROW ){
      rc = SQLITE_OK;
      CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT);
................................................................................
  int iCol                        /* Index of column to read value from */
){
  Fts5Config *pConfig = ((Fts5Table*)(pCursor->pVtab))->pConfig;
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int rc = SQLITE_OK;
  
  assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );

  if( pCsr->idxNum==FTS5_PLAN_SPECIAL ){
    if( iCol==pConfig->nCol ){
      sqlite3_result_text(pCtx, pCsr->zSpecial, -1, SQLITE_TRANSIENT);
    }
  }else

  if( iCol==pConfig->nCol ){
    if( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE ){
      fts5PoslistBlob(pCtx, pCsr);
    }else{
      /* User is requesting the value of the special column with the same name
      ** as the table. Return the cursor integer id number. This value is only

Changes to ext/fts5/fts5Int.h.

184
185
186
187
188
189
190
191

192
193
194
195
196
197
198
...
268
269
270
271
272
273
274


275
276
277
278
279
280
281
  int flags                       /* Mask of FTS5INDEX_QUERY_X flags */
);

/*
** Docid list iteration.
*/
int  sqlite3Fts5IterEof(Fts5IndexIter*);
void sqlite3Fts5IterNext(Fts5IndexIter*, i64 iMatch);

i64  sqlite3Fts5IterRowid(Fts5IndexIter*);

/*
** Obtain the position list that corresponds to the current position.
*/
const u8 *sqlite3Fts5IterPoslist(Fts5IndexIter*, int *pn);

................................................................................
/* Called during startup to register a UDF with SQLite */
int sqlite3Fts5IndexInit(sqlite3*);

void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz);

int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf);
int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);



/*
** End of interface to code in fts5_index.c.
**************************************************************************/

/**************************************************************************
** Interface to code in fts5_storage.c. fts5_storage.c contains contains 







|
>







 







>
>







184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
...
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
  int flags                       /* Mask of FTS5INDEX_QUERY_X flags */
);

/*
** Docid list iteration.
*/
int  sqlite3Fts5IterEof(Fts5IndexIter*);
void sqlite3Fts5IterNext(Fts5IndexIter*);
void sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
i64  sqlite3Fts5IterRowid(Fts5IndexIter*);

/*
** Obtain the position list that corresponds to the current position.
*/
const u8 *sqlite3Fts5IterPoslist(Fts5IndexIter*, int *pn);

................................................................................
/* Called during startup to register a UDF with SQLite */
int sqlite3Fts5IndexInit(sqlite3*);

void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz);

int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf);
int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);

int sqlite3Fts5IndexReads(Fts5Index *p);

/*
** End of interface to code in fts5_index.c.
**************************************************************************/

/**************************************************************************
** Interface to code in fts5_storage.c. fts5_storage.c contains contains 

Changes to ext/fts5/fts5_expr.c.

579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
...
608
609
610
611
612
613
614
615

616
617
618
619
620
621
622


623
624
625
626
627
628
629
630
631
632
633
634
...
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
  int rc = SQLITE_OK;             /* Return code */
  int i, j;                       /* Phrase and token index, respectively */

  for(i=0; i<pNear->nPhrase; i++){
    Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
    for(j=0; j<pPhrase->nTerm; j++){
      Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
      sqlite3Fts5IterNext(pIter, 0);
      if( sqlite3Fts5IterEof(pIter) ){
        *pbEof = 1;
        return rc;
      }
    }
  }

................................................................................
  int bAsc,                       /* True if iterator is "rowid ASC" */
  i64 *piLast,                    /* IN/OUT: Lastest rowid seen so far */
  int *pRc,                       /* OUT: Error code */
  int *pbEof                      /* OUT: Set to true if EOF */
){
  i64 iLast = *piLast;
  i64 iRowid;
  while( 1 ){

    iRowid = sqlite3Fts5IterRowid(pIter);
    if( (bAsc==0 && iRowid<=iLast) || (bAsc==1 && iRowid>=iLast) ) break;
    sqlite3Fts5IterNext(pIter, 0);
    if( sqlite3Fts5IterEof(pIter) ){
      *pbEof = 1;
      return 1;
    }


  }
  if( iRowid!=iLast ){
    assert( (bAsc==0 && iRowid<iLast) || (bAsc==1 && iRowid>iLast) );
    *piLast = iRowid;
  }

  return 0;
}

/*
** All individual term iterators in pNear are guaranteed to be valid when
** this function is called. This function checks if all term iterators
................................................................................
      }
    }
  }

  return SQLITE_OK;
}

/* fts3ExprNodeNext() calls fts5ExprNodeNextMatch(). And vice-versa. */
static int fts5ExprNodeNextMatch(Fts5Expr*, Fts5ExprNode*);

/*
** Compare the values currently indicated by the two nodes as follows:
**
**    res = (*p1) - (*p2)
**







|







 







<
>
|
|
|




>
>

<
<
|
<







 







|







579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
...
608
609
610
611
612
613
614

615
616
617
618
619
620
621
622
623
624
625


626

627
628
629
630
631
632
633
...
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
  int rc = SQLITE_OK;             /* Return code */
  int i, j;                       /* Phrase and token index, respectively */

  for(i=0; i<pNear->nPhrase; i++){
    Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
    for(j=0; j<pPhrase->nTerm; j++){
      Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
      sqlite3Fts5IterNext(pIter);
      if( sqlite3Fts5IterEof(pIter) ){
        *pbEof = 1;
        return rc;
      }
    }
  }

................................................................................
  int bAsc,                       /* True if iterator is "rowid ASC" */
  i64 *piLast,                    /* IN/OUT: Lastest rowid seen so far */
  int *pRc,                       /* OUT: Error code */
  int *pbEof                      /* OUT: Set to true if EOF */
){
  i64 iLast = *piLast;
  i64 iRowid;


  iRowid = sqlite3Fts5IterRowid(pIter);
  if( (bAsc==0 && iRowid>iLast) || (bAsc && iRowid<iLast) ){
    sqlite3Fts5IterNextFrom(pIter, iLast);
    if( sqlite3Fts5IterEof(pIter) ){
      *pbEof = 1;
      return 1;
    }
    iRowid = sqlite3Fts5IterRowid(pIter);
    assert( (bAsc==0 && iRowid<=iLast) || (bAsc==1 && iRowid>=iLast) );
  }


  *piLast = iRowid;


  return 0;
}

/*
** All individual term iterators in pNear are guaranteed to be valid when
** this function is called. This function checks if all term iterators
................................................................................
      }
    }
  }

  return SQLITE_OK;
}

/* fts5ExprNodeNext() calls fts5ExprNodeNextMatch(). And vice-versa. */
static int fts5ExprNodeNextMatch(Fts5Expr*, Fts5ExprNode*);

/*
** Compare the values currently indicated by the two nodes as follows:
**
**    res = (*p1) - (*p2)
**

Changes to ext/fts5/fts5_index.c.

302
303
304
305
306
307
308

309
310
311
312
313
314
315
...
698
699
700
701
702
703
704

705
706
707
708
709
710
711
....
1662
1663
1664
1665
1666
1667
1668

1669
1670


















1671
1672
1673
1674
1675
1676
1677
....
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769














3770
3771
3772
3773
3774
3775
3776
....
3835
3836
3837
3838
3839
3840
3841
3842








  /* Error state. */
  int rc;                         /* Current error code */

  /* State used by the fts5DataXXX() functions. */
  sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
  sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
  sqlite3_stmt *pDeleter;         /* "DELETE FROM %_data ... id>=? AND id<=?" */

};

struct Fts5DoclistIter {
  int bAsc;
  u8 *a;
  int n;
  int i;
................................................................................
        if( rc!=SQLITE_OK ){
          sqlite3_free(pRet);
          pRet = 0;
        }
      }
    }
    p->rc = rc;

  }

  return pRet;
}

/*
** Retrieve a record from the %_data table.
................................................................................

/*
** Return the rowid of the entry that the iterator currently points
** to. If the iterator points to EOF when this function is called the
** results are undefined.
*/
static i64 fts5MultiIterRowid(Fts5MultiSegIter *pIter){

  return pIter->aSeg[ pIter->aFirst[1] ].iRowid;
}



















/*
** Return a pointer to a buffer containing the term associated with the 
** entry that the iterator currently points to.
*/
static const u8 *fts5MultiIterTerm(Fts5MultiSegIter *pIter, int *pn){
  Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1] ];
................................................................................
    return fts5MultiIterEof(pIter->pIndex, pIter->pMulti);
  }
}

/*
** Move to the next matching rowid. 
*/
void sqlite3Fts5IterNext(Fts5IndexIter *pIter, i64 iMatch){
  if( pIter->pDoclist ){
    fts5DoclistIterNext(pIter->pDoclist);
  }else{
    fts5BufferZero(&pIter->poslist);
    fts5MultiIterNext(pIter->pIndex, pIter->pMulti);
  }
}















/*
** Return the current rowid.
*/
i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){
  if( pIter->pDoclist ){
    return pIter->pDoclist->iRowid;
................................................................................
** Replace the current "averages" record with the contents of the buffer 
** supplied as the second argument.
*/
int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){
  fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData);
  return p->rc;
}
















>







 







>







 







>


>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|







>
>
>
>
>
>
>
>
>
>
>
>
>
>







 








>
>
>
>
>
>
>
>
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
...
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
....
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
....
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
....
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
  /* Error state. */
  int rc;                         /* Current error code */

  /* State used by the fts5DataXXX() functions. */
  sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
  sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
  sqlite3_stmt *pDeleter;         /* "DELETE FROM %_data ... id>=? AND id<=?" */
  int nRead;                      /* Total number of blocks read */
};

struct Fts5DoclistIter {
  int bAsc;
  u8 *a;
  int n;
  int i;
................................................................................
        if( rc!=SQLITE_OK ){
          sqlite3_free(pRet);
          pRet = 0;
        }
      }
    }
    p->rc = rc;
    p->nRead++;
  }

  return pRet;
}

/*
** Retrieve a record from the %_data table.
................................................................................

/*
** Return the rowid of the entry that the iterator currently points
** to. If the iterator points to EOF when this function is called the
** results are undefined.
*/
static i64 fts5MultiIterRowid(Fts5MultiSegIter *pIter){
  assert( pIter->aSeg[ pIter->aFirst[1] ].pLeaf );
  return pIter->aSeg[ pIter->aFirst[1] ].iRowid;
}

/*
** Move the iterator to the next entry at or following iMatch.
*/
static void fts5MultiIterNextFrom(
  Fts5Index *p, 
  Fts5MultiSegIter *pIter, 
  i64 iMatch
){
  while( 1 ){
    i64 iRowid;
    fts5MultiIterNext(p, pIter);
    if( fts5MultiIterEof(p, pIter) ) break;
    iRowid = fts5MultiIterRowid(pIter);
    if( pIter->bRev==0 && iRowid<=iMatch ) break;
    if( pIter->bRev!=0 && iRowid>=iMatch ) break;
  }
}

/*
** Return a pointer to a buffer containing the term associated with the 
** entry that the iterator currently points to.
*/
static const u8 *fts5MultiIterTerm(Fts5MultiSegIter *pIter, int *pn){
  Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1] ];
................................................................................
    return fts5MultiIterEof(pIter->pIndex, pIter->pMulti);
  }
}

/*
** Move to the next matching rowid. 
*/
void sqlite3Fts5IterNext(Fts5IndexIter *pIter){
  if( pIter->pDoclist ){
    fts5DoclistIterNext(pIter->pDoclist);
  }else{
    fts5BufferZero(&pIter->poslist);
    fts5MultiIterNext(pIter->pIndex, pIter->pMulti);
  }
}

/*
** Move to the next matching rowid that occurs at or after iMatch. The
** definition of "at or after" depends on whether this iterator iterates
** in ascending or descending rowid order.
*/
void sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){
  if( pIter->pDoclist ){
    assert( 0 );
    /* fts5DoclistIterNextFrom(pIter->pDoclist, iMatch); */
  }else{
    fts5MultiIterNextFrom(pIter->pIndex, pIter->pMulti, iMatch);
  }
}

/*
** Return the current rowid.
*/
i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){
  if( pIter->pDoclist ){
    return pIter->pDoclist->iRowid;
................................................................................
** Replace the current "averages" record with the contents of the buffer 
** supplied as the second argument.
*/
int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){
  fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData);
  return p->rc;
}

/*
** Return the total number of blocks this module has read from the %_data
** table since it was created.
*/
int sqlite3Fts5IndexReads(Fts5Index *p){
  return p->nRead;
}

Changes to test/fts5aa.test.

280
281
282
283
284
285
286















287
288
289
#
do_catchsql_test 11.1 {
  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank);
} {1 {reserved fts5 column name: rank}}
do_catchsql_test 11.2 {
  CREATE VIRTUAL TABLE rank USING fts5(a, b, c);
} {1 {reserved fts5 table name: rank}}
















finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
#
do_catchsql_test 11.1 {
  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank);
} {1 {reserved fts5 column name: rank}}
do_catchsql_test 11.2 {
  CREATE VIRTUAL TABLE rank USING fts5(a, b, c);
} {1 {reserved fts5 table name: rank}}

#-------------------------------------------------------------------------
#
do_execsql_test 12.1 {
  CREATE VIRTUAL TABLE t2 USING fts5(x,y);
} {}

do_catchsql_test 12.2 {
  SELECT t2 FROM t2 WHERE t2 MATCH '*stuff'
} {1 {unknown special query: stuff}}

do_test 12.3 {
  set res [db one { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }]
  string is integer $res
} {1}

finish_test

Changes to test/fts5ah.test.

45
46
47
48
49
50
51















52
53
54
55
  SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x'
} [lsort -integer -decr $Y]

do_execsql_test 1.3 {
  INSERT INTO t1(t1) VALUES('integrity-check');
}


















finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>




45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
  SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x'
} [lsort -integer -decr $Y]

do_execsql_test 1.3 {
  INSERT INTO t1(t1) VALUES('integrity-check');
}

proc reads {} {
  db one {SELECT t1 FROM t1 WHERE t1 MATCH '*reads'}
}

do_test 1.4 {
  set nRead [reads]
  db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x' }
  set a [expr [reads] - $nRead]
} {}

do_test 1.5 {
  set nRead [reads]
  db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x + w' }
  set a [expr [reads] - $nRead]
} {}


finish_test