/ Check-in [39222761]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Allow the fts5vocab table to optionally provide data on a per-column basis.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 3922276135a7825d0ede8d9c757e9cfe492f803a
User & Date: dan 2015-05-09 18:28:27
Context
2015-05-13
17:15
Change fts5 doclist-index structures to be trees instead of flat lists. This only makes a difference for databases that contain millions of instances of the same token. check-in: aa34bf66 user: dan tags: fts5
2015-05-09
18:28
Allow the fts5vocab table to optionally provide data on a per-column basis. check-in: 39222761 user: dan tags: fts5
2015-05-08
20:21
Add the fts5vocab module, for direct access to the fts5 index. check-in: 6bf93e3b user: dan tags: fts5
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

1666
1667
1668
1669
1670
1671
1672



1673
1674




1675
1676
1677
1678
1679
1680
1681
1682
1683

1684
1685
1686
1687
1688
1689
1690
  }
}


/*
** Given cursor id iId, return a pointer to the corresponding Fts5Index 
** object. Or NULL If the cursor id does not exist.



*/
Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global *pGlobal, i64 iCsrId){




  Fts5Cursor *pCsr;
  Fts5Index *pIndex = 0;

  for(pCsr=pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
    if( pCsr->iCsrId==iCsrId ) break;
  }
  if( pCsr ){
    Fts5Table *pTab = (Fts5Table*)pCsr->base.pVtab;
    pIndex = pTab->pIndex;

  }

  return pIndex;
}

/*
** Return a "position-list blob" corresponding to the current position of







>
>
>

|
>
>
>
>









>







1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
  }
}


/*
** Given cursor id iId, return a pointer to the corresponding Fts5Index 
** object. Or NULL If the cursor id does not exist.
**
** If successful, set *pnCol to the number of indexed columns in the
** table before returning.
*/
Fts5Index *sqlite3Fts5IndexFromCsrid(
  Fts5Global *pGlobal, 
  i64 iCsrId, 
  int *pnCol
){
  Fts5Cursor *pCsr;
  Fts5Index *pIndex = 0;

  for(pCsr=pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
    if( pCsr->iCsrId==iCsrId ) break;
  }
  if( pCsr ){
    Fts5Table *pTab = (Fts5Table*)pCsr->base.pVtab;
    pIndex = pTab->pIndex;
    *pnCol = pTab->pConfig->nCol;
  }

  return pIndex;
}

/*
** Return a "position-list blob" corresponding to the current position of

Changes to ext/fts5/fts5Int.h.

391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
  const char **azArg,
  int nArg,
  Fts5Tokenizer**,
  fts5_tokenizer**,
  char **pzErr
);

Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64);

/*
** End of interface to code in fts5.c.
**************************************************************************/

/**************************************************************************
** Interface to code in fts5_hash.c. 







|







391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
  const char **azArg,
  int nArg,
  Fts5Tokenizer**,
  fts5_tokenizer**,
  char **pzErr
);

Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64, int*);

/*
** End of interface to code in fts5.c.
**************************************************************************/

/**************************************************************************
** Interface to code in fts5_hash.c. 

Changes to ext/fts5/fts5_vocab.c.

6
7
8
9
10
11
12
13


















14
15
16
17
18
19
20
..
23
24
25
26
27
28
29

30
31
32
33
34
35
36
37
38










39
40
41
42

43


































44
45
46
47
48
49
50
..
66
67
68
69
70
71
72



73







74
75
76
77
78
79
80
81
82
83




84
85

86


87
88
89
90
91
92
93

94
95

96



97

98
99
100
101
102

103
104
105
106
107
108
109
...
152
153
154
155
156
157
158


159
160



161
162
163
164
165
166
167
168
169
170
171
172
173

174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191


192







193
194
195
196
197
198
199
200
201
202
...
221
222
223
224
225
226
227

228
229









230
231
232
233
234
235
236
237
238
239



240
241

242
243
244
245


246
247
248


249
250
251
252








253



254
255
256
257
258
259
260








261
262
263
264
265
266
267
...
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an SQLite module implementing full-text search.


















*/

#if defined(SQLITE_ENABLE_FTS5)

#include "fts5Int.h"


................................................................................

struct Fts5VocabTable {
  sqlite3_vtab base;
  char *zFts5Tbl;                 /* Name of fts5 table */
  char *zFts5Db;                  /* Db containing fts5 table */
  sqlite3 *db;                    /* Database handle */
  Fts5Global *pGlobal;            /* FTS5 global object for this database */

};

struct Fts5VocabCursor {
  sqlite3_vtab_cursor base;
  sqlite3_stmt *pStmt;            /* Statement holding lock on pIndex */
  Fts5Index *pIndex;              /* Associated FTS5 index */

  Fts5IndexIter *pIter;           /* Iterator object */
  int bEof;                       /* True if this cursor is at EOF */










  Fts5Buffer term;                /* Current value of 'term' column */
  i64 nRow;                       /* Current value of 'row' column */
  i64 nInst;                      /* Current value of 'inst' column */
  i64 rowid;                      /* Current value of rowid column */

};




































/*
** The xDisconnect() virtual table method.
*/
static int fts5VocabDisconnectMethod(sqlite3_vtab *pVtab){
  Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab;
................................................................................
** methods of the FTS3 virtual table.
**
** The argv[] array contains the following:
**
**   argv[0]   -> module name  ("fts5vocab")
**   argv[1]   -> database name
**   argv[2]   -> table name



**   argv[3]   -> name of fts5 table







*/
static int fts5VocabInitVtab(
  sqlite3 *db,                    /* The SQLite database connection */
  void *pAux,                     /* Pointer to Fts5Global object */
  int argc,                       /* Number of elements in argv array */
  const char * const *argv,       /* xCreate/xConnect argument array */
  sqlite3_vtab **ppVTab,          /* Write the resulting vtab structure here */
  char **pzErr                    /* Write any error message here */
){
  const char *zSchema = "CREATE TABLE vvv(term, row, inst)";




  Fts5VocabTable *pRet = 0;
  int rc = SQLITE_OK;             /* Return code */




  if( argc!=4 ){
    *pzErr = sqlite3_mprintf("wrong number of vtable arguments");
    rc = SQLITE_ERROR;
  }else{
    int nByte;                      /* Bytes of space to allocate */
    const char *zDb = argv[1];
    const char *zTab = argv[3];

    int nDb = strlen(zDb) + 1;
    int nTab = strlen(zTab) + 1;





    rc = sqlite3_declare_vtab(db, zSchema);


    nByte = sizeof(Fts5VocabTable) + nDb + nTab;
    pRet = sqlite3Fts5MallocZero(&rc, nByte);
    if( pRet ){
      pRet->pGlobal = (Fts5Global*)pAux;

      pRet->db = db;
      pRet->zFts5Tbl = (char*)&pRet[1];
      pRet->zFts5Db = &pRet->zFts5Tbl[nTab];
      memcpy(pRet->zFts5Tbl, zTab, nTab);
      memcpy(pRet->zFts5Db, zDb, nDb);
    }
  }
................................................................................
** Implementation of xOpen method.
*/
static int fts5VocabOpenMethod(
  sqlite3_vtab *pVTab, 
  sqlite3_vtab_cursor **ppCsr
){
  Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab;


  Fts5VocabCursor *pCsr;
  int rc = SQLITE_OK;




  pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5VocabCursor));
  if( pCsr ){
    char *zSql = sqlite3_mprintf(
        "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'",
        pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl
    );
    if( zSql==0 ){
      rc = SQLITE_NOMEM;
    }else{
      rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
    }
    sqlite3_free(zSql);

    if( rc==SQLITE_OK && sqlite3_step(pCsr->pStmt)==SQLITE_ROW ){
      i64 iId = sqlite3_column_int64(pCsr->pStmt, 0);
      pCsr->pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId);
    }

    if( rc==SQLITE_OK && pCsr->pIndex==0 ){
      rc = sqlite3_finalize(pCsr->pStmt);
      pCsr->pStmt = 0;
      if( rc==SQLITE_OK ){
        pVTab->zErrMsg = sqlite3_mprintf(
            "no such fts5 table: %Q.%Q", pTab->zFts5Db, pTab->zFts5Tbl
        );
        rc = SQLITE_ERROR;
      }
    }

    if( rc!=SQLITE_OK ){
      sqlite3_free(pCsr);


      pCsr = 0;







    }
  }


  *ppCsr = (sqlite3_vtab_cursor*)pCsr;
  return rc;
}

static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){
  pCsr->rowid = 0;
................................................................................


/*
** Advance the cursor to the next row in the table.
*/
static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
  Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;

  int rc = SQLITE_OK;










  if( sqlite3Fts5IterEof(pCsr->pIter) ){
    pCsr->bEof = 1;
  }else{
    const char *zTerm;
    int nTerm;

    zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
    sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
    pCsr->nInst = 0;
    pCsr->nRow = 0;



    pCsr->rowid++;


    while( 1 ){
      const u8 *pPos; int nPos;   /* Position list */
      i64 dummy = 0;
      int iOff = 0;



      rc = sqlite3Fts5IterPoslist(pCsr->pIter, &pPos, &nPos);
      if( rc!=SQLITE_OK ) break;


      while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &dummy) ){
        pCsr->nInst++;
      }
      pCsr->nRow++;












      rc = sqlite3Fts5IterNextScan(pCsr->pIter);
      if( rc!=SQLITE_OK ) break;
      zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
      if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ) break;
      if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
    }
  }








  return rc;
}

/*
** This is the xFilter implementation for the virtual table.
*/
static int fts5VocabFilterMethod(
................................................................................
  switch( iCol ){
    case 0: /* term */
      sqlite3_result_text(
          pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT
      );
      break;

    case 1: /* row */
      sqlite3_result_int64(pCtx, pCsr->nRow);
      break;

    case 2: /* inst */
      sqlite3_result_int64(pCtx, pCsr->nInst);
      break;

    default:
      assert( 0 );
  }
  return SQLITE_OK;
}

/* 
** This is the xRowid method. The SQLite core calls this routine to
** retrieve the rowid for the current row of the result set. fts5







|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>







<

>
>
>
>
>
>
>
>
>
>

<
<
<
>

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>

>
>
>
>
>
>
>









|
>
>
>
>


>

>
>
|




|
|
>
|
|
>
|
>
>
>
|
>





>







 







>
>
|

>
>
>

<
<
|
|
|
|
|
|
|
|
|
|
>
|
|
|
|

|
|
|
|
|
|
|
|
|
|

<
<
>
>
|
>
>
>
>
>
>
>
|
<
<







 







>


>
>
>
>
>
>
>
>
>
|
|
|
|
|

|
|
<
<
>
>
>
|

>
|
|
<
<
>
>

|
|
>
>
|
|
|
|
>
>
>
>
>
>
>
>
|
>
>
>
|
|
|
|
|
|
|
>
>
>
>
>
>
>
>







 







|
|
<
<
<
|

<
<
<







6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
..
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

56
57
58
59
60
61
62
63
64
65
66
67



68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
...
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
...
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250


251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277


278
279
280
281
282
283
284
285
286
287
288


289
290
291
292
293
294
295
...
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340


341
342
343
344
345
346
347
348


349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
...
428
429
430
431
432
433
434
435
436



437
438



439
440
441
442
443
444
445
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an SQLite virtual table module implementing direct access to an
** existing FTS5 index. The module may create several different types of 
** tables:
**
** col:
**     CREATE TABLE vocab(term, col, doc, cnt, PRIMARY KEY(term, col));
**
**   One row for each term/column combination. The value of $doc is set to
**   the number of fts5 rows that contain at least one instance of term
**   $term within column $col. Field $cnt is set to the total number of 
**   instances of term $term in column $col (in any row of the fts5 table). 
**
** row:
**     CREATE TABLE vocab(term, doc, cnt, PRIMARY KEY(term));
**
**   One row for each term in the database. The value of $doc is set to
**   the number of fts5 rows that contain at least one instance of term
**   $term. Field $cnt is set to the total number of instances of term 
**   $term in the database.
*/

#if defined(SQLITE_ENABLE_FTS5)

#include "fts5Int.h"


................................................................................

struct Fts5VocabTable {
  sqlite3_vtab base;
  char *zFts5Tbl;                 /* Name of fts5 table */
  char *zFts5Db;                  /* Db containing fts5 table */
  sqlite3 *db;                    /* Database handle */
  Fts5Global *pGlobal;            /* FTS5 global object for this database */
  int eType;                      /* FTS5_VOCAB_COL or ROW */
};

struct Fts5VocabCursor {
  sqlite3_vtab_cursor base;
  sqlite3_stmt *pStmt;            /* Statement holding lock on pIndex */
  Fts5Index *pIndex;              /* Associated FTS5 index */


  int bEof;                       /* True if this cursor is at EOF */
  Fts5IndexIter *pIter;           /* Term/rowid iterator object */

  /* These are used by 'col' tables only */
  int nCol;
  int iCol;
  i64 *aCnt;
  i64 *aDoc;

  /* Output values */
  i64 rowid;                      /* This table's current rowid value */
  Fts5Buffer term;                /* Current value of 'term' column */



  i64 aVal[3];                    /* Up to three columns left of 'term' */
};

#define FTS5_VOCAB_COL    0
#define FTS5_VOCAB_ROW    1

#define FTS5_VOCAB_COL_SCHEMA  "term, col, doc, cnt"
#define FTS5_VOCAB_ROW_SCHEMA  "term, doc, cnt"

/*
** Translate a string containing an fts5vocab table type to an 
** FTS5_VOCAB_XXX constant. If successful, set *peType to the output
** value and return SQLITE_OK. Otherwise, set *pzErr to an error message
** and return SQLITE_ERROR.
*/
static int fts5VocabTableType(const char *zType, char **pzErr, int *peType){
  int rc = SQLITE_OK;
  char *zCopy = sqlite3Fts5Strndup(&rc, zType, -1);
  if( rc==SQLITE_OK ){
    sqlite3Fts5Dequote(zCopy);
    if( sqlite3_stricmp(zCopy, "col")==0 ){
      *peType = FTS5_VOCAB_COL;
    }else

    if( sqlite3_stricmp(zCopy, "row")==0 ){
      *peType = FTS5_VOCAB_ROW;
    }else
    {
      *pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy);
      rc = SQLITE_ERROR;
    }
    sqlite3_free(zCopy);
  }

  return rc;
}


/*
** The xDisconnect() virtual table method.
*/
static int fts5VocabDisconnectMethod(sqlite3_vtab *pVtab){
  Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab;
................................................................................
** methods of the FTS3 virtual table.
**
** The argv[] array contains the following:
**
**   argv[0]   -> module name  ("fts5vocab")
**   argv[1]   -> database name
**   argv[2]   -> table name
**
** then:
**
**   argv[3]   -> name of fts5 table
**   argv[4]   -> type of fts5vocab table
**
** or, for tables in the TEMP schema only.
**
**   argv[3]   -> name of fts5 tables database
**   argv[4]   -> name of fts5 table
**   argv[5]   -> type of fts5vocab table
*/
static int fts5VocabInitVtab(
  sqlite3 *db,                    /* The SQLite database connection */
  void *pAux,                     /* Pointer to Fts5Global object */
  int argc,                       /* Number of elements in argv array */
  const char * const *argv,       /* xCreate/xConnect argument array */
  sqlite3_vtab **ppVTab,          /* Write the resulting vtab structure here */
  char **pzErr                    /* Write any error message here */
){
  const char *azSchema[] = { 
    "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA  ")", 
    "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA  ")"
  };

  Fts5VocabTable *pRet = 0;
  int rc = SQLITE_OK;             /* Return code */
  int bDb;

  bDb = (argc==6 && strlen(argv[1])==4 && memcmp("temp", argv[1], 4)==0);

  if( argc!=5 && bDb==0 ){
    *pzErr = sqlite3_mprintf("wrong number of vtable arguments");
    rc = SQLITE_ERROR;
  }else{
    int nByte;                      /* Bytes of space to allocate */
    const char *zDb = bDb ? argv[3] : argv[1];
    const char *zTab = bDb ? argv[4] : argv[3];
    const char *zType = bDb ? argv[5] : argv[4];
    int nDb = strlen(zDb)+1; 
    int nTab = strlen(zTab)+1;
    int eType;
    
    rc = fts5VocabTableType(zType, pzErr, &eType);
    if( rc==SQLITE_OK ){
      assert( eType>=0 && eType<sizeof(azSchema)/sizeof(azSchema[0]) );
      rc = sqlite3_declare_vtab(db, azSchema[eType]);
    }

    nByte = sizeof(Fts5VocabTable) + nDb + nTab;
    pRet = sqlite3Fts5MallocZero(&rc, nByte);
    if( pRet ){
      pRet->pGlobal = (Fts5Global*)pAux;
      pRet->eType = eType;
      pRet->db = db;
      pRet->zFts5Tbl = (char*)&pRet[1];
      pRet->zFts5Db = &pRet->zFts5Tbl[nTab];
      memcpy(pRet->zFts5Tbl, zTab, nTab);
      memcpy(pRet->zFts5Db, zDb, nDb);
    }
  }
................................................................................
** Implementation of xOpen method.
*/
static int fts5VocabOpenMethod(
  sqlite3_vtab *pVTab, 
  sqlite3_vtab_cursor **ppCsr
){
  Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab;
  Fts5Index *pIndex = 0;
  int nCol = 0;
  Fts5VocabCursor *pCsr = 0;
  int rc = SQLITE_OK;
  sqlite3_stmt *pStmt = 0;
  char *zSql = 0;
  int nByte;



  zSql = sqlite3_mprintf(
      "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'",
      pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl
  );
  if( zSql==0 ){
    rc = SQLITE_NOMEM;
  }else{
    rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0);
  }
  sqlite3_free(zSql);

  if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
    i64 iId = sqlite3_column_int64(pStmt, 0);
    pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId, &nCol);
  }

  if( rc==SQLITE_OK && pIndex==0 ){
    rc = sqlite3_finalize(pStmt);
    pStmt = 0;
    if( rc==SQLITE_OK ){
      pVTab->zErrMsg = sqlite3_mprintf(
          "no such fts5 table: %Q.%Q", pTab->zFts5Db, pTab->zFts5Tbl
          );
      rc = SQLITE_ERROR;
    }
  }



  nByte = nCol * sizeof(i64) * 2 + sizeof(Fts5VocabCursor);
  pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte);
  if( pCsr ){
    pCsr->pIndex = pIndex;
    pCsr->pStmt = pStmt;
    pCsr->nCol = nCol;
    pCsr->aCnt = (i64*)&pCsr[1];
    pCsr->aDoc = &pCsr->aCnt[nCol];
  }else{
    sqlite3_finalize(pStmt);
  }



  *ppCsr = (sqlite3_vtab_cursor*)pCsr;
  return rc;
}

static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){
  pCsr->rowid = 0;
................................................................................


/*
** Advance the cursor to the next row in the table.
*/
static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
  Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
  Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
  int rc = SQLITE_OK;

  pCsr->rowid++;

  if( pTab->eType==FTS5_VOCAB_COL ){
    for(pCsr->iCol++; pCsr->iCol<pCsr->nCol; pCsr->iCol++){
      if( pCsr->aCnt[pCsr->iCol] ) break;
    }
  }

  if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=pCsr->nCol ){
    if( sqlite3Fts5IterEof(pCsr->pIter) ){
      pCsr->bEof = 1;
    }else{
      const char *zTerm;
      int nTerm;

      zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
      sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);


      memset(pCsr->aVal, 0, sizeof(pCsr->aVal));
      memset(pCsr->aCnt, 0, pCsr->nCol * sizeof(i64));
      memset(pCsr->aDoc, 0, pCsr->nCol * sizeof(i64));
      pCsr->iCol = 0;

      assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
      while( 1 ){
        const u8 *pPos; int nPos;   /* Position list */


        i64 iPos = 0;               /* 64-bit position read from poslist */
        int iOff = 0;               /* Current offset within position list */

        rc = sqlite3Fts5IterPoslist(pCsr->pIter, &pPos, &nPos);
        if( rc!=SQLITE_OK ) break;

        if( pTab->eType==FTS5_VOCAB_ROW ){
          while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
            pCsr->aVal[1]++;
          }
          pCsr->aVal[0]++;
        }else{
          int iCol = -1;
          while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
            int ii = FTS5_POS2COLUMN(iPos);
            pCsr->aCnt[ii]++;
            if( iCol!=ii ){
              pCsr->aDoc[ii]++;
              iCol = ii;
            }
          }
        }

        rc = sqlite3Fts5IterNextScan(pCsr->pIter);
        if( rc!=SQLITE_OK ) break;
        zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
        if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ) break;
        if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
      }
    }
  }

  if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
    while( pCsr->aCnt[pCsr->iCol]==0 ) pCsr->iCol++;
    pCsr->aVal[0] = pCsr->iCol;
    pCsr->aVal[1] = pCsr->aDoc[pCsr->iCol];
    pCsr->aVal[2] = pCsr->aCnt[pCsr->iCol];
  }
  return rc;
}

/*
** This is the xFilter implementation for the virtual table.
*/
static int fts5VocabFilterMethod(
................................................................................
  switch( iCol ){
    case 0: /* term */
      sqlite3_result_text(
          pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT
      );
      break;

    default:
      assert( iCol<4 && iCol>0 );



      sqlite3_result_int64(pCtx, pCsr->aVal[iCol-1]);
      break;



  }
  return SQLITE_OK;
}

/* 
** This is the xRowid method. The SQLite core calls this routine to
** retrieve the rowid for the current row of the result set. fts5

Changes to ext/fts5/test/fts5vocab.test.

12
13
14
15
16
17
18
19
20
21
22
23
24

25









26
27
28
29

30
31
32
33
34
35
36
37
38
39




40
41
42
43
44
45
46





47
48
49
50
51

52























53




















54
55
# The tests in this file focus on testing the fts5vocab module.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5vocab


do_execsql_test 1.1 {
  CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1);
  CREATE VIRTUAL TABLE v1 USING fts5vocab(t1);
  PRAGMA table_info = v1;
} {
  0 term {} 0 {} 0

  1 row {} 0 {} 0









  2 inst {} 0 {} 0
}

do_execsql_test 1.2 { SELECT * FROM v1 } { }


do_execsql_test 1.3 {
  INSERT INTO t1 VALUES('x y z');
  INSERT INTO t1 VALUES('x x x');
}

do_execsql_test 1.4 {
  SELECT * FROM v1;
} {x 2 4  y 1 1  z 1 1}





do_execsql_test 1.5 {
  BEGIN;
    INSERT INTO t1 VALUES('a b c');
    SELECT * FROM v1 WHERE term<'d';
  COMMIT;
} {a 1 1   b 1 1   c 1 1}






do_execsql_test 1.6 {
  DELETE FROM t1 WHERE one = 'a b c';
  SELECT * FROM v1;
} {x 2 4  y 1 1  z 1 1}















































finish_test








|

|



>
|
>
>
>
>
>
>
>
>
>
|


|
>






|



>
>
>
>
|



<


>
>
>
>
>





>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
# The tests in this file focus on testing the fts5vocab module.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5vocab


do_execsql_test 1.1.1 {
  CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1);
  CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row');
  PRAGMA table_info = v1;
} {
  0 term {} 0 {} 0
  1 doc {} 0 {} 0
  2 cnt {} 0 {} 0
}

do_execsql_test 1.1.2 {
  CREATE VIRTUAL TABLE v2 USING fts5vocab(t1, 'col');
  PRAGMA table_info = v2;
} {
  0 term {} 0 {} 0
  1 col {} 0 {} 0
  2 doc {} 0 {} 0
  3 cnt {} 0 {} 0
}

do_execsql_test 1.2.1 { SELECT * FROM v1 } { }
do_execsql_test 1.2.2 { SELECT * FROM v2 } { }

do_execsql_test 1.3 {
  INSERT INTO t1 VALUES('x y z');
  INSERT INTO t1 VALUES('x x x');
}

do_execsql_test 1.4.1 {
  SELECT * FROM v1;
} {x 2 4  y 1 1  z 1 1}

do_execsql_test 1.4.2 {
  SELECT * FROM v2;
} {x 0 2 4  y 0 1 1  z 0 1 1}

do_execsql_test 1.5.1 {
  BEGIN;
    INSERT INTO t1 VALUES('a b c');
    SELECT * FROM v1 WHERE term<'d';

} {a 1 1   b 1 1   c 1 1}

do_execsql_test 1.5.2 {
    SELECT * FROM v2 WHERE term<'d';
  COMMIT;
} {a 0 1 1  b 0 1 1  c 0 1 1}

do_execsql_test 1.6 {
  DELETE FROM t1 WHERE one = 'a b c';
  SELECT * FROM v1;
} {x 2 4  y 1 1  z 1 1}

#-------------------------------------------------------------------------
#
do_execsql_test 2.0 {
  CREATE VIRTUAL TABLE tt USING fts5(a, b);
  INSERT INTO tt VALUES('d g b f d f', 'f c e c d a');
  INSERT INTO tt VALUES('f a e a a b', 'e d c f d d');
  INSERT INTO tt VALUES('b c a a a b', 'f f c c b c');
  INSERT INTO tt VALUES('f d c a c e', 'd g d e g d');
  INSERT INTO tt VALUES('g d e f a g x', 'f f d a a b');
  INSERT INTO tt VALUES('g c f b c g', 'a g f d c b');
  INSERT INTO tt VALUES('c e c f g b', 'f e d b g a');
  INSERT INTO tt VALUES('g d e f d e', 'a c d b a g');
  INSERT INTO tt VALUES('e f a c c b', 'b f e a f d y');
  INSERT INTO tt VALUES('c c a a c f', 'd g a e b g');
  CREATE VIRTUAL TABLE tv USING fts5vocab(tt, 'col');
  SELECT * FROM tv;
} {
  a 0 6 11    a 1 7 9
  b 0 6 7     b 1 7 7 
  c 0 6 12    c 1 5 8 
  d 0 4 6     d 1 9 13 
  e 0 6 7     e 1 6 6 
  f 0 9 10    f 1 7 10 
  g 0 5 7     g 1 5 7
  x 0 1 1     y 1 1 1
}

do_execsql_test 2.1 {
  CREATE VIRTUAL TABLE temp.tv2 USING fts5vocab(main, tt, 'row');
  SELECT * FROM tv2;
} {
  a 10 20   b 9 14   c 9 20   d 9 19   
  e 8 13   f 10 20   g 7 14   x 1 1   
  y 1 1
}

#-------------------------------------------------------------------------
#
foreach {tn sql} {
  1 { CREATE VIRTUAL TABLE aa USING fts5vocab() }
  2 { CREATE VIRTUAL TABLE aa USING fts5vocab(x) }
  3 { CREATE VIRTUAL TABLE aa USING fts5vocab(x,y,z) }
  4 { CREATE VIRTUAL TABLE temp.aa USING fts5vocab(x,y,z,y) }
} {
  do_catchsql_test 3.$tn $sql {1 {wrong number of vtable arguments}}
}
finish_test