/ Check-in [8c6b0aff]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the xTokenize extension API.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 8c6b0aff3443fae4b7f0b9adcbf1514992b70653
User & Date: dan 2014-07-21 14:22:38
Context
2014-07-21
15:45
Fix DELETE and UPDATE operations on fts5 tables. check-in: d44d3a85 user: dan tags: fts5
14:22
Add the xTokenize extension API. check-in: 8c6b0aff user: dan tags: fts5
11:44
Fix the xColumnSize() extension API. check-in: 19504c41 user: dan tags: fts5
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

626
627
628
629
630
631
632
633
634

635
636
637
638
639
640
641

static int fts5ApiTokenize(
  Fts5Context *pCtx, 
  const char *pText, int nText, 
  void *pUserData,
  int (*xToken)(void*, const char*, int, int, int, int)
){
  assert( 0 );
  return SQLITE_OK;

}

static int fts5ApiPhraseCount(Fts5Context *pCtx){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  return sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
}








|
|
>







626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642

static int fts5ApiTokenize(
  Fts5Context *pCtx, 
  const char *pText, int nText, 
  void *pUserData,
  int (*xToken)(void*, const char*, int, int, int, int)
){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  return sqlite3Fts5Tokenize(pTab->pConfig, pText, nText, pUserData, xToken);
}

static int fts5ApiPhraseCount(Fts5Context *pCtx){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  return sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
}

Changes to ext/fts5/fts5.h.

64
65
66
67
68
69
70



71
72
73
74
75
76
77
**   are numbered starting from zero.
**
** xRowid:
**   Returns the rowid of the current row.
**
** xPoslist:
**   Iterate through instances of phrase iPhrase in the current row. 



*/
struct Fts5ExtensionApi {
  int iVersion;                   /* Currently always set to 1 */

  void *(*xUserData)(Fts5Context*);

  int (*xColumnCount)(Fts5Context*);







>
>
>







64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
**   are numbered starting from zero.
**
** xRowid:
**   Returns the rowid of the current row.
**
** xPoslist:
**   Iterate through instances of phrase iPhrase in the current row. 
**
** xTokenize:
**   Tokenize text using the tokenizer belonging to the FTS5 table.
*/
struct Fts5ExtensionApi {
  int iVersion;                   /* Currently always set to 1 */

  void *(*xUserData)(Fts5Context*);

  int (*xColumnCount)(Fts5Context*);

Changes to ext/fts5/fts5_aux.c.

18
19
20
21
22
23
24


















25
26
27
28
29
30
31
...
162
163
164
165
166
167
168



























169
170
171
172
173
174
175
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
  sqlite3_context *pCtx,          /* Context for returning result/error */
  int nVal,                       /* Number of values in apVal[] array */
  sqlite3_value **apVal           /* Array of trailing arguments */
){
  assert( 0 );
}



















static void fts5TestFunction(
  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
  sqlite3_context *pCtx,          /* Context for returning result/error */
  int nVal,                       /* Number of values in apVal[] array */
  sqlite3_value **apVal           /* Array of trailing arguments */
................................................................................
  if( zReq==0 ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, " rowid ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowid") ){
    iRowid = pApi->xRowid(pFts);
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", iRowid);
  }




























  if( rc==SQLITE_OK ){
    sqlite3_result_text(pCtx, (const char*)s.p, -1, SQLITE_TRANSIENT);
  }else{
    sqlite3_result_error_code(pCtx, rc);
  }
  sqlite3Fts5BufferFree(&s);







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
...
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
  sqlite3_context *pCtx,          /* Context for returning result/error */
  int nVal,                       /* Number of values in apVal[] array */
  sqlite3_value **apVal           /* Array of trailing arguments */
){
  assert( 0 );
}

static int fts5TestCallback(
  void *pContext,                 /* Pointer to Fts5Buffer object */
  const char *pToken,             /* Buffer containing token */
  int nToken,                     /* Size of token in bytes */
  int iStart,                     /* Start offset of token */
  int iEnd,                       /* End offset of token */
  int iPos                        /* Position offset of token */
){
  int rc = SQLITE_OK;
  Fts5Buffer *pBuf = (Fts5Buffer*)pContext;
  if( pBuf->n!=0 ){
    sqlite3Fts5BufferAppendString(&rc, pBuf, " ");
  }
  sqlite3Fts5BufferAppendListElem(&rc, pBuf, pToken, nToken);
  return rc;
}


static void fts5TestFunction(
  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
  sqlite3_context *pCtx,          /* Context for returning result/error */
  int nVal,                       /* Number of values in apVal[] array */
  sqlite3_value **apVal           /* Array of trailing arguments */
................................................................................
  if( zReq==0 ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, " rowid ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowid") ){
    iRowid = pApi->xRowid(pFts);
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", iRowid);
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, " tokenize ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "tokenize") ){
    Fts5Buffer buf;
    memset(&buf, 0, sizeof(buf));
    for(i=0; rc==SQLITE_OK && i<nCol; i++){
      const char *z;
      int n;
      rc = pApi->xColumnText(pFts, i, &z, &n);
      if( rc==SQLITE_OK ){
        Fts5Buffer buf1;
        memset(&buf1, 0, sizeof(Fts5Buffer));
        rc = pApi->xTokenize(pFts, z, n, (void*)&buf1, fts5TestCallback);
        if( i!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &buf, " ");
        sqlite3Fts5BufferAppendListElem(&rc, &buf, (const char*)buf1.p, buf1.n);
        sqlite3_free(buf1.p);
      }
    }
    if( zReq==0 ){
      sqlite3Fts5BufferAppendListElem(&rc, &s, (const char*)buf.p, buf.n);
    }else{
      sqlite3Fts5BufferAppendString(&rc, &s, (const char*)buf.p);
    }
    sqlite3_free(buf.p);
  }

  if( rc==SQLITE_OK ){
    sqlite3_result_text(pCtx, (const char*)s.p, -1, SQLITE_TRANSIENT);
  }else{
    sqlite3_result_error_code(pCtx, rc);
  }
  sqlite3Fts5BufferFree(&s);

Changes to test/fts5ae.test.

171
172
173
174
175
176
177


178













179
180
181
  ORDER BY rowid DESC;
} {
  3 {2 3}
  2 {2 3}
  1 {2 3}
}


















finish_test








>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>



171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
  ORDER BY rowid DESC;
} {
  3 {2 3}
  2 {2 3}
  1 {2 3}
}

#-------------------------------------------------------------------------
# Test the xTokenize() API
#
reset_db
do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE t6 USING fts5(x, y);
  INSERT INTO t6 VALUES('There are more', 'things in heaven and earth');
  INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.');
}

do_execsql_test 6.2 {
  SELECT rowid, fts5_test(t6, 'tokenize') FROM t6 WHERE t6 MATCH 't*'
} {
  2 {{horatio than are} {dreamt of in your philosophy}}
  1 {{there are more} {things in heaven and earth}}
}

finish_test