Index: ext/fts5/extract_api_docs.tcl
==================================================================
--- ext/fts5/extract_api_docs.tcl
+++ ext/fts5/extract_api_docs.tcl
@@ -12,10 +12,21 @@
#
# This script extracts the documentation for the API used by fts5 auxiliary
# functions from header file fts5.h. It outputs html text on stdout that
# is included in the documentation on the web.
#
+
+set ::fts5_docs_output ""
+if {[info commands hd_putsnl]==""} {
+ proc output {text} {
+ puts $text
+ }
+} else {
+ proc output {text} {
+ append ::fts5_docs_output $text
+ }
+}
set input_file [file join [file dir [info script]] fts5.h]
set fd [open $input_file]
set data [read $fd]
close $fd
@@ -100,32 +111,32 @@
foreach {hdr docs} $D {
if {[info exists M($hdr)]} {
set hdr $M($hdr)
}
- puts "
$hdr
"
+ output " $hdr
"
set mode ""
set bEmpty 1
foreach line [split [string trim $docs] "\n"] {
if {[string trim $line]==""} {
- if {$mode != ""} {puts "$mode>"}
+ if {$mode != ""} {output "$mode>"}
set mode ""
} elseif {$mode == ""} {
if {[regexp {^ } $line]} {
- set mode code
+ set mode codeblock
} else {
set mode p
}
- puts "<$mode>"
+ output "<$mode>"
}
- puts $line
+ output $line
}
- if {$mode != ""} {puts "$mode>"}
+ if {$mode != ""} {output "$mode>"}
}
-
+set ::fts5_docs_output
Index: ext/fts5/fts5.c
==================================================================
--- ext/fts5/fts5.c
+++ ext/fts5/fts5.c
@@ -21,10 +21,12 @@
typedef struct Fts5Cursor Fts5Cursor;
typedef struct Fts5Global Fts5Global;
typedef struct Fts5Auxiliary Fts5Auxiliary;
typedef struct Fts5Auxdata Fts5Auxdata;
+typedef struct Fts5TokenizerModule Fts5TokenizerModule;
+
/*
** NOTES ON TRANSACTIONS:
**
** SQLite invokes the following virtual table methods as transactions are
** opened and closed by the user:
@@ -63,13 +65,15 @@
** A single object of this type is allocated when the FTS5 module is
** registered with a database handle. It is used to store pointers to
** all registered FTS5 extensions - tokenizers and auxiliary functions.
*/
struct Fts5Global {
+ fts5_api api; /* User visible part of object (see fts5.h) */
sqlite3 *db; /* Associated database connection */
i64 iNextId; /* Used to allocate unique cursor ids */
Fts5Auxiliary *pAux; /* First in list of all aux. functions */
+ Fts5TokenizerModule *pTok; /* First in list of all tokenizer modules */
Fts5Cursor *pCsr; /* First in list of all open cursors */
};
/*
** Each auxiliary function registered with the FTS5 module is represented
@@ -82,10 +86,23 @@
void *pUserData; /* User-data pointer */
fts5_extension_function xFunc; /* Callback function */
void (*xDestroy)(void*); /* Destructor function */
Fts5Auxiliary *pNext; /* Next registered auxiliary function */
};
+
+/*
+** Each tokenizer module registered with the FTS5 module is represented
+** by an object of the following type. All such objects are stored as part
+** of the Fts5Global.pTok list.
+*/
+struct Fts5TokenizerModule {
+ char *zName; /* Name of tokenizer */
+ void *pUserData; /* User pointer passed to xCreate() */
+ fts5_tokenizer x; /* Tokenizer functions */
+ void (*xDestroy)(void*); /* Destructor function */
+ Fts5TokenizerModule *pNext; /* Next registered tokenizer module */
+};
/*
** Virtual-table object.
*/
struct Fts5Table {
@@ -279,16 +296,18 @@
int argc, /* Number of elements in argv array */
const char * const *argv, /* xCreate/xConnect argument array */
sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
char **pzErr /* Write any error message here */
){
+ Fts5Global *pGlobal = (Fts5Global*)pAux;
+ const char **azConfig = (const char**)argv;
int rc; /* Return code */
Fts5Config *pConfig; /* Results of parsing argc/argv */
Fts5Table *pTab = 0; /* New virtual table object */
/* Parse the arguments */
- rc = sqlite3Fts5ConfigParse(db, argc, (const char**)argv, &pConfig, pzErr);
+ rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr);
assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 );
/* Allocate the new vtab object */
if( rc==SQLITE_OK ){
pTab = (Fts5Table*)sqlite3_malloc(sizeof(Fts5Table));
@@ -295,11 +314,11 @@
if( pTab==0 ){
rc = SQLITE_NOMEM;
}else{
memset(pTab, 0, sizeof(Fts5Table));
pTab->pConfig = pConfig;
- pTab->pGlobal = (Fts5Global*)pAux;
+ pTab->pGlobal = pGlobal;
}
}
/* Open the index sub-system */
if( rc==SQLITE_OK ){
@@ -855,10 +874,14 @@
** INSERT INTO fts(fts) VALUES($pVal)
**
** Argument pVal is the value assigned to column "fts" by the INSERT
** statement. This function returns SQLITE_OK if successful, or an SQLite
** error code if an error occurs.
+**
+** The commands implemented by this function are documented in the "Special
+** INSERT Directives" section of the documentation. It should be updated if
+** more commands are added to this function.
*/
static int fts5SpecialCommand(Fts5Table *pTab, sqlite3_value *pVal){
const char *z = (const char*)sqlite3_value_text(pVal);
int n = sqlite3_value_bytes(pVal);
int rc = SQLITE_ERROR;
@@ -1385,17 +1408,18 @@
}
/*
** Register a new auxiliary function with global context pGlobal.
*/
-int sqlite3Fts5CreateAux(
- Fts5Global *pGlobal, /* Global context (one per db handle) */
+static int fts5CreateAux(
+ fts5_api *pApi, /* Global context (one per db handle) */
const char *zName, /* Name of new function */
void *pUserData, /* User data for aux. function */
fts5_extension_function xFunc, /* Aux. function implementation */
void(*xDestroy)(void*) /* Destructor for pUserData */
){
+ Fts5Global *pGlobal = (Fts5Global*)pApi;
int rc = sqlite3_overload_function(pGlobal->db, zName, -1);
if( rc==SQLITE_OK ){
Fts5Auxiliary *pAux;
int nByte; /* Bytes of space to allocate */
@@ -1417,24 +1441,135 @@
}
return rc;
}
-static void fts5ModuleDestroy(void *pCtx){
- Fts5Auxiliary *pAux;
- Fts5Auxiliary *pNext;
- Fts5Global *pGlobal = (Fts5Global*)pCtx;
- for(pAux=pGlobal->pAux; pAux; pAux=pNext){
- pNext = pAux->pNext;
- if( pAux->xDestroy ){
- pAux->xDestroy(pAux->pUserData);
- }
+/*
+** Register a new tokenizer. This is the implementation of the
+** fts5_api.xCreateTokenizer() method.
+*/
+static int fts5CreateTokenizer(
+ fts5_api *pApi, /* Global context (one per db handle) */
+ const char *zName, /* Name of new function */
+ void *pUserData, /* User data for aux. function */
+ fts5_tokenizer *pTokenizer, /* Tokenizer implementation */
+ void(*xDestroy)(void*) /* Destructor for pUserData */
+){
+ Fts5Global *pGlobal = (Fts5Global*)pApi;
+ Fts5TokenizerModule *pNew;
+ int nByte; /* Bytes of space to allocate */
+ int rc = SQLITE_OK;
+
+ nByte = sizeof(Fts5TokenizerModule) + strlen(zName) + 1;
+ pNew = (Fts5TokenizerModule*)sqlite3_malloc(nByte);
+ if( pNew ){
+ memset(pNew, 0, nByte);
+ pNew->zName = (char*)&pNew[1];
+ strcpy(pNew->zName, zName);
+ pNew->pUserData = pUserData;
+ pNew->x = *pTokenizer;
+ pNew->xDestroy = xDestroy;
+ pNew->pNext = pGlobal->pTok;
+ pGlobal->pTok = pNew;
+ }else{
+ rc = SQLITE_NOMEM;
+ }
+
+ return rc;
+}
+
+/*
+** Find a tokenizer. This is the implementation of the
+** fts5_api.xFindTokenizer() method.
+*/
+static int fts5FindTokenizer(
+ fts5_api *pApi, /* Global context (one per db handle) */
+ const char *zName, /* Name of new function */
+ fts5_tokenizer *pTokenizer /* Populate this object */
+){
+ Fts5Global *pGlobal = (Fts5Global*)pApi;
+ int rc = SQLITE_OK;
+ Fts5TokenizerModule *pTok;
+
+ for(pTok=pGlobal->pTok; pTok; pTok=pTok->pNext){
+ if( sqlite3_stricmp(zName, pTok->zName)==0 ) break;
+ }
+
+ if( pTok ){
+ *pTokenizer = pTok->x;
+ }else{
+ memset(pTokenizer, 0, sizeof(fts5_tokenizer));
+ rc = SQLITE_ERROR;
+ }
+
+ return rc;
+}
+
+int sqlite3Fts5GetTokenizer(
+ Fts5Global *pGlobal,
+ const char **azArg,
+ int nArg,
+ Fts5Tokenizer **ppTok,
+ fts5_tokenizer **ppTokApi
+){
+ Fts5TokenizerModule *pMod = 0;
+ int rc = SQLITE_OK;
+ if( nArg==0 ){
+ pMod = pGlobal->pTok;
+ }else{
+ for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){
+ if( sqlite3_stricmp(azArg[0], pMod->zName)==0 ) break;
+ }
+ }
+
+ if( pMod==0 ){
+ rc = SQLITE_ERROR;
+ }else{
+ rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok);
+ *ppTokApi = &pMod->x;
+ }
+
+ if( rc!=SQLITE_OK ){
+ *ppTokApi = 0;
+ *ppTok = 0;
+ }
+
+ return rc;
+}
+
+static void fts5ModuleDestroy(void *pCtx){
+ Fts5TokenizerModule *pTok, *pNextTok;
+ Fts5Auxiliary *pAux, *pNextAux;
+ Fts5Global *pGlobal = (Fts5Global*)pCtx;
+
+ for(pAux=pGlobal->pAux; pAux; pAux=pNextAux){
+ pNextAux = pAux->pNext;
+ if( pAux->xDestroy ) pAux->xDestroy(pAux->pUserData);
sqlite3_free(pAux);
}
+
+ for(pTok=pGlobal->pTok; pTok; pTok=pNextTok){
+ pNextTok = pTok->pNext;
+ if( pTok->xDestroy ) pTok->xDestroy(pTok->pUserData);
+ sqlite3_free(pTok);
+ }
+
sqlite3_free(pGlobal);
}
+static void fts5Fts5Func(
+ sqlite3_context *pCtx, /* Function call context */
+ int nArg, /* Number of args */
+ sqlite3_value **apVal /* Function arguments */
+){
+ Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx);
+ char buf[8];
+ assert( nArg==0 );
+ assert( sizeof(buf)>=sizeof(pGlobal) );
+ memcpy(buf, pGlobal, sizeof(pGlobal));
+ sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT);
+}
int sqlite3Fts5Init(sqlite3 *db){
static const sqlite3_module fts5Mod = {
/* iVersion */ 2,
/* xCreate */ fts5CreateMethod,
@@ -1469,14 +1604,24 @@
rc = SQLITE_NOMEM;
}else{
void *p = (void*)pGlobal;
memset(pGlobal, 0, sizeof(Fts5Global));
pGlobal->db = db;
+ pGlobal->api.iVersion = 1;
+ pGlobal->api.xCreateFunction = fts5CreateAux;
+ pGlobal->api.xCreateTokenizer = fts5CreateTokenizer;
+ pGlobal->api.xFindTokenizer = fts5FindTokenizer;
rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy);
if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db);
- if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(db);
- if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(pGlobal);
+ if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db);
+ if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api);
+ if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api);
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_create_function(
+ db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
+ );
+ }
}
return rc;
}
Index: ext/fts5/fts5.h
==================================================================
--- ext/fts5/fts5.h
+++ ext/fts5/fts5.h
@@ -258,8 +258,45 @@
};
/*
** END OF CUSTOM TOKENIZERS
*************************************************************************/
+
+/*************************************************************************
+** FTS5 EXTENSION REGISTRATION API
+*/
+typedef struct fts5_api fts5_api;
+struct fts5_api {
+ int iVersion; /* Currently always set to 1 */
+
+ /* Create a new tokenizer */
+ int (*xCreateTokenizer)(
+ fts5_api *pApi,
+ const char *zName,
+ void *pContext,
+ fts5_tokenizer *pTokenizer,
+ void (*xDestroy)(void*)
+ );
+
+ /* Find an existing tokenizer */
+ int (*xFindTokenizer)(
+ fts5_api *pApi,
+ const char *zName,
+ fts5_tokenizer *pTokenizer
+ );
+
+ /* Create a new auxiliary function */
+ int (*xCreateFunction)(
+ fts5_api *pApi,
+ const char *zName,
+ void *pContext,
+ fts5_extension_function xFunction,
+ void (*xDestroy)(void*)
+ );
+};
+
+/*
+** END OF REGISTRATION API
+*************************************************************************/
#endif /* _FTS5_H */
Index: ext/fts5/fts5Int.h
==================================================================
--- ext/fts5/fts5Int.h
+++ ext/fts5/fts5Int.h
@@ -29,10 +29,27 @@
#define FTS5_DEFAULT_NEARDIST 10
/* Name of rank column */
#define FTS5_RANK_NAME "rank"
+/**************************************************************************
+** Interface to code in fts5.c.
+*/
+typedef struct Fts5Global Fts5Global;
+
+int sqlite3Fts5GetTokenizer(
+ Fts5Global*,
+ const char **azArg,
+ int nArg,
+ Fts5Tokenizer**,
+ fts5_tokenizer**
+);
+
+/*
+** End of interface to code in fts5.c.
+**************************************************************************/
+
/**************************************************************************
** Interface to code in fts5_config.c. fts5_config.c contains contains code
** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
*/
@@ -48,14 +65,17 @@
char *zName; /* Name of FTS index */
int nCol; /* Number of columns */
char **azCol; /* Column names */
int nPrefix; /* Number of prefix indexes */
int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */
- sqlite3_tokenizer *pTokenizer; /* Tokenizer instance for this table */
+ Fts5Tokenizer *pTok;
+ fts5_tokenizer *pTokApi;
};
-int sqlite3Fts5ConfigParse(sqlite3*, int, const char**, Fts5Config**, char**);
+int sqlite3Fts5ConfigParse(
+ Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
+);
void sqlite3Fts5ConfigFree(Fts5Config*);
int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
int sqlite3Fts5Tokenize(
@@ -401,11 +421,11 @@
i64 sqlite3Fts5ExprRowid(Fts5Expr*);
void sqlite3Fts5ExprFree(Fts5Expr*);
/* Called during startup to register a UDF with SQLite */
-int sqlite3Fts5ExprInit(sqlite3*);
+int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
@@ -451,35 +471,28 @@
/*
** End of interface to code in fts5_expr.c.
**************************************************************************/
-/**************************************************************************
-** Interface to code in fts5.c.
-*/
-typedef struct Fts5Global Fts5Global;
-
-int sqlite3Fts5CreateAux(
- Fts5Global*,
- const char*,
- void*,
- fts5_extension_function,
- void(*)(void*)
-);
-/*
-** End of interface to code in fts5.c.
-**************************************************************************/
-
/**************************************************************************
** Interface to code in fts5_aux.c.
*/
-int sqlite3Fts5AuxInit(Fts5Global*);
+int sqlite3Fts5AuxInit(fts5_api*);
/*
** End of interface to code in fts5_aux.c.
**************************************************************************/
+
+/**************************************************************************
+** Interface to code in fts5_tokenizer.c.
+*/
+
+int sqlite3Fts5TokenizerInit(fts5_api*);
+/*
+** End of interface to code in fts5_tokenizer.c.
+**************************************************************************/
/**************************************************************************
** Interface to code in fts5_sorter.c.
*/
typedef struct Fts5Sorter Fts5Sorter;
Index: ext/fts5/fts5_aux.c
==================================================================
--- ext/fts5/fts5_aux.c
+++ ext/fts5/fts5_aux.c
@@ -954,11 +954,11 @@
sqlite3_result_error_code(pCtx, rc);
}
sqlite3Fts5BufferFree(&s);
}
-int sqlite3Fts5AuxInit(Fts5Global *pGlobal){
+int sqlite3Fts5AuxInit(fts5_api *pApi){
struct Builtin {
const char *zFunc; /* Function name (nul-terminated) */
void *pUserData; /* User-data pointer */
fts5_extension_function xFunc;/* Callback function */
void (*xDestroy)(void*); /* Destructor function */
@@ -971,11 +971,11 @@
int rc = SQLITE_OK; /* Return code */
int i; /* To iterate through builtin functions */
for(i=0; rc==SQLITE_OK && ixCreateFunction(pApi,
aBuiltin[i].zFunc,
aBuiltin[i].pUserData,
aBuiltin[i].xFunc,
aBuiltin[i].xDestroy
);
Index: ext/fts5/fts5_config.c
==================================================================
--- ext/fts5/fts5_config.c
+++ ext/fts5/fts5_config.c
@@ -111,30 +111,19 @@
*/
static char *fts5Strdup(const char *z){
return sqlite3_mprintf("%s", z);
}
-void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**);
-
/*
** Allocate an instance of the default tokenizer ("simple") at
** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
** code if an error occurs.
*/
-static int fts5ConfigDefaultTokenizer(Fts5Config *pConfig){
- const sqlite3_tokenizer_module *pMod; /* Tokenizer module "simple" */
- sqlite3_tokenizer *pTokenizer; /* Tokenizer instance */
- int rc; /* Return code */
-
- sqlite3Fts3SimpleTokenizerModule(&pMod);
- rc = pMod->xCreate(0, 0, &pTokenizer);
- if( rc==SQLITE_OK ){
- pTokenizer->pModule = pMod;
- pConfig->pTokenizer = pTokenizer;
- }
-
- return rc;
+static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
+ return sqlite3Fts5GetTokenizer(
+ pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi
+ );
}
/*
** Arguments nArg/azArg contain the string arguments passed to the xCreate
** or xConnect method of the virtual table. This function attempts to
@@ -146,10 +135,11 @@
** returned, *ppOut is set to NULL and an error message may be left in
** *pzErr. It is the responsibility of the caller to eventually free any
** such error message using sqlite3_free().
*/
int sqlite3Fts5ConfigParse(
+ Fts5Global *pGlobal,
sqlite3 *db,
int nArg, /* Number of arguments */
const char **azArg, /* Array of nArg CREATE VIRTUAL TABLE args */
Fts5Config **ppOut, /* OUT: Results of parse */
char **pzErr /* OUT: Error message */
@@ -204,12 +194,12 @@
}
}
}
}
- if( rc==SQLITE_OK && pRet->pTokenizer==0 ){
- rc = fts5ConfigDefaultTokenizer(pRet);
+ if( rc==SQLITE_OK && pRet->pTok==0 ){
+ rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
}
if( rc!=SQLITE_OK ){
sqlite3Fts5ConfigFree(pRet);
*ppOut = 0;
@@ -221,12 +211,12 @@
** Free the configuration object passed as the only argument.
*/
void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
if( pConfig ){
int i;
- if( pConfig->pTokenizer ){
- pConfig->pTokenizer->pModule->xDestroy(pConfig->pTokenizer);
+ if( pConfig->pTok && pConfig->pTokApi->xDelete ){
+ pConfig->pTokApi->xDelete(pConfig->pTok);
}
sqlite3_free(pConfig->zDb);
sqlite3_free(pConfig->zName);
for(i=0; inCol; i++){
sqlite3_free(pConfig->azCol[i]);
@@ -300,29 +290,9 @@
Fts5Config *pConfig, /* FTS5 Configuration object */
const char *pText, int nText, /* Text to tokenize */
void *pCtx, /* Context passed to xToken() */
int (*xToken)(void*, const char*, int, int, int, int) /* Callback */
){
- const sqlite3_tokenizer_module *pMod = pConfig->pTokenizer->pModule;
- sqlite3_tokenizer_cursor *pCsr = 0;
- int rc;
-
- rc = pMod->xOpen(pConfig->pTokenizer, pText, nText, &pCsr);
- assert( rc==SQLITE_OK || pCsr==0 );
- if( rc==SQLITE_OK ){
- const char *pToken; /* Pointer to token buffer */
- int nToken; /* Size of token in bytes */
- int iStart, iEnd, iPos; /* Start, end and position of token */
- pCsr->pTokenizer = pConfig->pTokenizer;
- for(rc = pMod->xNext(pCsr, &pToken, &nToken, &iStart, &iEnd, &iPos);
- rc==SQLITE_OK;
- rc = pMod->xNext(pCsr, &pToken, &nToken, &iStart, &iEnd, &iPos)
- ){
- if( (rc = xToken(pCtx, pToken, nToken, iStart, iEnd, iPos)) ) break;
- }
- if( rc==SQLITE_DONE ) rc = SQLITE_OK;
- pMod->xClose(pCsr);
- }
- return rc;
+ return pConfig->pTokApi->xTokenize(pConfig->pTok, pCtx, pText, nText, xToken);
}
Index: ext/fts5/fts5_expr.c
==================================================================
--- ext/fts5/fts5_expr.c
+++ ext/fts5/fts5_expr.c
@@ -1518,24 +1518,26 @@
return zRet;
}
/*
-** The implementation of user-defined scalar function fts5_expr().
+** The implementation of user-defined scalar functions fts5_expr() (bTcl==0)
+** and fts5_expr_tcl() (bTcl!=0).
*/
static void fts5ExprFunction(
sqlite3_context *pCtx, /* Function call context */
int nArg, /* Number of args */
- sqlite3_value **apVal /* Function arguments */
+ sqlite3_value **apVal, /* Function arguments */
+ int bTcl
){
+ Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx);
sqlite3 *db = sqlite3_context_db_handle(pCtx);
const char *zExpr = 0;
char *zErr = 0;
Fts5Expr *pExpr = 0;
int rc;
int i;
- int bTcl = sqlite3_user_data(pCtx)!=0;
const char **azConfig; /* Array of arguments for Fts5Config */
const char *zNearsetCmd = "nearset";
int nConfig; /* Size of azConfig[] */
Fts5Config *pConfig = 0;
@@ -1556,11 +1558,11 @@
for(i=1+bTcl; iz, -1, SQLITE_UTF8, p->p, p->x, 0, 0);
+ rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0);
}
return rc;
}
ADDED ext/fts5/fts5_tokenize.c
Index: ext/fts5/fts5_tokenize.c
==================================================================
--- /dev/null
+++ ext/fts5/fts5_tokenize.c
@@ -0,0 +1,145 @@
+/*
+** 2014 May 31
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+*/
+
+#include "fts5.h"
+
+
+/*
+** Create a "simple" tokenizer.
+*/
+static int fts5SimpleCreate(
+ void *pCtx,
+ const char **azArg, int nArg,
+ Fts5Tokenizer **ppOut
+){
+ *ppOut = 0;
+ return SQLITE_OK;
+}
+
+/*
+** Delete a "simple" tokenizer.
+*/
+static void fts5SimpleDelete(Fts5Tokenizer *p){
+ return;
+}
+
+/*
+** For tokenizers with no "unicode" modifier, the set of token characters
+** is the same as the set of ASCII range alphanumeric characters.
+*/
+static unsigned char aSimpleTokenChar[128] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00..0x0F */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x10..0x1F */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20..0x2F */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30..0x3F */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40..0x4F */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x50..0x5F */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60..0x6F */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x70..0x7F */
+};
+
+
+static void simpleFold(char *aOut, const char *aIn, int nByte){
+ int i;
+ for(i=0; i='A' && c<='Z' ) c += 32;
+ aOut[i] = c;
+ }
+}
+
+/*
+** Tokenize some text using the simple tokenizer.
+*/
+static int fts5SimpleTokenize(
+ Fts5Tokenizer *pTokenizer,
+ void *pCtx,
+ const char *pText, int nText,
+ int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos)
+){
+ int rc;
+ int ie;
+ int is = 0;
+ int iPos = 0;
+
+ char aFold[64];
+ int nFold = sizeof(aFold);
+ char *pFold = aFold;
+
+ do {
+ int nByte;
+
+ /* Skip any leading divider characters. */
+ while( isnFold ){
+ if( pFold!=aFold ) sqlite3_free(pFold);
+ pFold = sqlite3_malloc(nByte*2);
+ if( pFold==0 ){
+ rc = SQLITE_NOMEM;
+ break;
+ }
+ nFold = nByte*2;
+ }
+ simpleFold(pFold, &pText[is], nByte);
+
+ /* Invoke the token callback */
+ rc = xToken(pCtx, pFold, nByte, is, ie, iPos);
+ iPos++;
+ is = ie+1;
+ }while( isxCreateTokenizer(pApi,
+ aBuiltin[i].zName,
+ &aBuiltin[i].pUserData,
+ &aBuiltin[i].x,
+ 0
+ );
+ }
+
+ return SQLITE_OK;
+}
+
+
Index: main.mk
==================================================================
--- main.mk
+++ main.mk
@@ -78,10 +78,11 @@
LIBOBJ += fts5_config.o
LIBOBJ += fts5_expr.o
LIBOBJ += fts5_hash.o
LIBOBJ += fts5_index.o
LIBOBJ += fts5_storage.o
+LIBOBJ += fts5_tokenize.o
LIBOBJ += fts5parse.o
# All of the source code files.
@@ -234,11 +235,12 @@
$(TOP)/ext/fts5/fts5_config.c \
$(TOP)/ext/fts5/fts5_expr.c \
$(TOP)/ext/fts5/fts5_hash.c \
$(TOP)/ext/fts5/fts5_index.c \
fts5parse.c \
- $(TOP)/ext/fts5/fts5_storage.c
+ $(TOP)/ext/fts5/fts5_storage.c \
+ $(TOP)/ext/fts5/fts5_tokenize.c
# Generated source code files
#
SRC += \
@@ -608,14 +610,19 @@
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_index.c
fts5_storage.o: $(TOP)/ext/fts5/fts5_storage.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_storage.c
+fts5_tokenize.o: $(TOP)/ext/fts5/fts5_tokenize.c $(HDR) $(EXTHDR)
+ $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_tokenize.c
+
fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon
cp $(TOP)/ext/fts5/fts5parse.y .
rm -f fts5parse.h
./lemon $(OPTS) fts5parse.y
+ mv fts5parse.c fts5parse.c.orig
+ cat fts5parse.c.orig | sed 's/yy/fts5yy/g' | sed 's/YY/fts5YY/g' > fts5parse.c
# Rules for building test programs and for running tests
#
tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a
Index: src/main.c
==================================================================
--- src/main.c
+++ src/main.c
@@ -17,10 +17,13 @@
#include "sqliteInt.h"
#ifdef SQLITE_ENABLE_FTS3
# include "fts3.h"
#endif
+#ifdef SQLITE_ENABLE_FTS5
+int sqlite3Fts5Init(sqlite3*);
+#endif
#ifdef SQLITE_ENABLE_RTREE
# include "rtree.h"
#endif
#ifdef SQLITE_ENABLE_ICU
# include "sqliteicu.h"
@@ -2607,11 +2610,16 @@
#endif
#ifdef SQLITE_ENABLE_FTS3
if( !db->mallocFailed && rc==SQLITE_OK ){
rc = sqlite3Fts3Init(db);
- if( rc==SQLITE_OK ) rc = sqlite3Fts5Init(db);
+ }
+#endif
+
+#ifdef SQLITE_ENABLE_FTS5
+ if( !db->mallocFailed && rc==SQLITE_OK ){
+ rc = sqlite3Fts5Init(db);
}
#endif
#ifdef SQLITE_ENABLE_ICU
if( !db->mallocFailed && rc==SQLITE_OK ){
Index: src/test_config.c
==================================================================
--- src/test_config.c
+++ src/test_config.c
@@ -327,10 +327,16 @@
#ifdef SQLITE_ENABLE_FTS3
Tcl_SetVar2(interp, "sqlite_options", "fts3", "1", TCL_GLOBAL_ONLY);
#else
Tcl_SetVar2(interp, "sqlite_options", "fts3", "0", TCL_GLOBAL_ONLY);
#endif
+
+#ifdef SQLITE_ENABLE_FTS5
+ Tcl_SetVar2(interp, "sqlite_options", "fts5", "1", TCL_GLOBAL_ONLY);
+#else
+ Tcl_SetVar2(interp, "sqlite_options", "fts5", "0", TCL_GLOBAL_ONLY);
+#endif
#if defined(SQLITE_ENABLE_FTS3) && defined(SQLITE_ENABLE_FTS4_UNICODE61)
Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "1", TCL_GLOBAL_ONLY);
#else
Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "0", TCL_GLOBAL_ONLY);
Index: test/fts5aa.test
==================================================================
--- test/fts5aa.test
+++ test/fts5aa.test
@@ -15,11 +15,11 @@
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts5aa
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+ifcapable !fts5 {
finish_test
return
}
do_execsql_test 1.0 {
Index: test/fts5ab.test
==================================================================
--- test/fts5ab.test
+++ test/fts5ab.test
@@ -15,12 +15,12 @@
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts5ab
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
finish_test
return
}
do_execsql_test 1.0 {
Index: test/fts5ac.test
==================================================================
--- test/fts5ac.test
+++ test/fts5ac.test
@@ -15,12 +15,12 @@
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts5ac
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
finish_test
return
}
do_execsql_test 1.0 {
Index: test/fts5ad.test
==================================================================
--- test/fts5ad.test
+++ test/fts5ad.test
@@ -15,12 +15,12 @@
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts5ad
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
finish_test
return
}
do_execsql_test 1.0 {
Index: test/fts5ae.test
==================================================================
--- test/fts5ae.test
+++ test/fts5ae.test
@@ -15,12 +15,12 @@
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts5ae
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
finish_test
return
}
do_execsql_test 1.0 {
Index: test/fts5af.test
==================================================================
--- test/fts5af.test
+++ test/fts5af.test
@@ -17,12 +17,12 @@
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts5af
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
finish_test
return
}
@@ -136,10 +136,10 @@
3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
} {
- do_snippet_test 1.$tn $doc "X + Y" $res
+ do_snippet_test 2.$tn $doc "X + Y" $res
}
finish_test
Index: test/fts5ag.test
==================================================================
--- test/fts5ag.test
+++ test/fts5ag.test
@@ -14,12 +14,12 @@
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts5ag
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
finish_test
return
}
#-------------------------------------------------------------------------
Index: test/fts5ah.test
==================================================================
--- test/fts5ah.test
+++ test/fts5ah.test
@@ -14,12 +14,12 @@
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts5ah
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
finish_test
return
}
#-------------------------------------------------------------------------
Index: test/fts5ai.test
==================================================================
--- test/fts5ai.test
+++ test/fts5ai.test
@@ -16,12 +16,12 @@
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts5ai
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
finish_test
return
}
do_execsql_test 1.0 {
Index: test/fts5aj.test
==================================================================
--- test/fts5aj.test
+++ test/fts5aj.test
@@ -18,12 +18,12 @@
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts5aj
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
finish_test
return
}
proc doc {} {
Index: test/fts5ea.test
==================================================================
--- test/fts5ea.test
+++ test/fts5ea.test
@@ -12,12 +12,12 @@
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts5ea
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
finish_test
return
}
proc do_syntax_error_test {tn expr err} {
Index: tool/mksqlite3c.tcl
==================================================================
--- tool/mksqlite3c.tcl
+++ tool/mksqlite3c.tcl
@@ -333,13 +333,15 @@
fts5_aux.c
fts5_buffer.c
fts5.c
fts5_config.c
fts5_expr.c
+ fts5_hash.c
fts5_index.c
fts5parse.c
fts5_storage.c
+ fts5_tokenize.c
rtree.c
icu.c
fts3_icu.c
} {