Index: ext/fts5/extract_api_docs.tcl ================================================================== --- ext/fts5/extract_api_docs.tcl +++ ext/fts5/extract_api_docs.tcl @@ -12,10 +12,21 @@ # # This script extracts the documentation for the API used by fts5 auxiliary # functions from header file fts5.h. It outputs html text on stdout that # is included in the documentation on the web. # + +set ::fts5_docs_output "" +if {[info commands hd_putsnl]==""} { + proc output {text} { + puts $text + } +} else { + proc output {text} { + append ::fts5_docs_output $text + } +} set input_file [file join [file dir [info script]] fts5.h] set fd [open $input_file] set data [read $fd] close $fd @@ -100,32 +111,32 @@ foreach {hdr docs} $D { if {[info exists M($hdr)]} { set hdr $M($hdr) } - puts "

  $hdr

" + output "
  $hdr
" set mode "" set bEmpty 1 foreach line [split [string trim $docs] "\n"] { if {[string trim $line]==""} { - if {$mode != ""} {puts ""} + if {$mode != ""} {output ""} set mode "" } elseif {$mode == ""} { if {[regexp {^ } $line]} { - set mode code + set mode codeblock } else { set mode p } - puts "<$mode>" + output "<$mode>" } - puts $line + output $line } - if {$mode != ""} {puts ""} + if {$mode != ""} {output ""} } - +set ::fts5_docs_output Index: ext/fts5/fts5.c ================================================================== --- ext/fts5/fts5.c +++ ext/fts5/fts5.c @@ -21,10 +21,12 @@ typedef struct Fts5Cursor Fts5Cursor; typedef struct Fts5Global Fts5Global; typedef struct Fts5Auxiliary Fts5Auxiliary; typedef struct Fts5Auxdata Fts5Auxdata; +typedef struct Fts5TokenizerModule Fts5TokenizerModule; + /* ** NOTES ON TRANSACTIONS: ** ** SQLite invokes the following virtual table methods as transactions are ** opened and closed by the user: @@ -63,13 +65,15 @@ ** A single object of this type is allocated when the FTS5 module is ** registered with a database handle. It is used to store pointers to ** all registered FTS5 extensions - tokenizers and auxiliary functions. */ struct Fts5Global { + fts5_api api; /* User visible part of object (see fts5.h) */ sqlite3 *db; /* Associated database connection */ i64 iNextId; /* Used to allocate unique cursor ids */ Fts5Auxiliary *pAux; /* First in list of all aux. functions */ + Fts5TokenizerModule *pTok; /* First in list of all tokenizer modules */ Fts5Cursor *pCsr; /* First in list of all open cursors */ }; /* ** Each auxiliary function registered with the FTS5 module is represented @@ -82,10 +86,23 @@ void *pUserData; /* User-data pointer */ fts5_extension_function xFunc; /* Callback function */ void (*xDestroy)(void*); /* Destructor function */ Fts5Auxiliary *pNext; /* Next registered auxiliary function */ }; + +/* +** Each tokenizer module registered with the FTS5 module is represented +** by an object of the following type. All such objects are stored as part +** of the Fts5Global.pTok list. +*/ +struct Fts5TokenizerModule { + char *zName; /* Name of tokenizer */ + void *pUserData; /* User pointer passed to xCreate() */ + fts5_tokenizer x; /* Tokenizer functions */ + void (*xDestroy)(void*); /* Destructor function */ + Fts5TokenizerModule *pNext; /* Next registered tokenizer module */ +}; /* ** Virtual-table object. */ struct Fts5Table { @@ -279,16 +296,18 @@ int argc, /* Number of elements in argv array */ const char * const *argv, /* xCreate/xConnect argument array */ sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ char **pzErr /* Write any error message here */ ){ + Fts5Global *pGlobal = (Fts5Global*)pAux; + const char **azConfig = (const char**)argv; int rc; /* Return code */ Fts5Config *pConfig; /* Results of parsing argc/argv */ Fts5Table *pTab = 0; /* New virtual table object */ /* Parse the arguments */ - rc = sqlite3Fts5ConfigParse(db, argc, (const char**)argv, &pConfig, pzErr); + rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr); assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 ); /* Allocate the new vtab object */ if( rc==SQLITE_OK ){ pTab = (Fts5Table*)sqlite3_malloc(sizeof(Fts5Table)); @@ -295,11 +314,11 @@ if( pTab==0 ){ rc = SQLITE_NOMEM; }else{ memset(pTab, 0, sizeof(Fts5Table)); pTab->pConfig = pConfig; - pTab->pGlobal = (Fts5Global*)pAux; + pTab->pGlobal = pGlobal; } } /* Open the index sub-system */ if( rc==SQLITE_OK ){ @@ -855,10 +874,14 @@ ** INSERT INTO fts(fts) VALUES($pVal) ** ** Argument pVal is the value assigned to column "fts" by the INSERT ** statement. This function returns SQLITE_OK if successful, or an SQLite ** error code if an error occurs. +** +** The commands implemented by this function are documented in the "Special +** INSERT Directives" section of the documentation. It should be updated if +** more commands are added to this function. */ static int fts5SpecialCommand(Fts5Table *pTab, sqlite3_value *pVal){ const char *z = (const char*)sqlite3_value_text(pVal); int n = sqlite3_value_bytes(pVal); int rc = SQLITE_ERROR; @@ -1385,17 +1408,18 @@ } /* ** Register a new auxiliary function with global context pGlobal. */ -int sqlite3Fts5CreateAux( - Fts5Global *pGlobal, /* Global context (one per db handle) */ +static int fts5CreateAux( + fts5_api *pApi, /* Global context (one per db handle) */ const char *zName, /* Name of new function */ void *pUserData, /* User data for aux. function */ fts5_extension_function xFunc, /* Aux. function implementation */ void(*xDestroy)(void*) /* Destructor for pUserData */ ){ + Fts5Global *pGlobal = (Fts5Global*)pApi; int rc = sqlite3_overload_function(pGlobal->db, zName, -1); if( rc==SQLITE_OK ){ Fts5Auxiliary *pAux; int nByte; /* Bytes of space to allocate */ @@ -1417,24 +1441,135 @@ } return rc; } -static void fts5ModuleDestroy(void *pCtx){ - Fts5Auxiliary *pAux; - Fts5Auxiliary *pNext; - Fts5Global *pGlobal = (Fts5Global*)pCtx; - for(pAux=pGlobal->pAux; pAux; pAux=pNext){ - pNext = pAux->pNext; - if( pAux->xDestroy ){ - pAux->xDestroy(pAux->pUserData); - } +/* +** Register a new tokenizer. This is the implementation of the +** fts5_api.xCreateTokenizer() method. +*/ +static int fts5CreateTokenizer( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + fts5_tokenizer *pTokenizer, /* Tokenizer implementation */ + void(*xDestroy)(void*) /* Destructor for pUserData */ +){ + Fts5Global *pGlobal = (Fts5Global*)pApi; + Fts5TokenizerModule *pNew; + int nByte; /* Bytes of space to allocate */ + int rc = SQLITE_OK; + + nByte = sizeof(Fts5TokenizerModule) + strlen(zName) + 1; + pNew = (Fts5TokenizerModule*)sqlite3_malloc(nByte); + if( pNew ){ + memset(pNew, 0, nByte); + pNew->zName = (char*)&pNew[1]; + strcpy(pNew->zName, zName); + pNew->pUserData = pUserData; + pNew->x = *pTokenizer; + pNew->xDestroy = xDestroy; + pNew->pNext = pGlobal->pTok; + pGlobal->pTok = pNew; + }else{ + rc = SQLITE_NOMEM; + } + + return rc; +} + +/* +** Find a tokenizer. This is the implementation of the +** fts5_api.xFindTokenizer() method. +*/ +static int fts5FindTokenizer( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + fts5_tokenizer *pTokenizer /* Populate this object */ +){ + Fts5Global *pGlobal = (Fts5Global*)pApi; + int rc = SQLITE_OK; + Fts5TokenizerModule *pTok; + + for(pTok=pGlobal->pTok; pTok; pTok=pTok->pNext){ + if( sqlite3_stricmp(zName, pTok->zName)==0 ) break; + } + + if( pTok ){ + *pTokenizer = pTok->x; + }else{ + memset(pTokenizer, 0, sizeof(fts5_tokenizer)); + rc = SQLITE_ERROR; + } + + return rc; +} + +int sqlite3Fts5GetTokenizer( + Fts5Global *pGlobal, + const char **azArg, + int nArg, + Fts5Tokenizer **ppTok, + fts5_tokenizer **ppTokApi +){ + Fts5TokenizerModule *pMod = 0; + int rc = SQLITE_OK; + if( nArg==0 ){ + pMod = pGlobal->pTok; + }else{ + for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){ + if( sqlite3_stricmp(azArg[0], pMod->zName)==0 ) break; + } + } + + if( pMod==0 ){ + rc = SQLITE_ERROR; + }else{ + rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok); + *ppTokApi = &pMod->x; + } + + if( rc!=SQLITE_OK ){ + *ppTokApi = 0; + *ppTok = 0; + } + + return rc; +} + +static void fts5ModuleDestroy(void *pCtx){ + Fts5TokenizerModule *pTok, *pNextTok; + Fts5Auxiliary *pAux, *pNextAux; + Fts5Global *pGlobal = (Fts5Global*)pCtx; + + for(pAux=pGlobal->pAux; pAux; pAux=pNextAux){ + pNextAux = pAux->pNext; + if( pAux->xDestroy ) pAux->xDestroy(pAux->pUserData); sqlite3_free(pAux); } + + for(pTok=pGlobal->pTok; pTok; pTok=pNextTok){ + pNextTok = pTok->pNext; + if( pTok->xDestroy ) pTok->xDestroy(pTok->pUserData); + sqlite3_free(pTok); + } + sqlite3_free(pGlobal); } +static void fts5Fts5Func( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apVal /* Function arguments */ +){ + Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx); + char buf[8]; + assert( nArg==0 ); + assert( sizeof(buf)>=sizeof(pGlobal) ); + memcpy(buf, pGlobal, sizeof(pGlobal)); + sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT); +} int sqlite3Fts5Init(sqlite3 *db){ static const sqlite3_module fts5Mod = { /* iVersion */ 2, /* xCreate */ fts5CreateMethod, @@ -1469,14 +1604,24 @@ rc = SQLITE_NOMEM; }else{ void *p = (void*)pGlobal; memset(pGlobal, 0, sizeof(Fts5Global)); pGlobal->db = db; + pGlobal->api.iVersion = 1; + pGlobal->api.xCreateFunction = fts5CreateAux; + pGlobal->api.xCreateTokenizer = fts5CreateTokenizer; + pGlobal->api.xFindTokenizer = fts5FindTokenizer; rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy); if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db); - if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(db); - if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(pGlobal); + if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db); + if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api); + if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0 + ); + } } return rc; } Index: ext/fts5/fts5.h ================================================================== --- ext/fts5/fts5.h +++ ext/fts5/fts5.h @@ -258,8 +258,45 @@ }; /* ** END OF CUSTOM TOKENIZERS *************************************************************************/ + +/************************************************************************* +** FTS5 EXTENSION REGISTRATION API +*/ +typedef struct fts5_api fts5_api; +struct fts5_api { + int iVersion; /* Currently always set to 1 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_tokenizer *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer)( + fts5_api *pApi, + const char *zName, + fts5_tokenizer *pTokenizer + ); + + /* Create a new auxiliary function */ + int (*xCreateFunction)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_extension_function xFunction, + void (*xDestroy)(void*) + ); +}; + +/* +** END OF REGISTRATION API +*************************************************************************/ #endif /* _FTS5_H */ Index: ext/fts5/fts5Int.h ================================================================== --- ext/fts5/fts5Int.h +++ ext/fts5/fts5Int.h @@ -29,10 +29,27 @@ #define FTS5_DEFAULT_NEARDIST 10 /* Name of rank column */ #define FTS5_RANK_NAME "rank" +/************************************************************************** +** Interface to code in fts5.c. +*/ +typedef struct Fts5Global Fts5Global; + +int sqlite3Fts5GetTokenizer( + Fts5Global*, + const char **azArg, + int nArg, + Fts5Tokenizer**, + fts5_tokenizer** +); + +/* +** End of interface to code in fts5.c. +**************************************************************************/ + /************************************************************************** ** Interface to code in fts5_config.c. fts5_config.c contains contains code ** to parse the arguments passed to the CREATE VIRTUAL TABLE statement. */ @@ -48,14 +65,17 @@ char *zName; /* Name of FTS index */ int nCol; /* Number of columns */ char **azCol; /* Column names */ int nPrefix; /* Number of prefix indexes */ int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ - sqlite3_tokenizer *pTokenizer; /* Tokenizer instance for this table */ + Fts5Tokenizer *pTok; + fts5_tokenizer *pTokApi; }; -int sqlite3Fts5ConfigParse(sqlite3*, int, const char**, Fts5Config**, char**); +int sqlite3Fts5ConfigParse( + Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** +); void sqlite3Fts5ConfigFree(Fts5Config*); int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig); int sqlite3Fts5Tokenize( @@ -401,11 +421,11 @@ i64 sqlite3Fts5ExprRowid(Fts5Expr*); void sqlite3Fts5ExprFree(Fts5Expr*); /* Called during startup to register a UDF with SQLite */ -int sqlite3Fts5ExprInit(sqlite3*); +int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*); int sqlite3Fts5ExprPhraseCount(Fts5Expr*); int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); @@ -451,35 +471,28 @@ /* ** End of interface to code in fts5_expr.c. **************************************************************************/ -/************************************************************************** -** Interface to code in fts5.c. -*/ -typedef struct Fts5Global Fts5Global; - -int sqlite3Fts5CreateAux( - Fts5Global*, - const char*, - void*, - fts5_extension_function, - void(*)(void*) -); -/* -** End of interface to code in fts5.c. -**************************************************************************/ - /************************************************************************** ** Interface to code in fts5_aux.c. */ -int sqlite3Fts5AuxInit(Fts5Global*); +int sqlite3Fts5AuxInit(fts5_api*); /* ** End of interface to code in fts5_aux.c. **************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_tokenizer.c. +*/ + +int sqlite3Fts5TokenizerInit(fts5_api*); +/* +** End of interface to code in fts5_tokenizer.c. +**************************************************************************/ /************************************************************************** ** Interface to code in fts5_sorter.c. */ typedef struct Fts5Sorter Fts5Sorter; Index: ext/fts5/fts5_aux.c ================================================================== --- ext/fts5/fts5_aux.c +++ ext/fts5/fts5_aux.c @@ -954,11 +954,11 @@ sqlite3_result_error_code(pCtx, rc); } sqlite3Fts5BufferFree(&s); } -int sqlite3Fts5AuxInit(Fts5Global *pGlobal){ +int sqlite3Fts5AuxInit(fts5_api *pApi){ struct Builtin { const char *zFunc; /* Function name (nul-terminated) */ void *pUserData; /* User-data pointer */ fts5_extension_function xFunc;/* Callback function */ void (*xDestroy)(void*); /* Destructor function */ @@ -971,11 +971,11 @@ int rc = SQLITE_OK; /* Return code */ int i; /* To iterate through builtin functions */ for(i=0; rc==SQLITE_OK && ixCreateFunction(pApi, aBuiltin[i].zFunc, aBuiltin[i].pUserData, aBuiltin[i].xFunc, aBuiltin[i].xDestroy ); Index: ext/fts5/fts5_config.c ================================================================== --- ext/fts5/fts5_config.c +++ ext/fts5/fts5_config.c @@ -111,30 +111,19 @@ */ static char *fts5Strdup(const char *z){ return sqlite3_mprintf("%s", z); } -void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**); - /* ** Allocate an instance of the default tokenizer ("simple") at ** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error ** code if an error occurs. */ -static int fts5ConfigDefaultTokenizer(Fts5Config *pConfig){ - const sqlite3_tokenizer_module *pMod; /* Tokenizer module "simple" */ - sqlite3_tokenizer *pTokenizer; /* Tokenizer instance */ - int rc; /* Return code */ - - sqlite3Fts3SimpleTokenizerModule(&pMod); - rc = pMod->xCreate(0, 0, &pTokenizer); - if( rc==SQLITE_OK ){ - pTokenizer->pModule = pMod; - pConfig->pTokenizer = pTokenizer; - } - - return rc; +static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){ + return sqlite3Fts5GetTokenizer( + pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi + ); } /* ** Arguments nArg/azArg contain the string arguments passed to the xCreate ** or xConnect method of the virtual table. This function attempts to @@ -146,10 +135,11 @@ ** returned, *ppOut is set to NULL and an error message may be left in ** *pzErr. It is the responsibility of the caller to eventually free any ** such error message using sqlite3_free(). */ int sqlite3Fts5ConfigParse( + Fts5Global *pGlobal, sqlite3 *db, int nArg, /* Number of arguments */ const char **azArg, /* Array of nArg CREATE VIRTUAL TABLE args */ Fts5Config **ppOut, /* OUT: Results of parse */ char **pzErr /* OUT: Error message */ @@ -204,12 +194,12 @@ } } } } - if( rc==SQLITE_OK && pRet->pTokenizer==0 ){ - rc = fts5ConfigDefaultTokenizer(pRet); + if( rc==SQLITE_OK && pRet->pTok==0 ){ + rc = fts5ConfigDefaultTokenizer(pGlobal, pRet); } if( rc!=SQLITE_OK ){ sqlite3Fts5ConfigFree(pRet); *ppOut = 0; @@ -221,12 +211,12 @@ ** Free the configuration object passed as the only argument. */ void sqlite3Fts5ConfigFree(Fts5Config *pConfig){ if( pConfig ){ int i; - if( pConfig->pTokenizer ){ - pConfig->pTokenizer->pModule->xDestroy(pConfig->pTokenizer); + if( pConfig->pTok && pConfig->pTokApi->xDelete ){ + pConfig->pTokApi->xDelete(pConfig->pTok); } sqlite3_free(pConfig->zDb); sqlite3_free(pConfig->zName); for(i=0; inCol; i++){ sqlite3_free(pConfig->azCol[i]); @@ -300,29 +290,9 @@ Fts5Config *pConfig, /* FTS5 Configuration object */ const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ int (*xToken)(void*, const char*, int, int, int, int) /* Callback */ ){ - const sqlite3_tokenizer_module *pMod = pConfig->pTokenizer->pModule; - sqlite3_tokenizer_cursor *pCsr = 0; - int rc; - - rc = pMod->xOpen(pConfig->pTokenizer, pText, nText, &pCsr); - assert( rc==SQLITE_OK || pCsr==0 ); - if( rc==SQLITE_OK ){ - const char *pToken; /* Pointer to token buffer */ - int nToken; /* Size of token in bytes */ - int iStart, iEnd, iPos; /* Start, end and position of token */ - pCsr->pTokenizer = pConfig->pTokenizer; - for(rc = pMod->xNext(pCsr, &pToken, &nToken, &iStart, &iEnd, &iPos); - rc==SQLITE_OK; - rc = pMod->xNext(pCsr, &pToken, &nToken, &iStart, &iEnd, &iPos) - ){ - if( (rc = xToken(pCtx, pToken, nToken, iStart, iEnd, iPos)) ) break; - } - if( rc==SQLITE_DONE ) rc = SQLITE_OK; - pMod->xClose(pCsr); - } - return rc; + return pConfig->pTokApi->xTokenize(pConfig->pTok, pCtx, pText, nText, xToken); } Index: ext/fts5/fts5_expr.c ================================================================== --- ext/fts5/fts5_expr.c +++ ext/fts5/fts5_expr.c @@ -1518,24 +1518,26 @@ return zRet; } /* -** The implementation of user-defined scalar function fts5_expr(). +** The implementation of user-defined scalar functions fts5_expr() (bTcl==0) +** and fts5_expr_tcl() (bTcl!=0). */ static void fts5ExprFunction( sqlite3_context *pCtx, /* Function call context */ int nArg, /* Number of args */ - sqlite3_value **apVal /* Function arguments */ + sqlite3_value **apVal, /* Function arguments */ + int bTcl ){ + Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx); sqlite3 *db = sqlite3_context_db_handle(pCtx); const char *zExpr = 0; char *zErr = 0; Fts5Expr *pExpr = 0; int rc; int i; - int bTcl = sqlite3_user_data(pCtx)!=0; const char **azConfig; /* Array of arguments for Fts5Config */ const char *zNearsetCmd = "nearset"; int nConfig; /* Size of azConfig[] */ Fts5Config *pConfig = 0; @@ -1556,11 +1558,11 @@ for(i=1+bTcl; iz, -1, SQLITE_UTF8, p->p, p->x, 0, 0); + rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0); } return rc; } ADDED ext/fts5/fts5_tokenize.c Index: ext/fts5/fts5_tokenize.c ================================================================== --- /dev/null +++ ext/fts5/fts5_tokenize.c @@ -0,0 +1,145 @@ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +#include "fts5.h" + + +/* +** Create a "simple" tokenizer. +*/ +static int fts5SimpleCreate( + void *pCtx, + const char **azArg, int nArg, + Fts5Tokenizer **ppOut +){ + *ppOut = 0; + return SQLITE_OK; +} + +/* +** Delete a "simple" tokenizer. +*/ +static void fts5SimpleDelete(Fts5Tokenizer *p){ + return; +} + +/* +** For tokenizers with no "unicode" modifier, the set of token characters +** is the same as the set of ASCII range alphanumeric characters. +*/ +static unsigned char aSimpleTokenChar[128] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00..0x0F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x10..0x1F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20..0x2F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30..0x3F */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40..0x4F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x50..0x5F */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60..0x6F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x70..0x7F */ +}; + + +static void simpleFold(char *aOut, const char *aIn, int nByte){ + int i; + for(i=0; i='A' && c<='Z' ) c += 32; + aOut[i] = c; + } +} + +/* +** Tokenize some text using the simple tokenizer. +*/ +static int fts5SimpleTokenize( + Fts5Tokenizer *pTokenizer, + void *pCtx, + const char *pText, int nText, + int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos) +){ + int rc; + int ie; + int is = 0; + int iPos = 0; + + char aFold[64]; + int nFold = sizeof(aFold); + char *pFold = aFold; + + do { + int nByte; + + /* Skip any leading divider characters. */ + while( isnFold ){ + if( pFold!=aFold ) sqlite3_free(pFold); + pFold = sqlite3_malloc(nByte*2); + if( pFold==0 ){ + rc = SQLITE_NOMEM; + break; + } + nFold = nByte*2; + } + simpleFold(pFold, &pText[is], nByte); + + /* Invoke the token callback */ + rc = xToken(pCtx, pFold, nByte, is, ie, iPos); + iPos++; + is = ie+1; + }while( isxCreateTokenizer(pApi, + aBuiltin[i].zName, + &aBuiltin[i].pUserData, + &aBuiltin[i].x, + 0 + ); + } + + return SQLITE_OK; +} + + Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -78,10 +78,11 @@ LIBOBJ += fts5_config.o LIBOBJ += fts5_expr.o LIBOBJ += fts5_hash.o LIBOBJ += fts5_index.o LIBOBJ += fts5_storage.o +LIBOBJ += fts5_tokenize.o LIBOBJ += fts5parse.o # All of the source code files. @@ -234,11 +235,12 @@ $(TOP)/ext/fts5/fts5_config.c \ $(TOP)/ext/fts5/fts5_expr.c \ $(TOP)/ext/fts5/fts5_hash.c \ $(TOP)/ext/fts5/fts5_index.c \ fts5parse.c \ - $(TOP)/ext/fts5/fts5_storage.c + $(TOP)/ext/fts5/fts5_storage.c \ + $(TOP)/ext/fts5/fts5_tokenize.c # Generated source code files # SRC += \ @@ -608,14 +610,19 @@ $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_index.c fts5_storage.o: $(TOP)/ext/fts5/fts5_storage.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_storage.c +fts5_tokenize.o: $(TOP)/ext/fts5/fts5_tokenize.c $(HDR) $(EXTHDR) + $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_tokenize.c + fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon cp $(TOP)/ext/fts5/fts5parse.y . rm -f fts5parse.h ./lemon $(OPTS) fts5parse.y + mv fts5parse.c fts5parse.c.orig + cat fts5parse.c.orig | sed 's/yy/fts5yy/g' | sed 's/YY/fts5YY/g' > fts5parse.c # Rules for building test programs and for running tests # tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -17,10 +17,13 @@ #include "sqliteInt.h" #ifdef SQLITE_ENABLE_FTS3 # include "fts3.h" #endif +#ifdef SQLITE_ENABLE_FTS5 +int sqlite3Fts5Init(sqlite3*); +#endif #ifdef SQLITE_ENABLE_RTREE # include "rtree.h" #endif #ifdef SQLITE_ENABLE_ICU # include "sqliteicu.h" @@ -2607,11 +2610,16 @@ #endif #ifdef SQLITE_ENABLE_FTS3 if( !db->mallocFailed && rc==SQLITE_OK ){ rc = sqlite3Fts3Init(db); - if( rc==SQLITE_OK ) rc = sqlite3Fts5Init(db); + } +#endif + +#ifdef SQLITE_ENABLE_FTS5 + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3Fts5Init(db); } #endif #ifdef SQLITE_ENABLE_ICU if( !db->mallocFailed && rc==SQLITE_OK ){ Index: src/test_config.c ================================================================== --- src/test_config.c +++ src/test_config.c @@ -327,10 +327,16 @@ #ifdef SQLITE_ENABLE_FTS3 Tcl_SetVar2(interp, "sqlite_options", "fts3", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "fts3", "0", TCL_GLOBAL_ONLY); #endif + +#ifdef SQLITE_ENABLE_FTS5 + Tcl_SetVar2(interp, "sqlite_options", "fts5", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "fts5", "0", TCL_GLOBAL_ONLY); +#endif #if defined(SQLITE_ENABLE_FTS3) && defined(SQLITE_ENABLE_FTS4_UNICODE61) Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "0", TCL_GLOBAL_ONLY); Index: test/fts5aa.test ================================================================== --- test/fts5aa.test +++ test/fts5aa.test @@ -15,11 +15,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts5aa # If SQLITE_ENABLE_FTS3 is defined, omit this file. -ifcapable !fts3 { +ifcapable !fts5 { finish_test return } do_execsql_test 1.0 { Index: test/fts5ab.test ================================================================== --- test/fts5ab.test +++ test/fts5ab.test @@ -15,12 +15,12 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts5ab -# If SQLITE_ENABLE_FTS3 is defined, omit this file. -ifcapable !fts3 { +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { finish_test return } do_execsql_test 1.0 { Index: test/fts5ac.test ================================================================== --- test/fts5ac.test +++ test/fts5ac.test @@ -15,12 +15,12 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts5ac -# If SQLITE_ENABLE_FTS3 is defined, omit this file. -ifcapable !fts3 { +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { finish_test return } do_execsql_test 1.0 { Index: test/fts5ad.test ================================================================== --- test/fts5ad.test +++ test/fts5ad.test @@ -15,12 +15,12 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts5ad -# If SQLITE_ENABLE_FTS3 is defined, omit this file. -ifcapable !fts3 { +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { finish_test return } do_execsql_test 1.0 { Index: test/fts5ae.test ================================================================== --- test/fts5ae.test +++ test/fts5ae.test @@ -15,12 +15,12 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts5ae -# If SQLITE_ENABLE_FTS3 is defined, omit this file. -ifcapable !fts3 { +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { finish_test return } do_execsql_test 1.0 { Index: test/fts5af.test ================================================================== --- test/fts5af.test +++ test/fts5af.test @@ -17,12 +17,12 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts5af -# If SQLITE_ENABLE_FTS3 is defined, omit this file. -ifcapable !fts3 { +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { finish_test return } @@ -136,10 +136,10 @@ 3.6 {o o o o o X Y o o} {...o o o [X Y] o o} 3.7 {o o o o o o X Y o} {...o o o o [X Y] o} 3.8 {o o o o o o o X Y} {...o o o o o [X Y]} } { - do_snippet_test 1.$tn $doc "X + Y" $res + do_snippet_test 2.$tn $doc "X + Y" $res } finish_test Index: test/fts5ag.test ================================================================== --- test/fts5ag.test +++ test/fts5ag.test @@ -14,12 +14,12 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts5ag -# If SQLITE_ENABLE_FTS3 is defined, omit this file. -ifcapable !fts3 { +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { finish_test return } #------------------------------------------------------------------------- Index: test/fts5ah.test ================================================================== --- test/fts5ah.test +++ test/fts5ah.test @@ -14,12 +14,12 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts5ah -# If SQLITE_ENABLE_FTS3 is defined, omit this file. -ifcapable !fts3 { +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { finish_test return } #------------------------------------------------------------------------- Index: test/fts5ai.test ================================================================== --- test/fts5ai.test +++ test/fts5ai.test @@ -16,12 +16,12 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts5ai -# If SQLITE_ENABLE_FTS3 is defined, omit this file. -ifcapable !fts3 { +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { finish_test return } do_execsql_test 1.0 { Index: test/fts5aj.test ================================================================== --- test/fts5aj.test +++ test/fts5aj.test @@ -18,12 +18,12 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts5aj -# If SQLITE_ENABLE_FTS3 is defined, omit this file. -ifcapable !fts3 { +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { finish_test return } proc doc {} { Index: test/fts5ea.test ================================================================== --- test/fts5ea.test +++ test/fts5ea.test @@ -12,12 +12,12 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts5ea -# If SQLITE_ENABLE_FTS3 is defined, omit this file. -ifcapable !fts3 { +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { finish_test return } proc do_syntax_error_test {tn expr err} { Index: tool/mksqlite3c.tcl ================================================================== --- tool/mksqlite3c.tcl +++ tool/mksqlite3c.tcl @@ -333,13 +333,15 @@ fts5_aux.c fts5_buffer.c fts5.c fts5_config.c fts5_expr.c + fts5_hash.c fts5_index.c fts5parse.c fts5_storage.c + fts5_tokenize.c rtree.c icu.c fts3_icu.c } {