/* ** 2007 June 22 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** ** ** This is part of an SQLite module implementing full-text search. ** This particular file implements the generic tokenizer interface. */ /* ** The code in this file is only compiled if: ** ** * The FTS3 module is being built as an extension ** (in which case SQLITE_CORE is not defined), or ** ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include "sqlite3ext.h" #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT1 #endif #include "fts3Int.h" #include #include #include /* ** Implementation of the SQL scalar function for accessing the underlying ** hash table. This function may be called as follows: ** ** SELECT (); ** SELECT (, ); ** ** where is the name passed as the second argument ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer'). ** ** If the argument is specified, it must be a blob value ** containing a pointer to be stored as the hash data corresponding ** to the string . If is not specified, then ** the string must already exist in the has table. Otherwise, ** an error is returned. ** ** Whether or not the argument is specified, the value returned ** is a blob containing the pointer stored as the hash data corresponding ** to string (after the hash-table is updated, if applicable). */ static void scalarFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ Fts3Hash *pHash; void *pPtr = 0; const unsigned char *zName; int nName; assert( argc==1 || argc==2 ); pHash = (Fts3Hash *)sqlite3_user_data(context); zName = sqlite3_value_text(argv[0]); nName = sqlite3_value_bytes(argv[0])+1; if( argc==2 ){ void *pOld; int n = sqlite3_value_bytes(argv[1]); if( n!=sizeof(pPtr) ){ sqlite3_result_error(context, "argument type mismatch", -1); return; } pPtr = *(void **)sqlite3_value_blob(argv[1]); pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); if( pOld==pPtr ){ sqlite3_result_error(context, "out of memory", -1); return; } }else{ pPtr = sqlite3Fts3HashFind(pHash, zName, nName); if( !pPtr ){ char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); sqlite3_result_error(context, zErr, -1); sqlite3_free(zErr); return; } } sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); } static int fts3IsIdChar(char c){ static const char isFtsIdChar[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ }; return (c&0x80 || isFtsIdChar[(int)(c)]); } const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ const char *z1; const char *z2 = 0; /* Find the start of the next token. */ z1 = zStr; while( z2==0 ){ char c = *z1; switch( c ){ case '\0': return 0; /* No more tokens here */ case '\'': case '"': case '`': { z2 = z1; while( *++z2 && (*z2!=c || *++z2==c) ); break; } case '[': z2 = &z1[1]; while( *z2 && z2[0]!=']' ) z2++; if( *z2 ) z2++; break; default: if( fts3IsIdChar(*z1) ){ z2 = &z1[1]; while( fts3IsIdChar(*z2) ) z2++; }else{ z1++; } } } *pn = (int)(z2-z1); return z1; } int sqlite3Fts3InitTokenizer( Fts3Hash *pHash, /* Tokenizer hash table */ const char *zArg, /* Possible tokenizer specification */ sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ const char **pzTokenizer, /* OUT: Set to zArg if is tokenizer */ char **pzErr /* OUT: Set to malloced error message */ ){ int rc; char *z = (char *)zArg; int n; char *zCopy; char *zEnd; /* Pointer to nul-term of zCopy */ sqlite3_tokenizer_module *m; if( !z ){ zCopy = sqlite3_mprintf("simple"); }else{ if( sqlite3_strnicmp(z, "tokenize", 8) || fts3IsIdChar(z[8])){ return SQLITE_OK; } zCopy = sqlite3_mprintf("%s", &z[8]); *pzTokenizer = zArg; } if( !zCopy ){ return SQLITE_NOMEM; } zEnd = &zCopy[strlen(zCopy)]; z = (char *)sqlite3Fts3NextToken(zCopy, &n); z[n] = '\0'; sqlite3Fts3Dequote(z); m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, z, (int)strlen(z)+1); if( !m ){ *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z); rc = SQLITE_ERROR; }else{ char const **aArg = 0; int iArg = 0; z = &z[n+1]; while( zxCreate(iArg, aArg, ppTok); assert( rc!=SQLITE_OK || *ppTok ); if( rc!=SQLITE_OK ){ *pzErr = sqlite3_mprintf("unknown tokenizer"); }else{ (*ppTok)->pModule = m; } sqlite3_free((void *)aArg); } sqlite3_free(zCopy); return rc; } #ifdef SQLITE_TEST #include #include /* ** Implementation of a special SQL scalar function for testing tokenizers ** designed to be used in concert with the Tcl testing framework. This ** function must be called with two arguments: ** ** SELECT (, ); ** SELECT (, ); ** ** where is the name passed as the second argument ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer') ** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test'). ** ** The return value is a string that may be interpreted as a Tcl ** list. For each token in the , three elements are ** added to the returned list. The first is the token position, the ** second is the token text (folded, stemmed, etc.) and the third is the ** substring of associated with the token. For example, ** using the built-in "simple" tokenizer: ** ** SELECT fts_tokenizer_test('simple', 'I don't see how'); ** ** will return the string: ** ** "{0 i I 1 dont don't 2 see see 3 how how}" ** */ static void testFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ Fts3Hash *pHash; sqlite3_tokenizer_module *p; sqlite3_tokenizer *pTokenizer = 0; sqlite3_tokenizer_cursor *pCsr = 0; const char *zErr = 0; const char *zName; int nName; const char *zInput; int nInput; const char *zArg = 0; const char *zToken; int nToken; int iStart; int iEnd; int iPos; Tcl_Obj *pRet; assert( argc==2 || argc==3 ); nName = sqlite3_value_bytes(argv[0]); zName = (const char *)sqlite3_value_text(argv[0]); nInput = sqlite3_value_bytes(argv[argc-1]); zInput = (const char *)sqlite3_value_text(argv[argc-1]); if( argc==3 ){ zArg = (const char *)sqlite3_value_text(argv[1]); } pHash = (Fts3Hash *)sqlite3_user_data(context); p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); if( !p ){ char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); sqlite3_result_error(context, zErr, -1); sqlite3_free(zErr); return; } pRet = Tcl_NewObj(); Tcl_IncrRefCount(pRet); if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){ zErr = "error in xCreate()"; goto finish; } pTokenizer->pModule = p; if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){ zErr = "error in xOpen()"; goto finish; } pCsr->pTokenizer = pTokenizer; while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); zToken = &zInput[iStart]; nToken = iEnd-iStart; Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); } if( SQLITE_OK!=p->xClose(pCsr) ){ zErr = "error in xClose()"; goto finish; } if( SQLITE_OK!=p->xDestroy(pTokenizer) ){ zErr = "error in xDestroy()"; goto finish; } finish: if( zErr ){ sqlite3_result_error(context, zErr, -1); }else{ sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); } Tcl_DecrRefCount(pRet); } static int registerTokenizer( sqlite3 *db, char *zName, const sqlite3_tokenizer_module *p ){ int rc; sqlite3_stmt *pStmt; const char zSql[] = "SELECT fts3_tokenizer(?, ?)"; rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); if( rc!=SQLITE_OK ){ return rc; } sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); sqlite3_step(pStmt); return sqlite3_finalize(pStmt); } static int queryTokenizer( sqlite3 *db, char *zName, const sqlite3_tokenizer_module **pp ){ int rc; sqlite3_stmt *pStmt; const char zSql[] = "SELECT fts3_tokenizer(?)"; *pp = 0; rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); if( rc!=SQLITE_OK ){ return rc; } sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); if( SQLITE_ROW==sqlite3_step(pStmt) ){ if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); } } return sqlite3_finalize(pStmt); } void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); /* ** Implementation of the scalar function fts3_tokenizer_internal_test(). ** This function is used for testing only, it is not included in the ** build unless SQLITE_TEST is defined. ** ** The purpose of this is to test that the fts3_tokenizer() function ** can be used as designed by the C-code in the queryTokenizer and ** registerTokenizer() functions above. These two functions are repeated ** in the README.tokenizer file as an example, so it is important to ** test them. ** ** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar ** function with no arguments. An assert() will fail if a problem is ** detected. i.e.: ** ** SELECT fts3_tokenizer_internal_test(); ** */ static void intTestFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ int rc; const sqlite3_tokenizer_module *p1; const sqlite3_tokenizer_module *p2; sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); UNUSED_PARAMETER(argc); UNUSED_PARAMETER(argv); /* Test the query function */ sqlite3Fts3SimpleTokenizerModule(&p1); rc = queryTokenizer(db, "simple", &p2); assert( rc==SQLITE_OK ); assert( p1==p2 ); rc = queryTokenizer(db, "nosuchtokenizer", &p2); assert( rc==SQLITE_ERROR ); assert( p2==0 ); assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); /* Test the storage function */ rc = registerTokenizer(db, "nosuchtokenizer", p1); assert( rc==SQLITE_OK ); rc = queryTokenizer(db, "nosuchtokenizer", &p2); assert( rc==SQLITE_OK ); assert( p2==p1 ); sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); } #endif /* ** Set up SQL objects in database db used to access the contents of ** the hash table pointed to by argument pHash. The hash table must ** been initialised to use string keys, and to take a private copy ** of the key when a value is inserted. i.e. by a call similar to: ** ** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); ** ** This function adds a scalar function (see header comment above ** scalarFunc() in this file for details) and, if ENABLE_TABLE is ** defined at compilation time, a temporary virtual table (see header ** comment above struct HashTableVtab) to the database schema. Both ** provide read/write access to the contents of *pHash. ** ** The third argument to this function, zName, is used as the name ** of both the scalar and, if created, the virtual table. */ int sqlite3Fts3InitHashTable( sqlite3 *db, Fts3Hash *pHash, const char *zName ){ int rc = SQLITE_OK; void *p = (void *)pHash; const int any = SQLITE_ANY; char *zTest = 0; char *zTest2 = 0; #ifdef SQLITE_TEST void *pdb = (void *)db; zTest = sqlite3_mprintf("%s_test", zName); zTest2 = sqlite3_mprintf("%s_internal_test", zName); if( !zTest || !zTest2 ){ rc = SQLITE_NOMEM; } #endif if( SQLITE_OK!=rc || SQLITE_OK!=(rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0)) || SQLITE_OK!=(rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0)) #ifdef SQLITE_TEST || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0)) || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0)) || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0)) #endif ); sqlite3_free(zTest); sqlite3_free(zTest2); return rc; } #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */