Index: ext/wasm/GNUmakefile ================================================================== --- ext/wasm/GNUmakefile +++ ext/wasm/GNUmakefile @@ -347,10 +347,16 @@ EXPORTED_FUNCTIONS.api.main := $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) EXPORTED_FUNCTIONS.api.in := $(EXPORTED_FUNCTIONS.api.main) ifeq (1,$(SQLITE_C_IS_SEE)) EXPORTED_FUNCTIONS.api.in += $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-see) endif +ifeq (1,$(emcc.WASM_BIGINT)) + ifneq (,$(filter -DSQLITE_ENABLE_FTS5,$(SQLITE_OPT))) + EXPORTED_FUNCTIONS.api.in += $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-fts5) + endif +endif + EXPORTED_FUNCTIONS.api := $(dir.tmp)/EXPORTED_FUNCTIONS.api $(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(sqlite3.c) $(MAKEFILE) cat $(EXPORTED_FUNCTIONS.api.in) > $@ # sqlite3-license-version.js = generated JS file with the license @@ -373,10 +379,13 @@ sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.js sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.js sqlite3-api.jses += $(dir.api)/sqlite3-v-helper.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js +ifneq (,$(filter -DSQLITE_ENABLE_FTS5,$(SQLITE_OPT))) + sqlite3-api.jses += $(dir.api)/sqlite3-fts5-helper.js +endif sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js # SOAP.js is an external API file which is part of our distribution # but not part of the sqlite3-api.js amalgamation. SOAP.js := $(dir.api)/sqlite3-opfs-async-proxy.js ADDED ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-fts5 Index: ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-fts5 ================================================================== --- /dev/null +++ ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-fts5 @@ -0,0 +1,1 @@ +_fts5_api_from_db Index: ext/wasm/api/sqlite3-api-cleanup.js ================================================================== --- ext/wasm/api/sqlite3-api-cleanup.js +++ ext/wasm/api/sqlite3-api-cleanup.js @@ -50,13 +50,14 @@ throw e; }finally{ delete globalThis.sqlite3ApiBootstrap; delete globalThis.sqlite3ApiConfig; } - + delete sqlite3.__dbCleanupMap; Module.sqlite3 = sqlite3 /* Needed for customized sqlite3InitModule() to be able to pass the sqlite3 object off to the client. */; + sqlite3.wasm.xWrap.FuncPtrAdapter.warnOnUse = true; }else{ console.warn("This is not running in an Emscripten module context, so", "globalThis.sqlite3ApiBootstrap() is _not_ being called due to lack", "of config info for the WASM environment.", "It must be called manually."); Index: ext/wasm/api/sqlite3-api-glue.js ================================================================== --- ext/wasm/api/sqlite3-api-glue.js +++ ext/wasm/api/sqlite3-api-glue.js @@ -259,10 +259,22 @@ } } }), "*"/*pUserData*/ ]], + /** + Achtung: when specifying an xDestroy() method via + sqlite3_set_auxdata(), it is up to the client to re-set it to + 0/NULL at the end of its lifetime (e.g. in the associated UDF's + xFinal() impl), The C library will be able to call the + destructor but _not_ uninstall the temporary WASM-bound proxy + function because it does not have enough information to do + so. Alternately, clients may create the function pointer + themselves using wasm.createFunction() and pass that pointer + here, in which case they avoid creating a stranded "temporary" + function binding. + */ ["sqlite3_set_auxdata", undefined, [ "sqlite3_context*", "int", "*", new wasm.xWrap.FuncPtrAdapter({ name: 'xDestroyAuxData', signature: 'v(*)', @@ -419,12 +431,10 @@ ["sqlite3_vtab_rhs_value","int", "sqlite3_index_info*", "int", "**"] ]; // Add session/changeset APIs... if(wasm.bigIntEnabled && !!wasm.exports.sqlite3changegroup_add){ - /* ACHTUNG: 2022-12-23: the session/changeset API bindings are - COMPLETELY UNTESTED. */ /** FuncPtrAdapter options for session-related callbacks with the native signature "i(ps)". This proxy converts the 2nd argument from a C string to a JS string before passing the arguments on to the client-provided JS callback. @@ -595,10 +605,16 @@ '*' ]] ]); }/*session/changeset APIs*/ + if(wasm.bigIntEnabled && !!wasm.exports.fts5_api_from_db){ + wasm.bindingSignatures.int64.push( + ['fts5_api_from_db', 'fts5_api*', 'sqlite3*'] + ); + }/* fts5 APIs */ + /** Functions which are intended solely for API-internal use by the WASM components, not client code. These get installed into sqlite3.wasm. Some of them get exposed to clients via variants named sqlite3_js_...(). @@ -657,12 +673,12 @@ wasm.xWrap.argAdapter( 'string:static', function(v){ if(wasm.isPtr(v)) return v; v = ''+v; - let rc = this[v]; - return rc || (this[v] = wasm.allocCString(v)); + const rc = this[v]; + return (undefined===rc) ? (this[v] = wasm.allocCString(v)) : rc; }.bind(Object.create(null)) ); /** Add some descriptive xWrap() aliases for '*' intended to (A) @@ -715,11 +731,13 @@ return __xArgPtr((v instanceof (capi.sqlite3_vfs || nilType)) ? v.pointer : v); }); const __xRcPtr = wasm.xWrap.resultAdapter('*'); - wasm.xWrap.resultAdapter('sqlite3*', __xRcPtr) + wasm.xWrap.resultAdapter + ('fts5_api*', __xRcPtr) + ('sqlite3*', __xRcPtr) ('sqlite3_context*', __xRcPtr) ('sqlite3_stmt*', __xRcPtr) ('sqlite3_value*', __xRcPtr) ('sqlite3_vfs*', __xRcPtr) ('void*', __xRcPtr); @@ -821,10 +839,13 @@ 'stmtStatus', 'syncFlags', 'trace', 'txnState', 'udfFlags', 'version' ]; if(wasm.bigIntEnabled){ defineGroups.push('serialize', 'session', 'vtab'); + if(!!wasm.ctype.fts5){ + defineGroups.push('fts5'); + } } for(const t of defineGroups){ for(const e of Object.entries(wasm.ctype[t])){ // ^^^ [k,v] there triggers a buggy code transformation via // one of the Emscripten-driven optimizers. @@ -904,13 +925,13 @@ /** __dbCleanupMap is infrastructure for recording registration of UDFs and collations so that sqlite3_close_v2() can clean up any automated JS-to-WASM function conversions installed by those. */ - const __argPDb = (pDb)=>wasm.xWrap.argAdapter('sqlite3*')(pDb); + const __argPDb = wasm.xWrap.argAdapter('sqlite3*'); const __argStr = (str)=>wasm.isPtr(str) ? wasm.cstrToJs(str) : str; - const __dbCleanupMap = function( + const __dbCleanupMap = sqlite3.__dbCleanupMap = function( pDb, mode/*0=remove, >0=create if needed, <0=do not create if missing*/ ){ pDb = __argPDb(pDb); let m = this.dbMap.get(pDb); if(!mode){ @@ -971,10 +992,11 @@ https://sqlite.org/wasm/doc/trunk/api-c-style.md#convert-func-ptr */ __dbCleanupMap.cleanup = function(pDb){ pDb = __argPDb(pDb); + //console.warn("db cleanup",pDb); //wasm.xWrap.FuncPtrAdapter.debugFuncInstall = false; /** Installing NULL functions in the C API will remove those bindings. The FuncPtrAdapter which sits between us and the C API will also treat that as an opportunity to @@ -998,10 +1020,13 @@ WASM. */; try{ capi[name](...closeArgs) } catch(e){ console.warn("close-time call of",name+"(",closeArgs,") threw:",e); } + } + for(const callback of __dbCleanupMap.extraCallbacks){ + callback(pDb); } const m = __dbCleanupMap(pDb, 0); if(!m) return; if(m.collation){ for(const name of m.collation){ @@ -1035,19 +1060,43 @@ fmap.clear(); } delete m.udf; delete m.wudf; }/*__dbCleanupMap.cleanup()*/; + /** + Downstream code, namely sqlite3-fts5-helper.js, should add any custom + cleanup handlers to __dbCleanupMap.extraCallbacks. Each function in this + array will be called during sqlite3_close_v2() and passed a pointer to + the being-destroyed (sqlite3*) object. + */ + __dbCleanupMap.extraCallbacks = []; + /** + Downstream code, namely sqlite3-fts5-helper.js, should add any + custom cleanup handlers to __dbCleanupMap.postCloseCallbacks. + Each function in this array will be called during + sqlite3_close_v2(), AFTER the db is closed, and passed a pointer + to the being-destroyed (sqlite3*) object. The memory is NOT A + VALID OBJECT but its address is still valid as a lookup key. + */ + __dbCleanupMap.postCloseCallbacks = []; {/* Binding of sqlite3_close_v2() */ const __sqlite3CloseV2 = wasm.xWrap("sqlite3_close_v2", "int", "sqlite3*"); + const __xArgDb = wasm.xWrap.argAdapter('sqlite3*'); capi.sqlite3_close_v2 = function(pDb){ if(1!==arguments.length) return __dbArgcMismatch(pDb, 'sqlite3_close_v2', 1); + pDb = __xArgDb(pDb); if(pDb){ try{__dbCleanupMap.cleanup(pDb)} catch(e){/*ignored*/} } - return __sqlite3CloseV2(pDb); + const rc = __sqlite3CloseV2(pDb); + if(pDb/*noting that it's not valid anymore*/){ + for(const f of __dbCleanupMap.postCloseCallbacks){ + try{f(pDb)}catch(e){/*ingored*/} + } + } + return rc; }; }/*sqlite3_close_v2()*/ if(capi.sqlite3session_table_filter){ const __sqlite3SessionDelete = wasm.xWrap( @@ -1152,10 +1201,13 @@ }; /** JS proxies for the various sqlite3_create[_window]_function() callbacks, structured in a form usable by wasm.xWrap.FuncPtrAdapter. + + TODO: explore an API option which more closely resembles the + /ext/jni mapping, which is much friendlier at the client level. */ const __cfProxy = Object.assign(Object.create(null), { xInverseAndStep: { signature:'v(pip)', contextKey, callProxy: (callback)=>{ @@ -1663,7 +1715,6 @@ be used that way but it's not really intended to be. */ capi.sqlite3_vfs_unregister(pKvvfs); } }/*pKvvfs*/ - wasm.xWrap.FuncPtrAdapter.warnOnUse = true; }); Index: ext/wasm/api/sqlite3-api-prologue.js ================================================================== --- ext/wasm/api/sqlite3-api-prologue.js +++ ext/wasm/api/sqlite3-api-prologue.js @@ -1643,11 +1643,12 @@ 4 bytes (1 pointer each). The values those point to have 8-byte alignment but the individual argv entries do not. */ tgt.push(capi.sqlite3_value_to_js( - wasm.peekPtr(pArgv + (wasm.ptrSizeof * i)) + wasm.peekPtr(pArgv + (wasm.ptrSizeof * i)), + throwIfCannotConvert )); } return tgt; }; ADDED ext/wasm/api/sqlite3-fts5-helper.js Index: ext/wasm/api/sqlite3-fts5-helper.js ================================================================== --- /dev/null +++ ext/wasm/api/sqlite3-fts5-helper.js @@ -0,0 +1,347 @@ +/* +** 2023-08-03 +** +** The author disclaims copyright to this source code. In place of a +** legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +*/ + +/** + This file installs sqlite3.fts5, a namespace which exists to assist + in JavaScript-side extension of FTS5. +*/ +'use strict'; +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + const wasm = sqlite3.wasm, capi = sqlite3.capi, toss = sqlite3.util.toss3; + if(!capi.fts5_api_from_db){ + return /*this build does not have FTS5*/; + } + const fts = sqlite3.fts5 = Object.create(null); + const __xArgDb = wasm.xWrap.argAdapter('sqlite3*'); + + /** + Move FTS-specific APIs (installed via automation) from + sqlite3.capi to sqlite3.fts. + */ + for(const c of [ + 'Fts5ExtensionApi', 'Fts5PhraseIter', 'fts5_api', + 'fts5_api_from_db', 'fts5_tokenizer' + ]){ + fts[c] = capi[c] || toss("Cannot find capi."+c); + delete capi[c]; + } + + /** + Requires a JS Function intended to be used as an xFunction() + implementation. This function returns a proxy xFunction + wrapper which: + + - Converts all of its sqlite3_value arguments to an array + of JS values using sqlite3_values_to_js(). + + - Calls the given callback, passing it: + + (pFtsXApi, pFtsCx, pCtx, array-of-values) + + where the first 3 arguments are the first 3 pointers + in the xFunction interface. + + + The call is intended to set a result value into the db, and may + do so be either (A) explicitly returning non-undefined or (B) + using one of the sqlite3_result_XYZ() functions and returning + undefined. If the callback throws, its exception will be passed + to sqlite3_result_error_js(). + */ + fts.xFunctionProxy1 = function(callback){ + return (pFtsXApi, pFtsCx, pCtx, argc, pArgv)=>{ + try{ + capi.sqlite3_result_js(pCtx, callback( + pFtsXApi, pFtsCx, pCtx, + capi.sqlite3_values_to_js(argc, pArgv) + )); + }catch(e){ + capi.sqlite3_result_error_js(pCtx, e); + } + } + }; + + /** + Identical to xFunctionProxy1 except that the callback wrapper it + creates does _not_ perform sqlite3_value-to-JS conversion in + advance and calls the callback with: + + (pFtsXApi, pFtsCx, pCtx, array-of-ptr-to-sqlite3_value) + + It is up to the callback to use the sqlite3_value_XYZ() family of + functions to inspect or convert the values. + */ + fts.xFunctionProxy2 = function(callback){ + return (pFtsXApi, pFtsCx, pCtx, argc, pArgv)=>{ + try{ + const list = []; + let i; + for(i = 0; i < argc; ++i){ + list.push( wasm.peekPtr(pArgv + (wasm.ptrSizeof * i)) ); + } + capi.sqlite3_result_js(pCtx, callback( + pFtsXApi, pFtsCx, pCtx, list + )); + }catch(e){ + capi.sqlite3_result_error_js(pCtx, e); + } + } + }; + + /** + JS-to-WASM arg adapter for xCreateFunction()'s xFunction arg. + This binds JS impls of xFunction to WASM so that they can be + called from native code. Its context is limited to the + combination of ((fts5_api*) + functionNameCaseInsensitive), and + will replace any existing impl for subsequent invocations for the + same combination. + + The functions is creates are intended to set a result value into + the db, and may do so be either (A) explicitly returning + non-undefined or (B) using one of the sqlite3_result_XYZ() + functions and returning undefined. If the callback throws, its + exception will be passed to sqlite3_result_error_js(). + + PENDING DESIGN DECISION: this framework currently converts each + argument in its JS equivalent before passing them on to the + xFunction impl. We could, and possibly should, instead pass a JS + array of sqlite3_value pointers. The advantages would be: + + - No in-advance to-JS overhead which xFunction might not use. + + Disadvantages include: + + - xFunction would be required to call sqlite3_value_to_js(), + or one of the many sqlite3_value_XYZ() functions on their own. + This would be more cumbersome for most users. + + Regardless of which approach is chosen here, clients could + provide a function of their own which takes the _other_ approach, + install it with wasm.installFunction(), and then pass that + generated pointer to createFunction(), in which case this layer + does not proxying and passes all native-level arguments as-is to + the client-defined function. + */ + const xFunctionArgAdapter = new wasm.xWrap.FuncPtrAdapter({ + name: 'fts5_api::xCreateFunction(xFunction)', + signature: 'v(pppip)', + contextKey: (argv,argIndex)=>{ + return (argv[0]/*(fts5_api*)*/ + + wasm.cstrToJs(argv[1]).toLowerCase()/*name*/) + }, + callProxy: fts.xFunctionProxy1 + }); + + /** Map of (sqlite3*) to fts.fts5_api. */ + const __ftsApiToStruct = Object.create(null); + const __fts5_api_from_db = function(pDb, createIfNeeded){ + let rc = __ftsApiToStruct[pDb]; + if(!rc && createIfNeeded){ + const fapi = fts.fts5_api_from_db(pDb) + || toss("Internal error - cannot get FTS5 API object for db."); + rc = new fts.fts5_api(fapi); + __ftsApiToStruct[pDb] = rc; + } + return rc; + }; + + /** + Arrange for WASM functions dynamically created via this API to be + uninstalled when the db they were installed for is closed... */ + const __addCleanupForFunc = function(sfapi, name, pDestroy){ + if(!sfapi.$$cleanup){ + sfapi.$$cleanup = []; + } + sfapi.$$cleanup.push([name.toLowerCase(), pDestroy]); + }; + + /** + Callback to be invoked via the JS binding of sqlite3_close_v2(), + after the db has been closed, meaning that the argument to this + function is not a valid object. We use its address only as a + lookup key. + */ + sqlite3.__dbCleanupMap.postCloseCallbacks.push(function(pDb){ + const sfapi = __fts5_api_from_db(pDb, false); + if(sfapi){ + delete __ftsApiToStruct[pDb]; + if(sfapi.$$cleanup){ + const fapi = sfapi.pointer; + const scope = wasm.scopedAllocPush(); + //wasm.xWrap.FuncPtrAdapter.debugFuncInstall = true; + try{ + for(const [name, pDestroy] of sfapi.$$cleanup){ + try{ + /* Uninstall xFunctionArgAdapter's bindings via a + roundabout approach: its scoping rules uninstall each + new installation at the earliest opportunity, so we + simply need to fake a call with a 0-pointer for the + xFunction callback to uninstall the most recent + one. */ + const zName = wasm.scopedAllocCString(name); + const argv = [fapi, zName, 0, 0, 0]; + xFunctionArgAdapter.convertArg(argv[3], argv, 3); + /* xDestroy, on the other hand, requires some + hand-holding to ensure we don't prematurely + uninstall these when a function is replaced + (shadowed). */ + if(pDestroy) wasm.uninstallFunction(pDestroy); + }catch(e){ + sqlite3.config.warn("Could not remove FTS func",name,e); + } + } + }finally{ + wasm.scopedAllocPop(scope); + } + //wasm.xWrap.FuncPtrAdapter.debugFuncInstall = false; + } + sfapi.dispose(); + } + }); + + const __affirmDbArg = (arg)=>{ + arg = __xArgDb(arg); + if(!arg || !wasm.isPtr(arg)) toss("Invalid db argument."); + return arg; + }; + + /** + Convenience wrapper to fts5_api::xCreateFunction. + + Creates a new FTS5 function for the given database. The arguments are: + + - db must be either an sqlite3.oo1.DB instance or a WASM pointer + to (sqlite3*). + + - name: name (JS string) of the function + + - xFunction either a Function or a pointer to a WASM function. In + the former case a WASM-bound wrapper, behaving as documented + for fts5.xFunctionProxy1(), gets installed for the life of the + given db handle. In the latter case the function is + passed-through as-is, with no argument conversion or lifetime + tracking. In the former case the function is called as + documented for xFunctionProxy1() and in the latter it must + return void and is called with args (ptrToFts5ExtensionApi, + ptrToFts5Context, ptrToSqlite3Context, int argc, + C-array-of-sqlite3_value-pointers). + + - xDestroy optional Function or pointer to WASM function to call + when the binding is destroyed (when the db handle is + closed). The function will, in this context, always be passed 0 + as its only argument. A passed-in function must, however, + have one parameter so that type signature checks will pass. + It must return void and must not throw. + + The 2nd and subsequent aruguments may optionally be packed into + a single Object with like-named properties. + + This function throws on error, of which there are many potential + candidates. It returns `undefined`. + */ + fts.createFunction = function(db, name, xFunction, xDestroy = 0){ + db = __affirmDbArg(db); + if( 2 === arguments.length && 'string' !== typeof name){ + xDestroy = name.xDestroy || null; + xFunction = name.xFunction || null; + name = name.name; + } + if( !name || 'string' !== typeof name ) toss("Invalid name argument."); + const sfapi = __fts5_api_from_db(db, true); + let pDestroy = 0; + try{ + /** Because of how fts5_api::xCreateFunction() replaces + functions (by prepending new ones to a linked list but + retaining old ones), we cannot use a FuncPtrAdapter to + automatically convert xDestroy, lest we end up uninstalling + a bound-to-wasm JS function's wasm pointer before fts5 + cleans it up when the db is closed. */ + if(xDestroy instanceof Function){ + pDestroy = wasm.installFunction(xDestroy, 'v(p)'); + } + const xcf = sfapi.$$xCreateFunction || ( + sfapi.$$xCreateFunction = wasm.xWrap(sfapi.$xCreateFunction, 'int', [ + '*', 'string', '*', xFunctionArgAdapter, '*' + ]) + ); + const rc = xcf(sfapi.pointer, name, 0, xFunction || 0, pDestroy || xDestroy || 0 ); + if(rc) toss(rc,"FTS5::xCreateFunction() failed."); + __addCleanupForFunc(sfapi, name, pDestroy); + }catch(e){ + if(pDestroy) wasm.uninstallFunction(pDestroy); + sfapi.dispose(); + throw e; + } + }; + + /** + ! UNTESTED + + Convenience wrapper for fts5_api::xCreateTokenizer(). + + - db = the db to install the tokenizer into. + + - name = the JS string name of the tokenizer. + + - pTokenizer = the tokenizer instance, which must be a + fts5.fts5_tokenizer instance or a valid WASM pointer to one. + + - xDestroy = as documented for createFunction(). + + The C layer makes a bitwise copy of the tokenizer, so any + changes made to it after installation will have no effect. + + Throws on error. + */ + const createTokenizer = function(db, name, pTokenizer, xDestroy = 0){ + db = __affirmDbArg(db); + if( 2 === arguments.length && 'string' !== typeof name){ + pTokenizer = name.pTokenizer; + xDestroy = name.xDestroy || null; + name = name.name; + } + if( !name || 'string' !== typeof name ) toss("Invalid name argument."); + if(pTokenizer instanceof fts.fts5_tokenizer){ + pTokenizer = pTokenizer.pointer; + } + if(!pTokenizer || !wasm.isPtr(pTokenizer)){ + toss("Invalid pTokenizer argument - must be a valid fts5.fts5_tokenizer", + "instance or a WASM pointer to one."); + } + const sfapi = __fts5_api_from_db(db, true); + let pDestroy = 0; + const stackPos = wasm.pstack.pointer; + try{ + if(xDestroy instanceof Function){ + pDestroy = wasm.installFunction(xDestroy, 'v(p)'); + } + const xct = sfapi.$$xCreateTokenizer || ( + sfapi.$$xCreateTokenizer = wasm.xWrap(sfapi.$xCreateTokenizer, 'int', [ + '*', 'string', '*', '*', '*' + /* fts5_api*, const char *zName, void *pContext, + fts5_tokenizer *pTokenizer, void(*xDestroy)(void*) */ + ]) + ); + const outPtr = wasm.pstack.allocPtr(); + const rc = xct(fapi.pointer, name, 0, pTokenizer, pDestroy || xDestroy || 0 ); + if(rc) toss(rc,"FTS5::xCreateFunction() failed."); + if(pDestroy) __addCleanupForFunc(sfapi, name, pDestroy); + }catch(e){ + if(pDestroy) wasm.uninstallFunction(pDestroy); + sfapi.dispose(); + throw e; + }finally{ + wasm.pstack.restore(stackPost); + } + }; + //fts.createTokenizer = createTokenizer; + +}/*sqlite3ApiBootstrap.initializers.push()*/); Index: ext/wasm/api/sqlite3-v-helper.js ================================================================== --- ext/wasm/api/sqlite3-v-helper.js +++ ext/wasm/api/sqlite3-v-helper.js @@ -8,11 +8,11 @@ ** * May you find forgiveness for yourself and forgive others. ** * May you share freely, never taking more than you give. */ /** - This file installs sqlite3.vfs, and object which exists to assist + This file installs sqlite3.vfs, an object which exists to assist in the creation of JavaScript implementations of sqlite3_vfs, along with its virtual table counterpart, sqlite3.vtab. */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Index: ext/wasm/api/sqlite3-wasm.c ================================================================== --- ext/wasm/api/sqlite3-wasm.c +++ ext/wasm/api/sqlite3-wasm.c @@ -94,12 +94,12 @@ # define SQLITE_ENABLE_DBSTAT_VTAB 1 #endif #ifndef SQLITE_ENABLE_EXPLAIN_COMMENTS # define SQLITE_ENABLE_EXPLAIN_COMMENTS 1 #endif -#ifndef SQLITE_ENABLE_FTS4 -# define SQLITE_ENABLE_FTS4 1 +#ifndef SQLITE_ENABLE_FTS5 +# define SQLITE_ENABLE_FTS5 1 #endif #ifndef SQLITE_ENABLE_MATH_FUNCTIONS # define SQLITE_ENABLE_MATH_FUNCTIONS 1 #endif #ifndef SQLITE_ENABLE_OFFSET_SQL_FUNC @@ -361,10 +361,31 @@ } } return err_code; } +#ifdef SQLITE_ENABLE_FTS5 +/* +** Return a pointer to the fts5_api pointer for database connection db. +** If an error occurs, return NULL and leave an error in the database +** handle (accessible using sqlite3_errcode()/errmsg()). +** +** This function was taken verbatim from the /fts5.html docs. +*/ +SQLITE_WASM_EXPORT +fts5_api *fts5_api_from_db(sqlite3 *db){ + fts5_api *pRet = 0; + sqlite3_stmt *pStmt = 0; + if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0) ){ + sqlite3_bind_pointer(pStmt, 1, (void*)&pRet, "fts5_api_ptr", NULL); + sqlite3_step(pStmt); + } + sqlite3_finalize(pStmt); + return pRet; +} +#endif /*SQLITE_ENABLE_FTS5*/ + #if SQLITE_WASM_TESTS struct WasmTestStruct { int v4; void * ppV; const char * cstr; @@ -400,11 +421,14 @@ ** buffer is not large enough for the generated JSON and needs to be ** increased. In debug builds that will trigger an assert(). */ SQLITE_WASM_EXPORT const char * sqlite3_wasm_enum_json(void){ - static char aBuffer[1024 * 20] = {0} /* where the JSON goes */; + static char aBuffer[1024 * 22] = {0} + /* where the JSON goes. If this buffer is not large enough, this + function will assert (in debug builds) and return 0. When it does + so, this value needs to be increased. */; int n = 0, nChildren = 0, nStruct = 0 /* output counters for figuring out where commas go */; char * zPos = &aBuffer[1] /* skip first byte for now to help protect ** against a small race condition */; char const * const zEnd = &aBuffer[0] + sizeof(aBuffer) /* one-past-the-end */; @@ -648,10 +672,20 @@ DefInt(SQLITE_LOCK_SHARED); DefInt(SQLITE_LOCK_RESERVED); DefInt(SQLITE_LOCK_PENDING); DefInt(SQLITE_LOCK_EXCLUSIVE); } _DefGroup; + +#ifdef SQLITE_ENABLE_FTS5 + DefGroup(fts5) { + DefInt(FTS5_TOKENIZE_QUERY); + DefInt(FTS5_TOKENIZE_PREFIX); + DefInt(FTS5_TOKENIZE_DOCUMENT); + DefInt(FTS5_TOKENIZE_AUX); + DefInt(FTS5_TOKEN_COLOCATED); + } _DefGroup; +#endif DefGroup(ioCap) { DefInt(SQLITE_IOCAP_ATOMIC); DefInt(SQLITE_IOCAP_ATOMIC512); DefInt(SQLITE_IOCAP_ATOMIC1K); @@ -874,11 +908,11 @@ DefInt(SQLITE_STMTSTATUS_RUN); DefInt(SQLITE_STMTSTATUS_FILTER_MISS); DefInt(SQLITE_STMTSTATUS_FILTER_HIT); DefInt(SQLITE_STMTSTATUS_MEMUSED); } _DefGroup; - + DefGroup(syncFlags) { DefInt(SQLITE_SYNC_NORMAL); DefInt(SQLITE_SYNC_FULL); DefInt(SQLITE_SYNC_DATAONLY); } _DefGroup; @@ -1092,11 +1126,11 @@ M(xRollbackTo, "i(pi)"); // ^^^ v2. v3+ follows... M(xShadowName, "i(s)"); } _StructBinder; #undef CurrentStruct - + /** ** Workaround: in order to map the various inner structs from ** sqlite3_index_info, we have to uplift those into constructs we ** can access by type name. These structs _must_ match their ** in-sqlite3_index_info counterparts byte for byte. @@ -1169,10 +1203,116 @@ M(idxFlags, "i"); M(colUsed, "j"); } _StructBinder; #undef CurrentStruct +#ifdef SQLITE_ENABLE_FTS5 +#define CurrentStruct Fts5PhraseIter + StructBinder { + M(a, "p"); + M(b, "p"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct Fts5ExtensionApi + StructBinder { + M(iVersion, "i"); + M(xUserData, "p(p)");// void *(*)(Fts5Context*); + M(xColumnCount, "i(p)");// int (*)(Fts5Context*); + M(xRowCount, "i(pp)"); + //^^^ int (*)(Fts5Context*, sqlite3_int64 *pnRow); + M(xColumnTotalSize, "i(pip)"); + //^^^ int (*)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); + M(xTokenize, "i(ppipp)"); + //^^^ int (*)(Fts5Context*, + // const char *pText, int nText, /* Text to tokenize */ + // void *pCtx, /* Context passed to xToken() */ + // int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + //); + M(xPhraseCount, "i(p)"); // int (*)(Fts5Context*); + M(xPhraseSize, "i(pi)"); // int (*)(Fts5Context*, int iPhrase); + M(xInstCount, "i(pp)"); // int (*)(Fts5Context*, int *pnInst); + M(xInst, "i(pippp)"); + //^^^ int (*)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); + M(xRowid, "j(p)"); // sqlite3_int64 (*)(Fts5Context*); + M(xColumnText, "i(pipp)"); + //^^^ int (*)(Fts5Context*, int iCol, const char **pz, int *pn); + M(xColumnSize, "i(pip)"); + //^^^ int (*)(Fts5Context*, int iCol, int *pnToken); + M(xQueryPhrase, "i(pipp)"); + //^^^ int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, + // int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) + // ); + M(xSetAuxdata, "i(ppp)"); + //^^^ int (*)(Fts5Context*, void *pAux, void(*xDelete)(void*)); + M(xGetAuxdata, "p(pi)"); // void *(*)(Fts5Context*, int bClear); + M(xPhraseFirst, "i(pippp)"); + //^^^ int (*)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + M(xPhraseNext, "v(pppp)"); + //^^^ void (*)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); + M(xPhraseFirstColumn, "i(pipp)"); + //^^^ int (*)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); + M(xPhraseNextColumn, "v(ppp)"); + //^^^ void (*)(Fts5Context*, Fts5PhraseIter*, int *piCol); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct fts5_api + StructBinder { + M(iVersion, "i");/* Currently always 2 */ + M(xCreateTokenizer, "i(ppppp)"); + //^^^ int (*)( + // fts5_api *pApi, + // const char *zName, + // void *pContext, + // fts5_tokenizer *pTokenizer, + // void (*xDestroy)(void*) + // ); + M(xFindTokenizer, "i(pppp)"); + //^^^ int (*)( + // fts5_api *pApi, + // const char *zName, + // void **ppContext, + // fts5_tokenizer *pTokenizer + // ); + M(xCreateFunction, "i(ppppp)"); + //^^^ int (*)( + // fts5_api *pApi, + // const char *zName, + // void *pContext, + // fts5_extension_function xFunction, + // void (*xDestroy)(void*) + // ); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct fts5_tokenizer + StructBinder { + M(xCreate, "i(ppip)"); + //^^^ int (*)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + M(xDelete, "v(p)"); + //^^^ void(Fts5Tokenizer*) + M(xTokenize, "i(ppipip)"); + /**^^^ int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, + const char *pText, + int nText, + int (*xToken)( + void *pCtx, + int tflags, + const char *pToken, + int nToken, + int iStart, + int iEnd + ) + ); */ + } _StructBinder; +#undef CurrentStruct + +#endif /* SQLITE_ENABLE_FTS5 */ + #if SQLITE_WASM_TESTS #define CurrentStruct WasmTestStruct StructBinder { M(v4, "i"); M(cstr, "s"); Index: ext/wasm/common/whwasmutil.js ================================================================== --- ext/wasm/common/whwasmutil.js +++ ext/wasm/common/whwasmutil.js @@ -611,12 +611,10 @@ https://github.com/emscripten-core/emscripten/issues/17323 */ target.installFunction = (func, sig)=>__installFunction(func, sig, false); /** - EXPERIMENTAL! DO NOT USE IN CLIENT CODE! - Works exactly like installFunction() but requires that a scopedAllocPush() is active and uninstalls the given function when that alloc scope is popped via scopedAllocPop(). This is used for implementing JS/WASM function bindings which should only persist for the life of a call into a single @@ -1178,11 +1176,11 @@ if(n<0) toss("Invalid state object for scopedAllocPop()."); if(0===arguments.length) state = cache.scopedAlloc[n]; cache.scopedAlloc.splice(n,1); for(let p; (p = state.pop()); ){ if(target.functionEntry(p)){ - //console.warn("scopedAllocPop() uninstalling transient function",p); + //console.warn("scopedAllocPop() uninstalling function",p); target.uninstallFunction(p); } else target.dealloc(p); } }; @@ -1381,20 +1379,24 @@ target.exports. If found, it is called, passed all remaining arguments, and its return value is returned to xCall's caller. If not found, an exception is thrown. This function does no conversion of argument or return types, but see xWrap() and xCallWrapped() for variants which do. + + If the first argument is a function is is assumed to be + a WASM-bound function and is used as-is instead of looking up + the function via xGet(). As a special case, if passed only 1 argument after the name and that argument in an Array, that array's entries become the function arguments. (This is not an ambiguous case because it's not legal to pass an Array object to a WASM function.) */ target.xCall = function(fname, ...args){ - const f = target.xGet(fname); + const f = (fname instanceof Function) ? fname : target.xGet(fname); if(!(f instanceof Function)) toss("Exported symbol",fname,"is not a function."); - if(f.length!==args.length) __argcMismatch(fname,f.length) + if(f.length!==args.length) __argcMismatch(((f===fname) ? f.name : fname),f.length) /* This is arguably over-pedantic but we want to help clients keep from shooting themselves in the foot when calling C APIs. */; return (2===arguments.length && Array.isArray(arguments[1])) ? f.apply(null, arguments[1]) : f.apply(null, args); @@ -1537,11 +1539,11 @@ - signature: a function signature string compatible with jsFuncToWasm(). - bindScope (string): one of ('transient', 'context', - 'singleton'). Bind scopes are: + 'singleton', 'permanent'). Bind scopes are: - 'transient': it will convert JS functions to WASM only for the duration of the xWrap()'d function call, using scopedInstallFunction(). Before that call returns, the WASM-side binding will be uninstalled. @@ -1635,10 +1637,11 @@ if(xArg.FuncPtrAdapter.warnOnUse){ console.warn('xArg.FuncPtrAdapter is an internal-only API', 'and is not intended to be invoked from', 'client-level code. Invoked with:',opt); } + this.name = opt.name || "unnamed"; this.signature = opt.signature; if(opt.contextKey instanceof Function){ this.contextKey = opt.contextKey; if(!opt.bindScope) opt.bindScope = 'context'; } @@ -1696,18 +1699,20 @@ See the parent class's convertArg() docs for details on what exactly the 2nd and 3rd arguments are. */ convertArg(v,argv,argIndex){ - //FuncPtrAdapter.debugOut("FuncPtrAdapter.convertArg()",this.signature,this.transient,v); + //FuncPtrAdapter.debugOut("FuncPtrAdapter.convertArg()",this.name,this.signature,this.transient,v); let pair = this.singleton; if(!pair && this.isContext){ pair = this.contextMap(this.contextKey(argv,argIndex)); + //FuncPtrAdapter.debugOut(this.name, this.signature, "contextKey() =",this.contextKey(argv,argIndex), pair); } if(pair && pair[0]===v) return pair[1]; if(v instanceof Function){ /* Install a WASM binding and return its pointer. */ + //FuncPtrAdapter.debugOut("FuncPtrAdapter.convertArg()",this.name,this.signature,this.transient,v,pair); if(this.callProxy) v = this.callProxy(v); const fp = __installFunction(v, this.signature, this.isTransient); if(FuncPtrAdapter.debugFuncInstall){ FuncPtrAdapter.debugOut("FuncPtrAdapter installed", this, this.contextKey(argv,argIndex), '@'+fp, v); @@ -1717,25 +1722,37 @@ if(pair[1]){ if(FuncPtrAdapter.debugFuncInstall){ FuncPtrAdapter.debugOut("FuncPtrAdapter uninstalling", this, this.contextKey(argv,argIndex), '@'+pair[1], v); } - try{target.uninstallFunction(pair[1])} + try{ + /* Because the pending native call might rely on the + pointer we're replacing, e.g. as is normally the case + with sqlite3's xDestroy() methods, we don't + immediately uninstall but instead add its pointer to + the scopedAlloc stack, which will be cleared when the + xWrap() mechanism is done calling the native + function. We're relying very much here on xWrap() + having pushed an alloc scope. + */ + cache.scopedAlloc[cache.scopedAlloc.length-1].push(pair[1]); + } catch(e){/*ignored*/} } pair[0] = v; pair[1] = fp; } return fp; }else if(target.isPtr(v) || null===v || undefined===v){ + //FuncPtrAdapter.debugOut("FuncPtrAdapter.convertArg()",this.name,this.signature,this.transient,v,pair); if(pair && pair[1] && pair[1]!==v){ /* uninstall stashed mapping and replace stashed mapping with v. */ if(FuncPtrAdapter.debugFuncInstall){ FuncPtrAdapter.debugOut("FuncPtrAdapter uninstalling", this, this.contextKey(argv,argIndex), '@'+pair[1], v); } - try{target.uninstallFunction(pair[1])} + try{ cache.scopedAlloc[cache.scopedAlloc.length-1].push(pair[1]) } catch(e){/*ignored*/} pair[0] = pair[1] = (v | 0); } return v || 0; }else{ @@ -1772,15 +1789,33 @@ (t)=>xArg.get(t) || toss("Argument adapter not found:",t); const __xResultAdapterCheck = (t)=>xResult.get(t) || toss("Result adapter not found:",t); + /** + Fetches the xWrap() argument adapter mapped to t, calls it, + passing in all remaining arguments, and returns the result. + Throws if t is not mapped to an argument converter. + */ cache.xWrap.convertArg = (t,...args)=>__xArgAdapterCheck(t)(...args); + /** + Identical to convertArg() except that it does not perform + an is-defined check on the mapping to t before invoking it. + */ cache.xWrap.convertArgNoCheck = (t,...args)=>xArg.get(t)(...args); + /** + Fetches the xWrap() result adapter mapped to t, calls it, passing + it v, and returns the result. Throws if t is not mapped to an + argument converter. + */ cache.xWrap.convertResult = (t,v)=>(null===t ? v : (t ? __xResultAdapterCheck(t)(v) : undefined)); + /** + Identical to convertResult() except that it does not perform an + is-defined check on the mapping to t before invoking it. + */ cache.xWrap.convertResultNoCheck = (t,v)=>(null===t ? v : (t ? xResult.get(t)(v) : undefined)); /** Creates a wrapper for another function which converts the arguments Index: ext/wasm/tester1.c-pp.js ================================================================== --- ext/wasm/tester1.c-pp.js +++ ext/wasm/tester1.c-pp.js @@ -2897,10 +2897,62 @@ db2.close(); } } })/*session API sanity tests*/ ;/*end of session API group*/; + //////////////////////////////////////////////////////////////////////// + T.g('FTS5') + .t({ + name: "Sanity checks", + predicate: (sqlite3)=>sqlite3.fts5 || "Missing sqlite3.fts5 namespace.", + test: function(sqlite3){ + const db = new sqlite3.oo1.DB(); + db.exec([ + "create virtual table ft using fts5(a,b);", + "insert into ft(a,b) values", + "('a1','b1'),", + "('a2','b2'),", + "('a3','b3');" + ]); + const fts = sqlite3.fts5; + let pApi = fts.fts5_api_from_db(db); + T.assert( !!pApi ); + let fApi = new fts.fts5_api(pApi); + T.assert( fApi.$iVersion >= 2 ); + fApi.dispose(); + fApi = undefined; + let destroyCalled = false; + fts.createFunction(db, { + name: 'mymatch', + xFunction: function(pFtsX, pFtsCx, pCtx, argv){ + // Both of these return-value approaches are equivalent: + const rv = "MY+"+argv.join(':'); + if(0){ + return rv; + }else{ + capi.sqlite3_result_text(pCtx, rv, -1, capi.SQLITE_TRANSIENT); + // implicit return of undefined + } + }, + xDestroy: function(){ + destroyCalled = true; + } + }); + let list = db.selectValues( + "select mymatch(ft,a,b) from ft where b match 'b2'" + ); + T.assert( 1 === list.length ) + .assert( 'MY+a2:b2' === list[0] ); + + //const fTok = new fts.fts5_tokenizer(); + //fTok.installMethods({}); + + db.close(); + T.assert( destroyCalled ); + //toss("Testing"); + } + })/*FTS5*/ //////////////////////////////////////////////////////////////////////// T.g('OPFS: Origin-Private File System', (sqlite3)=>(sqlite3.capi.sqlite3_vfs_find("opfs") || 'requires "opfs" VFS'))