Index: VERSION ================================================================== --- VERSION +++ VERSION @@ -1,1 +1,1 @@ -3.46.0 +3.46.1 Index: autoconf/tea/configure.ac ================================================================== --- autoconf/tea/configure.ac +++ autoconf/tea/configure.ac @@ -17,11 +17,11 @@ # so you can encode the package version directly into the source files. # This will also define a special symbol for Windows (BUILD_ # so that we create the export library with the dll. #----------------------------------------------------------------------- -AC_INIT([sqlite],[3.46.0]) +AC_INIT([sqlite],[3.46.1]) #-------------------------------------------------------------------- # Call TEA_INIT as the first TEA_ macro to set up initial vars. # This will define a ${TEA_PLATFORM} variable == "unix" or "windows" # as well as PKG_LIB_FILE and PKG_STUB_LIB_FILE. Index: configure ================================================================== --- configure +++ configure @@ -1,8 +1,8 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.46.0. +# Generated by GNU Autoconf 2.69 for sqlite 3.46.1. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. # # @@ -724,12 +724,12 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.46.0' -PACKAGE_STRING='sqlite 3.46.0' +PACKAGE_VERSION='3.46.1' +PACKAGE_STRING='sqlite 3.46.1' PACKAGE_BUGREPORT='' PACKAGE_URL='' # Factoring default headers for most tests. ac_includes_default="\ @@ -1470,11 +1470,11 @@ # if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.46.0 to adapt to many kinds of systems. +\`configure' configures sqlite 3.46.1 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... To assign environment variables (e.g., CC, CFLAGS...), specify them as VAR=VALUE. See below for descriptions of some of the useful variables. @@ -1535,11 +1535,11 @@ _ACEOF fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.46.0:";; + short | recursive ) echo "Configuration of sqlite 3.46.1:";; esac cat <<\_ACEOF Optional Features: --disable-option-checking ignore unrecognized --enable/--with options @@ -1666,11 +1666,11 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.46.0 +sqlite configure 3.46.1 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it. @@ -2085,11 +2085,11 @@ } # ac_fn_c_check_header_mongrel cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.46.0, which was +It was created by sqlite $as_me 3.46.1, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ _ACEOF @@ -12479,11 +12479,11 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # Save the log message, to keep $0 and so on meaningful, and to # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.46.0, which was +This file was extended by sqlite $as_me 3.46.1, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS CONFIG_LINKS = $CONFIG_LINKS @@ -12545,11 +12545,11 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.46.0 +sqlite config.status 3.46.1 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" Copyright (C) 2012 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation Index: ext/consio/console_io.c ================================================================== --- ext/consio/console_io.c +++ ext/consio/console_io.c @@ -51,15 +51,10 @@ # endif #else # define CIO_WIN_WC_XLATE 0 /* Not exposing translation routines at all */ #endif -#if CIO_WIN_WC_XLATE -/* Character used to represent a known-incomplete UTF-8 char group (�) */ -static WCHAR cBadGroup = 0xfffd; -#endif - #if CIO_WIN_WC_XLATE static HANDLE handleOfFile(FILE *pf){ int fileDesc = _fileno(pf); union { intptr_t osfh; HANDLE fh; } fid = { (fileDesc>=0)? _get_osfhandle(fileDesc) : (intptr_t)INVALID_HANDLE_VALUE Index: ext/expert/expert1.test ================================================================== --- ext/expert/expert1.test +++ ext/expert/expert1.test @@ -463,10 +463,17 @@ t2 t2_idx_00000063 {100 20} t2 t2_idx_00000064 {100 5} t2 t2_idx_0001295b {100 20 5} } +do_catchsql_test 5.4 { + SELECT sqlite_expert_rem(123, 123); +} {1 {no such function: sqlite_expert_rem}} +do_catchsql_test 5.5 { + SELECT sqlite_expert_sample(); +} {1 {no such function: sqlite_expert_sample}} + if 0 { do_test expert1-6.0 { catchcmd :memory: { .expert select base64(''); Index: ext/expert/sqlite3expert.c ================================================================== --- ext/expert/sqlite3expert.c +++ ext/expert/sqlite3expert.c @@ -624,11 +624,11 @@ (void)argv; rc = sqlite3_finalize(pCsr->pData); pCsr->pData = 0; if( rc==SQLITE_OK ){ rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg, - "SELECT * FROM main.%Q WHERE sample()", pVtab->pTab->zName + "SELECT * FROM main.%Q WHERE sqlite_expert_sample()", pVtab->pTab->zName ); } if( rc==SQLITE_OK ){ rc = expertNext(cur); @@ -1498,11 +1498,11 @@ char *z; /* SQLITE_TEXT/BLOB value */ } aSlot[1]; }; /* -** Implementation of scalar function rem(). +** Implementation of scalar function sqlite_expert_rem(). */ static void idxRemFunc( sqlite3_context *pCtx, int argc, sqlite3_value **argv @@ -1511,11 +1511,11 @@ struct IdxRemSlot *pSlot; int iSlot; assert( argc==2 ); iSlot = sqlite3_value_int(argv[0]); - assert( iSlot<=p->nSlot ); + assert( iSlotnSlot ); pSlot = &p->aSlot[iSlot]; switch( pSlot->eType ){ case SQLITE_NULL: /* no-op */ @@ -1622,11 +1622,12 @@ while( SQLITE_OK==rc && SQLITE_ROW==sqlite3_step(pIndexXInfo) ){ const char *zComma = zCols==0 ? "" : ", "; const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0); const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1); zCols = idxAppendText(&rc, zCols, - "%sx.%Q IS rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl + "%sx.%Q IS sqlite_expert_rem(%d, x.%Q) COLLATE %s", + zComma, zName, nCol, zName, zColl ); zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol); } sqlite3_reset(pIndexXInfo); if( rc==SQLITE_OK ){ @@ -1755,17 +1756,17 @@ pCtx = (struct IdxRemCtx*)idxMalloc(&rc, nByte); } if( rc==SQLITE_OK ){ sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv); - rc = sqlite3_create_function( - dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0 + rc = sqlite3_create_function(dbrem, "sqlite_expert_rem", + 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0 ); } if( rc==SQLITE_OK ){ - rc = sqlite3_create_function( - p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0 + rc = sqlite3_create_function(p->db, "sqlite_expert_sample", + 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0 ); } if( rc==SQLITE_OK ){ pCtx->nSlot = nMax+1; @@ -1813,10 +1814,13 @@ if( rc==SQLITE_OK ){ rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_schema", 0, 0, 0); } + sqlite3_create_function(p->db, "sqlite_expert_rem", 2, SQLITE_UTF8, 0,0,0,0); + sqlite3_create_function(p->db, "sqlite_expert_sample", 0,SQLITE_UTF8,0,0,0,0); + sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0); return rc; } /* Index: ext/fts5/fts5_expr.c ================================================================== --- ext/fts5/fts5_expr.c +++ ext/fts5/fts5_expr.c @@ -322,11 +322,15 @@ }else{ sqlite3Fts5ParseNodeFree(sParse.pExpr); } sqlite3_free(sParse.apPhrase); - *pzErr = sParse.zErr; + if( 0==*pzErr ){ + *pzErr = sParse.zErr; + }else{ + sqlite3_free(sParse.zErr); + } return sParse.rc; } /* ** Assuming that buffer z is at least nByte bytes in size and contains a @@ -2450,10 +2454,11 @@ || pLeft->eType==FTS5_AND ); assert( pRight->eType==FTS5_STRING || pRight->eType==FTS5_TERM || pRight->eType==FTS5_EOF + || (pRight->eType==FTS5_AND && pParse->bPhraseToAnd) ); if( pLeft->eType==FTS5_AND ){ pPrev = pLeft->apChild[pLeft->nChild-1]; }else{ Index: ext/fts5/fts5_main.c ================================================================== --- ext/fts5/fts5_main.c +++ ext/fts5/fts5_main.c @@ -1698,10 +1698,11 @@ "'delete' may not be used with a contentless_delete=1 table" ); rc = SQLITE_ERROR; }else{ rc = fts5SpecialDelete(pTab, apVal); + bUpdateOrDelete = 1; } }else{ rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); } }else{ @@ -2872,18 +2873,20 @@ pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]); if( pMod==0 ){ assert( nArg>0 ); rc = SQLITE_ERROR; - *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); + if( pzErr ) *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); }else{ rc = pMod->x.xCreate( pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->pTok ); pConfig->pTokApi = &pMod->x; if( rc!=SQLITE_OK ){ - if( pzErr ) *pzErr = sqlite3_mprintf("error in tokenizer constructor"); + if( pzErr && rc!=SQLITE_NOMEM ){ + *pzErr = sqlite3_mprintf("error in tokenizer constructor"); + } }else{ pConfig->ePattern = sqlite3Fts5TokenizerPattern( pMod->x.xCreate, pConfig->pTok ); } @@ -2973,21 +2976,27 @@ Fts5FullTable *pTab = (Fts5FullTable*)pVtab; int rc; assert( pzErr!=0 && *pzErr==0 ); UNUSED_PARAM(isQuick); + assert( pTab->p.pConfig->pzErrmsg==0 ); + pTab->p.pConfig->pzErrmsg = pzErr; rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0); - if( (rc&0xff)==SQLITE_CORRUPT ){ - *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", - zSchema, zTabname); - rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM; - }else if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("unable to validate the inverted index for" - " FTS5 table %s.%s: %s", - zSchema, zTabname, sqlite3_errstr(rc)); - } - sqlite3Fts5IndexCloseReader(pTab->p.pIndex); + if( *pzErr==0 && rc!=SQLITE_OK ){ + if( (rc&0xff)==SQLITE_CORRUPT ){ + *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", + zSchema, zTabname); + rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM; + }else{ + *pzErr = sqlite3_mprintf("unable to validate the inverted index for" + " FTS5 table %s.%s: %s", + zSchema, zTabname, sqlite3_errstr(rc)); + } + } + + sqlite3Fts5IndexCloseReader(pTab->p.pIndex); + pTab->p.pConfig->pzErrmsg = 0; return rc; } static int fts5Init(sqlite3 *db){ Index: ext/fts5/fts5_tokenize.c ================================================================== --- ext/fts5/fts5_tokenize.c +++ ext/fts5/fts5_tokenize.c @@ -77,11 +77,11 @@ rc = SQLITE_NOMEM; }else{ int i; memset(p, 0, sizeof(AsciiTokenizer)); memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar)); - for(i=0; rc==SQLITE_OK && iaFold==0 ){ rc = SQLITE_NOMEM; } /* Search for a "categories" argument */ - for(i=0; rc==SQLITE_OK && ibFold = 1; pNew->iFoldParam = 0; - for(i=0; rc==SQLITE_OK && iiFoldParam!=0 && pNew->bFold==0 ){ rc = SQLITE_ERROR; } Index: ext/fts5/test/fts5aux.test ================================================================== --- ext/fts5/test/fts5aux.test +++ ext/fts5/test/fts5aux.test @@ -374,7 +374,31 @@ SELECT fts5_collist(t1, 2) FROM t1('one AND two'); } {1 SQLITE_RANGE} do_catchsql_test 12.3.3 { SELECT fts5_collist(t1, 1) FROM t1('one AND two'); } {0 1} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 13.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii); + INSERT INTO t1 VALUES('a b c'), ('d e f'); + PRAGMA integrity_check; +} {ok} + +do_catchsql_test 13.2 { + SELECT highlight(t1, 0, '[', ']') FROM t1 +} {0 {{a b c} {d e f}}} + +do_execsql_test 13.3 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)' + WHERE name = 't1'; +} + +db close +sqlite3 db test.db +do_catchsql_test 13.4 { + SELECT highlight(t1, 0, '[', ']') FROM t1 +} {1 {no such tokenizer: blah}} finish_test Index: ext/fts5/test/fts5integrity.test ================================================================== --- ext/fts5/test/fts5integrity.test +++ ext/fts5/test/fts5integrity.test @@ -377,8 +377,35 @@ } do_execsql_test 12.3 { PRAGMA integrity_check } {ok} + +#------------------------------------------------------------------- +reset_db +do_execsql_test 13.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii); + INSERT INTO t1 VALUES('a b c'), ('d e f'); + PRAGMA integrity_check; +} {ok} + +db close +sqlite3 db test.db +do_catchsql_test 13.2 { + PRAGMA integrity_check; +} {0 ok} + +do_execsql_test 13.3 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)' + WHERE name = 't1'; +} + +db close +sqlite3 db test.db +breakpoint +do_catchsql_test 13.4 { + PRAGMA integrity_check; +} {1 {no such tokenizer: blah}} finish_test Index: ext/fts5/test/fts5secure8.test ================================================================== --- ext/fts5/test/fts5secure8.test +++ ext/fts5/test/fts5secure8.test @@ -39,10 +39,26 @@ } do_execsql_test 1.2 { PRAGMA integrity_check; } {ok} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE xyz USING fts5 ( + name, + content='' + ); + + INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 1); + INSERT INTO xyz (rowid, name) VALUES(1, 'A'); + INSERT INTO xyz (rowid, name) VALUES(2, 'A'); + INSERT INTO xyz(xyz, rowid, name) VALUES('delete', 2, 'A'); +} + +do_execsql_test 2.1 { + pragma quick_check; +} {ok} Index: ext/fts5/test/fts5tokenizer2.test ================================================================== --- ext/fts5/test/fts5tokenizer2.test +++ ext/fts5/test/fts5tokenizer2.test @@ -82,8 +82,28 @@ } {AAdontBB>mess<} do_execsql_test 1.7 { SELECT highlight(t1, 0, '>', '<') FROM t1('BB mess'); } {AAdont>BBmess<} + +# 2024-08-06 https://sqlite.org/forum/forumpost/171bcc2bcd +# Error handling of tokenize= arguments. +# +foreach {n tkz} { + 1 {ascii none} + 2 {unicode61 none} + 3 {porter none} + 4 {trigram none} + 5 {ascii none 0} + 6 {unicode61 none 0} + 7 {porter none 0} + 8 {trigram none 0} +} { + db eval {DROP TABLE IF EXISTS t2;} + do_catchsql_test 2.$n " + DROP TABLE IF EXISTS t2; + CREATE VIRTUAL TABLE t2 USING fts5(a,b,c,tokenize='$tkz'); + " {1 {error in tokenizer constructor}} +} finish_test Index: ext/fts5/test/fts5trigram.test ================================================================== --- ext/fts5/test/fts5trigram.test +++ ext/fts5/test/fts5trigram.test @@ -67,10 +67,13 @@ do_execsql_test 2.0 { CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize="trigram case_sensitive 1"); INSERT INTO t1 VALUES('abcdefghijklm'); INSERT INTO t1 VALUES('กรุงเทพมหานคร'); } +do_catchsql_test 2.0.1 { + CREATE VIRTUAL TABLE t2 USING fts5(z, tokenize='trigram case_sensitive'); +} {1 {error in tokenizer constructor}} foreach {tn s res} { 1 abc "(abc)defghijklm" 2 defgh "abc(defgh)ijklm" 3 abcdefghijklm "(abcdefghijklm)" @@ -204,11 +207,11 @@ INSERT INTO f (rowid, filename) VALUES (10, "giraffe.png"), (20, "жираф.png"), (30, "cat.png"), (40, "кот.png"), - (50, "misic-🎵-.mp3"); + (50, "misic-🎵-.mp3"); } do_execsql_test 7.1 { SELECT rowid FROM f WHERE +filename GLOB '*ир*'; } {20} do_execsql_test 7.2 { Index: ext/fts5/test/fts5trigram2.test ================================================================== --- ext/fts5/test/fts5trigram2.test +++ ext/fts5/test/fts5trigram2.test @@ -19,10 +19,13 @@ do_execsql_test 1.0 " CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize='trigram remove_diacritics 1'); INSERT INTO t1 VALUES('abc\u0303defghijklm'); INSERT INTO t1 VALUES('a\u0303b\u0303c\u0303defghijklm'); " +do_catchsql_test 1.0.1 { + CREATE VIRTUAL TABLE t2 USING fts5(z, tokenize='trigram remove_diacritics'); +} {1 {error in tokenizer constructor}} do_execsql_test 1.1 { SELECT highlight(t1, 0, '(', ')') FROM t1('abc'); } [list \ "(abc\u0303)defghijklm" \ Index: ext/recover/sqlite3recover.c ================================================================== --- ext/recover/sqlite3recover.c +++ ext/recover/sqlite3recover.c @@ -361,12 +361,12 @@ char *z = 0; va_list ap; va_start(ap, zFmt); if( zFmt ){ z = sqlite3_vmprintf(zFmt, ap); - va_end(ap); } + va_end(ap); sqlite3_free(p->zErrMsg); p->zErrMsg = z; p->errCode = errCode; return errCode; } ADDED ext/session/sessionblob.test Index: ext/session/sessionblob.test ================================================================== --- /dev/null +++ ext/session/sessionblob.test @@ -0,0 +1,55 @@ +# 2024 November 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +if {$::tcl_platform(pointerSize)<8} { + finish_test + return +} + +set testprefix sessionblob + +forcedelete test.db2 +sqlite3 db2 test.db2 + +set NBLOB 2000000 + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(123, zeroblob($NBLOB)); +} + +do_test 1.1 { + sqlite3session S db main + S attach t1 +} {} + +set b2 [string repeat x 1000] +do_test 1.2 { + set ::blob [db incrblob t1 b 123] + for {set ii 0} {$ii < $NBLOB} {incr ii [string length $b2]} { + seek $::blob $ii + puts -nonewline $::blob $b2 + } + close $::blob +} {} + +S delete + +finish_test + Index: ext/wasm/api/sqlite3-opfs-async-proxy.js ================================================================== --- ext/wasm/api/sqlite3-opfs-async-proxy.js +++ ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -85,39 +85,10 @@ if(state.verbose>level) loggers[level]("OPFS asyncer:",...args); }; const log = (...args)=>logImpl(2, ...args); const warn = (...args)=>logImpl(1, ...args); const error = (...args)=>logImpl(0, ...args); - const metrics = Object.create(null); - metrics.reset = ()=>{ - let k; - const r = (m)=>(m.count = m.time = m.wait = 0); - for(k in state.opIds){ - r(metrics[k] = Object.create(null)); - } - let s = metrics.s11n = Object.create(null); - s = s.serialize = Object.create(null); - s.count = s.time = 0; - s = metrics.s11n.deserialize = Object.create(null); - s.count = s.time = 0; - }; - metrics.dump = ()=>{ - let k, n = 0, t = 0, w = 0; - for(k in state.opIds){ - const m = metrics[k]; - n += m.count; - t += m.time; - w += m.wait; - m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; - } - console.log(globalThis?.location?.href, - "metrics for",globalThis?.location?.href,":\n", - metrics, - "\nTotal of",n,"op(s) for",t,"ms", - "approx",w,"ms spent waiting on OPFS APIs."); - console.log("Serialization metrics:",metrics.s11n); - }; /** __openFiles is a map of sqlite3_file pointers (integers) to metadata related to a given OPFS file handles. The pointers are, in this side of the interface, opaque file handle IDs provided by the @@ -263,27 +234,38 @@ cause: errorObject }); this.name = 'GetSyncHandleError'; } }; + + /** + Attempts to find a suitable SQLITE_xyz result code for Error + object e. Returns either such a translation or rc if if it does + not know how to translate the exception. + */ GetSyncHandleError.convertRc = (e,rc)=>{ - if(1){ - return ( - e instanceof GetSyncHandleError - && ((e.cause.name==='NoModificationAllowedError') - /* Inconsistent exception.name from Chrome/ium with the - same exception.message text: */ - || (e.cause.name==='DOMException' - && 0===e.cause.message.indexOf('Access Handles cannot'))) - ) ? ( - /*console.warn("SQLITE_BUSY",e),*/ - state.sq3Codes.SQLITE_BUSY - ) : rc; - }else{ - return rc; - } - } + if( e instanceof GetSyncHandleError ){ + if( e.cause.name==='NoModificationAllowedError' + /* Inconsistent exception.name from Chrome/ium with the + same exception.message text: */ + || (e.cause.name==='DOMException' + && 0===e.cause.message.indexOf('Access Handles cannot')) ){ + return state.sq3Codes.SQLITE_BUSY; + }else if( 'NotFoundError'===e.cause.name ){ + /** + Maintenance reminder: SQLITE_NOTFOUND, though it looks like + a good match, has different semantics than NotFoundError + and is not suitable here. + */ + return state.sq3Codes.SQLITE_CANTOPEN; + } + }else if( 'NotFoundError'===e?.name ){ + return state.sq3Codes.SQLITE_CANTOPEN; + } + return rc; + }; + /** Returns the sync access handle associated with the given file handle object (which must be a valid handle object, as created by xOpen()), lazily opening it if needed. @@ -345,41 +327,10 @@ */ const affirmNotRO = function(opName,fh){ if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs); }; - /** - We track 2 different timers: the "metrics" timer records how much - time we spend performing work. The "wait" timer records how much - time we spend waiting on the underlying OPFS timer. See the calls - to mTimeStart(), mTimeEnd(), wTimeStart(), and wTimeEnd() - throughout this file to see how they're used. - */ - const __mTimer = Object.create(null); - __mTimer.op = undefined; - __mTimer.start = undefined; - const mTimeStart = (op)=>{ - __mTimer.start = performance.now(); - __mTimer.op = op; - //metrics[op] || toss("Maintenance required: missing metrics for",op); - ++metrics[op].count; - }; - const mTimeEnd = ()=>( - metrics[__mTimer.op].time += performance.now() - __mTimer.start - ); - const __wTimer = Object.create(null); - __wTimer.op = undefined; - __wTimer.start = undefined; - const wTimeStart = (op)=>{ - __wTimer.start = performance.now(); - __wTimer.op = op; - //metrics[op] || toss("Maintenance required: missing metrics for",op); - }; - const wTimeEnd = ()=>( - metrics[__wTimer.op].wait += performance.now() - __wTimer.start - ); - /** Gets set to true by the 'opfs-async-shutdown' command to quit the wait loop. This is only intended for debugging purposes: we cannot inspect this file's state while the tight waitLoop() is running and need a way to stop that loop for introspection purposes. @@ -386,41 +337,28 @@ */ let flagAsyncShutdown = false; /** Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods - methods, as well as helpers like mkdir(). Maintenance reminder: - members are in alphabetical order to simplify finding them. + methods, as well as helpers like mkdir(). */ const vfsAsyncImpls = { - 'opfs-async-metrics': async ()=>{ - mTimeStart('opfs-async-metrics'); - metrics.dump(); - storeAndNotify('opfs-async-metrics', 0); - mTimeEnd(); - }, 'opfs-async-shutdown': async ()=>{ flagAsyncShutdown = true; storeAndNotify('opfs-async-shutdown', 0); }, mkdir: async (dirname)=>{ - mTimeStart('mkdir'); let rc = 0; - wTimeStart('mkdir'); try { await getDirForFilename(dirname+"/filepart", true); }catch(e){ state.s11n.storeException(2,e); rc = state.sq3Codes.SQLITE_IOERR; - }finally{ - wTimeEnd(); } storeAndNotify('mkdir', rc); - mTimeEnd(); }, xAccess: async (filename)=>{ - mTimeStart('xAccess'); /* OPFS cannot support the full range of xAccess() queries sqlite3 calls for. We can essentially just tell if the file is accessible, but if it is then it's automatically writable (unless it's locked, which we cannot(?) know without trying to open it). OPFS does not have the notion of read-only. @@ -429,30 +367,24 @@ xAccess semantics because we are limited in what we can communicate back to our synchronous communication partner: 0 = accessible, non-0 means not accessible. */ let rc = 0; - wTimeStart('xAccess'); try{ const [dh, fn] = await getDirForFilename(filename); await dh.getFileHandle(fn); }catch(e){ state.s11n.storeException(2,e); rc = state.sq3Codes.SQLITE_IOERR; - }finally{ - wTimeEnd(); } storeAndNotify('xAccess', rc); - mTimeEnd(); }, xClose: async function(fid/*sqlite3_file pointer*/){ const opName = 'xClose'; - mTimeStart(opName); __implicitLocks.delete(fid); const fh = __openFiles[fid]; let rc = 0; - wTimeStart(opName); if(fh){ delete __openFiles[fid]; await closeSyncHandle(fh); if(fh.deleteOnClose){ try{ await fh.dirHandle.removeEntry(fh.filenamePart) } @@ -460,19 +392,15 @@ } }else{ state.s11n.serialize(); rc = state.sq3Codes.SQLITE_NOTFOUND; } - wTimeEnd(); storeAndNotify(opName, rc); - mTimeEnd(); }, xDelete: async function(...args){ - mTimeStart('xDelete'); const rc = await vfsAsyncImpls.xDeleteNoWait(...args); storeAndNotify('xDelete', rc); - mTimeEnd(); }, xDeleteNoWait: async function(filename, syncDir = 0, recursive = false){ /* The syncDir flag is, for purposes of the VFS API's semantics, ignored here. However, if it has the value 0x1234 then: after deleting the given file, recursively try to delete any empty @@ -484,11 +412,10 @@ however, a "recursive" flag which defaults to false, so presumably it will fail if the dir is not empty and that flag is false. */ let rc = 0; - wTimeStart('xDelete'); try { while(filename){ const [hDir, filenamePart] = await getDirForFilename(filename, false); if(!filenamePart) break; await hDir.removeEntry(filenamePart, {recursive}); @@ -500,68 +427,55 @@ } }catch(e){ state.s11n.storeException(2,e); rc = state.sq3Codes.SQLITE_IOERR_DELETE; } - wTimeEnd(); return rc; }, xFileSize: async function(fid/*sqlite3_file pointer*/){ - mTimeStart('xFileSize'); const fh = __openFiles[fid]; let rc = 0; - wTimeStart('xFileSize'); try{ const sz = await (await getSyncHandle(fh,'xFileSize')).getSize(); state.s11n.serialize(Number(sz)); }catch(e){ state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR); } await releaseImplicitLock(fh); - wTimeEnd(); storeAndNotify('xFileSize', rc); - mTimeEnd(); }, xLock: async function(fid/*sqlite3_file pointer*/, lockType/*SQLITE_LOCK_...*/){ - mTimeStart('xLock'); const fh = __openFiles[fid]; let rc = 0; const oldLockType = fh.xLock; fh.xLock = lockType; if( !fh.syncHandle ){ - wTimeStart('xLock'); try { await getSyncHandle(fh,'xLock'); __implicitLocks.delete(fid); }catch(e){ state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); fh.xLock = oldLockType; } - wTimeEnd(); } storeAndNotify('xLock',rc); - mTimeEnd(); }, xOpen: async function(fid/*sqlite3_file pointer*/, filename, flags/*SQLITE_OPEN_...*/, opfsFlags/*OPFS_...*/){ const opName = 'xOpen'; - mTimeStart(opName); const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); - wTimeStart('xOpen'); try{ let hDir, filenamePart; try { [hDir, filenamePart] = await getDirForFilename(filename, !!create); }catch(e){ state.s11n.storeException(1,e); storeAndNotify(opName, state.sq3Codes.SQLITE_NOTFOUND); - mTimeEnd(); - wTimeEnd(); return; } if( state.opfsFlags.OPFS_UNLINK_BEFORE_OPEN & opfsFlags ){ try{ await hDir.removeEntry(filenamePart); @@ -569,131 +483,95 @@ /* ignoring */ //warn("Ignoring failed Unlink of",filename,":",e); } } const hFile = await hDir.getFileHandle(filenamePart, {create}); - wTimeEnd(); const fh = Object.assign(Object.create(null),{ fid: fid, filenameAbs: filename, filenamePart: filenamePart, dirHandle: hDir, fileHandle: hFile, sabView: state.sabFileBufView, - readOnly: create - ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), + readOnly: !create && !!(state.sq3Codes.SQLITE_OPEN_READONLY & flags), deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags) }); fh.releaseImplicitLocks = (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) || state.opfsFlags.defaultUnlockAsap; - if(0 /* this block is modelled after something wa-sqlite - does but it leads to immediate contention on journal files. - Update: this approach reportedly only works for DELETE journal - mode. */ - && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ - /* sqlite does not lock these files, so go ahead and grab an OPFS - lock. */ - fh.xLock = "xOpen"/* Truthy value to keep entry from getting - flagged as auto-locked. String value so - that we can easily distinguish is later - if needed. */; - await getSyncHandle(fh,'xOpen'); - } __openFiles[fid] = fh; storeAndNotify(opName, 0); }catch(e){ - wTimeEnd(); error(opName,e); state.s11n.storeException(1,e); storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR); } - mTimeEnd(); }, xRead: async function(fid/*sqlite3_file pointer*/,n,offset64){ - mTimeStart('xRead'); let rc = 0, nRead; const fh = __openFiles[fid]; try{ - wTimeStart('xRead'); nRead = (await getSyncHandle(fh,'xRead')).read( fh.sabView.subarray(0, n), {at: Number(offset64)} ); - wTimeEnd(); if(nRead < n){/* Zero-fill remaining bytes */ fh.sabView.fill(0, nRead, n); rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; } }catch(e){ - if(undefined===nRead) wTimeEnd(); error("xRead() failed",e,fh); state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); } await releaseImplicitLock(fh); storeAndNotify('xRead',rc); - mTimeEnd(); }, xSync: async function(fid/*sqlite3_file pointer*/,flags/*ignored*/){ - mTimeStart('xSync'); const fh = __openFiles[fid]; let rc = 0; if(!fh.readOnly && fh.syncHandle){ try { - wTimeStart('xSync'); await fh.syncHandle.flush(); }catch(e){ state.s11n.storeException(2,e); rc = state.sq3Codes.SQLITE_IOERR_FSYNC; } - wTimeEnd(); } storeAndNotify('xSync',rc); - mTimeEnd(); }, xTruncate: async function(fid/*sqlite3_file pointer*/,size){ - mTimeStart('xTruncate'); let rc = 0; const fh = __openFiles[fid]; - wTimeStart('xTruncate'); try{ affirmNotRO('xTruncate', fh); await (await getSyncHandle(fh,'xTruncate')).truncate(size); }catch(e){ error("xTruncate():",e,fh); state.s11n.storeException(2,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); } await releaseImplicitLock(fh); - wTimeEnd(); storeAndNotify('xTruncate',rc); - mTimeEnd(); }, xUnlock: async function(fid/*sqlite3_file pointer*/, lockType/*SQLITE_LOCK_...*/){ - mTimeStart('xUnlock'); let rc = 0; const fh = __openFiles[fid]; if( state.sq3Codes.SQLITE_LOCK_NONE===lockType && fh.syncHandle ){ - wTimeStart('xUnlock'); try { await closeSyncHandle(fh) } catch(e){ state.s11n.storeException(1,e); rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; } - wTimeEnd(); } storeAndNotify('xUnlock',rc); - mTimeEnd(); }, xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){ - mTimeStart('xWrite'); let rc; const fh = __openFiles[fid]; - wTimeStart('xWrite'); try{ affirmNotRO('xWrite', fh); rc = ( n === (await getSyncHandle(fh,'xWrite')) .write(fh.sabView.subarray(0, n), @@ -703,13 +581,11 @@ error("xWrite():",e,fh); state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); } await releaseImplicitLock(fh); - wTimeEnd(); storeAndNotify('xWrite',rc); - mTimeEnd(); } }/*vfsAsyncImpls*/; const initS11n = ()=>{ /** @@ -739,12 +615,10 @@ case TypeIds.string.id: return TypeIds.string; default: toss("Invalid type ID:",tid); } }; state.s11n.deserialize = function(clear=false){ - ++metrics.s11n.deserialize.count; - const t = performance.now(); const argc = viewU8[0]; const rc = argc ? [] : null; if(argc){ const typeIds = []; let offset = 1, i, n, v; @@ -765,16 +639,13 @@ rc.push(v); } } if(clear) viewU8[0] = 0; //log("deserialize:",argc, rc); - metrics.s11n.deserialize.time += performance.now() - t; return rc; }; state.s11n.serialize = function(...args){ - const t = performance.now(); - ++metrics.s11n.serialize.count; if(args.length){ //log("serialize():",args); const typeIds = []; let i = 0, offset = 1; viewU8[0] = args.length & 0xff /* header = # of args */; @@ -801,11 +672,10 @@ } //log("serialize() result:",viewU8.slice(0,offset)); }else{ viewU8[0] = 0; } - metrics.s11n.serialize.time += performance.now() - t; }; state.s11n.storeException = state.asyncS11nExceptions ? ((priority,e)=>{ if(priority<=state.asyncS11nExceptions){ @@ -883,11 +753,10 @@ if(!Number.isFinite(state.opIds[k])){ toss("Maintenance required: missing state.opIds[",k,"]"); } }); initS11n(); - metrics.reset(); log("init state",state); wPost('opfs-async-inited'); waitLoop(); break; } @@ -896,13 +765,10 @@ warn("Restarting after opfs-async-shutdown. Might or might not work."); flagAsyncShutdown = false; waitLoop(); } break; - case 'opfs-async-metrics': - metrics.dump(); - break; } }; wPost('opfs-async-loaded'); }).catch((e)=>error("error initializing OPFS asyncer:",e)); }/*installAsyncProxy()*/; Index: ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js ================================================================== --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -55,11 +55,11 @@ */ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; const toss = sqlite3.util.toss; const toss3 = sqlite3.util.toss3; - const initPromises = Object.create(null); + const initPromises = Object.create(null) /* cache of (name:result) of VFS init results */; const capi = sqlite3.capi; const util = sqlite3.util; const wasm = sqlite3.wasm; // Config opts for the VFS... const SECTOR_SIZE = 4096; @@ -841,10 +841,11 @@ */ async removeVfs(){ if(!this.#cVfs.pointer || !this.#dhOpaque) return false; capi.sqlite3_vfs_unregister(this.#cVfs.pointer); this.#cVfs.dispose(); + delete initPromises[this.vfsName]; try{ this.releaseAccessHandles(); await this.#dhVfsRoot.removeEntry(OPAQUE_DIR_NAME, {recursive: true}); this.#dhOpaque = undefined; await this.#dhVfsParent.removeEntry( Index: ext/wasm/api/sqlite3-vfs-opfs.c-pp.js ================================================================== --- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -390,10 +390,11 @@ state.sq3Codes = Object.create(null); [ 'SQLITE_ACCESS_EXISTS', 'SQLITE_ACCESS_READWRITE', 'SQLITE_BUSY', + 'SQLITE_CANTOPEN', 'SQLITE_ERROR', 'SQLITE_IOERR', 'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE', 'SQLITE_IOERR_DELETE', @@ -442,11 +443,11 @@ instance results in Undefined Behavior. */ OPFS_UNLINK_BEFORE_OPEN: 0x02, /** If true, any async routine which implicitly acquires a sync - access handle (i.e. an OPFS lock) will release that locks at + access handle (i.e. an OPFS lock) will release that lock at the end of the call which acquires it. If false, such "autolocks" are not released until the VFS is idle for some brief amount of time. The benefit of enabling this is much higher concurrency. The @@ -469,13 +470,26 @@ Atomics.store(state.sabOPView, state.opIds.rc, -1); Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); Atomics.notify(state.sabOPView, state.opIds.whichOp) /* async thread will take over here */; const t = performance.now(); - Atomics.wait(state.sabOPView, state.opIds.rc, -1) - /* When this wait() call returns, the async half will have - completed the operation and reported its results. */; + while('not-equal'!==Atomics.wait(state.sabOPView, state.opIds.rc, -1)){ + /* + The reason for this loop is buried in the details of a long + discussion at: + + https://github.com/sqlite/sqlite-wasm/issues/12 + + Summary: in at least one browser flavor, under high loads, + the wait()/notify() pairings can get out of sync. Calling + wait() here until it returns 'not-equal' gets them back in + sync. + */ + } + /* When the above wait() call returns 'not-equal', the async + half will have completed the operation and reported its results + in the state.opIds.rc slot of the SAB. */ const rc = Atomics.load(state.sabOPView, state.opIds.rc); metrics[op].wait += performance.now() - t; if(rc && state.asyncS11nExceptions){ const err = state.s11n.deserialize(); if(err) error(op+"() async error:",...err); @@ -718,13 +732,22 @@ connection has a lock except by trying to obtain and (on success) release a sync-handle for it, but doing so would involve an inherent race condition. For the time being, pending a better solution, we simply report whether the given pFile is open. + + Update 2024-06-12: based on forum discussions, this + function now always sets pOut to 0 (false): + + https://sqlite.org/forum/forumpost/a2f573b00cda1372 */ - const f = __openFiles[pFile]; - wasm.poke(pOut, f.lockType ? 1 : 0, 'i32'); + if(1){ + wasm.poke(pOut, 0, 'i32'); + }else{ + const f = __openFiles[pFile]; + wasm.poke(pOut, f.lockType ? 1 : 0, 'i32'); + } return 0; }, xClose: function(pFile){ mTimeStart('xClose'); let rc = 0; @@ -736,11 +759,10 @@ } mTimeEnd(); return rc; }, xDeviceCharacteristics: function(pFile){ - //debug("xDeviceCharacteristics(",pFile,")"); return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; }, xFileControl: function(pFile, opId, pArg){ /*mTimeStart('xFileControl'); mTimeEnd();*/ @@ -905,10 +927,12 @@ const fh = Object.create(null); fh.fid = pFile; fh.filename = zName; fh.sab = new SharedArrayBuffer(state.fileBufferSize); fh.flags = flags; + fh.readOnly = !(sqlite3.SQLITE_OPEN_CREATE & flags) + && !!(flags & capi.SQLITE_OPEN_READONLY); const rc = opRun('xOpen', pFile, zName, flags, opfsFlags); if(!rc){ /* Recall that sqlite3_vfs::xClose() will be called, even on error, unless pFile->pMethods is NULL. */ if(fh.readOnly){ Index: ext/wasm/tester1.c-pp.js ================================================================== --- ext/wasm/tester1.c-pp.js +++ ext/wasm/tester1.c-pp.js @@ -3222,10 +3222,59 @@ T.assert(2===rc.length) .assert('abcghi'===rc.join('')); //log('rc =',rc); db.close(); } + }) + .t({ + name: 'r/o connection recovery from write op error', + predicate: ()=>hasOpfs() || "Requires OPFS to reproduce", + //predicate: ()=>false, + test: async function(sqlite3){ + /* https://sqlite.org/forum/forumpost/cf37d5ff1182c31081 + + The "opfs" VFS (but not SAHPool) was formerly misbehaving + after a write attempt was made on a db opened with + mode=ro. This test ensures that that behavior is fixed and + compares that behavior with other VFSes. */ + const tryOne = function(vfsName,descr){ + const uri = 'file:///foo.db'; + let db = new sqlite3.oo1.DB(uri + (vfsName ? '?vfs='+vfsName : '')); + db.exec([ + "drop table if exists t;", + "create table t(a);", + "insert into t(a) values('abc'),('def'),('ghi');" + ]); + db.close(); + db = new sqlite3.oo1.DB(uri+'?mode=ro'+ + (vfsName ? '&vfs='+vfsName : '')); + let err; + try { + db.exec('insert into t(a) values(1)'); + }catch(e){ + err = e; + } + T.assert(err && (err.message.indexOf('SQLITE_READONLY')===0)); + try{ + db.exec('select a from t'); + }finally{ + db.close(); + } + }; + const poolConfig = JSON.parse(JSON.stringify(sahPoolConfig)); + poolConfig.name = 'opfs-sahpool-cf37d5ff11'; + let poolUtil; + await sqlite3.installOpfsSAHPoolVfs(poolConfig).then(p=>poolUtil=p); + T.assert(!!sqlite3.capi.sqlite3_vfs_find(poolConfig.name), "Expecting to find just-registered VFS"); + try{ + tryOne(false, "Emscripten filesystem"); + tryOne(poolConfig.name); + tryOne('opfs'); + }finally{ + await poolUtil.removeVfs(); + } + } }) ;/*end of Bug Reports group*/; //////////////////////////////////////////////////////////////////////// log("Loading and initializing sqlite3 WASM module..."); Index: src/alter.c ================================================================== --- src/alter.c +++ src/alter.c @@ -1318,11 +1318,11 @@ ); pParse->eTriggerOp = pNew->op; /* ALWAYS() because if the table of the trigger does not exist, the ** error would have been hit before this point */ if( ALWAYS(pParse->pTriggerTab) ){ - rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab); + rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab)!=0; } /* Resolve symbols in WHEN clause */ if( rc==SQLITE_OK && pNew->pWhen ){ rc = sqlite3ResolveExprNames(&sNC, pNew->pWhen); Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -5987,11 +5987,11 @@ } if( pCur->iPage>0 && indexCellCompare(pCur, 0, pIdxKey, xRecordCompare)<=0 && pIdxKey->errCode==SQLITE_OK ){ - pCur->curFlags &= ~BTCF_ValidOvfl; + pCur->curFlags &= ~(BTCF_ValidOvfl|BTCF_AtLast); if( !pCur->pPage->isInit ){ return SQLITE_CORRUPT_BKPT; } goto bypass_moveto_root; /* Start search on the current page */ } Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -3062,12 +3062,13 @@ #endif /* SQLITE_OMIT_VIEW */ #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) /* ** The Table structure pTable is really a VIEW. Fill in the names of -** the columns of the view in the pTable structure. Return the number -** of errors. If an error is seen leave an error message in pParse->zErrMsg. +** the columns of the view in the pTable structure. Return non-zero if +** there are errors. If an error is seen an error message is left +** in pParse->zErrMsg. */ static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ Table *pSelTab; /* A fake table from which we get the result set */ Select *pSel; /* Copy of the SELECT that implements the view */ int nErr = 0; /* Number of errors encountered */ @@ -3186,11 +3187,11 @@ pTable->pSchema->schemaFlags |= DB_UnresetViews; if( db->mallocFailed ){ sqlite3DeleteColumnNames(db, pTable); } #endif /* SQLITE_OMIT_VIEW */ - return nErr; + return nErr + pParse->nErr; } int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ assert( pTable!=0 ); if( !IsVirtual(pTable) && pTable->nCol>0 ) return 0; return viewGetColumnNames(pParse, pTable); Index: src/date.c ================================================================== --- src/date.c +++ src/date.c @@ -269,12 +269,12 @@ } if( M<=2 ){ Y--; M += 12; } - A = Y/100; - B = 2 - A + (A/4); + A = (Y+4800)/100; + B = 38 - A + (A/4); X1 = 36525*(Y+4716)/100; X2 = 306001*(M+1)/10000; p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000); p->validJD = 1; if( p->validHMS ){ @@ -454,11 +454,11 @@ /* ** Compute the Year, Month, and Day from the julian day number. */ static void computeYMD(DateTime *p){ - int Z, A, B, C, D, E, X1; + int Z, alpha, A, B, C, D, E, X1; if( p->validYMD ) return; if( !p->validJD ){ p->Y = 2000; p->M = 1; p->D = 1; @@ -465,12 +465,12 @@ }else if( !validJulianDay(p->iJD) ){ datetimeError(p); return; }else{ Z = (int)((p->iJD + 43200000)/86400000); - A = (int)((Z - 1867216.25)/36524.25); - A = Z + 1 + A - (A/4); + alpha = (int)((Z + 32044.75)/36524.25) - 52; + A = Z + 1 + alpha - ((alpha+100)/4) + 25; B = A + 1524; C = (int)((B - 122.1)/365.25); D = (36525*(C&32767))/100; E = (int)((B-D)/30.6001); X1 = (int)(30.6001*E); Index: src/func.c ================================================================== --- src/func.c +++ src/func.c @@ -2204,10 +2204,12 @@ StrAccum *pAccum = &pGCC->str; if( pAccum->accError==SQLITE_TOOBIG ){ sqlite3_result_error_toobig(context); }else if( pAccum->accError==SQLITE_NOMEM ){ sqlite3_result_error_nomem(context); + }else if( pGCC->nAccum>0 && pAccum->nChar==0 ){ + sqlite3_result_text(context, "", 1, SQLITE_STATIC); }else{ const char *zText = sqlite3_str_value(pAccum); sqlite3_result_text(context, zText, pAccum->nChar, SQLITE_TRANSIENT); } } Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -715,10 +715,11 @@ if( pRet ){ SelectDest dest; pRet->pSrc->nSrc = 1; pRet->pPrior = pLeft->pPrior; pRet->op = pLeft->op; + if( pRet->pPrior ) pRet->selFlags |= SF_Values; pLeft->pPrior = 0; pLeft->op = TK_SELECT; assert( pLeft->pNext==0 ); assert( pRet->pNext==0 ); p = &pRet->pSrc->a[0]; Index: src/json.c ================================================================== --- src/json.c +++ src/json.c @@ -2845,11 +2845,13 @@ int rawKey = 1; x = pParse->aBlob[iRoot]; zPath++; if( zPath[0]=='"' ){ zKey = zPath + 1; - for(i=1; zPath[i] && zPath[i]!='"'; i++){} + for(i=1; zPath[i] && zPath[i]!='"'; i++){ + if( zPath[i]=='\\' && zPath[i+1]!=0 ) i++; + } nKey = i-1; if( zPath[i] ){ i++; }else{ return JSON_LOOKUP_PATHERROR; Index: src/parse.y ================================================================== --- src/parse.y +++ src/parse.y @@ -528,13 +528,13 @@ pLoop->pOrderBy!=0 ? "ORDER BY" : "LIMIT", sqlite3SelectOpName(pNext->op)); break; } } - if( (p->selFlags & SF_MultiValue)==0 && - (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 && - cnt>mxSelect + if( (p->selFlags & (SF_MultiValue|SF_Values))==0 + && (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 + && cnt>mxSelect ){ sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); } } } Index: src/resolve.c ================================================================== --- src/resolve.c +++ src/resolve.c @@ -226,11 +226,11 @@ ** Return TRUE (non-zero) if zTab is a valid name for the schema table pTab. */ static SQLITE_NOINLINE int isValidSchemaTableName( const char *zTab, /* Name as it appears in the SQL */ Table *pTab, /* The schema table we are trying to match */ - Schema *pSchema /* non-NULL if a database qualifier is present */ + const char *zDb /* non-NULL if a database qualifier is present */ ){ const char *zLegacy; assert( pTab!=0 ); assert( pTab->tnum==1 ); if( sqlite3StrNICmp(zTab, "sqlite_", 7)!=0 ) return 0; @@ -237,11 +237,11 @@ zLegacy = pTab->zName; if( strcmp(zLegacy+7, &LEGACY_TEMP_SCHEMA_TABLE[7])==0 ){ if( sqlite3StrICmp(zTab+7, &PREFERRED_TEMP_SCHEMA_TABLE[7])==0 ){ return 1; } - if( pSchema==0 ) return 0; + if( zDb==0 ) return 0; if( sqlite3StrICmp(zTab+7, &LEGACY_SCHEMA_TABLE[7])==0 ) return 1; if( sqlite3StrICmp(zTab+7, &PREFERRED_SCHEMA_TABLE[7])==0 ) return 1; }else{ if( sqlite3StrICmp(zTab+7, &PREFERRED_SCHEMA_TABLE[7])==0 ) return 1; } @@ -420,11 +420,11 @@ if( sqlite3StrICmp(zTab, pItem->zAlias)!=0 ){ continue; } }else if( sqlite3StrICmp(zTab, pTab->zName)!=0 ){ if( pTab->tnum!=1 ) continue; - if( !isValidSchemaTableName(zTab, pTab, pSchema) ) continue; + if( !isValidSchemaTableName(zTab, pTab, zDb) ) continue; } assert( ExprUseYTab(pExpr) ); if( IN_RENAME_OBJECT && pItem->zAlias ){ sqlite3RenameTokenRemap(pParse, 0, (void*)&pExpr->y.pTab); } @@ -1294,11 +1294,11 @@ assert( pExpr->pLeft->op==TK_ORDER ); assert( ExprUseXList(pExpr->pLeft) ); sqlite3WalkExprList(pWalker, pExpr->pLeft->x.pList); } #ifndef SQLITE_OMIT_WINDOWFUNC - if( pWin ){ + if( pWin && pParse->nErr==0 ){ Select *pSel = pNC->pWinSelect; assert( pWin==0 || (ExprUseYWin(pExpr) && pWin==pExpr->y.pWin) ); if( IN_RENAME_OBJECT==0 ){ sqlite3WindowUpdate(pParse, pSel ? pSel->pWinDefn : 0, pWin, pDef); if( pParse->db->mallocFailed ) break; @@ -2152,19 +2152,22 @@ /* ** Resolve all names for all expression in an expression list. This is ** just like sqlite3ResolveExprNames() except that it works for an expression ** list rather than a single expression. +** +** The return value is SQLITE_OK (0) for success or SQLITE_ERROR (1) for a +** failure. */ int sqlite3ResolveExprListNames( NameContext *pNC, /* Namespace to resolve expressions in. */ ExprList *pList /* The expression list to be analyzed. */ ){ int i; int savedHasAgg = 0; Walker w; - if( pList==0 ) return WRC_Continue; + if( pList==0 ) return SQLITE_OK; w.pParse = pNC->pParse; w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; w.xSelectCallback2 = 0; w.u.pNC = pNC; @@ -2174,11 +2177,11 @@ Expr *pExpr = pList->a[i].pExpr; if( pExpr==0 ) continue; #if SQLITE_MAX_EXPR_DEPTH>0 w.pParse->nHeight += pExpr->nHeight; if( sqlite3ExprCheckHeight(w.pParse, w.pParse->nHeight) ){ - return WRC_Abort; + return SQLITE_ERROR; } #endif sqlite3WalkExprNN(&w, pExpr); #if SQLITE_MAX_EXPR_DEPTH>0 w.pParse->nHeight -= pExpr->nHeight; @@ -2191,14 +2194,14 @@ ExprSetProperty(pExpr, pNC->ncFlags & (NC_HasAgg|NC_HasWin) ); savedHasAgg |= pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); } - if( w.pParse->nErr>0 ) return WRC_Abort; + if( w.pParse->nErr>0 ) return SQLITE_ERROR; } pNC->ncFlags |= savedHasAgg; - return WRC_Continue; + return SQLITE_OK; } /* ** Resolve all names in all expressions of a SELECT and in all ** descendants of the SELECT, including compounds off of p->pPrior, Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -6733,10 +6733,11 @@ int i; struct AggInfo_func *pF; for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); + if( pParse->nErr ) return; pList = pF->pFExpr->x.pList; if( pF->iOBTab>=0 ){ /* For an ORDER BY aggregate, calls to OP_AggStep were deferred. Inputs ** were stored in emphermal table pF->iOBTab. Here, we extract those ** inputs (in ORDER BY order) and make all calls to OP_AggStep @@ -6942,19 +6943,21 @@ sqlite3ReleaseTempRange(pParse, regAgg, nArg); } if( addrNext ){ sqlite3VdbeResolveLabel(v, addrNext); } + if( pParse->nErr ) return; } if( regHit==0 && pAggInfo->nAccumulator ){ regHit = regAcc; } if( regHit ){ addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v); } for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ sqlite3ExprCode(pParse, pC->pCExpr, AggInfoColumnReg(pAggInfo,i)); + if( pParse->nErr ) return; } pAggInfo->directMode = 0; if( addrHitTest ){ sqlite3VdbeJumpHereOrPopInst(v, addrHitTest); Index: src/shell.c.in ================================================================== --- src/shell.c.in +++ src/shell.c.in @@ -8975,11 +8975,10 @@ zTable, zSchema); if( zSql==0 ){ import_cleanup(&sCtx); shell_out_of_memory(); } - nByte = strlen(zSql); rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); zSql = 0; if( rc ){ if (pStmt) sqlite3_finalize(pStmt); @@ -8994,28 +8993,34 @@ nCol = 0; } sqlite3_finalize(pStmt); pStmt = 0; if( nCol==0 ) return 0; /* no columns, no error */ - zSql = sqlite3_malloc64( nByte*2 + 20 + nCol*2 ); + + nByte = 64 /* space for "INSERT INTO", "VALUES(", ")\0" */ + + (zSchema ? strlen(zSchema)*2 + 2: 0) /* Quoted schema name */ + + strlen(zTable)*2 + 2 /* Quoted table name */ + + nCol*2; /* Space for ",?" for each column */ + zSql = sqlite3_malloc64( nByte ); if( zSql==0 ){ import_cleanup(&sCtx); shell_out_of_memory(); } if( zSchema ){ - sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", zSchema, zTable); }else{ - sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); } j = strlen30(zSql); for(i=1; i=2 ){ oputf("Insert using: %s\n", zSql); } rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -3386,11 +3386,11 @@ #define WHERE_WANT_DISTINCT 0x0100 /* All output needs to be distinct */ #define WHERE_SORTBYGROUP 0x0200 /* Support sqlite3WhereIsSorted() */ #define WHERE_AGG_DISTINCT 0x0400 /* Query is "SELECT agg(DISTINCT ...)" */ #define WHERE_ORDERBY_LIMIT 0x0800 /* ORDERBY+LIMIT on the inner loop */ #define WHERE_RIGHT_JOIN 0x1000 /* Processing a RIGHT JOIN */ - /* 0x2000 not currently used */ +#define WHERE_KEEP_ALL_JOINS 0x2000 /* Do not do the omit-noop-join opt */ #define WHERE_USE_LIMIT 0x4000 /* Use the LIMIT in cost estimates */ /* 0x8000 not currently used */ /* Allowed return values from sqlite3WhereIsDistinct() */ Index: src/vdbeInt.h ================================================================== --- src/vdbeInt.h +++ src/vdbeInt.h @@ -538,10 +538,11 @@ UnpackedRecord *pNewUnpacked; /* Unpacked version of new.* record */ int iNewReg; /* Register for new.* values */ int iBlobWrite; /* Value returned by preupdate_blobwrite() */ i64 iKey1; /* First key value passed to hook */ i64 iKey2; /* Second key value passed to hook */ + Mem oldipk; /* Memory cell holding "old" IPK value */ Mem *aNew; /* Array of new.* values */ Table *pTab; /* Schema object being updated */ Index *pPk; /* PK index if pTab is WITHOUT ROWID */ }; Index: src/vdbeapi.c ================================================================== --- src/vdbeapi.c +++ src/vdbeapi.c @@ -2178,41 +2178,44 @@ if( iIdx>=p->pCsr->nField || iIdx<0 ){ rc = SQLITE_RANGE; goto preupdate_old_out; } - /* If the old.* record has not yet been loaded into memory, do so now. */ - if( p->pUnpacked==0 ){ - u32 nRec; - u8 *aRec; - - assert( p->pCsr->eCurType==CURTYPE_BTREE ); - nRec = sqlite3BtreePayloadSize(p->pCsr->uc.pCursor); - aRec = sqlite3DbMallocRaw(db, nRec); - if( !aRec ) goto preupdate_old_out; - rc = sqlite3BtreePayload(p->pCsr->uc.pCursor, 0, nRec, aRec); - if( rc==SQLITE_OK ){ - p->pUnpacked = vdbeUnpackRecord(&p->keyinfo, nRec, aRec); - if( !p->pUnpacked ) rc = SQLITE_NOMEM; - } - if( rc!=SQLITE_OK ){ - sqlite3DbFree(db, aRec); - goto preupdate_old_out; - } - p->aRecord = aRec; - } - - pMem = *ppValue = &p->pUnpacked->aMem[iIdx]; if( iIdx==p->pTab->iPKey ){ + pMem = *ppValue = &p->oldipk; sqlite3VdbeMemSetInt64(pMem, p->iKey1); - }else if( iIdx>=p->pUnpacked->nField ){ - *ppValue = (sqlite3_value *)columnNullValue(); - }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){ - if( pMem->flags & (MEM_Int|MEM_IntReal) ){ - testcase( pMem->flags & MEM_Int ); - testcase( pMem->flags & MEM_IntReal ); - sqlite3VdbeMemRealify(pMem); + }else{ + /* If the old.* record has not yet been loaded into memory, do so now. */ + if( p->pUnpacked==0 ){ + u32 nRec; + u8 *aRec; + + assert( p->pCsr->eCurType==CURTYPE_BTREE ); + nRec = sqlite3BtreePayloadSize(p->pCsr->uc.pCursor); + aRec = sqlite3DbMallocRaw(db, nRec); + if( !aRec ) goto preupdate_old_out; + rc = sqlite3BtreePayload(p->pCsr->uc.pCursor, 0, nRec, aRec); + if( rc==SQLITE_OK ){ + p->pUnpacked = vdbeUnpackRecord(&p->keyinfo, nRec, aRec); + if( !p->pUnpacked ) rc = SQLITE_NOMEM; + } + if( rc!=SQLITE_OK ){ + sqlite3DbFree(db, aRec); + goto preupdate_old_out; + } + p->aRecord = aRec; + } + + pMem = *ppValue = &p->pUnpacked->aMem[iIdx]; + if( iIdx>=p->pUnpacked->nField ){ + *ppValue = (sqlite3_value *)columnNullValue(); + }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){ + if( pMem->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_IntReal ); + sqlite3VdbeMemRealify(pMem); + } } } preupdate_old_out: sqlite3Error(db, rc); Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -5334,11 +5334,12 @@ */ sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff){ assert( iVar>0 ); if( v ){ Mem *pMem = &v->aVar[iVar-1]; - assert( (v->db->flags & SQLITE_EnableQPSG)==0 ); + assert( (v->db->flags & SQLITE_EnableQPSG)==0 + || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 ); if( 0==(pMem->flags & MEM_Null) ){ sqlite3_value *pRet = sqlite3ValueNew(v->db); if( pRet ){ sqlite3VdbeMemCopy((Mem *)pRet, pMem); sqlite3ValueApplyAffinity(pRet, aff, SQLITE_UTF8); @@ -5354,11 +5355,12 @@ ** to sqlite3_reoptimize() that re-preparing the statement may result ** in a better query plan. */ void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ assert( iVar>0 ); - assert( (v->db->flags & SQLITE_EnableQPSG)==0 ); + assert( (v->db->flags & SQLITE_EnableQPSG)==0 + || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 ); if( iVar>=32 ){ v->expmask |= 0x80000000; }else{ v->expmask |= ((u32)1 << (iVar-1)); } @@ -5521,14 +5523,15 @@ db->xPreUpdateCallback(db->pPreUpdateArg, db, op, zDb, zTbl, iKey1, iKey2); db->pPreUpdate = 0; sqlite3DbFree(db, preupdate.aRecord); vdbeFreeUnpacked(db, preupdate.keyinfo.nKeyField+1, preupdate.pUnpacked); vdbeFreeUnpacked(db, preupdate.keyinfo.nKeyField+1, preupdate.pNewUnpacked); + sqlite3VdbeMemRelease(&preupdate.oldipk); if( preupdate.aNew ){ int i; for(i=0; inField; i++){ sqlite3VdbeMemRelease(&preupdate.aNew[i]); } sqlite3DbNNFreeNN(db, preupdate.aNew); } } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -3957,11 +3957,13 @@ WHERETRACE(0x200, ("-> %s might be a covering expression index" " according to whereIsCoveringIndex()\n", pProbe->zName)); } } - }else if( m==0 ){ + }else if( m==0 + && (HasRowid(pTab) || pWInfo->pSelect!=0 || sqlite3FaultSim(700)) + ){ WHERETRACE(0x200, ("-> %s a covering index according to bitmasks\n", pProbe->zName, m==0 ? "is" : "is not")); pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; } @@ -5846,10 +5848,14 @@ ** might move from the right side to the left side of the RIGHT JOIN. ** Note: Due to (2), this condition can only arise if the table is ** the right-most table of a subquery that was flattened into the ** main query and that subquery was the right-hand operand of an ** inner join that held an ON or USING clause. +** 6) The ORDER BY clause has 63 or fewer terms +** 7) The omit-noop-join optimization is enabled. +** +** Items (1), (6), and (7) are checked by the caller. ** ** For example, given: ** ** CREATE TABLE t1(ipk INTEGER PRIMARY KEY, v1); ** CREATE TABLE t2(ipk INTEGER PRIMARY KEY, v2); @@ -5891,10 +5897,11 @@ hasRightJoin = (pWInfo->pTabList->a[0].fg.jointype & JT_LTORJ)!=0; for(i=pWInfo->nLevel-1; i>=1; i--){ WhereTerm *pTerm, *pEnd; SrcItem *pItem; WhereLoop *pLoop; + Bitmask m1; pLoop = pWInfo->a[i].pWLoop; pItem = &pWInfo->pTabList->a[pLoop->iTab]; if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ) continue; if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)==0 && (pLoop->wsFlags & WHERE_ONEROW)==0 @@ -5918,10 +5925,12 @@ break; /* restriction (5) */ } } if( pTerm drop loop %c not used\n", pLoop->cId)); + m1 = MASKBIT(i)-1; + pWInfo->revMask = (m1 & pWInfo->revMask) | ((pWInfo->revMask>>1) & ~m1); notReady &= ~pLoop->maskSelf; for(pTerm=pWInfo->sWC.a; pTermprereqAll & pLoop->maskSelf)!=0 ){ pTerm->wtFlags |= TERM_CODED; } @@ -6259,10 +6268,11 @@ /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */ testcase( pOrderBy && pOrderBy->nExpr==BMS-1 ); if( pOrderBy && pOrderBy->nExpr>=BMS ){ pOrderBy = 0; wctrlFlags &= ~WHERE_WANT_DISTINCT; + wctrlFlags |= WHERE_KEEP_ALL_JOINS; /* Disable omit-noop-join opt */ } /* The number of tables in the FROM clause is limited by the number of ** bits in a Bitmask */ @@ -6559,14 +6569,14 @@ ** some C-compiler optimizers from in-lining the ** sqlite3WhereCodeOneLoopStart() procedure, and it is important to ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons. */ notReady = ~(Bitmask)0; - if( pWInfo->nLevel>=2 - && pResultSet!=0 /* these two combine to guarantee */ - && 0==(wctrlFlags & WHERE_AGG_DISTINCT) /* condition (1) above */ - && OptimizationEnabled(db, SQLITE_OmitNoopJoin) + if( pWInfo->nLevel>=2 /* Must be a join, or this opt8n is pointless */ + && pResultSet!=0 /* Condition (1) */ + && 0==(wctrlFlags & (WHERE_AGG_DISTINCT|WHERE_KEEP_ALL_JOINS)) /* (1),(6) */ + && OptimizationEnabled(db, SQLITE_OmitNoopJoin) /* (7) */ ){ notReady = whereOmitNoopJoin(pWInfo, notReady); nTabList = pWInfo->nLevel; assert( nTabList>0 ); } @@ -6882,30 +6892,10 @@ if( (db->flags & SQLITE_VdbeAddopTrace)==0 ) return; sqlite3VdbePrintOp(0, pc, pOp); } #endif -#ifdef SQLITE_DEBUG -/* -** Return true if cursor iCur is opened by instruction k of the -** bytecode. Used inside of assert() only. -*/ -static int cursorIsOpen(Vdbe *v, int iCur, int k){ - while( k>=0 ){ - VdbeOp *pOp = sqlite3VdbeGetOp(v,k--); - if( pOp->p1!=iCur ) continue; - if( pOp->opcode==OP_Close ) return 0; - if( pOp->opcode==OP_OpenRead ) return 1; - if( pOp->opcode==OP_OpenWrite ) return 1; - if( pOp->opcode==OP_OpenDup ) return 1; - if( pOp->opcode==OP_OpenAutoindex ) return 1; - if( pOp->opcode==OP_OpenEphemeral ) return 1; - } - return 0; -} -#endif /* SQLITE_DEBUG */ - /* ** Generate the end of the WHERE loop. See comments on ** sqlite3WhereBegin() for additional information. */ void sqlite3WhereEnd(WhereInfo *pWInfo){ @@ -7201,20 +7191,14 @@ }else{ /* Unable to translate the table reference into an index ** reference. Verify that this is harmless - that the ** table being referenced really is open. */ -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 - || cursorIsOpen(v,pOp->p1,k) - || pOp->opcode==OP_Offset - ); -#else - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 - || cursorIsOpen(v,pOp->p1,k) - ); -#endif + if( pLoop->wsFlags & WHERE_IDX_ONLY ){ + sqlite3ErrorMsg(pParse, "internal query planner error"); + pParse->rc = SQLITE_INTERNAL; + } } }else if( pOp->opcode==OP_Rowid ){ pOp->p1 = pLevel->iIdxCur; pOp->opcode = OP_IdxRowid; OpcodeRewriteTrace(db, k, pOp); Index: src/whereexpr.c ================================================================== --- src/whereexpr.c +++ src/whereexpr.c @@ -211,16 +211,30 @@ }else if( op==TK_STRING ){ assert( !ExprHasProperty(pRight, EP_IntValue) ); z = (u8*)pRight->u.zToken; } if( z ){ - - /* Count the number of prefix characters prior to the first wildcard */ + /* Count the number of prefix bytes prior to the first wildcard. + ** or U+fffd character. If the underlying database has a UTF16LE + ** encoding, then only consider ASCII characters. Note that the + ** encoding of z[] is UTF8 - we are dealing with only UTF8 here in + ** this code, but the database engine itself might be processing + ** content using a different encoding. */ cnt = 0; while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ cnt++; - if( c==wc[3] && z[cnt]!=0 ) cnt++; + if( c==wc[3] && z[cnt]!=0 ){ + cnt++; + }else if( c>=0x80 ){ + const u8 *z2 = z+cnt-1; + if( sqlite3Utf8Read(&z2)==0xfffd || ENC(db)==SQLITE_UTF16LE ){ + cnt--; + break; + }else{ + cnt = (int)(z2-z); + } + } } /* The optimization is possible only if (1) the pattern does not begin ** with a wildcard and if (2) the non-wildcard prefix does not end with ** an (illegal 0xff) character, or (3) the pattern does not consist of @@ -227,15 +241,15 @@ ** a single escape character. The second condition is necessary so ** that we can increment the prefix key to find an upper bound for the ** range search. The third is because the caller assumes that the pattern ** consists of at least one character after all escapes have been ** removed. */ - if( (cnt>1 || (cnt>0 && z[0]!=wc[3])) && 255!=(u8)z[cnt-1] ){ + if( (cnt>1 || (cnt>0 && z[0]!=wc[3])) && ALWAYS(255!=(u8)z[cnt-1]) ){ Expr *pPrefix; /* A "complete" match if the pattern ends with "*" or "%" */ - *pisComplete = c==wc[0] && z[cnt+1]==0; + *pisComplete = c==wc[0] && z[cnt+1]==0 && ENC(db)!=SQLITE_UTF16LE; /* Get the pattern prefix. Remove all escapes from the prefix. */ pPrefix = sqlite3Expr(db, TK_STRING, (char*)z); if( pPrefix ){ int iFrom, iTo; ADDED test/date5.test Index: test/date5.test ================================================================== --- /dev/null +++ test/date5.test @@ -0,0 +1,86 @@ +# 2024-08-19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# https://sqlite.org/forum/forumpost/eaa0a09786c6368b +# +# Apparently SQLite has been miscomputing leap-year dates before +# the year 0400. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Skip this whole file if date and time functions are omitted +# at compile-time +# +ifcapable {!datetime} { + finish_test + return +} + +# Data sources: +# 1-10 https://ssd.jpl.nasa.gov/tools/jdc/#/cd +# 11 Jean Meeus, Astronomical Algorithms, ISBN 0-943396-61-1, p.59 +# 12 https://en.wikipedia.org/wiki/Julian_day +# +# ID YEAR MONTH DAY JD +set date5data { + 1 2024 2 29 2460369.5 + 2 2024 3 1 2460370.5 + 3 2023 2 28 2460003.5 + 4 2023 3 1 2460004.5 + 5 2000 2 29 2451603.5 + 6 2000 3 1 2451604.5 + 7 1900 2 28 2415078.5 + 8 1900 3 1 2415079.5 + 9 1712 2 29 2346413.5 + 10 1712 3 1 2346414.5 + 11 1977 4 26 2443259.5 + 12 2013 1 1 2456293.5 +} + +foreach {id y m d jd} $date5data { + set date [format %04d-%02d-%02d $y $m $d] + do_execsql_test date5-jd$jd { + SELECT date($::jd); + } $date + do_execsql_test date5-cal/$date { + SELECT julianday($::date); + } $jd + for {set i 1} {$y+400*$i<=9999} {incr i} { + set y2 [expr {$y+400*$i}] + set date2 [format %04d-%02d-%02d $y2 $m $d] + set jd2 [expr {$jd+146097*$i}] + do_execsql_test date5-jd$jd2 { + SELECT date($::jd2); + } $date2 + do_execsql_test date5-cal/$date2 { + SELECT julianday($::date2); + } $jd2 + } + for {set i 1} {$y-400*$i>=-4712} {incr i} { + set y2 [expr {$y-400*$i}] + if {$y2<0} { + set date2 [format -%04d-%02d-%02d [expr {-$y2}] $m $d] + } else { + set date2 [format %04d-%02d-%02d $y2 $m $d] + } + set jd2 [expr {$jd-146097*$i}] + do_execsql_test date5-jd$jd2 { + SELECT date($::jd2); + } $date2 + do_execsql_test date5-cal/$date2 { + SELECT julianday($::date2); + } $jd2 + } +} + +finish_test Index: test/fuzzinvariants.c ================================================================== --- test/fuzzinvariants.c +++ test/fuzzinvariants.c @@ -35,10 +35,35 @@ sqlite3_stmt *pTest, /* The alternative test query with a missing row */ int iRow, /* Row number in pOrig */ unsigned int dbOpt, /* Optimization flags on pOrig */ int noOpt /* True if opt flags inverted for pTest */ ); + +/* +** Special parameter binding, for testing and debugging purposes. +** +** $int_NNN -> integer value NNN +** $text_TTTT -> floating point value TTT with destructor +*/ +static void bindDebugParameters(sqlite3_stmt *pStmt){ + int nVar = sqlite3_bind_parameter_count(pStmt); + int i; + for(i=0; i=2 ){ char *zSql = sqlite3_expanded_sql(pCk); printf("invariant-validity-check #2:\n%s\n", zSql); sqlite3_free(zSql); } + bindDebugParameters(pCk); while( (rc = sqlite3_step(pCk))==SQLITE_ROW ){ for(i=0; i=nCol ) break; @@ -197,10 +224,11 @@ printf("invariant-validity-check #3:\n%s\n", zSql); sqlite3_free(zSql); } sqlite3_reset(pTestStmt); + bindDebugParameters(pCk); while( (rc = sqlite3_step(pTestStmt))==SQLITE_ROW ){ for(i=0; i=nCol ){ @@ -296,10 +324,11 @@ rc = sqlite3_prepare_v2(db, sqlite3_str_value(pTest), -1, &pBase, 0); if( rc ){ sqlite3_finalize(pBase); pBase = pStmt; } + bindDebugParameters(pBase); for(i=0; i