Index: ext/fts3/fts3_snippet.c ================================================================== --- ext/fts3/fts3_snippet.c +++ ext/fts3/fts3_snippet.c @@ -444,11 +444,11 @@ }else{ iScore += 1000; } mCover |= mPhrase; - for(j=0; jnToken; j++){ + for(j=0; jnToken && jnSnippet; j++){ mHighlight |= (mPos>>j); } if( 0==(*pCsr & 0x0FE) ) break; fts3GetDeltaPosition(&pCsr, &iCsr); Index: ext/recover/dbdata.c ================================================================== --- ext/recover/dbdata.c +++ ext/recover/dbdata.c @@ -492,10 +492,19 @@ } } } } +/* This macro is a copy of the MX_CELL() macro in the SQLite core. Given +** a page-size, it returns the maximum number of cells that may be present +** on the page. */ +#define DBDATA_MX_CELL(pgsz) ((pgsz-8)/6) + +/* Maximum number of fields that may appear in a single record. This is +** the "hard-limit", according to comments in sqliteLimit.h. */ +#define DBDATA_MX_FIELD 32676 + /* ** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry. */ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; @@ -520,10 +529,13 @@ } assert( iOff+3+2<=pCsr->nPage ); pCsr->iCell = pTab->bPtr ? -2 : 0; pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); + if( pCsr->nCell>DBDATA_MX_CELL(pCsr->nPage) ){ + pCsr->nCell = DBDATA_MX_CELL(pCsr->nPage); + } } if( pTab->bPtr ){ if( pCsr->aPage[iOff]!=0x02 && pCsr->aPage[iOff]!=0x05 ){ pCsr->iCell = pCsr->nCell; @@ -564,23 +576,23 @@ } if( pCsr->iCell>=pCsr->nCell ){ bNextPage = 1; }else{ + int iCellPtr = iOff + 8 + nPointer + pCsr->iCell*2; - iOff += 8 + nPointer + pCsr->iCell*2; - if( iOff>pCsr->nPage ){ + if( iCellPtr>pCsr->nPage ){ bNextPage = 1; }else{ - iOff = get_uint16(&pCsr->aPage[iOff]); + iOff = get_uint16(&pCsr->aPage[iCellPtr]); } /* For an interior node cell, skip past the child-page number */ iOff += nPointer; /* Load the "byte of payload including overflow" field */ - if( bNextPage || iOff>pCsr->nPage ){ + if( bNextPage || iOff>pCsr->nPage || iOff<=iCellPtr ){ bNextPage = 1; }else{ iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload); if( nPayload>0x7fffff00 ) nPayload &= 0x3fff; } @@ -659,11 +671,13 @@ } }else{ pCsr->iField++; if( pCsr->iField>0 ){ sqlite3_int64 iType; - if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){ + if( pCsr->pHdrPtr>=&pCsr->pRec[pCsr->nRec] + || pCsr->iField>=DBDATA_MX_FIELD + ){ bNextPage = 1; }else{ int szField = 0; pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType); szField = dbdataValueBytes(iType); Index: ext/recover/recovercorrupt2.test ================================================================== --- ext/recover/recovercorrupt2.test +++ ext/recover/recovercorrupt2.test @@ -521,8 +521,36 @@ do_test 7.1 { set R [sqlite3_recover_init db main test.db2] catch { $R run } list [catch { $R finish } msg] $msg } {1 {file is not a database}} + +reset_db +breakpoint +do_test 8.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 8192 pagesize 4096 filename db.sqlite +| page 1 offset 0 +| 0: ac ae b3 76 74 65 20 66 6f 72 6d 61 74 20 33 00 ...vte format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 01 00 00 00 02 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ................ +| 96: 00 2e 76 8a 0d ff ff ff 1e 0f cb 00 0f cb 00 00 ..v............. +| 4032: 00 00 00 00 00 00 00 00 00 00 00 33 01 06 17 19 ...........3.... +| 4048: 19 01 43 74 61 62 6c 65 54 61 62 6c 65 30 54 61 ..CtableTable0Ta +| 4064: 62 6c 65 30 02 43 52 45 41 54 45 20 54 41 42 4c ble0.CREATE TABL +| 4080: 45 20 54 61 62 6c 65 30 20 28 43 6f 6c 30 20 29 E Table0 (Col0 ) +| page 2 offset 4096 +| 0: 0d 00 00 00 00 10 00 00 00 00 00 00 00 00 00 00 ................ +| end db.sqlite +}]} {} + +do_test 8.1 { + set R [sqlite3_recover_init db main test.db2] + catch { $R run } + list [catch { $R finish } msg] $msg +} {0 {}} finish_test Index: ext/recover/sqlite3recover.c ================================================================== --- ext/recover/sqlite3recover.c +++ ext/recover/sqlite3recover.c @@ -1187,11 +1187,11 @@ if( rc==SQLITE_OK ){ recoverSqlCallback(p, zSql); if( bTable && !bVirtual ){ if( SQLITE_ROW==sqlite3_step(pTblname) ){ const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0); - recoverAddTable(p, zTbl, iRoot); + if( zTbl ) recoverAddTable(p, zTbl, iRoot); } recoverReset(p, pTblname); } }else if( rc!=SQLITE_ERROR ){ recoverDbError(p, p->dbOut); Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -5617,10 +5617,14 @@ } } memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); pCur->curFlags |= BTCF_ValidOvfl; }else{ + /* Sanity check the validity of the overflow page cache */ + assert( pCur->aOverflow[0]==nextPage || pCur->aOverflow[0]==0 ); + assert( pCur->aOverflow[0]!=0 || pCur->aOverflow[offset/ovflSize]==0 ); + /* If the overflow page-list cache has been allocated and the ** entry for the first required overflow page is valid, skip ** directly to it. */ if( pCur->aOverflow[offset/ovflSize] ){ @@ -6105,10 +6109,27 @@ *pRes = 1; rc = SQLITE_OK; } return rc; } + +#ifdef SQLITE_DEBUG +/* The cursors is CURSOR_VALID and has BTCF_AtLast set. Verify that +** this flags are true for a consistent database. +** +** This routine is is called from within assert() statements only. +** It is an internal verification routine and does not appear in production +** builds. +*/ +static int cursorIsAtLastEntry(BtCursor *pCur){ + int ii; + for(ii=0; iiiPage; ii++){ + if( pCur->aiIdx[ii]!=pCur->apPage[ii]->nCell ) return 0; + } + return pCur->ix==pCur->pPage->nCell-1 && pCur->pPage->leaf!=0; +} +#endif /* Move the cursor to the last entry in the table. Return SQLITE_OK ** on success. Set *pRes to 0 if the cursor actually points to something ** or set *pRes to 1 if the table is empty. */ @@ -6134,22 +6155,11 @@ assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); /* If the cursor already points to the last entry, this is a no-op. */ if( CURSOR_VALID==pCur->eState && (pCur->curFlags & BTCF_AtLast)!=0 ){ -#ifdef SQLITE_DEBUG - /* This block serves to assert() that the cursor really does point - ** to the last entry in the b-tree. */ - int ii; - for(ii=0; iiiPage; ii++){ - assert( pCur->aiIdx[ii]==pCur->apPage[ii]->nCell ); - } - assert( pCur->ix==pCur->pPage->nCell-1 || CORRUPT_DB ); - testcase( pCur->ix!=pCur->pPage->nCell-1 ); - /* ^-- dbsqlfuzz b92b72e4de80b5140c30ab71372ca719b8feb618 */ - assert( pCur->pPage->leaf ); -#endif + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); *pRes = 0; return SQLITE_OK; } return btreeLast(pCur, pRes); } @@ -6198,10 +6208,11 @@ *pRes = 0; return SQLITE_OK; } if( pCur->info.nKeycurFlags & BTCF_AtLast)!=0 ){ + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); *pRes = -1; return SQLITE_OK; } /* If the requested key is one more than the previous key, then ** try to get there using sqlite3BtreeNext() rather than a full @@ -10017,11 +10028,11 @@ dropCell(pPage, idx, info.nSize, &rc); if( rc ) goto end_insert; }else if( loc<0 && pPage->nCell>0 ){ assert( pPage->leaf ); idx = ++pCur->ix; - pCur->curFlags &= ~BTCF_ValidNKey; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); }else{ assert( pPage->leaf ); } rc = insertCellFast(pPage, idx, newCell, szNew); assert( pPage->nOverflow==0 || rc==SQLITE_OK ); @@ -10047,11 +10058,11 @@ ** larger than the largest existing key, it is possible to insert the ** row without seeking the cursor. This can be a big performance boost. */ if( pPage->nOverflow ){ assert( rc==SQLITE_OK ); - pCur->curFlags &= ~(BTCF_ValidNKey); + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); rc = balance(pCur); /* Must make sure nOverflow is reset to zero even if the balance() ** fails. Internal data structure corruption will result otherwise. ** Also, set the cursor state to invalid. This stops saveCursorPosition() Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -3180,11 +3180,14 @@ ** further downstream. */ return 0; /* Corrupt schema - two indexes on the same btree */ } } #ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ + if( pDest->pCheck + && (db->mDbFlags & DBFLAG_Vacuum)==0 + && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) + ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif #ifndef SQLITE_OMIT_FOREIGN_KEY /* Disallow the transfer optimization if the destination table contains Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -7256,11 +7256,11 @@ /* ** Return the file handle for the journal file (if it exists). ** This will be either the rollback journal or the WAL file. */ sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ -#if SQLITE_OMIT_WAL +#ifdef SQLITE_OMIT_WAL return pPager->jfd; #else return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; #endif } Index: test/fts3snippet2.test ================================================================== --- test/fts3snippet2.test +++ test/fts3snippet2.test @@ -52,8 +52,19 @@ } do_execsql_test 2.2 { SELECT snippet(t0) FROM t0 WHERE t0 MATCH '(def AND (one NEAR abc)) OR one' } {one} + +#------------------------------------------------------------------------- + +do_execsql_test 3.0 { +CREATE VIRTUAL TABLE f USING fts3(a,b); +INSERT INTO f VALUES (101,x'056522650565056505650d051e056505650565286505650565056505056505650565056505650565056505650565056505650565056505656505650565056505650d05650505656505650565ef65056505844c746e65650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056f05650565056505650565056505650565050565056505640565056505650565056505651e05650565056505650565056505650505656565056505650565056505651e0565056505650565056505650565052265056505650569056505650565056505650565056505650565056505650500406505650565056505650565056505000101e5c501014b010101c501c5c501010101f5010201010101014101017373737373737373737373737373737373737373737373737373737330737373737373737373737373737365056505650d051e05650565056528056505650d05650505656505650565650565056505650565056505e505650565056505656505650565056505650d05650505656505650565ef65056505844c746e65650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565286505c705650565050565059494949494949494949494949494949494949494949494949494949494949494949494650565056505650565056505650565056505650565056505656505650565056505650d05650505656505650565ef650565058405056505650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056505650565056505650505650565056505650565056505650565056500000000000000000000000000000000000000000000000000000000000000000565056505656505650565056505650d056500000000000000000000000000000000000000000000000000000000000000000100000000000000ed0000000000ffffffffffffffffffffff0007ffffff0001c5c50001c5c50001c5c50001c5c50001c5c50001c5c50001e5c50001c5c50001c5c50001c5c50001c5c50001c5c5000100000014720000000000000016dac5c50001c5c50001c5c50001c5c50001c5c50001c5c50d0505656505650565ef650565058405056505650565056505650565056505650565056505650565058405800465056505650565056505651e650565056505650565056505650d05650505656505650565050565650584058005650565056505650565056505650565056522650565056505650d051e056505650561286505c70565056505056505650565056505650565056505650565056505650565056505656505650565056505650d05650505656505650565ef650565058405056505650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056505650565056505650565226505737373737373737373737373737373737373737373737373737373737373737373737373737373737373737373737373733a73737373737373737373737373737373737373737373737373737373737373737373737373737373737c7373737365056505650d051e05650565056528650565056505650505650565056505650565056505650565056505e505650565056505656505650565056505650d05650505656505650565ef65056505'); +} + +do_execsql_test 3.1 { + SELECT length(snippet(f)) FROM f WHERE b MATCH x'0565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056505650565056505650565056505650565050565056505640565056505650565056505651e05650565056522650565056505650d051e056505650565286505650565056505056505650565056505650565056505650565056505650565056505656505650565056505650d05650505656505650565ef65056505844c746e65650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056f05650565056505650565056505650565050565056505640565056505650565056505651e05650565056505650565056505650505656565056505650565056505651e0565056505650565056505650565052265056505650569056505650565056505650565056505650565056505650500406505650565056505650565056505000101e5c501014b010101c501c5c501010101f50102010101010141010141010001017bf15905000000000017'; +} {192} set sqlite_fts3_enable_parentheses 0 finish_test Index: test/returning1.test ================================================================== --- test/returning1.test +++ test/returning1.test @@ -225,25 +225,25 @@ SELECT * FROM log; } {} } else { # Note: The values returned by the RETURNING clauses of the following # two statements are the rowid columns of views. These values are not - # well defined, so the INSERT returns -1, and the UPDATE returns 1, 2 - # and 3. These match the values used for new.rowid expressions, but - # not much else. + # well defined, so the INSERT returns -1, and the UPDATE returns NULL. + # These match the values used for new.rowid expressions, but not much + # else. do_catchsql_test 10.3a { INSERT INTO t1(a, b) VALUES(1234, 5678) RETURNING rowid; } {0 -1} do_catchsql_test 10.3b { UPDATE t1 SET a='z' WHERE b='y' RETURNING rowid; - } {0 {1 2 3}} + } {0 {{} {} {}}} do_execsql_test 10.4 { SELECT * FROM log; } { - insert -1 1234 5678 update 1 z y update 2 z y update 3 z y + insert -1 1234 5678 update {} z y update {} z y update {} z y } } # 2021-04-27 dbsqlfuzz 78b9400770ef8cc7d9427dfba26f4fcf46ea7dc2 # Returning clauses on TEMP tables with triggers. Index: test/testrunner.tcl ================================================================== --- test/testrunner.tcl +++ test/testrunner.tcl @@ -268,11 +268,11 @@ priority INTEGER NOT NULL, -- higher priority jobs may run earlier /* Fields updated as jobs run */ starttime INTEGER, endtime INTEGER, - state TEXT CHECK( state IN ('', 'ready', 'running', 'done', 'failed') ), + state TEXT CHECK( state IN ('','ready','running','done','failed','omit') ), output TEXT ); CREATE TABLE config( name TEXT COLLATE nocase PRIMARY KEY, @@ -444,10 +444,14 @@ puts "Failures: " mydb eval { SELECT * FROM jobs WHERE state='failed' ORDER BY starttime } job { display_job [array get job] + } + set nOmit [db one {SELECT count(*) FROM jobs WHERE state='omit'}] + if {$nOmit} { + puts "$nOmit jobs omitted due to failures" } } mydb close exit @@ -973,15 +977,20 @@ } proc mark_job_as_finished {jobid output state endtm} { r_write_db { + if {$state=="failed"} { + set childstate omit + } else { + set childstate ready + } trdb eval { UPDATE jobs SET output=$output, state=$state, endtime=$endtm WHERE jobid=$jobid; - UPDATE jobs SET state='ready' WHERE depid=$jobid; + UPDATE jobs SET state=$childstate WHERE depid=$jobid; } } } proc script_input_ready {fd iJob jobid} { @@ -1199,10 +1208,14 @@ trdb eval { SELECT displayname FROM jobs WHERE state='failed' } { puts "FAILED: $displayname" } + } + set nOmit [trdb one {SELECT count(*) FROM jobs WHERE state='omit'}] + if {$nOmit>0} { + puts "$nOmit jobs skipped due to prior failures" } } puts "\nTest database is $TRG(dbname)" puts "Test log is $TRG(logname)" Index: test/upsert1.test ================================================================== --- test/upsert1.test +++ test/upsert1.test @@ -265,7 +265,31 @@ } sqlite3_db_config db ENABLE_QPSG 1 do_catchsql_test upsert1-1210 { INSERT INTO t1(a,b) VALUES(1,2) ON CONFLICT(b+?1) DO NOTHING; } {1 {ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint}} + +# 2024-04-11 https://sqlite.org/forum/forumpost/284955a3cd454a15 +# Incorrect value passed into a trigger that fires as the result of +# an upsert. +# +reset_db +do_execsql_test upsert1-1300 { + CREATE TABLE t1(x INT, y TEXT); + INSERT INTO t1 VALUES + (11, printf('%.9000c','a')), + (11, printf('%.9000c','a')), + (33, printf('%.9000c','b')), + (33, printf('%.9000c','b')); + CREATE TABLE t2(x INT UNIQUE, y TEXT); + CREATE TRIGGER r1 BEFORE UPDATE ON t2 BEGIN + SELECT raise(ABORT,'Incorrect old.y value passed to trigger!') + WHERE old.y != new.y; + /* ^^^ This trigger will fire and cause the ABORT if the problem has + ** not been fixed, or if there is a regression. */ + END; + INSERT INTO t2(x, y) SELECT x, y FROM t1 + WHERE true + ON CONFLICT (x) DO UPDATE SET y = excluded.y; +} {} finish_test Index: test/vacuum-into.test ================================================================== --- test/vacuum-into.test +++ test/vacuum-into.test @@ -24,17 +24,40 @@ return } forcedelete out.db do_execsql_test vacuum-into-100 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t1( + a INTEGER PRIMARY KEY, + b ANY, + c INT AS (b+1), --- See "2024-04-09" block + CHECK( typeof(b)!='integer' OR b>a-5 ) --- comment below + ); WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) INSERT INTO t1(a,b) SELECT x, randomblob(600) FROM c; CREATE INDEX t1b ON t1(b); DELETE FROM t1 WHERE a%2; SELECT count(*), sum(a), sum(length(b)) FROM t1; } {50 2550 30000} + +# Update 2024-04-09 for forum post eec177d68fe7fa2c. +# +# VACUUM INTO is sensitive to tables holding both generated columns +# and CHECK constraints. +# +# CHECK constraints are ignored for read-only databases in order to save +# memory (see check-in 34ddf02d3d21151b on 2014-05-21). But the xfer +# optimization normally only works if CHECK constraints match between the +# source and destination tables. So the xfer optimization was not +# working for VACUUM INTO when the source was a read-only database and the +# table held CHECK constraints. But if the table has generated columns, +# then the xfer optimization is required or else VACUUM will raise an +# error. +# +# Fix this by ignoring CHECK constraints when determining whether or not +# the xfer optimization can run while doing VACUUM. + do_execsql_test vacuum-into-110 { VACUUM main INTO 'out.db'; } {} sqlite3 db2 out.db do_test vacuum-into-120 { @@ -86,15 +109,25 @@ VACUUM INTO target2() } {1 {no such function: target2}} # The ability to VACUUM INTO a read-only database db close +if {$tcl_platform(platform)=="windows"} { + file attributes test.db -readonly 1 +} else { + file attributes test.db -permissions 292 ;# 292 == 0444 +} sqlite3 db test.db -readonly 1 forcedelete test.db2 do_execsql_test vacuum-into-500 { VACUUM INTO 'test.db2'; } +if {$tcl_platform(platform)=="windows"} { + file attributes test.db -readonly 0 +} else { + file attributes test.db -permissions 420 ;# 420 = 0644 +} sqlite3 db2 test.db2 do_test vacuum-into-510 { db2 eval {SELECT name FROM sqlite_master ORDER BY 1} } {t1 t1b t2} db2 close