Index: ext/fts5/fts5Int.h ================================================================== --- ext/fts5/fts5Int.h +++ ext/fts5/fts5Int.h @@ -471,11 +471,11 @@ ** Called during virtual module initialization to register UDF ** fts5_decode() with SQLite */ int sqlite3Fts5IndexInit(sqlite3*); -int sqlite3Fts5IndexSetCookie(Fts5Index*, int); +int sqlite3Fts5IndexIncrCookie(Fts5Index*); /* ** Return the total number of entries read from the %_data table by ** this connection since it was created. */ @@ -482,13 +482,15 @@ int sqlite3Fts5IndexReads(Fts5Index *p); int sqlite3Fts5IndexReinit(Fts5Index *p); int sqlite3Fts5IndexOptimize(Fts5Index *p); int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); -int sqlite3Fts5IndexReset(Fts5Index *p); +int sqlite3Fts5IndexNewTrans(Fts5Index *p); int sqlite3Fts5IndexLoadConfig(Fts5Index *p); + +void sqlite3Fts5IndexCloseReader(Fts5Index*); /* ** End of interface to code in fts5_index.c. **************************************************************************/ Index: ext/fts5/fts5_index.c ================================================================== --- ext/fts5/fts5_index.c +++ ext/fts5/fts5_index.c @@ -10,23 +10,24 @@ ** ****************************************************************************** ** ** Low level access to the FTS index stored in the database file. The ** routines in this file file implement all read and write access to the -** %_data table. Other parts of the system access this functionality via -** the interface defined in fts5Int.h. +** %_data and %_idx tables. Other parts of the system access this +** functionality via the interface defined in fts5Int.h. */ #include "fts5Int.h" /* ** Overview: ** -** The %_data table contains all the FTS indexes for an FTS5 virtual table. -** As well as the main term index, there may be up to 31 prefix indexes. -** The format is similar to FTS3/4, except that: +** The %_data table contains the FTS index for an FTS5 virtual table. +** All entries, for terms and prefixes, are stored in a single data +** structure. The format is similar to FTS3/4, but differs in the +** following ways: ** ** * all segment b-tree leaf data is stored in fixed size page records ** (e.g. 1000 bytes). A single doclist may span multiple pages. Care is ** taken to ensure it is possible to iterate in either direction through ** the entries in a doclist, or to seek to a specific entry within a @@ -37,35 +38,23 @@ ** the doclist. This is used to speed up seek operations, and merges of ** large doclists with very small doclists. ** ** * extra fields in the "structure record" record the state of ongoing ** incremental merge operations. -** */ -#define FTS5_OPT_WORK_UNIT 1000 /* Number of leaf pages per optimize step */ -#define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */ - -#define FTS5_MIN_DLIDX_SIZE 4 /* Add dlidx if this many empty pages */ - -#define FTS5_MAIN_PREFIX '0' - -#if FTS5_MAX_PREFIX_INDEXES > 31 -# error "FTS5_MAX_PREFIX_INDEXES is too large" -#endif - /* -** Details: +** Contents of %_data table: ** ** The %_data table managed by this module, ** ** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB); ** -** , contains the following 5 types of records. See the comments surrounding +** , contains the following 4 types of records. See the comments surrounding ** the FTS5_*_ROWID macros below for a description of how %_data rowids are -** assigned to each fo them. +** assigned to each of them. ** ** 1. Structure Records: ** ** The set of segments that make up an index - the index structure - are ** recorded in a single record within the %_data table. The record consists @@ -174,11 +163,11 @@ ** very first term of the segment: ** ** varint : size of first term ** blob: first term data ** -** 5. Segment doclist indexes: +** 4. Segment doclist indexes: ** ** Doclist indexes are themselves b-trees, however they usually consist of ** a single leaf record only. The format of each doclist index leaf page ** is: ** @@ -204,10 +193,25 @@ ** ** * A list of delta-encoded varints - the first rowid on each subsequent ** child page. ** */ + + +#define FTS5_OPT_WORK_UNIT 1000 /* Number of leaf pages per optimize step */ +#define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */ +#define FTS5_MIN_DLIDX_SIZE 4 /* Add dlidx if this many empty pages */ + +/* All entries for regular terms in the FTS index are prefixed with '0'. +** Entries for the first prefix index are prefixed with '1'. And so on, +** up to ('0'+31). */ +#define FTS5_MAIN_PREFIX '0' + +#if FTS5_MAX_PREFIX_INDEXES > 31 +# error "FTS5_MAX_PREFIX_INDEXES is too large" +#endif + /* ** Rowids for the averages and structure records in the %_data table. */ #define FTS5_AVERAGES_ROWID 1 /* Rowid used for the averages record */ @@ -303,11 +307,12 @@ sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */ sqlite3_stmt *pIdxSelect; int nRead; /* Total number of blocks read */ - sqlite3_stmt *pDataVersion; + /* In-memory cache of the 'structure' record */ + sqlite3_stmt *pDataVersion; /* PRAGMA .data_version */ i64 iStructVersion; /* data_version when pStruct read */ Fts5Structure *pStruct; /* Current db structure (or NULL) */ }; struct Fts5DoclistIter { @@ -618,11 +623,11 @@ } /* ** Close the read-only blob handle, if it is open. */ -static void fts5CloseReader(Fts5Index *p){ +void sqlite3Fts5IndexCloseReader(Fts5Index *p){ if( p->pReader ){ sqlite3_blob *pReader = p->pReader; p->pReader = 0; sqlite3_blob_close(pReader); } @@ -648,11 +653,11 @@ p->pReader = 0; rc = sqlite3_blob_reopen(pBlob, iRowid); assert( p->pReader==0 ); p->pReader = pBlob; if( rc!=SQLITE_OK ){ - fts5CloseReader(p); + sqlite3Fts5IndexCloseReader(p); } if( rc==SQLITE_ABORT ) rc = SQLITE_OK; } /* If the blob handle is not open at this point, open it and seek @@ -992,11 +997,11 @@ if( p->rc==SQLITE_OK ){ if( p->pDataVersion==0 ){ p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb) - ); + ); if( p->rc ) return 0; } if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){ iVersion = sqlite3_column_int64(p->pDataVersion, 0); @@ -1004,10 +1009,23 @@ p->rc = sqlite3_reset(p->pDataVersion); } return iVersion; } + +/* +** If there is currently no cache of the index structure in memory, load +** one from the database. +*/ +static void fts5StructureCache(Fts5Index *p){ + if( p->pStruct==0 ){ + p->iStructVersion = fts5IndexDataVersion(p); + if( p->rc==SQLITE_OK ){ + p->pStruct = fts5StructureReadUncached(p); + } + } +} /* ** Read, deserialize and return the structure record. ** ** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array @@ -1017,17 +1035,11 @@ ** If an error occurs, NULL is returned and an error code left in the ** Fts5Index handle. If an error has already occurred when this function ** is called, it is a no-op. */ static Fts5Structure *fts5StructureRead(Fts5Index *p){ - - if( p->pStruct==0 ){ - p->iStructVersion = fts5IndexDataVersion(p); - if( p->rc==SQLITE_OK ){ - p->pStruct = fts5StructureReadUncached(p); - } - } + fts5StructureCache(p); #if 0 else{ Fts5Structure *pTest = fts5StructureReadUncached(p); if( pTest ){ @@ -4394,16 +4406,10 @@ int rc = p->rc; p->rc = SQLITE_OK; return rc; } -typedef struct Fts5FlushCtx Fts5FlushCtx; -struct Fts5FlushCtx { - Fts5Index *pIdx; - Fts5SegWriter writer; -}; - /* ** Buffer aBuf[] contains a list of varints, all small enough to fit ** in a 32-bit integer. Return the size of the largest prefix of this ** list nMax bytes or less in size. */ @@ -5102,11 +5108,11 @@ ** Commit data to disk. */ int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){ assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); - if( bCommit ) fts5CloseReader(p); + if( bCommit ) sqlite3Fts5IndexCloseReader(p); return fts5IndexReturn(p); } /* ** Discard any data stored in the in-memory hash tables. Do not write it @@ -5113,13 +5119,14 @@ ** to the database. Additionally, assume that the contents of the %_data ** table may have changed on disk. So any in-memory caches of %_data ** records must be invalidated. */ int sqlite3Fts5IndexRollback(Fts5Index *p){ - fts5CloseReader(p); + sqlite3Fts5IndexCloseReader(p); fts5IndexDiscardData(p); fts5StructureInvalidate(p); + p->pConfig->iCookie = -1; /* assert( p->rc==SQLITE_OK ); */ return SQLITE_OK; } /* @@ -5350,11 +5357,11 @@ } if( p->rc ){ sqlite3Fts5IterClose(&pRet->base); pRet = 0; - fts5CloseReader(p); + sqlite3Fts5IndexCloseReader(p); } *ppIter = &pRet->base; sqlite3Fts5BufferFree(&buf); } @@ -5423,11 +5430,11 @@ void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){ if( pIndexIter ){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5Index *pIndex = pIter->pIndex; fts5MultiIterFree(pIter); - fts5CloseReader(pIndex); + sqlite3Fts5IndexCloseReader(pIndex); } } /* ** Read and decode the "averages" record from the database. @@ -5465,49 +5472,46 @@ return fts5IndexReturn(p); } /* ** Return the total number of blocks this module has read from the %_data -** table since it was created. +** table (since it was created by sqlite3Fts5IndexOpen). */ int sqlite3Fts5IndexReads(Fts5Index *p){ return p->nRead; } /* -** Set the 32-bit cookie value stored at the start of all structure -** records to the value passed as the second argument. -** -** Return SQLITE_OK if successful, or an SQLite error code if an error -** occurs. +** Increment the value of the configuration cookie stored as the first +** 32-bits of the structure record in the database. This is done after +** modifying the contents of the %_config table. */ -int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){ - int rc; /* Return code */ - Fts5Config *pConfig = p->pConfig; /* Configuration object */ - u8 aCookie[4]; /* Binary representation of iNew */ - sqlite3_blob *pBlob = 0; - - assert( p->rc==SQLITE_OK ); - sqlite3Fts5Put32(aCookie, iNew); - - rc = sqlite3_blob_open(pConfig->db, pConfig->zDb, p->zDataTbl, - "block", FTS5_STRUCTURE_ROWID, 1, &pBlob - ); - if( rc==SQLITE_OK ){ - sqlite3_blob_write(pBlob, aCookie, 4, 0); - rc = sqlite3_blob_close(pBlob); - } - - return rc; -} - -int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ +int sqlite3Fts5IndexIncrCookie(Fts5Index *p){ Fts5Structure *pStruct; pStruct = fts5StructureRead(p); + p->pConfig->iCookie++; + fts5StructureWrite(p, pStruct); fts5StructureRelease(pStruct); return fts5IndexReturn(p); } + +/* +** Ensure the contents of the %_config table have been loaded into memory. +*/ +int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ + fts5StructureCache(p); + return fts5IndexReturn(p); +} + +int sqlite3Fts5IndexNewTrans(Fts5Index *p){ + assert( p->pStruct==0 || p->iStructVersion!=0 ); + if( p->pConfig->iCookie<0 || fts5IndexDataVersion(p)!=p->iStructVersion ){ + fts5StructureInvalidate(p); + } + return fts5IndexReturn(p); +} + /************************************************************************* ************************************************************************** ** Below this point is the implementation of the integrity-check @@ -6449,13 +6453,5 @@ ); } return rc; } - -int sqlite3Fts5IndexReset(Fts5Index *p){ - assert( p->pStruct==0 || p->iStructVersion!=0 ); - if( fts5IndexDataVersion(p)!=p->iStructVersion ){ - fts5StructureInvalidate(p); - } - return fts5IndexReturn(p); -} Index: ext/fts5/fts5_main.c ================================================================== --- ext/fts5/fts5_main.c +++ ext/fts5/fts5_main.c @@ -600,11 +600,11 @@ static int fts5NewTransaction(Fts5Table *pTab){ Fts5Cursor *pCsr; for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK; } - return sqlite3Fts5StorageReset(pTab->pStorage); + return sqlite3Fts5IndexNewTrans(pTab->pIndex); } /* ** Implementation of xOpen method. */ @@ -614,10 +614,13 @@ Fts5Cursor *pCsr = 0; /* New cursor object */ int nByte; /* Bytes of space to allocate */ int rc; /* Return code */ rc = fts5NewTransaction(pTab); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex); + } if( rc==SQLITE_OK ){ nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int); pCsr = (Fts5Cursor*)sqlite3_malloc(nByte); if( pCsr ){ Fts5Global *pGlobal = pTab->pGlobal; @@ -628,10 +631,13 @@ pCsr->iCsrId = ++pGlobal->iNextId; }else{ rc = SQLITE_NOMEM; } } + if( rc!=SQLITE_OK ){ + sqlite3Fts5IndexCloseReader(pTab->pIndex); + } *ppCsr = (sqlite3_vtab_cursor*)pCsr; return rc; } static int fts5StmtType(Fts5Cursor *pCsr){ @@ -708,10 +714,11 @@ /* Remove the cursor from the Fts5Global.pCsr list */ for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext); *pp = pCsr->pNext; sqlite3_free(pCsr); + sqlite3Fts5IndexCloseReader(pTab->pIndex); } return SQLITE_OK; } static int fts5SorterNext(Fts5Cursor *pCsr){ @@ -1587,13 +1594,20 @@ /* ** Implementation of xBegin() method. */ static int fts5BeginMethod(sqlite3_vtab *pVtab){ - fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0); - fts5NewTransaction((Fts5Table*)pVtab); - return SQLITE_OK; + Fts5Table *pTab = (Fts5Table*)pVtab; + int rc; + rc = fts5NewTransaction(pTab); + if( rc!=SQLITE_OK ){ + sqlite3Fts5IndexCloseReader(pTab->pIndex); + } +#ifdef SQLITE_DEBUG + if( rc==SQLITE_OK ) fts5CheckTransactionState(pTab, FTS5_BEGIN, 0); +#endif + return rc; } /* ** Implementation of xCommit() method. This is a no-op. The contents of ** the pending-terms hash-table have already been flushed into the database Index: ext/fts5/fts5_storage.c ================================================================== --- ext/fts5/fts5_storage.c +++ ext/fts5/fts5_storage.c @@ -639,14 +639,10 @@ int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){ return sqlite3Fts5IndexMerge(p->pIndex, nMerge); } -int sqlite3Fts5StorageReset(Fts5Storage *p){ - return sqlite3Fts5IndexReset(p->pIndex); -} - /* ** Allocate a new rowid. This is used for "external content" tables when ** a NULL value is inserted into the rowid column. The new rowid is allocated ** by inserting a dummy row into the %_docsize table. The dummy will be ** overwritten later. @@ -1118,13 +1114,9 @@ } sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); } if( rc==SQLITE_OK && pVal ){ - int iNew = p->pConfig->iCookie + 1; - rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew); - if( rc==SQLITE_OK ){ - p->pConfig->iCookie = iNew; - } + rc = sqlite3Fts5IndexIncrCookie(p->pIndex); } return rc; } Index: ext/fts5/test/fts5fault4.test ================================================================== --- ext/fts5/test/fts5fault4.test +++ ext/fts5/test/fts5fault4.test @@ -84,11 +84,11 @@ set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}] do_faultsim_test 4 -faults oom-* -body { db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'} } -test { - faultsim_test_result {0 {0 {} 3}} + faultsim_test_result {0 {0 {} 4}} } #------------------------------------------------------------------------- # An OOM within a query that uses a custom rank function. # ADDED ext/fts5/test/fts5faultC.test Index: ext/fts5/test/fts5faultC.test ================================================================== --- ext/fts5/test/fts5faultC.test +++ ext/fts5/test/fts5faultC.test @@ -0,0 +1,79 @@ +# 2016 March 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultC +return_if_no_fts5 + +#-------------------------------------------------------------------------- +# Test that if an OOM error occurs while trying to set a configuration +# option, the in-memory and on-disk configurations are not left in an +# inconsistent state. +# + + + +proc posrowid {cmd} { $cmd xRowid } +proc negrowid {cmd} { expr -1 * [$cmd xRowid] } + +sqlite3_fts5_create_function db posrowid posrowid +sqlite3_fts5_create_function db negrowid negrowid + +do_execsql_test 1.0.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('d a d'); + INSERT INTO t1 VALUES('c b a'); +} +do_execsql_test 1.0.1 { + INSERT INTO t1(t1, rank) VALUES('rank', 'posrowid()'); + SELECT rowid FROM t1('a') ORDER BY rank; +} {1 2 3} +do_execsql_test 1.0.2 { + INSERT INTO t1(t1, rank) VALUES('rank', 'negrowid()'); + SELECT rowid FROM t1('a') ORDER BY rank; +} {3 2 1} + +faultsim_save_and_close +do_faultsim_test 1.1 -faults oom-* -prep { + faultsim_restore_and_reopen + sqlite3_fts5_create_function db posrowid posrowid + sqlite3_fts5_create_function db negrowid negrowid + execsql { SELECT * FROM t1('*reads') } +} -body { + execsql { INSERT INTO t1(t1, rank) VALUES('rank', 'posrowid()') } +} -test { + + faultsim_test_result [list 0 {}] + sqlite3 db2 test.db + set ex [db2 one { SELECT v FROM t1_config WHERE k='rank' }] + switch -- $ex { + "posrowid()" { set ex {1 2 3} } + "negrowid()" { set ex {3 2 1} } + default { error 1 } + } + + set res [db eval { SELECT rowid FROM t1('a') ORDER BY rank }] + if {$res != $ex} { + error "2: expected {$ex} got {$res}" + } + db2 close +} + + + + +finish_test + Index: ext/fts5/test/fts5simple.test ================================================================== --- ext/fts5/test/fts5simple.test +++ ext/fts5/test/fts5simple.test @@ -348,11 +348,11 @@ INSERT INTO x1 SELECT rnddoc(5) FROM ii; } do_execsql_test 14.4 { SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads' -} {0 {} 3} +} {0 {} 4} #------------------------------------------------------------------------- reset_db do_execsql_test 15.0 { CREATE VIRTUAL TABLE x2 USING fts5(x, prefix=1);