Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -619,10 +619,11 @@ p->nColumn = nCol; p->nPendingData = 0; p->azColumn = (char **)&p[1]; p->pTokenizer = pTokenizer; p->nNodeSize = 1000; + p->nMaxPendingData = FTS3_MAX_PENDING_DATA; zCsr = (char *)&p->azColumn[nCol]; fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); /* Fill in the zName and zDb fields of the vtab structure. */ @@ -2265,10 +2266,14 @@ #endif ){ rc = SQLITE_NOMEM; } } + +#ifdef SQLITE_TEST + sqlite3Fts3ExprInitTestInterface(db); +#endif /* Create the virtual table wrapper around the hash-table and overload ** the two scalar functions. If this is successful, register the ** module with sqlite. */ Index: ext/fts3/fts3Int.h ================================================================== --- ext/fts3/fts3Int.h +++ ext/fts3/fts3Int.h @@ -118,14 +118,15 @@ int nNodeSize; /* Soft limit for node size */ /* The following hash table is used to buffer pending index updates during ** transactions. Variable nPendingData estimates the memory size of the ** pending data, including hash table overhead, but not malloc overhead. - ** When nPendingData exceeds FTS3_MAX_PENDING_DATA, the buffer is flushed + ** When nPendingData exceeds nMaxPendingData, the buffer is flushed ** automatically. Variable iPrevDocid is the docid of the most recently ** inserted record. */ + int nMaxPendingData; int nPendingData; sqlite_int64 iPrevDocid; Fts3Hash pendingTerms; }; Index: ext/fts3/fts3_write.c ================================================================== --- ext/fts3/fts3_write.c +++ ext/fts3/fts3_write.c @@ -292,12 +292,12 @@ return (rc==SQLITE_DONE ? SQLITE_CORRUPT : rc); } *pnBlock = sqlite3_column_bytes(pStmt, 0); *pzBlock = (char *)sqlite3_column_blob(pStmt, 0); - if( !*pzBlock ){ - return SQLITE_NOMEM; + if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ + return SQLITE_CORRUPT; } } return SQLITE_OK; } @@ -508,11 +508,11 @@ ** forced-flush would provide better performance. I suspect that if ** we ordered the doclists by size and flushed the largest until the ** buffer was half empty, that would let the less frequent terms ** generate longer doclists. */ - if( iDocid<=p->iPrevDocid || p->nPendingData>FTS3_MAX_PENDING_DATA ){ + if( iDocid<=p->iPrevDocid || p->nPendingData>p->nMaxPendingData ){ int rc = sqlite3Fts3PendingTermsFlush(p); if( rc!=SQLITE_OK ) return rc; } p->iPrevDocid = iDocid; return SQLITE_OK; @@ -2218,10 +2218,13 @@ sqlite3Fts3PendingTermsClear(p); } #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); + rc = SQLITE_OK; + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ + p->nMaxPendingData = atoi(&zVal[11]); rc = SQLITE_OK; #endif }else{ rc = SQLITE_ERROR; } Index: src/test_hexio.c ================================================================== --- src/test_hexio.c +++ src/test_hexio.c @@ -328,16 +328,16 @@ return (int) (q - (unsigned char *)p); } /* -** USAGE: read_varint BLOB VARNAME +** USAGE: read_fts3varint BLOB VARNAME ** ** Read a varint from the start of BLOB. Set variable VARNAME to contain ** the interpreted value. Return the number of bytes of BLOB consumed. */ -static int read_varint( +static int read_fts3varint( void * clientData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[] ){ @@ -371,13 +371,13 @@ { "hexio_write", hexio_write }, { "hexio_get_int", hexio_get_int }, { "hexio_render_int16", hexio_render_int16 }, { "hexio_render_int32", hexio_render_int32 }, { "utf8_to_utf8", utf8_to_utf8 }, - { "read_varint", read_varint }, + { "read_fts3varint", read_fts3varint }, }; int i; for(i=0; i0} {incr iTest} { + + if {$isRestart} { sqlite3 db test.db } + + sqlite3_memdebug_fail [expr $iFail-$iTest] -repeat $nRepeat + set res [uplevel [list catchsql $sql]] + if {[lsearch -exact $answers $res]>=0} { set res $str } + set testname "$name.$zName.$iFail" + do_test "$name.$zName.$iLimit.$iFail" [list set {} $res] $str - set res [uplevel [list catchsql $sql]] - if {[lsearch -exact $answers $res]>=0} { - set res $str + set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign] } - set testname "$name.$zName.$iFail" - if {$zName == ""} { set testname $name } - do_test $testname [list set {} $res] $str - set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign] - if {$nFail==0} break } } } ADDED test/fts3cov.test Index: test/fts3cov.test ================================================================== --- test/fts3cov.test +++ test/fts3cov.test @@ -0,0 +1,168 @@ +# 2009 December 03 +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The tests in this file are structural coverage tests. They are designed +# to complement the tests in fts3rnd.test and fts3doc.test. Between them, +# the three files should provide full coverage of the fts3 extension code. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If this build does not include FTS3, skip the tests in this file. +# +ifcapable !fts3 { finish_test ; return } +source $testdir/fts3_common.tcl + +set DO_MALLOC_TEST 0 + +#-------------------------------------------------------------------------- +# When it first needs to read a block from the %_segments table, the FTS3 +# module compiles an SQL statement for that purpose. The statement is +# stored and reused each subsequent time a block is read. This test case +# tests the effects of an OOM error occuring while compiling the statement. +# +# Similarly, when FTS3 first needs to scan through a set of segment leaves +# to find a set of documents that matches a term, it allocates a string +# containing the text of the required SQL, and compiles one or more +# statements to traverse the leaves. This test case tests that OOM errors +# that occur while allocating this string and statement are handled correctly +# also. +# +do_test fts3cov-1.1 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts3(x); + INSERT INTO t1(t1) VALUES('nodesize=24'); + BEGIN; + INSERT INTO t1 VALUES('Is the night chilly and dark?'); + INSERT INTO t1 VALUES('The night is chilly, but not dark.'); + INSERT INTO t1 VALUES('The thin gray cloud is spread on high,'); + INSERT INTO t1 VALUES('It covers but not hides the sky.'); + COMMIT; + SELECT count(*)>0 FROM t1_segments; + } +} {1} + +set DO_MALLOC_TEST 1 +do_restart_select_test fts3cov-1.2 { + SELECT docid FROM t1 WHERE t1 MATCH 'chilly'; +} {1 2} +set DO_MALLOC_TEST 0 + +#-------------------------------------------------------------------------- +# When querying the full-text index, if an expected internal node block is +# missing from the %_segments table, or if a NULL value is stored in the +# %_segments table instead of a binary blob, database corruption should be +# reported. +# +# Even with tiny 24 byte nodes, it takes a fair bit of data to produce a +# segment b-tree that uses the %_segments table to store internal nodes. +# +do_test fts3cov-2.1 { + execsql { + INSERT INTO t1(t1) VALUES('nodesize=24'); + BEGIN; + INSERT INTO t1 VALUES('The moon is behind, and at the full;'); + INSERT INTO t1 VALUES('And yet she looks both small and dull.'); + INSERT INTO t1 VALUES('The night is chill, the cloud is gray:'); + INSERT INTO t1 VALUES('''T is a month before the month of May,'); + INSERT INTO t1 VALUES('And the Spring comes slowly up this way.'); + INSERT INTO t1 VALUES('The lovely lady, Christabel,'); + INSERT INTO t1 VALUES('Whom her father loves so well,'); + INSERT INTO t1 VALUES('What makes her in the wood so late,'); + INSERT INTO t1 VALUES('A furlong from the castle gate?'); + INSERT INTO t1 VALUES('She had dreams all yesternight'); + INSERT INTO t1 VALUES('Of her own betrothed knight;'); + INSERT INTO t1 VALUES('And she in the midnight wood will pray'); + INSERT INTO t1 VALUES('For the weal of her lover that''s far away.'); + COMMIT; + + INSERT INTO t1(t1) VALUES('optimize'); + SELECT substr(hex(root), 1, 2) FROM t1_segdir; + } +} {03} + +# Test the "missing entry" case: +do_test fts3cov-2.1 { + set root [db one {SELECT root FROM t1_segdir}] + read_fts3varint [string range $root 1 end] left_child + execsql { DELETE FROM t1_segments WHERE blockid = $left_child } +} {} +do_error_test fts3cov-2.2 { + SELECT * FROM t1 WHERE t1 MATCH 'c*' +} {database disk image is malformed} + +# Test the "replaced with NULL" case: +do_test fts3cov-2.3 { + execsql { INSERT INTO t1_segments VALUES($left_child, NULL) } +} {} +do_error_test fts3cov-2.4 { + SELECT * FROM t1 WHERE t1 MATCH 'cloud' +} {database disk image is malformed} + +#-------------------------------------------------------------------------- +# The following tests are to test the effects of OOM errors while storing +# terms in the pending-hash table. Specifically, while creating doclist +# blobs to store in the table. More specifically, to test OOM errors while +# appending column numbers to doclists. For example, if a doclist consists +# of: +# +# 0x01 +# +# The following tests check that malloc errors encountered while appending +# the "0x01 " data to the dynamically growable blob used to +# accumulate the doclist in memory are handled correctly. +# +do_test fts3cov-3.1 { + set cols [list] + set vals [list] + for {set i 0} {$i < 120} {incr i} { + lappend cols "col$i" + lappend vals "'word'" + } + execsql "CREATE VIRTUAL TABLE t2 USING fts3([join $cols ,])" +} {} +set DO_MALLOC_TEST 1 +do_write_test fts3cov-3.2 t2_content " + INSERT INTO t2(docid, [join $cols ,]) VALUES(1, [join $vals ,]) +" +do_write_test fts3cov-3.3 t2_content " + INSERT INTO t2(docid, [join $cols ,]) VALUES(200, [join $vals ,]) +" +do_write_test fts3cov-3.4 t2_content " + INSERT INTO t2(docid, [join $cols ,]) VALUES(60000, [join $vals ,]) +" + +#------------------------------------------------------------------------- +# If too much data accumulates in the pending-terms hash table, it is +# flushed to the database automatically, even if the transaction has not +# finished. The following tests check the effects of encountering an OOM +# while doing this. +# +do_test fts3cov-4.1 { + execsql { + CREATE VIRTUAL TABLE t3 USING fts3(x); + INSERT INTO t3(t3) VALUES('nodesize=24'); + INSERT INTO t3(t3) VALUES('maxpending=100'); + } +} {} +set DO_MALLOC_TEST 1 +do_write_test fts3cov-4.2 t3_content { + INSERT INTO t3(docid, x) + SELECT 1, 'Then Christabel stretched forth her hand,' UNION ALL + SELECT 3, 'And comforted fair Geraldine:' UNION ALL + SELECT 4, '''O well, bright dame, may you command' UNION ALL + SELECT 5, 'The service of Sir Leoline;' UNION ALL + SELECT 2, 'And gladly our stout chivalry' UNION ALL + SELECT 7, 'Will he send forth, and friends withal,' UNION ALL + SELECT 8, 'To guide and guard you safe and free' UNION ALL + SELECT 6, 'Home to your noble father''s hall.''' +} + +finish_test +