Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Add coverage test cases for fts3. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA1: |
8fcb0478c82507403165719724b62a30 |
| User & Date: | dan 2009-12-12 09:51:25 |
Context
|
2009-12-12
| ||
| 13:16 | Extra tests for coverage of fts3 code. check-in: eee921a9 user: dan tags: trunk | |
| 09:51 | Add coverage test cases for fts3. check-in: 8fcb0478 user: dan tags: trunk | |
|
2009-12-11
| ||
| 23:11 | Additional changes to C-language interface documentation. check-in: 1342916f user: drh tags: trunk | |
Changes
Changes to ext/fts3/fts3.c.
617 617 618 618 p->db = db; 619 619 p->nColumn = nCol; 620 620 p->nPendingData = 0; 621 621 p->azColumn = (char **)&p[1]; 622 622 p->pTokenizer = pTokenizer; 623 623 p->nNodeSize = 1000; 624 + p->nMaxPendingData = FTS3_MAX_PENDING_DATA; 624 625 zCsr = (char *)&p->azColumn[nCol]; 625 626 626 627 fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); 627 628 628 629 /* Fill in the zName and zDb fields of the vtab structure. */ 629 630 p->zName = zCsr; 630 631 memcpy(zCsr, argv[2], nName); ................................................................................ 2263 2264 #ifdef SQLITE_ENABLE_ICU 2264 2265 || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) 2265 2266 #endif 2266 2267 ){ 2267 2268 rc = SQLITE_NOMEM; 2268 2269 } 2269 2270 } 2271 + 2272 +#ifdef SQLITE_TEST 2273 + sqlite3Fts3ExprInitTestInterface(db); 2274 +#endif 2270 2275 2271 2276 /* Create the virtual table wrapper around the hash-table and overload 2272 2277 ** the two scalar functions. If this is successful, register the 2273 2278 ** module with sqlite. 2274 2279 */ 2275 2280 if( SQLITE_OK==rc 2276 2281 && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
Changes to ext/fts3/fts3Int.h.
116 116 sqlite3_stmt **aLeavesStmt; /* Array of prepared zSelectLeaves stmts */ 117 117 118 118 int nNodeSize; /* Soft limit for node size */ 119 119 120 120 /* The following hash table is used to buffer pending index updates during 121 121 ** transactions. Variable nPendingData estimates the memory size of the 122 122 ** pending data, including hash table overhead, but not malloc overhead. 123 - ** When nPendingData exceeds FTS3_MAX_PENDING_DATA, the buffer is flushed 123 + ** When nPendingData exceeds nMaxPendingData, the buffer is flushed 124 124 ** automatically. Variable iPrevDocid is the docid of the most recently 125 125 ** inserted record. 126 126 */ 127 + int nMaxPendingData; 127 128 int nPendingData; 128 129 sqlite_int64 iPrevDocid; 129 130 Fts3Hash pendingTerms; 130 131 }; 131 132 132 133 /* 133 134 ** When the core wants to read from the virtual table, it creates a
Changes to ext/fts3/fts3_write.c.
290 290 rc = sqlite3_step(pStmt); 291 291 if( rc!=SQLITE_ROW ){ 292 292 return (rc==SQLITE_DONE ? SQLITE_CORRUPT : rc); 293 293 } 294 294 295 295 *pnBlock = sqlite3_column_bytes(pStmt, 0); 296 296 *pzBlock = (char *)sqlite3_column_blob(pStmt, 0); 297 - if( !*pzBlock ){ 298 - return SQLITE_NOMEM; 297 + if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ 298 + return SQLITE_CORRUPT; 299 299 } 300 300 } 301 301 return SQLITE_OK; 302 302 } 303 303 304 304 /* 305 305 ** Set *ppStmt to a statement handle that may be used to iterate through ................................................................................ 506 506 static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ 507 507 /* TODO(shess) Explore whether partially flushing the buffer on 508 508 ** forced-flush would provide better performance. I suspect that if 509 509 ** we ordered the doclists by size and flushed the largest until the 510 510 ** buffer was half empty, that would let the less frequent terms 511 511 ** generate longer doclists. 512 512 */ 513 - if( iDocid<=p->iPrevDocid || p->nPendingData>FTS3_MAX_PENDING_DATA ){ 513 + if( iDocid<=p->iPrevDocid || p->nPendingData>p->nMaxPendingData ){ 514 514 int rc = sqlite3Fts3PendingTermsFlush(p); 515 515 if( rc!=SQLITE_OK ) return rc; 516 516 } 517 517 p->iPrevDocid = iDocid; 518 518 return SQLITE_OK; 519 519 } 520 520 ................................................................................ 2216 2216 if( rc==SQLITE_DONE || rc==SQLITE_OK ){ 2217 2217 rc = SQLITE_OK; 2218 2218 sqlite3Fts3PendingTermsClear(p); 2219 2219 } 2220 2220 #ifdef SQLITE_TEST 2221 2221 }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ 2222 2222 p->nNodeSize = atoi(&zVal[9]); 2223 + rc = SQLITE_OK; 2224 + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ 2225 + p->nMaxPendingData = atoi(&zVal[11]); 2223 2226 rc = SQLITE_OK; 2224 2227 #endif 2225 2228 }else{ 2226 2229 rc = SQLITE_ERROR; 2227 2230 } 2228 2231 2229 2232 return rc;
Changes to src/test_hexio.c.
326 326 x += y * (*q++); 327 327 *v = (sqlite_int64) x; 328 328 return (int) (q - (unsigned char *)p); 329 329 } 330 330 331 331 332 332 /* 333 -** USAGE: read_varint BLOB VARNAME 333 +** USAGE: read_fts3varint BLOB VARNAME 334 334 ** 335 335 ** Read a varint from the start of BLOB. Set variable VARNAME to contain 336 336 ** the interpreted value. Return the number of bytes of BLOB consumed. 337 337 */ 338 -static int read_varint( 338 +static int read_fts3varint( 339 339 void * clientData, 340 340 Tcl_Interp *interp, 341 341 int objc, 342 342 Tcl_Obj *CONST objv[] 343 343 ){ 344 344 int nBlob; 345 345 unsigned char *zBlob; ................................................................................ 369 369 } aObjCmd[] = { 370 370 { "hexio_read", hexio_read }, 371 371 { "hexio_write", hexio_write }, 372 372 { "hexio_get_int", hexio_get_int }, 373 373 { "hexio_render_int16", hexio_render_int16 }, 374 374 { "hexio_render_int32", hexio_render_int32 }, 375 375 { "utf8_to_utf8", utf8_to_utf8 }, 376 - { "read_varint", read_varint }, 376 + { "read_fts3varint", read_fts3varint }, 377 377 }; 378 378 int i; 379 379 for(i=0; i<sizeof(aObjCmd)/sizeof(aObjCmd[0]); i++){ 380 380 Tcl_CreateObjCommand(interp, aObjCmd[i].zName, aObjCmd[i].xProc, 0, 0); 381 381 } 382 382 return TCL_OK; 383 383 }
Changes to test/fts3_common.tcl.
204 204 join $lDoc " " 205 205 } 206 206 207 207 ########################################################################### 208 208 209 209 proc gobble_varint {varname} { 210 210 upvar $varname blob 211 - set n [read_varint $blob ret] 211 + set n [read_fts3varint $blob ret] 212 212 set blob [string range $blob $n end] 213 213 return $ret 214 214 } 215 215 proc gobble_string {varname nLength} { 216 216 upvar $varname blob 217 217 set ret [string range $blob 0 [expr $nLength-1]] 218 218 set blob [string range $blob $nLength end] ................................................................................ 306 306 # by parameter $result, or (b) TCL throws an "out of memory" error. 307 307 # 308 308 # If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement 309 309 # is executed just once. In this case the test case passes if the results 310 310 # match the expected results passed via parameter $result. 311 311 # 312 312 proc do_select_test {name sql result} { 313 - uplevel [list doPassiveTest $name $sql [list 0 $result]] 313 + uplevel [list doPassiveTest 0 $name $sql [list 0 $result]] 314 +} 315 + 316 +proc do_restart_select_test {name sql result} { 317 + uplevel [list doPassiveTest 1 $name $sql [list 0 $result]] 314 318 } 315 319 316 320 proc do_error_test {name sql error} { 317 - uplevel [list doPassiveTest $name $sql [list 1 $error]] 321 + uplevel [list doPassiveTest 0 $name $sql [list 1 $error]] 318 322 } 319 323 320 -proc doPassiveTest {name sql catchres} { 324 +proc doPassiveTest {isRestart name sql catchres} { 321 325 if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 } 322 326 323 - if {$::DO_MALLOC_TEST} { 324 - set answers [list {1 {out of memory}} $catchres] 325 - set modes [list 100000 transient 1 persistent] 326 - } else { 327 - set answers [list $catchres] 328 - set modes [list 0 ""] 327 + switch $::DO_MALLOC_TEST { 328 + 0 { # No malloc failures. 329 + do_test $name [list catchsql $sql] $catchres 330 + return 331 + } 332 + 1 { # Simulate transient failures. 333 + set nRepeat 1 334 + set zName "transient" 335 + set nStartLimit 100000 336 + set nBackup 1 337 + } 338 + 2 { # Simulate persistent failures. 339 + set nRepeat 1 340 + set zName "persistent" 341 + set nStartLimit 100000 342 + set nBackup 1 343 + } 344 + 3 { # Simulate transient failures with extra brute force. 345 + set nRepeat 100000 346 + set zName "ridiculous" 347 + set nStartLimit 1 348 + set nBackup 10 349 + } 329 350 } 351 + 352 + # The set of acceptable results from running [catchsql $sql]. 353 + # 354 + set answers [list {1 {out of memory}} $catchres] 330 355 set str [join $answers " OR "] 331 356 332 - foreach {nRepeat zName} $modes { 333 - for {set iFail 1} 1 {incr iFail} { 334 - if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat} 357 + set nFail 1 358 + for {set iLimit $nStartLimit} {$nFail} {incr iLimit} { 359 + for {set iFail 1} {$nFail && $iFail<=$iLimit} {incr iFail} { 360 + for {set iTest 0} {$iTest<$nBackup && ($iFail-$iTest)>0} {incr iTest} { 361 + 362 + if {$isRestart} { sqlite3 db test.db } 363 + 364 + sqlite3_memdebug_fail [expr $iFail-$iTest] -repeat $nRepeat 365 + set res [uplevel [list catchsql $sql]] 366 + if {[lsearch -exact $answers $res]>=0} { set res $str } 367 + set testname "$name.$zName.$iFail" 368 + do_test "$name.$zName.$iLimit.$iFail" [list set {} $res] $str 335 369 336 - set res [uplevel [list catchsql $sql]] 337 - if {[lsearch -exact $answers $res]>=0} { 338 - set res $str 370 + set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign] 339 371 } 340 - set testname "$name.$zName.$iFail" 341 - if {$zName == ""} { set testname $name } 342 - do_test $testname [list set {} $res] $str 343 - set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign] 344 - if {$nFail==0} break 345 372 } 346 373 } 347 374 } 348 375 349 376 350 377 #------------------------------------------------------------------------- 351 378 # Test a single write to the database. In this case a "write" is a
Added test/fts3cov.test.
1 +# 2009 December 03 2 +# 3 +# May you do good and not evil. 4 +# May you find forgiveness for yourself and forgive others. 5 +# May you share freely, never taking more than you give. 6 +# 7 +#*********************************************************************** 8 +# 9 +# The tests in this file are structural coverage tests. They are designed 10 +# to complement the tests in fts3rnd.test and fts3doc.test. Between them, 11 +# the three files should provide full coverage of the fts3 extension code. 12 +# 13 + 14 +set testdir [file dirname $argv0] 15 +source $testdir/tester.tcl 16 + 17 +# If this build does not include FTS3, skip the tests in this file. 18 +# 19 +ifcapable !fts3 { finish_test ; return } 20 +source $testdir/fts3_common.tcl 21 + 22 +set DO_MALLOC_TEST 0 23 + 24 +#-------------------------------------------------------------------------- 25 +# When it first needs to read a block from the %_segments table, the FTS3 26 +# module compiles an SQL statement for that purpose. The statement is 27 +# stored and reused each subsequent time a block is read. This test case 28 +# tests the effects of an OOM error occuring while compiling the statement. 29 +# 30 +# Similarly, when FTS3 first needs to scan through a set of segment leaves 31 +# to find a set of documents that matches a term, it allocates a string 32 +# containing the text of the required SQL, and compiles one or more 33 +# statements to traverse the leaves. This test case tests that OOM errors 34 +# that occur while allocating this string and statement are handled correctly 35 +# also. 36 +# 37 +do_test fts3cov-1.1 { 38 + execsql { 39 + CREATE VIRTUAL TABLE t1 USING fts3(x); 40 + INSERT INTO t1(t1) VALUES('nodesize=24'); 41 + BEGIN; 42 + INSERT INTO t1 VALUES('Is the night chilly and dark?'); 43 + INSERT INTO t1 VALUES('The night is chilly, but not dark.'); 44 + INSERT INTO t1 VALUES('The thin gray cloud is spread on high,'); 45 + INSERT INTO t1 VALUES('It covers but not hides the sky.'); 46 + COMMIT; 47 + SELECT count(*)>0 FROM t1_segments; 48 + } 49 +} {1} 50 + 51 +set DO_MALLOC_TEST 1 52 +do_restart_select_test fts3cov-1.2 { 53 + SELECT docid FROM t1 WHERE t1 MATCH 'chilly'; 54 +} {1 2} 55 +set DO_MALLOC_TEST 0 56 + 57 +#-------------------------------------------------------------------------- 58 +# When querying the full-text index, if an expected internal node block is 59 +# missing from the %_segments table, or if a NULL value is stored in the 60 +# %_segments table instead of a binary blob, database corruption should be 61 +# reported. 62 +# 63 +# Even with tiny 24 byte nodes, it takes a fair bit of data to produce a 64 +# segment b-tree that uses the %_segments table to store internal nodes. 65 +# 66 +do_test fts3cov-2.1 { 67 + execsql { 68 + INSERT INTO t1(t1) VALUES('nodesize=24'); 69 + BEGIN; 70 + INSERT INTO t1 VALUES('The moon is behind, and at the full;'); 71 + INSERT INTO t1 VALUES('And yet she looks both small and dull.'); 72 + INSERT INTO t1 VALUES('The night is chill, the cloud is gray:'); 73 + INSERT INTO t1 VALUES('''T is a month before the month of May,'); 74 + INSERT INTO t1 VALUES('And the Spring comes slowly up this way.'); 75 + INSERT INTO t1 VALUES('The lovely lady, Christabel,'); 76 + INSERT INTO t1 VALUES('Whom her father loves so well,'); 77 + INSERT INTO t1 VALUES('What makes her in the wood so late,'); 78 + INSERT INTO t1 VALUES('A furlong from the castle gate?'); 79 + INSERT INTO t1 VALUES('She had dreams all yesternight'); 80 + INSERT INTO t1 VALUES('Of her own betrothed knight;'); 81 + INSERT INTO t1 VALUES('And she in the midnight wood will pray'); 82 + INSERT INTO t1 VALUES('For the weal of her lover that''s far away.'); 83 + COMMIT; 84 + 85 + INSERT INTO t1(t1) VALUES('optimize'); 86 + SELECT substr(hex(root), 1, 2) FROM t1_segdir; 87 + } 88 +} {03} 89 + 90 +# Test the "missing entry" case: 91 +do_test fts3cov-2.1 { 92 + set root [db one {SELECT root FROM t1_segdir}] 93 + read_fts3varint [string range $root 1 end] left_child 94 + execsql { DELETE FROM t1_segments WHERE blockid = $left_child } 95 +} {} 96 +do_error_test fts3cov-2.2 { 97 + SELECT * FROM t1 WHERE t1 MATCH 'c*' 98 +} {database disk image is malformed} 99 + 100 +# Test the "replaced with NULL" case: 101 +do_test fts3cov-2.3 { 102 + execsql { INSERT INTO t1_segments VALUES($left_child, NULL) } 103 +} {} 104 +do_error_test fts3cov-2.4 { 105 + SELECT * FROM t1 WHERE t1 MATCH 'cloud' 106 +} {database disk image is malformed} 107 + 108 +#-------------------------------------------------------------------------- 109 +# The following tests are to test the effects of OOM errors while storing 110 +# terms in the pending-hash table. Specifically, while creating doclist 111 +# blobs to store in the table. More specifically, to test OOM errors while 112 +# appending column numbers to doclists. For example, if a doclist consists 113 +# of: 114 +# 115 +# <docid> <column 0 offset-list> 0x01 <column N> <column N offset-list> 116 +# 117 +# The following tests check that malloc errors encountered while appending 118 +# the "0x01 <column N>" data to the dynamically growable blob used to 119 +# accumulate the doclist in memory are handled correctly. 120 +# 121 +do_test fts3cov-3.1 { 122 + set cols [list] 123 + set vals [list] 124 + for {set i 0} {$i < 120} {incr i} { 125 + lappend cols "col$i" 126 + lappend vals "'word'" 127 + } 128 + execsql "CREATE VIRTUAL TABLE t2 USING fts3([join $cols ,])" 129 +} {} 130 +set DO_MALLOC_TEST 1 131 +do_write_test fts3cov-3.2 t2_content " 132 + INSERT INTO t2(docid, [join $cols ,]) VALUES(1, [join $vals ,]) 133 +" 134 +do_write_test fts3cov-3.3 t2_content " 135 + INSERT INTO t2(docid, [join $cols ,]) VALUES(200, [join $vals ,]) 136 +" 137 +do_write_test fts3cov-3.4 t2_content " 138 + INSERT INTO t2(docid, [join $cols ,]) VALUES(60000, [join $vals ,]) 139 +" 140 + 141 +#------------------------------------------------------------------------- 142 +# If too much data accumulates in the pending-terms hash table, it is 143 +# flushed to the database automatically, even if the transaction has not 144 +# finished. The following tests check the effects of encountering an OOM 145 +# while doing this. 146 +# 147 +do_test fts3cov-4.1 { 148 + execsql { 149 + CREATE VIRTUAL TABLE t3 USING fts3(x); 150 + INSERT INTO t3(t3) VALUES('nodesize=24'); 151 + INSERT INTO t3(t3) VALUES('maxpending=100'); 152 + } 153 +} {} 154 +set DO_MALLOC_TEST 1 155 +do_write_test fts3cov-4.2 t3_content { 156 + INSERT INTO t3(docid, x) 157 + SELECT 1, 'Then Christabel stretched forth her hand,' UNION ALL 158 + SELECT 3, 'And comforted fair Geraldine:' UNION ALL 159 + SELECT 4, '''O well, bright dame, may you command' UNION ALL 160 + SELECT 5, 'The service of Sir Leoline;' UNION ALL 161 + SELECT 2, 'And gladly our stout chivalry' UNION ALL 162 + SELECT 7, 'Will he send forth, and friends withal,' UNION ALL 163 + SELECT 8, 'To guide and guard you safe and free' UNION ALL 164 + SELECT 6, 'Home to your noble father''s hall.''' 165 +} 166 + 167 +finish_test 168 +