/ Check-in [8fcb0478]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add coverage test cases for fts3.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 8fcb0478c82507403165719724b62a308cb83b57
User & Date: dan 2009-12-12 09:51:25
Context
2009-12-12
13:16
Extra tests for coverage of fts3 code. check-in: eee921a9 user: dan tags: trunk
09:51
Add coverage test cases for fts3. check-in: 8fcb0478 user: dan tags: trunk
2009-12-11
23:11
Additional changes to C-language interface documentation. check-in: 1342916f user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

   617    617   
   618    618     p->db = db;
   619    619     p->nColumn = nCol;
   620    620     p->nPendingData = 0;
   621    621     p->azColumn = (char **)&p[1];
   622    622     p->pTokenizer = pTokenizer;
   623    623     p->nNodeSize = 1000;
          624  +  p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
   624    625     zCsr = (char *)&p->azColumn[nCol];
   625    626   
   626    627     fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1);
   627    628   
   628    629     /* Fill in the zName and zDb fields of the vtab structure. */
   629    630     p->zName = zCsr;
   630    631     memcpy(zCsr, argv[2], nName);
................................................................................
  2263   2264   #ifdef SQLITE_ENABLE_ICU
  2264   2265        || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
  2265   2266   #endif
  2266   2267       ){
  2267   2268         rc = SQLITE_NOMEM;
  2268   2269       }
  2269   2270     }
         2271  +
         2272  +#ifdef SQLITE_TEST
         2273  +  sqlite3Fts3ExprInitTestInterface(db);
         2274  +#endif
  2270   2275   
  2271   2276     /* Create the virtual table wrapper around the hash-table and overload 
  2272   2277     ** the two scalar functions. If this is successful, register the
  2273   2278     ** module with sqlite.
  2274   2279     */
  2275   2280     if( SQLITE_OK==rc 
  2276   2281      && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))

Changes to ext/fts3/fts3Int.h.

   116    116     sqlite3_stmt **aLeavesStmt;     /* Array of prepared zSelectLeaves stmts */
   117    117   
   118    118     int nNodeSize;                  /* Soft limit for node size */
   119    119   
   120    120     /* The following hash table is used to buffer pending index updates during
   121    121     ** transactions. Variable nPendingData estimates the memory size of the 
   122    122     ** pending data, including hash table overhead, but not malloc overhead. 
   123         -  ** When nPendingData exceeds FTS3_MAX_PENDING_DATA, the buffer is flushed 
          123  +  ** When nPendingData exceeds nMaxPendingData, the buffer is flushed 
   124    124     ** automatically. Variable iPrevDocid is the docid of the most recently
   125    125     ** inserted record.
   126    126     */
          127  +  int nMaxPendingData;
   127    128     int nPendingData;
   128    129     sqlite_int64 iPrevDocid;
   129    130     Fts3Hash pendingTerms;
   130    131   };
   131    132   
   132    133   /*
   133    134   ** When the core wants to read from the virtual table, it creates a

Changes to ext/fts3/fts3_write.c.

   290    290       rc = sqlite3_step(pStmt); 
   291    291       if( rc!=SQLITE_ROW ){
   292    292         return (rc==SQLITE_DONE ? SQLITE_CORRUPT : rc);
   293    293       }
   294    294     
   295    295       *pnBlock = sqlite3_column_bytes(pStmt, 0);
   296    296       *pzBlock = (char *)sqlite3_column_blob(pStmt, 0);
   297         -    if( !*pzBlock ){
   298         -      return SQLITE_NOMEM;
          297  +    if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){
          298  +      return SQLITE_CORRUPT;
   299    299       }
   300    300     }
   301    301     return SQLITE_OK;
   302    302   }
   303    303   
   304    304   /*
   305    305   ** Set *ppStmt to a statement handle that may be used to iterate through
................................................................................
   506    506   static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){
   507    507     /* TODO(shess) Explore whether partially flushing the buffer on
   508    508     ** forced-flush would provide better performance.  I suspect that if
   509    509     ** we ordered the doclists by size and flushed the largest until the
   510    510     ** buffer was half empty, that would let the less frequent terms
   511    511     ** generate longer doclists.
   512    512     */
   513         -  if( iDocid<=p->iPrevDocid || p->nPendingData>FTS3_MAX_PENDING_DATA ){
          513  +  if( iDocid<=p->iPrevDocid || p->nPendingData>p->nMaxPendingData ){
   514    514       int rc = sqlite3Fts3PendingTermsFlush(p);
   515    515       if( rc!=SQLITE_OK ) return rc;
   516    516     }
   517    517     p->iPrevDocid = iDocid;
   518    518     return SQLITE_OK;
   519    519   }
   520    520   
................................................................................
  2216   2216       if( rc==SQLITE_DONE || rc==SQLITE_OK ){
  2217   2217         rc = SQLITE_OK;
  2218   2218         sqlite3Fts3PendingTermsClear(p);
  2219   2219       }
  2220   2220   #ifdef SQLITE_TEST
  2221   2221     }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
  2222   2222       p->nNodeSize = atoi(&zVal[9]);
         2223  +    rc = SQLITE_OK;
         2224  +  }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
         2225  +    p->nMaxPendingData = atoi(&zVal[11]);
  2223   2226       rc = SQLITE_OK;
  2224   2227   #endif
  2225   2228     }else{
  2226   2229       rc = SQLITE_ERROR;
  2227   2230     }
  2228   2231   
  2229   2232     return rc;

Changes to src/test_hexio.c.

   326    326     x += y * (*q++);
   327    327     *v = (sqlite_int64) x;
   328    328     return (int) (q - (unsigned char *)p);
   329    329   }
   330    330   
   331    331   
   332    332   /*
   333         -** USAGE:  read_varint BLOB VARNAME
          333  +** USAGE:  read_fts3varint BLOB VARNAME
   334    334   **
   335    335   ** Read a varint from the start of BLOB. Set variable VARNAME to contain
   336    336   ** the interpreted value. Return the number of bytes of BLOB consumed.
   337    337   */
   338         -static int read_varint(
          338  +static int read_fts3varint(
   339    339     void * clientData,
   340    340     Tcl_Interp *interp,
   341    341     int objc,
   342    342     Tcl_Obj *CONST objv[]
   343    343   ){
   344    344     int nBlob;
   345    345     unsigned char *zBlob;
................................................................................
   369    369     } aObjCmd[] = {
   370    370        { "hexio_read",                   hexio_read            },
   371    371        { "hexio_write",                  hexio_write           },
   372    372        { "hexio_get_int",                hexio_get_int         },
   373    373        { "hexio_render_int16",           hexio_render_int16    },
   374    374        { "hexio_render_int32",           hexio_render_int32    },
   375    375        { "utf8_to_utf8",                 utf8_to_utf8          },
   376         -     { "read_varint",                  read_varint           },
          376  +     { "read_fts3varint",              read_fts3varint       },
   377    377     };
   378    378     int i;
   379    379     for(i=0; i<sizeof(aObjCmd)/sizeof(aObjCmd[0]); i++){
   380    380       Tcl_CreateObjCommand(interp, aObjCmd[i].zName, aObjCmd[i].xProc, 0, 0);
   381    381     }
   382    382     return TCL_OK;
   383    383   }

Changes to test/fts3_common.tcl.

   204    204     join $lDoc " "
   205    205   }
   206    206   
   207    207   ###########################################################################
   208    208   
   209    209   proc gobble_varint {varname} {
   210    210     upvar $varname blob
   211         -  set n [read_varint $blob ret]
          211  +  set n [read_fts3varint $blob ret]
   212    212     set blob [string range $blob $n end]
   213    213     return $ret
   214    214   }
   215    215   proc gobble_string {varname nLength} {
   216    216     upvar $varname blob
   217    217     set ret [string range $blob 0 [expr $nLength-1]]
   218    218     set blob [string range $blob $nLength end]
................................................................................
   306    306   # by parameter $result, or (b) TCL throws an "out of memory" error.
   307    307   #
   308    308   # If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement
   309    309   # is executed just once. In this case the test case passes if the results
   310    310   # match the expected results passed via parameter $result.
   311    311   #
   312    312   proc do_select_test {name sql result} {
   313         -  uplevel [list doPassiveTest $name $sql [list 0 $result]]
          313  +  uplevel [list doPassiveTest 0 $name $sql [list 0 $result]]
          314  +}
          315  +
          316  +proc do_restart_select_test {name sql result} {
          317  +  uplevel [list doPassiveTest 1 $name $sql [list 0 $result]]
   314    318   }
   315    319   
   316    320   proc do_error_test {name sql error} {
   317         -  uplevel [list doPassiveTest $name $sql [list 1 $error]]
          321  +  uplevel [list doPassiveTest 0 $name $sql [list 1 $error]]
   318    322   }
   319    323   
   320         -proc doPassiveTest {name sql catchres} {
          324  +proc doPassiveTest {isRestart name sql catchres} {
   321    325     if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }
   322    326   
   323         -  if {$::DO_MALLOC_TEST} {
   324         -    set answers [list {1 {out of memory}} $catchres]
   325         -    set modes [list 100000 transient 1 persistent]
   326         -  } else {
   327         -    set answers [list $catchres]
   328         -    set modes [list 0 ""]
          327  +  switch $::DO_MALLOC_TEST {
          328  +    0 { # No malloc failures.
          329  +      do_test $name [list catchsql $sql] $catchres
          330  +      return
          331  +    }
          332  +    1 { # Simulate transient failures.
          333  +      set nRepeat 1
          334  +      set zName "transient"
          335  +      set nStartLimit 100000
          336  +      set nBackup 1
          337  +    }
          338  +    2 { # Simulate persistent failures.
          339  +      set nRepeat 1
          340  +      set zName "persistent"
          341  +      set nStartLimit 100000
          342  +      set nBackup 1
          343  +    }
          344  +    3 { # Simulate transient failures with extra brute force.
          345  +      set nRepeat 100000
          346  +      set zName "ridiculous"
          347  +      set nStartLimit 1
          348  +      set nBackup 10
          349  +    }
   329    350     }
          351  +
          352  +  # The set of acceptable results from running [catchsql $sql].
          353  +  #
          354  +  set answers [list {1 {out of memory}} $catchres]
   330    355     set str [join $answers " OR "]
   331    356   
   332         -  foreach {nRepeat zName} $modes {
   333         -    for {set iFail 1} 1 {incr iFail} {
   334         -      if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat}
          357  +  set nFail 1
          358  +  for {set iLimit $nStartLimit} {$nFail} {incr iLimit} {
          359  +    for {set iFail 1} {$nFail && $iFail<=$iLimit} {incr iFail} {
          360  +      for {set iTest 0} {$iTest<$nBackup && ($iFail-$iTest)>0} {incr iTest} {
          361  +
          362  +        if {$isRestart} { sqlite3 db test.db }
          363  +
          364  +        sqlite3_memdebug_fail [expr $iFail-$iTest] -repeat $nRepeat
          365  +        set res [uplevel [list catchsql $sql]]
          366  +        if {[lsearch -exact $answers $res]>=0} { set res $str }
          367  +        set testname "$name.$zName.$iFail"
          368  +        do_test "$name.$zName.$iLimit.$iFail" [list set {} $res] $str
   335    369   
   336         -      set res [uplevel [list catchsql $sql]]
   337         -      if {[lsearch -exact $answers $res]>=0} {
   338         -        set res $str
          370  +        set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
   339    371         }
   340         -      set testname "$name.$zName.$iFail"
   341         -      if {$zName == ""} { set testname $name }
   342         -      do_test $testname [list set {} $res] $str
   343         -      set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
   344         -      if {$nFail==0} break
   345    372       }
   346    373     }
   347    374   }
   348    375   
   349    376   
   350    377   #-------------------------------------------------------------------------
   351    378   # Test a single write to the database. In this case a  "write" is a 

Added test/fts3cov.test.

            1  +# 2009 December 03
            2  +#
            3  +#    May you do good and not evil.
            4  +#    May you find forgiveness for yourself and forgive others.
            5  +#    May you share freely, never taking more than you give.
            6  +#
            7  +#***********************************************************************
            8  +#
            9  +# The tests in this file are structural coverage tests. They are designed
           10  +# to complement the tests in fts3rnd.test and fts3doc.test. Between them,
           11  +# the three files should provide full coverage of the fts3 extension code.
           12  +#
           13  +
           14  +set testdir [file dirname $argv0]
           15  +source $testdir/tester.tcl
           16  +
           17  +# If this build does not include FTS3, skip the tests in this file.
           18  +#
           19  +ifcapable !fts3 { finish_test ; return }
           20  +source $testdir/fts3_common.tcl
           21  +
           22  +set DO_MALLOC_TEST 0
           23  +
           24  +#--------------------------------------------------------------------------
           25  +# When it first needs to read a block from the %_segments table, the FTS3 
           26  +# module compiles an SQL statement for that purpose. The statement is 
           27  +# stored and reused each subsequent time a block is read. This test case 
           28  +# tests the effects of an OOM error occuring while compiling the statement.
           29  +#
           30  +# Similarly, when FTS3 first needs to scan through a set of segment leaves
           31  +# to find a set of documents that matches a term, it allocates a string
           32  +# containing the text of the required SQL, and compiles one or more 
           33  +# statements to traverse the leaves. This test case tests that OOM errors
           34  +# that occur while allocating this string and statement are handled correctly
           35  +# also.
           36  +#
           37  +do_test fts3cov-1.1 {
           38  +  execsql { 
           39  +    CREATE VIRTUAL TABLE t1 USING fts3(x);
           40  +    INSERT INTO t1(t1) VALUES('nodesize=24');
           41  +    BEGIN;
           42  +      INSERT INTO t1 VALUES('Is the night chilly and dark?');
           43  +      INSERT INTO t1 VALUES('The night is chilly, but not dark.');
           44  +      INSERT INTO t1 VALUES('The thin gray cloud is spread on high,');
           45  +      INSERT INTO t1 VALUES('It covers but not hides the sky.');
           46  +    COMMIT;
           47  +    SELECT count(*)>0 FROM t1_segments;
           48  +  }
           49  +} {1}
           50  +
           51  +set DO_MALLOC_TEST 1
           52  +do_restart_select_test fts3cov-1.2 {
           53  +  SELECT docid FROM t1 WHERE t1 MATCH 'chilly';
           54  +} {1 2}
           55  +set DO_MALLOC_TEST 0
           56  +
           57  +#--------------------------------------------------------------------------
           58  +# When querying the full-text index, if an expected internal node block is 
           59  +# missing from the %_segments table, or if a NULL value is stored in the 
           60  +# %_segments table instead of a binary blob, database corruption should be 
           61  +# reported.
           62  +#
           63  +# Even with tiny 24 byte nodes, it takes a fair bit of data to produce a
           64  +# segment b-tree that uses the %_segments table to store internal nodes. 
           65  +#
           66  +do_test fts3cov-2.1 {
           67  +  execsql {
           68  +    INSERT INTO t1(t1) VALUES('nodesize=24');
           69  +    BEGIN;
           70  +      INSERT INTO t1 VALUES('The moon is behind, and at the full;');
           71  +      INSERT INTO t1 VALUES('And yet she looks both small and dull.');
           72  +      INSERT INTO t1 VALUES('The night is chill, the cloud is gray:');
           73  +      INSERT INTO t1 VALUES('''T is a month before the month of May,');
           74  +      INSERT INTO t1 VALUES('And the Spring comes slowly up this way.');
           75  +      INSERT INTO t1 VALUES('The lovely lady, Christabel,');
           76  +      INSERT INTO t1 VALUES('Whom her father loves so well,');
           77  +      INSERT INTO t1 VALUES('What makes her in the wood so late,');
           78  +      INSERT INTO t1 VALUES('A furlong from the castle gate?');
           79  +      INSERT INTO t1 VALUES('She had dreams all yesternight');
           80  +      INSERT INTO t1 VALUES('Of her own betrothed knight;');
           81  +      INSERT INTO t1 VALUES('And she in the midnight wood will pray');
           82  +      INSERT INTO t1 VALUES('For the weal of her lover that''s far away.');
           83  +    COMMIT;
           84  +
           85  +    INSERT INTO t1(t1) VALUES('optimize');
           86  +    SELECT substr(hex(root), 1, 2) FROM t1_segdir;
           87  +  }
           88  +} {03}
           89  +
           90  +# Test the "missing entry" case:
           91  +do_test fts3cov-2.1 {
           92  +  set root [db one {SELECT root FROM t1_segdir}]
           93  +  read_fts3varint [string range $root 1 end] left_child
           94  +  execsql { DELETE FROM t1_segments WHERE blockid = $left_child }
           95  +} {}
           96  +do_error_test fts3cov-2.2 {
           97  +  SELECT * FROM t1 WHERE t1 MATCH 'c*'
           98  +} {database disk image is malformed}
           99  +
          100  +# Test the "replaced with NULL" case:
          101  +do_test fts3cov-2.3 {
          102  +  execsql { INSERT INTO t1_segments VALUES($left_child, NULL) }
          103  +} {}
          104  +do_error_test fts3cov-2.4 {
          105  +  SELECT * FROM t1 WHERE t1 MATCH 'cloud'
          106  +} {database disk image is malformed}
          107  +
          108  +#--------------------------------------------------------------------------
          109  +# The following tests are to test the effects of OOM errors while storing
          110  +# terms in the pending-hash table. Specifically, while creating doclist
          111  +# blobs to store in the table. More specifically, to test OOM errors while
          112  +# appending column numbers to doclists. For example, if a doclist consists
          113  +# of:
          114  +#
          115  +#   <docid> <column 0 offset-list> 0x01 <column N> <column N offset-list>
          116  +#
          117  +# The following tests check that malloc errors encountered while appending
          118  +# the "0x01 <column N>" data to the dynamically growable blob used to 
          119  +# accumulate the doclist in memory are handled correctly.
          120  +#
          121  +do_test fts3cov-3.1 {
          122  +  set cols [list]
          123  +  set vals [list]
          124  +  for {set i 0} {$i < 120} {incr i} {
          125  +    lappend cols "col$i"
          126  +    lappend vals "'word'"
          127  +  }
          128  +  execsql "CREATE VIRTUAL TABLE t2 USING fts3([join $cols ,])"
          129  +} {}
          130  +set DO_MALLOC_TEST 1 
          131  +do_write_test fts3cov-3.2 t2_content "
          132  +  INSERT INTO t2(docid, [join $cols ,]) VALUES(1, [join $vals ,])
          133  +"
          134  +do_write_test fts3cov-3.3 t2_content "
          135  +  INSERT INTO t2(docid, [join $cols ,]) VALUES(200, [join $vals ,])
          136  +"
          137  +do_write_test fts3cov-3.4 t2_content "
          138  +  INSERT INTO t2(docid, [join $cols ,]) VALUES(60000, [join $vals ,])
          139  +"
          140  +
          141  +#-------------------------------------------------------------------------
          142  +# If too much data accumulates in the pending-terms hash table, it is
          143  +# flushed to the database automatically, even if the transaction has not
          144  +# finished. The following tests check the effects of encountering an OOM 
          145  +# while doing this.
          146  +#
          147  +do_test fts3cov-4.1 {
          148  +  execsql {
          149  +    CREATE VIRTUAL TABLE t3 USING fts3(x);
          150  +    INSERT INTO t3(t3) VALUES('nodesize=24');
          151  +    INSERT INTO t3(t3) VALUES('maxpending=100');
          152  +  }
          153  +} {}
          154  +set DO_MALLOC_TEST 1 
          155  +do_write_test fts3cov-4.2 t3_content {
          156  +  INSERT INTO t3(docid, x)
          157  +    SELECT 1, 'Then Christabel stretched forth her hand,' UNION ALL
          158  +    SELECT 3, 'And comforted fair Geraldine:'             UNION ALL
          159  +    SELECT 4, '''O well, bright dame, may you command'    UNION ALL
          160  +    SELECT 5, 'The service of Sir Leoline;'               UNION ALL
          161  +    SELECT 2, 'And gladly our stout chivalry'             UNION ALL
          162  +    SELECT 7, 'Will he send forth, and friends withal,'   UNION ALL
          163  +    SELECT 8, 'To guide and guard you safe and free'      UNION ALL
          164  +    SELECT 6, 'Home to your noble father''s hall.'''
          165  +}
          166  +
          167  +finish_test
          168  +