/ Check-in [11eb8e87]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix the fts5 "prefix=" option to match the documentation (space separated list, multiple prefix= options supported). The undocumented comma-separated format (compatible with fts4) still works.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 11eb8e877e2ba859ef6b44318f286597186dfaf2
User & Date: dan 2015-11-25 11:56:24
Context
2015-11-25
14:00
Simplification of the error code translator in os_unix.c. Code cleanup only. The logic is unchanged. check-in: 2a20f793 user: drh tags: trunk
11:56
Fix the fts5 "prefix=" option to match the documentation (space separated list, multiple prefix= options supported). The undocumented comma-separated format (compatible with fts4) still works. check-in: 11eb8e87 user: dan tags: trunk
01:57
Enhancement the virtual table interface to support LIKE, GLOB, and REGEXP operators. Also add the sqlite3_strlike() interface, which might be useful as part of the implementation of LIKE on some virtual tables. check-in: a6bfd469 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_config.c.

   212    212     char **pzErr                    /* OUT: Error message */
   213    213   ){
   214    214     int rc = SQLITE_OK;
   215    215     int nCmd = (int)strlen(zCmd);
   216    216     if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){
   217    217       const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
   218    218       const char *p;
   219         -    if( pConfig->aPrefix ){
   220         -      *pzErr = sqlite3_mprintf("multiple prefix=... directives");
   221         -      rc = SQLITE_ERROR;
   222         -    }else{
          219  +    int bFirst = 1;
          220  +    if( pConfig->aPrefix==0 ){
   223    221         pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte);
          222  +      if( rc ) return rc;
   224    223       }
          224  +
   225    225       p = zArg;
   226         -    while( rc==SQLITE_OK && p[0] ){
          226  +    while( 1 ){
   227    227         int nPre = 0;
          228  +
   228    229         while( p[0]==' ' ) p++;
          230  +      if( bFirst==0 && p[0]==',' ){
          231  +        p++;
          232  +        while( p[0]==' ' ) p++;
          233  +      }else if( p[0]=='\0' ){
          234  +        break;
          235  +      }
          236  +      if( p[0]<'0' || p[0]>'9' ){
          237  +        *pzErr = sqlite3_mprintf("malformed prefix=... directive");
          238  +        rc = SQLITE_ERROR;
          239  +        break;
          240  +      }
          241  +
          242  +      if( pConfig->nPrefix==FTS5_MAX_PREFIX_INDEXES ){
          243  +        *pzErr = sqlite3_mprintf(
          244  +            "too many prefix indexes (max %d)", FTS5_MAX_PREFIX_INDEXES
          245  +        );
          246  +        rc = SQLITE_ERROR;
          247  +        break;
          248  +      }
          249  +
   229    250         while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
   230    251           nPre = nPre*10 + (p[0] - '0');
   231    252           p++;
   232    253         }
   233         -      while( p[0]==' ' ) p++;
   234         -      if( p[0]==',' ){
   235         -        p++;
   236         -      }else if( p[0] ){
   237         -        *pzErr = sqlite3_mprintf("malformed prefix=... directive");
          254  +
          255  +      if( rc==SQLITE_OK && (nPre<=0 || nPre>=1000) ){
          256  +        *pzErr = sqlite3_mprintf("prefix length out of range (max 999)");
   238    257           rc = SQLITE_ERROR;
          258  +        break;
   239    259         }
   240         -      if( rc==SQLITE_OK && (nPre==0 || nPre>=1000) ){
   241         -        *pzErr = sqlite3_mprintf("prefix length out of range: %d", nPre);
   242         -        rc = SQLITE_ERROR;
   243         -      }
          260  +
   244    261         pConfig->aPrefix[pConfig->nPrefix] = nPre;
   245    262         pConfig->nPrefix++;
          263  +      bFirst = 0;
   246    264       }
          265  +    assert( pConfig->nPrefix<=FTS5_MAX_PREFIX_INDEXES );
   247    266       return rc;
   248    267     }
   249    268   
   250    269     if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
   251    270       const char *p = (const char*)zArg;
   252    271       int nArg = (int)strlen(zArg) + 1;
   253    272       char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);

Changes to ext/fts5/test/fts5config.test.

    38     38   #-------------------------------------------------------------------------
    39     39   # Syntax errors in the prefix= option.
    40     40   #
    41     41   foreach {tn opt} {
    42     42     1 {prefix=x}  
    43     43     2 {prefix='x'}
    44     44     3 {prefix='$'}
           45  +  4 {prefix='1,2,'}
           46  +  5 {prefix=',1'}
    45     47   } {
    46     48     set res [list 1 {malformed prefix=... directive}]
    47     49     do_catchsql_test 2.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
    48     50   }
    49     51   
    50     52   #-------------------------------------------------------------------------
    51     53   # Syntax errors in the 'rank' option.
................................................................................
   114    116   do_catchsql_test 5.3 {
   115    117     CREATE VIRTUAL TABLE yy USING fts5(x, [y]]);
   116    118   } {1 {unrecognized token: "]"}}
   117    119   
   118    120   #-------------------------------------------------------------------------
   119    121   # Errors in prefix= directives.
   120    122   #
   121         -do_catchsql_test 6.1 {
   122         -  CREATE VIRTUAL TABLE abc USING fts5(a, prefix=1, prefix=2);
   123         -} {1 {multiple prefix=... directives}}
   124    123   do_catchsql_test 6.2 {
   125    124     CREATE VIRTUAL TABLE abc USING fts5(a, prefix='1, 2, 1001');
   126         -} {1 {prefix length out of range: 1001}}
          125  +} {1 {prefix length out of range (max 999)}}
   127    126   do_catchsql_test 6.3 {
   128    127     CREATE VIRTUAL TAbLE abc USING fts5(a, prefix='1, 2, 0000');
   129         -} {1 {prefix length out of range: 0}}
          128  +} {1 {prefix length out of range (max 999)}}
   130    129   do_catchsql_test 6.4 {
   131    130     CREATE VIRTUAL TABLE abc USING fts5(a, prefix='1  , 1000000');
   132         -} {1 {malformed prefix=... directive}}
          131  +} {1 {prefix length out of range (max 999)}}
   133    132   
   134    133   #-------------------------------------------------------------------------
   135    134   # Duplicate tokenize= and other options.
   136    135   #
   137    136   do_catchsql_test 7.1 {
   138    137     CREATE VIRTUAL TABLE abc USING fts5(a, tokenize=porter, tokenize=ascii);
   139    138   } {1 {multiple tokenize=... directives}}
................................................................................
   199    198   do_execsql_test 9.3.4 {
   200    199     INSERT INTO abc(abc, rank) VALUES('crisismerge', 50000000);
   201    200   } {}
   202    201   
   203    202   do_catchsql_test 9.4.1 {
   204    203     INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
   205    204   } {1 {SQL logic error or missing database}}
          205  +
          206  +#-------------------------------------------------------------------------
          207  +# Too many prefix indexes. Maximum allowed is 31.
          208  +#
          209  +foreach {tn spec} {
          210  +  1 {prefix="1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32"}
          211  +  2 {prefix="1 2 3 4", prefix="5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32"}
          212  +} {
          213  +  set sql "CREATE VIRTUAL TABLE xyz USING fts5(x, $spec)"
          214  +  do_catchsql_test 10.$tn $sql {1 {too many prefix indexes (max 31)}}
          215  +}
   206    216   
   207    217   finish_test
   208    218   

Changes to ext/fts5/test/fts5prefix.test.

   293    293     5 x {xb*}
   294    294     6 x {xc*}
   295    295   } {
   296    296     set res [db eval "SELECT rowid FROM t5 WHERE gmatch($col, \$pattern)"]
   297    297     set query "$col : $pattern"
   298    298     do_execsql_test 6.$tn { SELECT rowid FROM t5($query) } $res
   299    299   }
          300  +
          301  +#-------------------------------------------------------------------------
          302  +# Check that the various ways of creating prefix indexes produce the
          303  +# same database on disk.
          304  +#
          305  +save_prng_state
          306  +foreach {tn create} {
          307  +  1 { CREATE VIRTUAL TABLE tt USING fts5(x, y, prefix="1,2,3") }
          308  +  2 { CREATE VIRTUAL TABLE tt USING fts5(x, y, prefix="1 2 3") }
          309  +  3 { CREATE VIRTUAL TABLE tt USING fts5(x, y, prefix=1, prefix=2, prefix=3) }
          310  +  4 { CREATE VIRTUAL TABLE tt USING fts5(x, y, prefix="1 2", prefix=3) }
          311  +} {
          312  +  execsql { DROP TABLE IF EXISTS tt }
          313  +  restore_prng_state
          314  +  execsql $create
          315  +  execsql {
          316  +    INSERT INTO tt VALUES('cc b ggg ccc aa eee hh', 'aa g b hh a e');
          317  +    INSERT INTO tt VALUES('cc bb cc gg j g cc', 'ii jjj ggg jjj cc cc');
          318  +    INSERT INTO tt VALUES('h eee cc h iii', 'aaa iii dd iii dd');
          319  +    INSERT INTO tt VALUES('jjj hh eee c e b gg', 'j bbb jj ddd jj');
          320  +    INSERT INTO tt VALUES('ii hhh aaa ff c hhh iii', 'j cc hh bb e');
          321  +    INSERT INTO tt VALUES('e fff hhh i aaa', 'g b aa gg c aa dd');
          322  +    INSERT INTO tt VALUES('i aaa ccc gg hhh aa h', 'j bbb bbb d ff');
          323  +    INSERT INTO tt VALUES('g f gg ff ff jjj d', 'jjj d j fff fff ee j');
          324  +    INSERT INTO tt VALUES('a cc e ccc jjj c', 'ccc iii d bb a eee g');
          325  +    INSERT INTO tt VALUES('jj hh hh bb bbb gg', 'j c jjj bb iii f');
          326  +    INSERT INTO tt VALUES('a ggg g cc ccc aa', 'jjj j j aaa c');
          327  +    INSERT INTO tt VALUES('ddd j dd b i', 'aaa bbb iii ggg ff ccc ddd');
          328  +    INSERT INTO tt VALUES('jj ii hh c ii h gg', 'hhh bbb ddd bbb hh g ggg');
          329  +    INSERT INTO tt VALUES('aa hhh ccc h ggg ccc', 'iii d jj a ff ii');
          330  +  }
          331  +
          332  +  #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM tt_data} {puts $r}
          333  +
          334  +  if {$tn==1} {
          335  +    set ::checksum [execsql {SELECT md5sum(id, block) FROM tt_data}]
          336  +  } else {
          337  +    do_execsql_test 7.$tn {
          338  +      SELECT md5sum(id, block) FROM tt_data
          339  +    } [list $::checksum]
          340  +  }
          341  +}
   300    342   
   301    343   finish_test
   302    344   
   303    345