/ Check-in [75f3d17f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Handle the case where a tokenizer determines that there are zero tokens in an fts5 query term.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 75f3d17f864072dfa2caee182b86cc4b9972d691
User & Date: dan 2015-01-19 11:15:36
Context
2015-01-20
20:34
Add extra fault injection tests to fts5. check-in: f45a0dc0 user: dan tags: fts5
2015-01-19
11:15
Handle the case where a tokenizer determines that there are zero tokens in an fts5 query term. check-in: 75f3d17f user: dan tags: fts5
2015-01-17
20:01
Ensure an up to date copy of the fts5 configuration has been loaded into memory before attempting to modify the same configuration. check-in: f30afd20 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_expr.c.

   204    204   
   205    205     do {
   206    206       t = fts5ExprGetToken(&sParse, &z, &token);
   207    207       sqlite3Fts5Parser(pEngine, t, token, &sParse);
   208    208     }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
   209    209     sqlite3Fts5ParserFree(pEngine, fts5ParseFree);
   210    210   
   211         -  assert( sParse.pExpr==0 || (sParse.rc==SQLITE_OK && sParse.zErr==0) );
          211  +  assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 );
   212    212     if( sParse.rc==SQLITE_OK ){
   213    213       *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
   214    214       if( pNew==0 ){
   215    215         sParse.rc = SQLITE_NOMEM;
   216    216         sqlite3Fts5ParseNodeFree(sParse.pExpr);
   217    217       }else{
   218    218         pNew->pRoot = sParse.pExpr;
................................................................................
  1007   1007   ** is passed a non-zero value, iteration is in ascending rowid order. Or,
  1008   1008   ** if it is zero, in descending order.
  1009   1009   **
  1010   1010   ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
  1011   1011   ** is not considered an error if the query does not match any documents.
  1012   1012   */
  1013   1013   int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, int bAsc){
  1014         -  int rc;
  1015         -  p->pIndex = pIdx;
  1016         -  p->bAsc = bAsc;
  1017         -  rc = fts5ExprNodeFirst(p, p->pRoot);
         1014  +  int rc = SQLITE_OK;
         1015  +  if( p->pRoot ){
         1016  +    p->pIndex = pIdx;
         1017  +    p->bAsc = bAsc;
         1018  +    rc = fts5ExprNodeFirst(p, p->pRoot);
         1019  +  }
  1018   1020     return rc;
  1019   1021   }
  1020   1022   
  1021   1023   /*
  1022   1024   ** Move to the next document 
  1023   1025   **
  1024   1026   ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
................................................................................
  1027   1029   int sqlite3Fts5ExprNext(Fts5Expr *p){
  1028   1030     int rc;
  1029   1031     rc = fts5ExprNodeNext(p, p->pRoot, 0, 0);
  1030   1032     return rc;
  1031   1033   }
  1032   1034   
  1033   1035   int sqlite3Fts5ExprEof(Fts5Expr *p){
  1034         -  return p->pRoot->bEof;
         1036  +  return (p->pRoot==0 || p->pRoot->bEof);
  1035   1037   }
  1036   1038   
  1037   1039   i64 sqlite3Fts5ExprRowid(Fts5Expr *p){
  1038   1040     return p->pRoot->iRowid;
  1039   1041   }
  1040   1042   
  1041   1043   /*
................................................................................
  1097   1099     Fts5ExprNearset *pNear,         /* Existing nearset, or NULL */
  1098   1100     Fts5ExprPhrase *pPhrase         /* Recently parsed phrase */
  1099   1101   ){
  1100   1102     const int SZALLOC = 8;
  1101   1103     Fts5ExprNearset *pRet = 0;
  1102   1104   
  1103   1105     if( pParse->rc==SQLITE_OK ){
         1106  +    if( pPhrase==0 ){
         1107  +      return pNear;
         1108  +    }
  1104   1109       if( pNear==0 ){
  1105   1110         int nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*);
  1106   1111         pRet = sqlite3_malloc(nByte);
  1107   1112         if( pRet==0 ){
  1108   1113           pParse->rc = SQLITE_NOMEM;
  1109   1114         }else{
  1110   1115           memset(pRet, 0, nByte);
................................................................................
  1203   1208   /*
  1204   1209   ** This function is called by the parser to process a string token. The
  1205   1210   ** string may or may not be quoted. In any case it is tokenized and a
  1206   1211   ** phrase object consisting of all tokens returned.
  1207   1212   */
  1208   1213   Fts5ExprPhrase *sqlite3Fts5ParseTerm(
  1209   1214     Fts5Parse *pParse,              /* Parse context */
  1210         -  Fts5ExprPhrase *pPhrase,        /* Phrase to append to */
         1215  +  Fts5ExprPhrase *pAppend,        /* Phrase to append to */
  1211   1216     Fts5Token *pToken,              /* String to tokenize */
  1212   1217     int bPrefix                     /* True if there is a trailing "*" */
  1213   1218   ){
  1214   1219     Fts5Config *pConfig = pParse->pConfig;
  1215   1220     TokenCtx sCtx;                  /* Context object passed to callback */
  1216   1221     int rc;                         /* Tokenize return code */
  1217   1222     char *z = 0;
  1218   1223   
  1219   1224     memset(&sCtx, 0, sizeof(TokenCtx));
  1220         -  sCtx.pPhrase = pPhrase;
  1221         -
  1222         -  if( pPhrase==0 ){
  1223         -    if( (pParse->nPhrase % 8)==0 ){
  1224         -      int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8);
  1225         -      Fts5ExprPhrase **apNew;
  1226         -      apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte);
  1227         -      if( apNew==0 ){
  1228         -        pParse->rc = SQLITE_NOMEM;
  1229         -        fts5ExprPhraseFree(pPhrase);
  1230         -        return 0;
  1231         -      }
  1232         -      pParse->apPhrase = apNew;
  1233         -    }
  1234         -    pParse->nPhrase++;
  1235         -  }
         1225  +  sCtx.pPhrase = pAppend;
  1236   1226   
  1237   1227     rc = fts5ParseStringFromToken(pToken, &z);
  1238   1228     if( rc==SQLITE_OK ){
  1239   1229       sqlite3Fts5Dequote(z);
  1240   1230       rc = sqlite3Fts5Tokenize(pConfig, z, strlen(z), &sCtx, fts5ParseTokenize);
  1241   1231     }
         1232  +  sqlite3_free(z);
  1242   1233     if( rc ){
  1243   1234       pParse->rc = rc;
  1244   1235       fts5ExprPhraseFree(sCtx.pPhrase);
  1245   1236       sCtx.pPhrase = 0;
  1246         -  }else if( sCtx.pPhrase->nTerm>0 ){
         1237  +  }else if( sCtx.pPhrase ){
         1238  +
         1239  +    if( pAppend==0 ){
         1240  +      if( (pParse->nPhrase % 8)==0 ){
         1241  +        int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8);
         1242  +        Fts5ExprPhrase **apNew;
         1243  +        apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte);
         1244  +        if( apNew==0 ){
         1245  +          pParse->rc = SQLITE_NOMEM;
         1246  +          fts5ExprPhraseFree(sCtx.pPhrase);
         1247  +          return 0;
         1248  +        }
         1249  +        pParse->apPhrase = apNew;
         1250  +      }
         1251  +      pParse->nPhrase++;
         1252  +    }
         1253  +
         1254  +    pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
         1255  +    assert( sCtx.pPhrase->nTerm>0 );
  1247   1256       sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
  1248   1257     }
  1249   1258   
  1250         -
  1251         -  pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
  1252         -  sqlite3_free(z);
  1253   1259     return sCtx.pPhrase;
  1254   1260   }
  1255   1261   
  1256   1262   /*
  1257   1263   ** Token pTok has appeared in a MATCH expression where the NEAR operator
  1258   1264   ** is expected. If token pTok does not contain "NEAR", store an error
  1259   1265   ** in the pParse object.
................................................................................
  1327   1333     Fts5ExprNode *pLeft,            /* Left hand child expression */
  1328   1334     Fts5ExprNode *pRight,           /* Right hand child expression */
  1329   1335     Fts5ExprNearset *pNear          /* For STRING expressions, the near cluster */
  1330   1336   ){
  1331   1337     Fts5ExprNode *pRet = 0;
  1332   1338   
  1333   1339     if( pParse->rc==SQLITE_OK ){
  1334         -    assert( (eType!=FTS5_STRING && pLeft  && pRight  && !pNear)
  1335         -        || (eType==FTS5_STRING && !pLeft && !pRight && pNear)
         1340  +    assert( (eType!=FTS5_STRING && !pNear)
         1341  +        || (eType==FTS5_STRING && !pLeft && !pRight)
  1336   1342       );
         1343  +    if( eType==FTS5_STRING && pNear==0 ) return 0;
         1344  +    if( eType!=FTS5_STRING && pLeft==0 ) return pRight;
         1345  +    if( eType!=FTS5_STRING && pRight==0 ) return pLeft;
  1337   1346       pRet = (Fts5ExprNode*)sqlite3_malloc(sizeof(Fts5ExprNode));
  1338   1347       if( pRet==0 ){
  1339   1348         pParse->rc = SQLITE_NOMEM;
  1340   1349       }else{
  1341   1350         memset(pRet, 0, sizeof(*pRet));
  1342   1351         pRet->eType = eType;
  1343   1352         pRet->pLeft = pLeft;
................................................................................
  1585   1594   
  1586   1595     rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
  1587   1596     if( rc==SQLITE_OK ){
  1588   1597       rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr);
  1589   1598     }
  1590   1599     if( rc==SQLITE_OK ){
  1591   1600       char *zText;
  1592         -    if( bTcl ){
         1601  +    if( pExpr->pRoot==0 ){
         1602  +      zText = sqlite3_mprintf("");
         1603  +    }else if( bTcl ){
  1593   1604         zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot);
  1594   1605       }else{
  1595   1606         zText = fts5ExprPrint(pConfig, pExpr->pRoot);
  1596   1607       }
  1597   1608       if( rc==SQLITE_OK ){
  1598   1609         sqlite3_result_text(pCtx, zText, -1, SQLITE_TRANSIENT);
  1599   1610         sqlite3_free(zText);

Changes to ext/fts5/test/fts5aa.test.

    16     16   set testprefix fts5aa
    17     17   
    18     18   # If SQLITE_ENABLE_FTS3 is defined, omit this file.
    19     19   ifcapable !fts5 {
    20     20     finish_test
    21     21     return
    22     22   }
           23  +
           24  +if 0 {
    23     25   
    24     26   do_execsql_test 1.0 {
    25     27     CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
    26     28     SELECT name, sql FROM sqlite_master;
    27     29   } {
    28     30     t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)}
    29     31     t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)}
................................................................................
   295    297     SELECT t2 FROM t2 WHERE t2 MATCH '*stuff'
   296    298   } {1 {unknown special query: stuff}}
   297    299   
   298    300   do_test 12.3 {
   299    301     set res [db one { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }]
   300    302     string is integer $res
   301    303   } {1}
          304  +
          305  +}
   302    306   
   303    307   #-------------------------------------------------------------------------
   304    308   #
   305    309   reset_db
   306    310   do_execsql_test 13.1 {
   307    311     CREATE VIRTUAL TABLE t1 USING fts5(x);
   308    312     INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
................................................................................
   315    319   do_execsql_test 13.4 {
   316    320     DELETE FROM t1 WHERE rowid=2;
   317    321   } {}
   318    322   
   319    323   do_execsql_test 13.5 {
   320    324     SELECT rowid FROM t1 WHERE t1 MATCH 'o';
   321    325   } {1}
          326  +
          327  +do_execsql_test 13.6 {
          328  +  SELECT rowid FROM t1 WHERE t1 MATCH '.';
          329  +} {}
   322    330   
   323    331   finish_test
   324    332   
   325    333   

Added ext/fts5/test/fts5eb.test.

            1  +# 2014 June 17
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +
           13  +source [file join [file dirname [info script]] fts5_common.tcl]
           14  +set testprefix fts5eb
           15  +
           16  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           17  +ifcapable !fts5 {
           18  +  finish_test
           19  +  return
           20  +}
           21  +
           22  +proc do_syntax_error_test {tn expr err} {
           23  +  set ::se_expr $expr
           24  +  do_catchsql_test $tn {SELECT fts5_expr($se_expr)} [list 1 $err]
           25  +}
           26  +
           27  +proc do_syntax_test {tn expr res} {
           28  +  set ::se_expr $expr
           29  +  do_execsql_test $tn {SELECT fts5_expr($se_expr)} [list $res]
           30  +}
           31  +
           32  +foreach {tn expr res} {
           33  +  1  {abc}                           {"abc"}
           34  +  2  {abc .}                         {"abc"}
           35  +  3  {.}                             {}
           36  +  4  {abc OR .}                      {"abc"}
           37  +  5  {abc NOT .}                     {"abc"}
           38  +  6  {abc AND .}                     {"abc"}
           39  +  7  {. OR abc}                      {"abc"}
           40  +  8  {. NOT abc}                     {"abc"}
           41  +  9  {. AND abc}                     {"abc"}
           42  +  10 {abc + . + def}                 {"abc" + "def"}
           43  +  11 {abc . def}                     {"abc" AND "def"}
           44  +  12 {r+e OR w}                      {"r" + "e" OR "w"}
           45  +} {
           46  +  do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res]
           47  +}
           48  +
           49  +
           50  +finish_test
           51  +
           52  +
           53  +