/ Check-in [f1e942a1]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix an fts4 matchinf() problem triggered by deferred tokens that are part of phrases that are part of AND expressions.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: f1e942a1dda496a509741e9cc2a17e8b4dac63a3
User & Date: dan 2015-05-25 10:57:13
Context
2015-05-25
15:03
Fix an assert() in btree routine freeSpace() that may be false if the database is corrupt. check-in: 00a473c5 user: dan tags: trunk
10:57
Fix an fts4 matchinf() problem triggered by deferred tokens that are part of phrases that are part of AND expressions. check-in: f1e942a1 user: dan tags: trunk
10:29
Fix an obsolete comment - no changes to code. check-in: 4e4228c4 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

  5070   5070   **      is advanced to the next row that contains an instance of "A * C", 
  5071   5071   **      where "*" may match any single token. The position list in this case
  5072   5072   **      is populated as for "A * C" before returning.
  5073   5073   **
  5074   5074   **   2. NEAR is treated as AND. If the expression is "x NEAR y", it is 
  5075   5075   **      advanced to point to the next row that matches "x AND y".
  5076   5076   ** 
  5077         -** See fts3EvalTestDeferredAndNear() for details on testing if a row is
         5077  +** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is
  5078   5078   ** really a match, taking into account deferred tokens and NEAR operators.
  5079   5079   */
  5080   5080   static void fts3EvalNextRow(
  5081   5081     Fts3Cursor *pCsr,               /* FTS Cursor handle */
  5082   5082     Fts3Expr *pExpr,                /* Expr. to advance to next matching row */
  5083   5083     int *pRc                        /* IN/OUT: Error code */
  5084   5084   ){
................................................................................
  5290   5290       }
  5291   5291     }
  5292   5292   
  5293   5293     return res;
  5294   5294   }
  5295   5295   
  5296   5296   /*
  5297         -** This function is a helper function for fts3EvalTestDeferredAndNear().
         5297  +** This function is a helper function for sqlite3Fts3EvalTestDeferred().
  5298   5298   ** Assuming no error occurs or has occurred, It returns non-zero if the
  5299   5299   ** expression passed as the second argument matches the row that pCsr 
  5300   5300   ** currently points to, or zero if it does not.
  5301   5301   **
  5302   5302   ** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
  5303   5303   ** If an error occurs during execution of this function, *pRc is set to 
  5304   5304   ** the appropriate SQLite error code. In this case the returned value is 
................................................................................
  5411   5411   **
  5412   5412   **   2. After scanning the current FTS table row for the deferred tokens,
  5413   5413   **      it is determined that the row does *not* match the query.
  5414   5414   **
  5415   5415   ** Or, if no error occurs and it seems the current row does match the FTS
  5416   5416   ** query, return 0.
  5417   5417   */
  5418         -static int fts3EvalTestDeferredAndNear(Fts3Cursor *pCsr, int *pRc){
         5418  +int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){
  5419   5419     int rc = *pRc;
  5420   5420     int bMiss = 0;
  5421   5421     if( rc==SQLITE_OK ){
  5422   5422   
  5423   5423       /* If there are one or more deferred tokens, load the current row into
  5424   5424       ** memory and scan it to determine the position list for each deferred
  5425   5425       ** token. Then, see if this row is really a match, considering deferred
................................................................................
  5458   5458         }
  5459   5459         assert( sqlite3_data_count(pCsr->pStmt)==0 );
  5460   5460         fts3EvalNextRow(pCsr, pExpr, &rc);
  5461   5461         pCsr->isEof = pExpr->bEof;
  5462   5462         pCsr->isRequireSeek = 1;
  5463   5463         pCsr->isMatchinfoNeeded = 1;
  5464   5464         pCsr->iPrevId = pExpr->iDocid;
  5465         -    }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) );
         5465  +    }while( pCsr->isEof==0 && sqlite3Fts3EvalTestDeferred(pCsr, &rc) );
  5466   5466     }
  5467   5467   
  5468   5468     /* Check if the cursor is past the end of the docid range specified
  5469   5469     ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag.  */
  5470   5470     if( rc==SQLITE_OK && (
  5471   5471           (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid)
  5472   5472        || (pCsr->bDesc!=0 && pCsr->iPrevId<pCsr->iMinDocid)
................................................................................
  5619   5619           fts3EvalNextRow(pCsr, pRoot, &rc);
  5620   5620           pCsr->isEof = pRoot->bEof;
  5621   5621           pCsr->isRequireSeek = 1;
  5622   5622           pCsr->isMatchinfoNeeded = 1;
  5623   5623           pCsr->iPrevId = pRoot->iDocid;
  5624   5624         }while( pCsr->isEof==0 
  5625   5625              && pRoot->eType==FTSQUERY_NEAR 
  5626         -           && fts3EvalTestDeferredAndNear(pCsr, &rc) 
         5626  +           && sqlite3Fts3EvalTestDeferred(pCsr, &rc) 
  5627   5627         );
  5628   5628   
  5629   5629         if( rc==SQLITE_OK && pCsr->isEof==0 ){
  5630   5630           fts3EvalUpdateCounts(pRoot);
  5631   5631         }
  5632   5632       }
  5633   5633   
................................................................................
  5644   5644         **   do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK );
  5645   5645         */
  5646   5646         fts3EvalRestart(pCsr, pRoot, &rc);
  5647   5647         do {
  5648   5648           fts3EvalNextRow(pCsr, pRoot, &rc);
  5649   5649           assert( pRoot->bEof==0 );
  5650   5650         }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
  5651         -      fts3EvalTestDeferredAndNear(pCsr, &rc);
  5652   5651       }
  5653   5652     }
  5654   5653     return rc;
  5655   5654   }
  5656   5655   
  5657   5656   /*
  5658   5657   ** This function is used by the matchinfo() module to query a phrase 

Changes to ext/fts3/fts3Int.h.

   547    547   int sqlite3Fts3GetVarint32(const char *, int *);
   548    548   int sqlite3Fts3VarintLen(sqlite3_uint64);
   549    549   void sqlite3Fts3Dequote(char *);
   550    550   void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
   551    551   int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
   552    552   int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *);
   553    553   void sqlite3Fts3CreateStatTable(int*, Fts3Table*);
          554  +int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc);
   554    555   
   555    556   /* fts3_tokenizer.c */
   556    557   const char *sqlite3Fts3NextToken(const char *, int *);
   557    558   int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
   558    559   int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, 
   559    560       sqlite3_tokenizer **, char **
   560    561   );

Changes to ext/fts3/fts3_snippet.c.

  1284   1284           if( rc!=SQLITE_OK ) break;
  1285   1285           if( bGlobal ){
  1286   1286             if( pCsr->pDeferred ){
  1287   1287               rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0);
  1288   1288               if( rc!=SQLITE_OK ) break;
  1289   1289             }
  1290   1290             rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
         1291  +          sqlite3Fts3EvalTestDeferred(pCsr, &rc);
  1291   1292             if( rc!=SQLITE_OK ) break;
  1292   1293           }
  1293   1294           (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
  1294   1295           break;
  1295   1296         }
  1296   1297       }
  1297   1298   

Changes to test/fts3defer2.test.

   117    117     do_execsql_test 2.2.$tn.3 {
   118    118       SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g OR (g z)';
   119    119     } [list                                       \
   120    120       [list 1 2 2  1 2 2   1 54 54]               \
   121    121       [list 1 2 2  1 2 2   0 54 54]               \
   122    122     ]
   123    123     set sqlite_fts3_enable_parentheses 0
          124  +
          125  +  do_execsql_test 2.2.$tn.4 {
          126  +    SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'e "g z"';
          127  +  } [list                                       \
          128  +    [list 1 2 2  1 2 2]                         \
          129  +  ]
   124    130   }
   125    131   
   126    132   do_execsql_test 2.3.1 {
   127    133     CREATE VIRTUAL TABLE t3 USING fts4;
   128    134     INSERT INTO t3 VALUES('a b c d e f');
   129    135     INSERT INTO t3 VALUES('x b c d e f');
   130    136     INSERT INTO t3 VALUES('d e f a b c');