SQLite

Check-in [f1e942a1dd]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix an fts4 matchinf() problem triggered by deferred tokens that are part of phrases that are part of AND expressions.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: f1e942a1dda496a509741e9cc2a17e8b4dac63a3
User & Date: dan 2015-05-25 10:57:13.025
Context
2015-05-25
15:03
Fix an assert() in btree routine freeSpace() that may be false if the database is corrupt. (check-in: 00a473c561 user: dan tags: trunk)
10:57
Fix an fts4 matchinf() problem triggered by deferred tokens that are part of phrases that are part of AND expressions. (check-in: f1e942a1dd user: dan tags: trunk)
10:29
Fix an obsolete comment - no changes to code. (check-in: 4e4228c47f user: drh tags: trunk)
Changes
Side-by-Side Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3.c.
5070
5071
5072
5073
5074
5075
5076
5077

5078
5079
5080
5081
5082
5083
5084
5070
5071
5072
5073
5074
5075
5076

5077
5078
5079
5080
5081
5082
5083
5084







-
+







**      is advanced to the next row that contains an instance of "A * C", 
**      where "*" may match any single token. The position list in this case
**      is populated as for "A * C" before returning.
**
**   2. NEAR is treated as AND. If the expression is "x NEAR y", it is 
**      advanced to point to the next row that matches "x AND y".
** 
** See fts3EvalTestDeferredAndNear() for details on testing if a row is
** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is
** really a match, taking into account deferred tokens and NEAR operators.
*/
static void fts3EvalNextRow(
  Fts3Cursor *pCsr,               /* FTS Cursor handle */
  Fts3Expr *pExpr,                /* Expr. to advance to next matching row */
  int *pRc                        /* IN/OUT: Error code */
){
5290
5291
5292
5293
5294
5295
5296
5297

5298
5299
5300
5301
5302
5303
5304
5290
5291
5292
5293
5294
5295
5296

5297
5298
5299
5300
5301
5302
5303
5304







-
+







    }
  }

  return res;
}

/*
** This function is a helper function for fts3EvalTestDeferredAndNear().
** This function is a helper function for sqlite3Fts3EvalTestDeferred().
** Assuming no error occurs or has occurred, It returns non-zero if the
** expression passed as the second argument matches the row that pCsr 
** currently points to, or zero if it does not.
**
** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
** If an error occurs during execution of this function, *pRc is set to 
** the appropriate SQLite error code. In this case the returned value is 
5411
5412
5413
5414
5415
5416
5417
5418

5419
5420
5421
5422
5423
5424
5425
5411
5412
5413
5414
5415
5416
5417

5418
5419
5420
5421
5422
5423
5424
5425







-
+







**
**   2. After scanning the current FTS table row for the deferred tokens,
**      it is determined that the row does *not* match the query.
**
** Or, if no error occurs and it seems the current row does match the FTS
** query, return 0.
*/
static int fts3EvalTestDeferredAndNear(Fts3Cursor *pCsr, int *pRc){
int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){
  int rc = *pRc;
  int bMiss = 0;
  if( rc==SQLITE_OK ){

    /* If there are one or more deferred tokens, load the current row into
    ** memory and scan it to determine the position list for each deferred
    ** token. Then, see if this row is really a match, considering deferred
5458
5459
5460
5461
5462
5463
5464
5465

5466
5467
5468
5469
5470
5471
5472
5458
5459
5460
5461
5462
5463
5464

5465
5466
5467
5468
5469
5470
5471
5472







-
+







      }
      assert( sqlite3_data_count(pCsr->pStmt)==0 );
      fts3EvalNextRow(pCsr, pExpr, &rc);
      pCsr->isEof = pExpr->bEof;
      pCsr->isRequireSeek = 1;
      pCsr->isMatchinfoNeeded = 1;
      pCsr->iPrevId = pExpr->iDocid;
    }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) );
    }while( pCsr->isEof==0 && sqlite3Fts3EvalTestDeferred(pCsr, &rc) );
  }

  /* Check if the cursor is past the end of the docid range specified
  ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag.  */
  if( rc==SQLITE_OK && (
        (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid)
     || (pCsr->bDesc!=0 && pCsr->iPrevId<pCsr->iMinDocid)
5619
5620
5621
5622
5623
5624
5625
5626

5627
5628
5629
5630
5631
5632
5633
5619
5620
5621
5622
5623
5624
5625

5626
5627
5628
5629
5630
5631
5632
5633







-
+







        fts3EvalNextRow(pCsr, pRoot, &rc);
        pCsr->isEof = pRoot->bEof;
        pCsr->isRequireSeek = 1;
        pCsr->isMatchinfoNeeded = 1;
        pCsr->iPrevId = pRoot->iDocid;
      }while( pCsr->isEof==0 
           && pRoot->eType==FTSQUERY_NEAR 
           && fts3EvalTestDeferredAndNear(pCsr, &rc) 
           && sqlite3Fts3EvalTestDeferred(pCsr, &rc) 
      );

      if( rc==SQLITE_OK && pCsr->isEof==0 ){
        fts3EvalUpdateCounts(pRoot);
      }
    }

5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5644
5645
5646
5647
5648
5649
5650

5651
5652
5653
5654
5655
5656
5657







-







      **   do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK );
      */
      fts3EvalRestart(pCsr, pRoot, &rc);
      do {
        fts3EvalNextRow(pCsr, pRoot, &rc);
        assert( pRoot->bEof==0 );
      }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
      fts3EvalTestDeferredAndNear(pCsr, &rc);
    }
  }
  return rc;
}

/*
** This function is used by the matchinfo() module to query a phrase 
Changes to ext/fts3/fts3Int.h.
547
548
549
550
551
552
553

554
555
556
557
558
559
560
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561







+







int sqlite3Fts3GetVarint32(const char *, int *);
int sqlite3Fts3VarintLen(sqlite3_uint64);
void sqlite3Fts3Dequote(char *);
void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *);
void sqlite3Fts3CreateStatTable(int*, Fts3Table*);
int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc);

/* fts3_tokenizer.c */
const char *sqlite3Fts3NextToken(const char *, int *);
int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, 
    sqlite3_tokenizer **, char **
);
Changes to ext/fts3/fts3_snippet.c.
1284
1285
1286
1287
1288
1289
1290

1291
1292
1293
1294
1295
1296
1297
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298







+







        if( rc!=SQLITE_OK ) break;
        if( bGlobal ){
          if( pCsr->pDeferred ){
            rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0);
            if( rc!=SQLITE_OK ) break;
          }
          rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
          sqlite3Fts3EvalTestDeferred(pCsr, &rc);
          if( rc!=SQLITE_OK ) break;
        }
        (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
        break;
      }
    }

Changes to test/fts3defer2.test.
117
118
119
120
121
122
123






124
125
126
127
128
129
130
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136







+
+
+
+
+
+







  do_execsql_test 2.2.$tn.3 {
    SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g OR (g z)';
  } [list                                       \
    [list 1 2 2  1 2 2   1 54 54]               \
    [list 1 2 2  1 2 2   0 54 54]               \
  ]
  set sqlite_fts3_enable_parentheses 0

  do_execsql_test 2.2.$tn.4 {
    SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'e "g z"';
  } [list                                       \
    [list 1 2 2  1 2 2]                         \
  ]
}

do_execsql_test 2.3.1 {
  CREATE VIRTUAL TABLE t3 USING fts4;
  INSERT INTO t3 VALUES('a b c d e f');
  INSERT INTO t3 VALUES('x b c d e f');
  INSERT INTO t3 VALUES('d e f a b c');