Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
| Comment: | Fix a problem with NEAR queries executed inside a transaction that writes the FTS table. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA1: |
051c756c367837908f6691c0a36108e0 |
| User & Date: | dan 2011-06-16 16:06:05 |
|
2011-06-17
| ||
| 07:07 | Add Microsoft nmake compatible makefile; update a few test cases for Windows. (check-in: a7590af6 user: shaneh tags: trunk) | |
|
2011-06-16
| ||
| 16:06 | Fix a problem with NEAR queries executed inside a transaction that writes the FTS table. (check-in: 051c756c user: dan tags: trunk) | |
| 00:54 | Changes to #ifdefs so that the build goes correctly if the only FTS macro defined is SQLITE_ENABLE_FTS4. (check-in: a0b43a32 user: drh tags: trunk) | |
Changes to ext/fts3/fts3.c.
3489 3489 ** edited in place by fts3EvalNearTrim2(), then pIter may not actually 3490 3490 ** point to the start of the next docid value. The following line deals 3491 3491 ** with this case by advancing pIter past the zero-padding added by 3492 3492 ** fts3EvalNearTrim2(). */ 3493 3493 while( pIter<pEnd && *pIter==0 ) pIter++; 3494 3494 3495 3495 pDL->pNextDocid = pIter; 3496 - assert( *pIter || pIter>=&pDL->aAll[pDL->nAll] ); 3496 + assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); 3497 3497 *pbEof = 0; 3498 3498 } 3499 3499 } 3500 3500 3501 3501 return rc; 3502 3502 } 3503 3503 ................................................................................ 4127 4127 ){ 4128 4128 if( pExpr && *pRc==SQLITE_OK ){ 4129 4129 Fts3Phrase *pPhrase = pExpr->pPhrase; 4130 4130 4131 4131 if( pPhrase ){ 4132 4132 fts3EvalZeroPoslist(pPhrase); 4133 4133 if( pPhrase->bIncr ){ 4134 - sqlite3Fts3EvalPhraseCleanup(pPhrase); 4135 - memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); 4136 - *pRc = sqlite3Fts3EvalStart(pCsr, pExpr, 0); 4137 - }else{ 4134 + assert( pPhrase->nToken==1 ); 4135 + assert( pPhrase->aToken[0].pSegcsr ); 4136 + sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr); 4137 + *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); 4138 + } 4139 + 4138 4140 pPhrase->doclist.pNextDocid = 0; 4139 4141 pPhrase->doclist.iDocid = 0; 4140 4142 } 4141 - } 4142 4143 4143 4144 pExpr->iDocid = 0; 4144 4145 pExpr->bEof = 0; 4145 4146 pExpr->bStart = 0; 4146 4147 4147 4148 fts3EvalRestart(pCsr, pExpr->pLeft, pRc); 4148 4149 fts3EvalRestart(pCsr, pExpr->pRight, pRc);
Changes to ext/fts3/fts3Int.h.
435 435 int nSegment; /* Size of apSegment array */ 436 436 int nAdvance; /* How many seg-readers to advance */ 437 437 Fts3SegFilter *pFilter; /* Pointer to filter object */ 438 438 char *aBuffer; /* Buffer to merge doclists in */ 439 439 int nBuffer; /* Allocated size of aBuffer[] in bytes */ 440 440 441 441 int iColFilter; /* If >=0, filter for this column */ 442 + int bRestart; 442 443 443 444 /* Used by fts3.c only. */ 444 445 int nCost; /* Cost of running iterator */ 445 446 int bLookup; /* True if a lookup of a single entry. */ 446 447 447 448 /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ 448 449 char *zTerm; /* Pointer to term buffer */
Changes to ext/fts3/fts3_write.c.
1243 1243 ); 1244 1244 if( bEof ){ 1245 1245 pReader->pOffsetList = 0; 1246 1246 }else{ 1247 1247 pReader->pOffsetList = p; 1248 1248 } 1249 1249 }else{ 1250 + char *pEnd = &pReader->aDoclist[pReader->nDoclist]; 1250 1251 1251 1252 /* Pointer p currently points at the first byte of an offset list. The 1252 1253 ** following block advances it to point one byte past the end of 1253 1254 ** the same offset list. */ 1254 1255 while( 1 ){ 1255 1256 1256 1257 /* The following line of code (and the "p++" below the while() loop) is ................................................................................ 1271 1272 /* If required, populate the output variables with a pointer to and the 1272 1273 ** size of the previous offset-list. 1273 1274 */ 1274 1275 if( ppOffsetList ){ 1275 1276 *ppOffsetList = pReader->pOffsetList; 1276 1277 *pnOffsetList = (int)(p - pReader->pOffsetList - 1); 1277 1278 } 1279 + 1280 + while( p<pEnd && *p==0 ) p++; 1278 1281 1279 1282 /* If there are no more entries in the doclist, set pOffsetList to 1280 1283 ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and 1281 1284 ** Fts3SegReader.pOffsetList to point to the next offset list before 1282 1285 ** returning. 1283 1286 */ 1284 - if( p>=&pReader->aDoclist[pReader->nDoclist] ){ 1287 + if( p>=pEnd ){ 1285 1288 pReader->pOffsetList = 0; 1286 1289 }else{ 1287 1290 rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); 1288 1291 if( rc==SQLITE_OK ){ 1289 1292 sqlite3_int64 iDelta; 1290 1293 pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); 1291 1294 if( pTab->bDescIdx ){ ................................................................................ 2261 2264 p += sqlite3Fts3GetVarint32(p, &iCurrent); 2262 2265 } 2263 2266 2264 2267 *ppList = pList; 2265 2268 *pnList = nList; 2266 2269 } 2267 2270 2268 -int sqlite3Fts3MsrIncrStart( 2269 - Fts3Table *p, /* Virtual table handle */ 2270 - Fts3MultiSegReader *pCsr, /* Cursor object */ 2271 - int iCol, /* Column to match on. */ 2272 - const char *zTerm, /* Term to iterate through a doclist for */ 2273 - int nTerm /* Number of bytes in zTerm */ 2274 -){ 2275 - int i; 2276 - int nSegment = pCsr->nSegment; 2277 - int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( 2278 - p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp 2279 - ); 2280 - 2281 - assert( pCsr->pFilter==0 ); 2282 - assert( zTerm && nTerm>0 ); 2283 - 2284 - /* Advance each segment iterator until it points to the term zTerm/nTerm. */ 2285 - for(i=0; i<nSegment; i++){ 2286 - Fts3SegReader *pSeg = pCsr->apSegment[i]; 2287 - do { 2288 - int rc = fts3SegReaderNext(p, pSeg, 1); 2289 - if( rc!=SQLITE_OK ) return rc; 2290 - }while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); 2291 - } 2292 - fts3SegReaderSort(pCsr->apSegment, nSegment, nSegment, fts3SegReaderCmp); 2293 - 2294 - /* Determine how many of the segments actually point to zTerm/nTerm. */ 2295 - for(i=0; i<nSegment; i++){ 2296 - Fts3SegReader *pSeg = pCsr->apSegment[i]; 2297 - if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ 2298 - break; 2299 - } 2300 - } 2301 - pCsr->nAdvance = i; 2302 - 2303 - /* Advance each of the segments to point to the first docid. */ 2304 - for(i=0; i<pCsr->nAdvance; i++){ 2305 - int rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); 2306 - if( rc!=SQLITE_OK ) return rc; 2307 - } 2308 - fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); 2309 - 2310 - assert( iCol<0 || iCol<p->nColumn ); 2311 - pCsr->iColFilter = iCol; 2312 - 2271 +/* 2272 +** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any 2273 +** existing data). Grow the buffer if required. 2274 +** 2275 +** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered 2276 +** trying to resize the buffer, return SQLITE_NOMEM. 2277 +*/ 2278 +static int fts3MsrBufferData( 2279 + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ 2280 + char *pList, 2281 + int nList 2282 +){ 2283 + if( nList>pMsr->nBuffer ){ 2284 + char *pNew; 2285 + pMsr->nBuffer = nList*2; 2286 + pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); 2287 + if( !pNew ) return SQLITE_NOMEM; 2288 + pMsr->aBuffer = pNew; 2289 + } 2290 + 2291 + memcpy(pMsr->aBuffer, pList, nList); 2313 2292 return SQLITE_OK; 2314 2293 } 2315 2294 2316 2295 int sqlite3Fts3MsrIncrNext( 2317 2296 Fts3Table *p, /* Virtual table handle */ 2318 2297 Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ 2319 2298 sqlite3_int64 *piDocid, /* OUT: Docid value */ ................................................................................ 2359 2338 fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); 2360 2339 2361 2340 if( pMsr->iColFilter>=0 ){ 2362 2341 fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); 2363 2342 } 2364 2343 2365 2344 if( nList>0 ){ 2366 - *piDocid = iDocid; 2345 + if( fts3SegReaderIsPending(apSegment[0]) ){ 2346 + rc = fts3MsrBufferData(pMsr, pList, nList+1); 2347 + if( rc!=SQLITE_OK ) return rc; 2348 + *paPoslist = pMsr->aBuffer; 2349 + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); 2350 + }else{ 2367 2351 *paPoslist = pList; 2352 + } 2353 + *piDocid = iDocid; 2368 2354 *pnPoslist = nList; 2369 2355 break; 2370 2356 } 2371 2357 } 2358 + } 2372 2359 2360 + return SQLITE_OK; 2373 2361 } 2374 2362 2363 +static int fts3SegReaderStart( 2364 + Fts3Table *p, /* Virtual table handle */ 2365 + Fts3MultiSegReader *pCsr, /* Cursor object */ 2366 + const char *zTerm, /* Term searched for (or NULL) */ 2367 + int nTerm /* Length of zTerm in bytes */ 2368 +){ 2369 + int i; 2370 + int nSeg = pCsr->nSegment; 2371 + 2372 + /* If the Fts3SegFilter defines a specific term (or term prefix) to search 2373 + ** for, then advance each segment iterator until it points to a term of 2374 + ** equal or greater value than the specified term. This prevents many 2375 + ** unnecessary merge/sort operations for the case where single segment 2376 + ** b-tree leaf nodes contain more than one term. 2377 + */ 2378 + for(i=0; pCsr->bRestart==0 && i<pCsr->nSegment; i++){ 2379 + Fts3SegReader *pSeg = pCsr->apSegment[i]; 2380 + do { 2381 + int rc = fts3SegReaderNext(p, pSeg, 0); 2382 + if( rc!=SQLITE_OK ) return rc; 2383 + }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); 2384 + } 2385 + fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); 2386 + 2375 2387 return SQLITE_OK; 2376 2388 } 2377 2389 2378 2390 int sqlite3Fts3SegReaderStart( 2379 2391 Fts3Table *p, /* Virtual table handle */ 2380 2392 Fts3MultiSegReader *pCsr, /* Cursor object */ 2381 2393 Fts3SegFilter *pFilter /* Restrictions on range of iteration */ 2382 2394 ){ 2383 - int i; 2384 - 2385 - /* Initialize the cursor object */ 2386 2395 pCsr->pFilter = pFilter; 2396 + return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); 2397 +} 2387 2398 2388 - /* If the Fts3SegFilter defines a specific term (or term prefix) to search 2389 - ** for, then advance each segment iterator until it points to a term of 2390 - ** equal or greater value than the specified term. This prevents many 2391 - ** unnecessary merge/sort operations for the case where single segment 2392 - ** b-tree leaf nodes contain more than one term. 2393 - */ 2394 - for(i=0; i<pCsr->nSegment; i++){ 2395 - int nTerm = pFilter->nTerm; 2396 - const char *zTerm = pFilter->zTerm; 2399 +int sqlite3Fts3MsrIncrStart( 2400 + Fts3Table *p, /* Virtual table handle */ 2401 + Fts3MultiSegReader *pCsr, /* Cursor object */ 2402 + int iCol, /* Column to match on. */ 2403 + const char *zTerm, /* Term to iterate through a doclist for */ 2404 + int nTerm /* Number of bytes in zTerm */ 2405 +){ 2406 + int i; 2407 + int rc; 2408 + int nSegment = pCsr->nSegment; 2409 + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( 2410 + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp 2411 + ); 2412 + 2413 + assert( pCsr->pFilter==0 ); 2414 + assert( zTerm && nTerm>0 ); 2415 + 2416 + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ 2417 + rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); 2418 + if( rc!=SQLITE_OK ) return rc; 2419 + 2420 + /* Determine how many of the segments actually point to zTerm/nTerm. */ 2421 + for(i=0; i<nSegment; i++){ 2397 2422 Fts3SegReader *pSeg = pCsr->apSegment[i]; 2398 - do { 2399 - int rc = fts3SegReaderNext(p, pSeg, 0); 2423 + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ 2424 + break; 2425 + } 2426 + } 2427 + pCsr->nAdvance = i; 2428 + 2429 + /* Advance each of the segments to point to the first docid. */ 2430 + for(i=0; i<pCsr->nAdvance; i++){ 2431 + rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); 2400 2432 if( rc!=SQLITE_OK ) return rc; 2401 - }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); 2402 2433 } 2403 - fts3SegReaderSort( 2404 - pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp); 2434 + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); 2435 + 2436 + assert( iCol<0 || iCol<p->nColumn ); 2437 + pCsr->iColFilter = iCol; 2438 + 2439 + return SQLITE_OK; 2440 +} 2441 + 2442 +/* 2443 +** This function is called on a MultiSegReader that has been started using 2444 +** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also 2445 +** have been made. Calling this function puts the MultiSegReader in such 2446 +** a state that if the next two calls are: 2447 +** 2448 +** sqlite3Fts3SegReaderStart() 2449 +** sqlite3Fts3SegReaderStep() 2450 +** 2451 +** then the entire doclist for the term is available in 2452 +** MultiSegReader.aDoclist/nDoclist. 2453 +*/ 2454 +int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ 2455 + int i; /* Used to iterate through segment-readers */ 2456 + 2457 + assert( pCsr->zTerm==0 ); 2458 + assert( pCsr->nTerm==0 ); 2459 + assert( pCsr->aDoclist==0 ); 2460 + assert( pCsr->nDoclist==0 ); 2461 + 2462 + pCsr->nAdvance = 0; 2463 + pCsr->bRestart = 1; 2464 + for(i=0; i<pCsr->nSegment; i++){ 2465 + pCsr->apSegment[i]->pOffsetList = 0; 2466 + pCsr->apSegment[i]->nOffsetList = 0; 2467 + pCsr->apSegment[i]->iDocid = 0; 2468 + } 2405 2469 2406 2470 return SQLITE_OK; 2407 2471 } 2472 + 2408 2473 2409 2474 int sqlite3Fts3SegReaderStep( 2410 2475 Fts3Table *p, /* Virtual table handle */ 2411 2476 Fts3MultiSegReader *pCsr /* Cursor object */ 2412 2477 ){ 2413 2478 int rc = SQLITE_OK; 2414 2479 ................................................................................ 2474 2539 } 2475 2540 2476 2541 assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); 2477 2542 if( nMerge==1 2478 2543 && !isIgnoreEmpty 2479 2544 && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) 2480 2545 ){ 2481 - pCsr->aDoclist = apSegment[0]->aDoclist; 2482 2546 pCsr->nDoclist = apSegment[0]->nDoclist; 2483 - rc = SQLITE_ROW; 2547 + if( fts3SegReaderIsPending(apSegment[0]) ){ 2548 + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); 2549 + pCsr->aDoclist = pCsr->aBuffer; 2550 + }else{ 2551 + pCsr->aDoclist = apSegment[0]->aDoclist; 2552 + } 2553 + if( rc==SQLITE_OK ) rc = SQLITE_ROW; 2484 2554 }else{ 2485 2555 int nDoclist = 0; /* Size of doclist */ 2486 2556 sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ 2487 2557 2488 2558 /* The current term of the first nMerge entries in the array 2489 2559 ** of Fts3SegReader objects is the same. The doclists must be merged 2490 2560 ** and a single term returned with the merged doclist.
Changes to test/fts3auto.test.
517 517 expr {[fts3_zero_long_segments t1 $limit]>0} 518 518 } {1} 519 519 520 520 do_fts3query_test 4.$tn.3.1 -deferred five t1 {one AND five} 521 521 do_fts3query_test 4.$tn.3.2 -deferred five t1 {one NEAR five} 522 522 do_fts3query_test 4.$tn.3.3 -deferred five t1 {one NEAR/1 five} 523 523 do_fts3query_test 4.$tn.3.4 -deferred five t1 {one NEAR/2 five} 524 + 524 525 do_fts3query_test 4.$tn.3.5 -deferred five t1 {one NEAR/3 five} 525 526 526 527 do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*} 527 528 do_fts3query_test 4.$tn.4.2 -deferred fi* t1 {on* NEAR fi*} 528 529 do_fts3query_test 4.$tn.4.3 -deferred fi* t1 {on* NEAR/1 fi*} 529 530 do_fts3query_test 4.$tn.4.4 -deferred fi* t1 {on* NEAR/2 fi*} 530 531 do_fts3query_test 4.$tn.4.5 -deferred fi* t1 {on* NEAR/3 fi*} 531 532 } 532 533 533 534 #-------------------------------------------------------------------------- 534 535 # The following test cases - fts3auto-5.* - focus on using prefix indexes. 535 536 # 536 537 set chunkconfig [fts3_configure_incr_load 1 1] 537 -foreach {tn create} { 538 - 1 "fts4(a, b)" 539 - 2 "fts4(a, b, order=DESC, prefix=1)" 540 - 3 "fts4(a, b, order=ASC, prefix=1,3)" 541 - 4 "fts4(a, b, order=DESC, prefix=2,4)" 538 +foreach {tn create pending} { 539 + 2 "fts4(a, b, order=ASC, prefix=1)" 1 540 + 541 + 1 "fts4(a, b)" 1 542 + 3 "fts4(a, b, order=ASC, prefix=1,3)" 0 543 + 4 "fts4(a, b, order=DESC, prefix=2,4)" 0 544 + 5 "fts4(a, b, order=DESC, prefix=1)" 0 545 + 6 "fts4(a, b, order=ASC, prefix=1,3)" 0 542 546 } { 543 547 544 548 execsql [subst { 545 - DROP TABLE t1; 549 + DROP TABLE IF EXISTS t1; 546 550 CREATE VIRTUAL TABLE t1 USING $create; 547 551 }] 552 + 553 + if {$pending} {execsql BEGIN} 548 554 549 555 foreach {a b} { 550 556 "the song of songs which is solomons" 551 557 "let him kiss me with the kisses of his mouth for thy love is better than wine" 552 558 "because of the savour of thy good ointments thy name is as ointment poured forth therefore do the virgins love thee" 553 559 "draw me we will run after thee the king hath brought me into his chambers we will be glad and rejoice in thee we will remember thy love more than wine the upright love thee" 554 560 "i am black but comely o ye daughters of jerusalem as the tents of kedar as the curtains of solomon" ................................................................................ 563 569 "my beloved is unto me as a cluster of camphire in the vineyards of en gedi" 564 570 "behold thou art fair my love behold thou art fair thou hast doves eyes" 565 571 "behold thou art fair my beloved yea pleasant also our bed is green" 566 572 "the beams of our house are cedar and our rafters of fir" 567 573 } { 568 574 execsql {INSERT INTO t1(a, b) VALUES($a, $b)} 569 575 } 576 + 570 577 571 578 do_fts3query_test 5.$tn.1.1 t1 {s*} 572 579 do_fts3query_test 5.$tn.1.2 t1 {so*} 573 580 do_fts3query_test 5.$tn.1.3 t1 {"s* o*"} 574 581 do_fts3query_test 5.$tn.1.4 t1 {b* NEAR/3 a*} 575 - do_fts3query_test 5.$tn.1.5 t1 {th* NEAR/5 a* NEAR/5 w*} 576 - do_fts3query_test 5.$tn.1.6 t1 {"b* th* art* fair*"} 582 + do_fts3query_test 5.$tn.1.5 t1 {a*} 583 + do_fts3query_test 5.$tn.1.6 t1 {th* NEAR/5 a* NEAR/5 w*} 584 + do_fts3query_test 5.$tn.1.7 t1 {"b* th* art* fair*"} 585 + 586 + if {$pending} {execsql COMMIT} 577 587 } 578 588 eval fts3_configure_incr_load $chunkconfig 579 589 580 590 set sqlite_fts3_enable_parentheses $sfep 581 591 finish_test 582 592