Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Improve the performance of fts3/4 queries that use the OR operator and at least one auxiliary fts function. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
245e8730451fbdc1c729beff7295c452 |
User & Date: | dan 2015-01-27 18:43:02.971 |
References
2015-02-12
| ||
17:15 | Improve the performance of fts3/4 queries that use the OR operator and at least one auxiliary fts function. Cherrypick of [245e8730451f]. (check-in: b20824628f user: dan tags: branch-3.8.8) | |
Context
2015-02-12
| ||
17:15 | Improve the performance of fts3/4 queries that use the OR operator and at least one auxiliary fts function. Cherrypick of [245e8730451f]. (check-in: b20824628f user: dan tags: branch-3.8.8) | |
2015-01-27
| ||
19:01 | Fix a bug in the fts3 snippet() function causing it to omit leading separator characters from snippets that begin with the first token in a column. (check-in: adc9283dd9 user: dan tags: trunk) | |
18:43 | Improve the performance of fts3/4 queries that use the OR operator and at least one auxiliary fts function. (check-in: 245e873045 user: dan tags: trunk) | |
13:17 | Fix a (almost always harmless) read past the end of a memory allocation that comes about because the Expr.pTab field is checked on an EXPR_REDUCEDSIZE Expr object before checking the Expr.op field to know that the Expr.pTab field is meaningless. (check-in: e098de6910 user: drh tags: trunk) | |
Changes
Changes to ext/fts3/fts3.c.
︙ | ︙ | |||
5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 | fts3EvalNextRow(pCsr, pLeft, pRc); }else{ fts3EvalNextRow(pCsr, pRight, pRc); } } pExpr->iDocid = pLeft->iDocid; pExpr->bEof = (pLeft->bEof || pRight->bEof); } break; } case FTSQUERY_OR: { Fts3Expr *pLeft = pExpr->pLeft; Fts3Expr *pRight = pExpr->pRight; | > > > > > > > > > > > > > > > > | 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 | fts3EvalNextRow(pCsr, pLeft, pRc); }else{ fts3EvalNextRow(pCsr, pRight, pRc); } } pExpr->iDocid = pLeft->iDocid; pExpr->bEof = (pLeft->bEof || pRight->bEof); if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){ if( pRight->pPhrase && pRight->pPhrase->doclist.aAll ){ Fts3Doclist *pDl = &pRight->pPhrase->doclist; while( *pRc==SQLITE_OK && pRight->bEof==0 ){ memset(pDl->pList, 0, pDl->nList); fts3EvalNextRow(pCsr, pRight, pRc); } } if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){ Fts3Doclist *pDl = &pLeft->pPhrase->doclist; while( *pRc==SQLITE_OK && pLeft->bEof==0 ){ memset(pDl->pList, 0, pDl->nList); fts3EvalNextRow(pCsr, pLeft, pRc); } } } } break; } case FTSQUERY_OR: { Fts3Expr *pLeft = pExpr->pLeft; Fts3Expr *pRight = pExpr->pRight; |
︙ | ︙ | |||
5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 | sqlite3Fts3MsrIncrRestart(pToken->pSegcsr); } } *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); } pPhrase->doclist.pNextDocid = 0; pPhrase->doclist.iDocid = 0; } pExpr->iDocid = 0; pExpr->bEof = 0; pExpr->bStart = 0; fts3EvalRestart(pCsr, pExpr->pLeft, pRc); | > | 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 | sqlite3Fts3MsrIncrRestart(pToken->pSegcsr); } } *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); } pPhrase->doclist.pNextDocid = 0; pPhrase->doclist.iDocid = 0; pPhrase->pOrPoslist = 0; } pExpr->iDocid = 0; pExpr->bEof = 0; pExpr->bStart = 0; fts3EvalRestart(pCsr, pExpr->pLeft, pRc); |
︙ | ︙ | |||
5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 | if( (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol) ){ return SQLITE_OK; } iDocid = pExpr->iDocid; pIter = pPhrase->doclist.pList; if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ int iMul; /* +1 if csr dir matches index dir, else -1 */ int bOr = 0; u8 bEof = 0; u8 bTreeEof = 0; Fts3Expr *p; /* Used to iterate from pExpr to root */ Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ | > | 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 | if( (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol) ){ return SQLITE_OK; } iDocid = pExpr->iDocid; pIter = pPhrase->doclist.pList; if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ int rc = SQLITE_OK; int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ int iMul; /* +1 if csr dir matches index dir, else -1 */ int bOr = 0; u8 bEof = 0; u8 bTreeEof = 0; Fts3Expr *p; /* Used to iterate from pExpr to root */ Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ |
︙ | ︙ | |||
5658 5659 5660 5661 5662 5663 5664 | } if( bOr==0 ) return SQLITE_OK; /* This is the descendent of an OR node. In this case we cannot use ** an incremental phrase. Load the entire doclist for the phrase ** into memory in this case. */ if( pPhrase->bIncr ){ | < | | | | | < < < < | | < < < | < < < | | < < < | < < < < < < < < < < < < < < < < < < | < < < < < < < < | | < > > | | | | | | > > > > > > > > > > | > > > | 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 | } if( bOr==0 ) return SQLITE_OK; /* This is the descendent of an OR node. In this case we cannot use ** an incremental phrase. Load the entire doclist for the phrase ** into memory in this case. */ if( pPhrase->bIncr ){ int bEofSave = pNear->bEof; fts3EvalRestart(pCsr, pNear, &rc); while( rc==SQLITE_OK && !pNear->bEof ){ fts3EvalNextRow(pCsr, pNear, &rc); if( bEofSave==0 && pNear->iDocid==iDocid ) break; } assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); } if( bTreeEof ){ while( rc==SQLITE_OK && !pNear->bEof ){ fts3EvalNextRow(pCsr, pNear, &rc); } } if( rc!=SQLITE_OK ) return rc; pIter = pPhrase->pOrPoslist; iDocid = pPhrase->iOrDocid; if( pCsr->bDesc==bDescDoclist ){ bEof = (pIter >= (pPhrase->doclist.aAll + pPhrase->doclist.nAll)); while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ sqlite3Fts3DoclistNext( bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, &pIter, &iDocid, &bEof ); } }else{ bEof = !pPhrase->doclist.nAll || (pIter && pIter<=pPhrase->doclist.aAll); while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ int dummy; sqlite3Fts3DoclistPrev( bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, &pIter, &iDocid, &dummy, &bEof ); } } pPhrase->pOrPoslist = pIter; pPhrase->iOrDocid = iDocid; if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0; } if( pIter==0 ) return SQLITE_OK; if( *pIter==0x01 ){ pIter++; pIter += fts3GetVarint32(pIter, &iThis); }else{ iThis = 0; } while( iThis<iCol ){ fts3ColumnlistCopy(0, &pIter); if( *pIter==0x00 ) return SQLITE_OK; pIter++; pIter += fts3GetVarint32(pIter, &iThis); } if( *pIter==0x00 ){ pIter = 0; } *ppOut = ((iCol==iThis)?pIter:0); return SQLITE_OK; } /* |
︙ | ︙ |
Changes to ext/fts3/fts3Int.h.
︙ | ︙ | |||
370 371 372 373 374 375 376 377 378 379 380 381 382 383 | }; struct Fts3Phrase { /* Cache of doclist for this phrase. */ Fts3Doclist doclist; int bIncr; /* True if doclist is loaded incrementally */ int iDoclistToken; /* Variables below this point are populated by fts3_expr.c when parsing ** a MATCH expression. Everything above is part of the evaluation phase. */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ | > > > > > | 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 | }; struct Fts3Phrase { /* Cache of doclist for this phrase. */ Fts3Doclist doclist; int bIncr; /* True if doclist is loaded incrementally */ int iDoclistToken; /* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an ** OR condition. */ char *pOrPoslist; i64 iOrDocid; /* Variables below this point are populated by fts3_expr.c when parsing ** a MATCH expression. Everything above is part of the evaluation phase. */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ |
︙ | ︙ |
Changes to ext/fts3/fts3_snippet.c.
︙ | ︙ | |||
438 439 440 441 442 443 444 | ** the set of phrases in the expression to populate the aPhrase[] array. */ sIter.pCsr = pCsr; sIter.iCol = iCol; sIter.nSnippet = nSnippet; sIter.nPhrase = nList; sIter.iCurrent = -1; | | > | | | | | | | | | | | | | | | | | | | | | | | | < | > > | | 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 | ** the set of phrases in the expression to populate the aPhrase[] array. */ sIter.pCsr = pCsr; sIter.iCol = iCol; sIter.nSnippet = nSnippet; sIter.nPhrase = nList; sIter.iCurrent = -1; rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void *)&sIter); if( rc==SQLITE_OK ){ /* Set the *pmSeen output variable. */ for(i=0; i<nList; i++){ if( sIter.aPhrase[i].pHead ){ *pmSeen |= (u64)1 << i; } } /* Loop through all candidate snippets. Store the best snippet in ** *pFragment. Store its associated 'score' in iBestScore. */ pFragment->iCol = iCol; while( !fts3SnippetNextCandidate(&sIter) ){ int iPos; int iScore; u64 mCover; u64 mHighlite; fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover,&mHighlite); assert( iScore>=0 ); if( iScore>iBestScore ){ pFragment->iPos = iPos; pFragment->hlmask = mHighlite; pFragment->covered = mCover; iBestScore = iScore; } } *piScore = iBestScore; } sqlite3_free(sIter.aPhrase); return rc; } /* ** Append a string to the string-buffer passed as the first argument. ** ** If nAppend is negative, then the length of the string zAppend is |
︙ | ︙ |