Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Have fts5 store rowids in ascending order. Query speed is virtually the same regardless of rowid order, and ascending order makes some insert optimizations easier. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts5 |
| Files: | files | file ages | folders |
| SHA1: |
5206ca6005bfa9dfc7346d4b89430c97 |
| User & Date: | dan 2015-01-24 19:57:03 |
Context
|
2015-01-27
| ||
| 20:41 | Fix a problem with fts5 doclist-indexes that occured if the first rowid of the first non-term page of a doclist is zero. (check-in: f704bc05 user: dan tags: fts5) | |
|
2015-01-24
| ||
| 19:57 | Have fts5 store rowids in ascending order. Query speed is virtually the same regardless of rowid order, and ascending order makes some insert optimizations easier. (check-in: 5206ca60 user: dan tags: fts5) | |
|
2015-01-23
| ||
| 17:43 | Fix compression of keys stored on internal segment b-tree nodes by fts5. (check-in: 51444f67 user: dan tags: fts5) | |
Changes
Changes to ext/fts5/fts5.c.
501 501 } 502 502 *ppCsr = (sqlite3_vtab_cursor*)pCsr; 503 503 return rc; 504 504 } 505 505 506 506 static int fts5StmtType(int idxNum){ 507 507 if( FTS5_PLAN(idxNum)==FTS5_PLAN_SCAN ){ 508 - return (idxNum&FTS5_ORDER_ASC) ? FTS5_STMT_SCAN_ASC : FTS5_STMT_SCAN_DESC; 508 + return (idxNum&FTS5_ORDER_DESC) ? FTS5_STMT_SCAN_DESC : FTS5_STMT_SCAN_ASC; 509 509 } 510 510 return FTS5_STMT_LOOKUP; 511 511 } 512 512 513 513 /* 514 514 ** This function is called after the cursor passed as the only argument 515 515 ** is moved to point at a different row. It clears all cached data ................................................................................ 648 648 } 649 649 break; 650 650 } 651 651 652 652 return rc; 653 653 } 654 654 655 -static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bAsc){ 655 +static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){ 656 656 Fts5Config *pConfig = pTab->pConfig; 657 657 Fts5Sorter *pSorter; 658 658 int nPhrase; 659 659 int nByte; 660 660 int rc = SQLITE_OK; 661 661 char *zSql; 662 662 const char *zRank = pCsr->zRank; ................................................................................ 676 676 ** table, saving it creates a circular reference. 677 677 ** 678 678 ** If SQLite a built-in statement cache, this wouldn't be a problem. */ 679 679 zSql = sqlite3_mprintf("SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s", 680 680 pConfig->zDb, pConfig->zName, zRank, pConfig->zName, 681 681 (zRankArgs ? ", " : ""), 682 682 (zRankArgs ? zRankArgs : ""), 683 - bAsc ? "ASC" : "DESC" 683 + bDesc ? "DESC" : "ASC" 684 684 ); 685 685 if( zSql==0 ){ 686 686 rc = SQLITE_NOMEM; 687 687 }else{ 688 688 rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pSorter->pStmt, 0); 689 689 sqlite3_free(zSql); 690 690 } ................................................................................ 702 702 sqlite3_free(pSorter); 703 703 pCsr->pSorter = 0; 704 704 } 705 705 706 706 return rc; 707 707 } 708 708 709 -static int fts5CursorFirst(Fts5Table *pTab, Fts5Cursor *pCsr, int bAsc){ 709 +static int fts5CursorFirst(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){ 710 710 int rc; 711 - rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, bAsc); 711 + rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, bDesc); 712 712 if( sqlite3Fts5ExprEof(pCsr->pExpr) ){ 713 713 CsrFlagSet(pCsr, FTS5CSR_EOF); 714 714 } 715 715 fts5CsrNewrow(pCsr); 716 716 return rc; 717 717 } 718 718 ................................................................................ 869 869 int idxNum, /* Strategy index */ 870 870 const char *idxStr, /* Unused */ 871 871 int nVal, /* Number of elements in apVal */ 872 872 sqlite3_value **apVal /* Arguments for the indexing scheme */ 873 873 ){ 874 874 Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); 875 875 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; 876 - int bAsc = ((idxNum & FTS5_ORDER_ASC) ? 1 : 0); 876 + int bDesc = ((idxNum & FTS5_ORDER_DESC) ? 1 : 0); 877 877 int rc = SQLITE_OK; 878 878 879 879 assert( nVal<=2 ); 880 880 assert( pCsr->pStmt==0 ); 881 881 assert( pCsr->pExpr==0 ); 882 882 assert( pCsr->csrflags==0 ); 883 883 assert( pCsr->pRank==0 ); ................................................................................ 890 890 ** set to FTS5_PLAN_SORTED_MATCH). pSortCsr is the cursor that will 891 891 ** return results to the user for this query. The current cursor 892 892 ** (pCursor) is used to execute the query issued by function 893 893 ** fts5CursorFirstSorted() above. */ 894 894 assert( FTS5_PLAN(idxNum)==FTS5_PLAN_SCAN ); 895 895 pCsr->idxNum = FTS5_PLAN_SOURCE; 896 896 pCsr->pExpr = pTab->pSortCsr->pExpr; 897 - rc = fts5CursorFirst(pTab, pCsr, bAsc); 897 + rc = fts5CursorFirst(pTab, pCsr, bDesc); 898 898 }else{ 899 899 int ePlan = FTS5_PLAN(idxNum); 900 900 pCsr->idxNum = idxNum; 901 901 if( ePlan==FTS5_PLAN_MATCH || ePlan==FTS5_PLAN_SORTED_MATCH ){ 902 902 const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); 903 903 904 904 rc = fts5CursorParseRank(pTab->pConfig, pCsr, (nVal==2 ? apVal[1] : 0)); ................................................................................ 909 909 ** but a request for an internal parameter. */ 910 910 rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]); 911 911 }else{ 912 912 char **pzErr = &pTab->base.zErrMsg; 913 913 rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr); 914 914 if( rc==SQLITE_OK ){ 915 915 if( ePlan==FTS5_PLAN_MATCH ){ 916 - rc = fts5CursorFirst(pTab, pCsr, bAsc); 916 + rc = fts5CursorFirst(pTab, pCsr, bDesc); 917 917 }else{ 918 - rc = fts5CursorFirstSorted(pTab, pCsr, bAsc); 918 + rc = fts5CursorFirstSorted(pTab, pCsr, bDesc); 919 919 } 920 920 } 921 921 } 922 922 } 923 923 }else{ 924 924 /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup 925 925 ** by rowid (ePlan==FTS5_PLAN_ROWID). */
Changes to ext/fts5/fts5Int.h.
225 225 typedef struct Fts5Index Fts5Index; 226 226 typedef struct Fts5IndexIter Fts5IndexIter; 227 227 228 228 /* 229 229 ** Values used as part of the flags argument passed to IndexQuery(). 230 230 */ 231 231 #define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ 232 -#define FTS5INDEX_QUERY_ASC 0x0002 /* Docs in ascending rowid order */ 232 +#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ 233 233 234 234 /* 235 235 ** Create/destroy an Fts5Index object. 236 236 */ 237 237 int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**); 238 238 int sqlite3Fts5IndexClose(Fts5Index *p, int bDestroy); 239 239 ................................................................................ 361 361 **************************************************************************/ 362 362 363 363 /************************************************************************** 364 364 ** Interface to code in fts5_hash.c. 365 365 */ 366 366 typedef struct Fts5Hash Fts5Hash; 367 367 368 +typedef struct Fts5Data Fts5Data; 369 +struct Fts5Data { 370 + u8 *p; /* Pointer to buffer containing record */ 371 + int n; /* Size of record in bytes */ 372 + int nRef; /* Ref count */ 373 +}; 374 + 368 375 /* 369 376 ** Create a hash table, free a hash table. 370 377 */ 371 378 int sqlite3Fts5HashNew(Fts5Hash**, int *pnSize); 372 379 void sqlite3Fts5HashFree(Fts5Hash*); 373 380 374 381 int sqlite3Fts5HashWrite( ................................................................................ 391 398 Fts5Hash*, 392 399 void *pCtx, 393 400 int (*xTerm)(void*, const char*, int), 394 401 int (*xEntry)(void*, i64, const u8*, int), 395 402 int (*xTermDone)(void*) 396 403 ); 397 404 405 +int sqlite3Fts5HashQuery( 406 + Fts5Hash*, /* Hash table to query */ 407 + const char *pTerm, int nTerm, /* Query term */ 408 + Fts5Data **ppData /* OUT: Query result */ 409 +); 398 410 399 411 400 412 /* 401 413 ** End of interface to code in fts5_hash.c. 402 414 **************************************************************************/ 403 415 404 416 /************************************************************************** ................................................................................ 466 478 Fts5Config *pConfig, 467 479 const char *zExpr, 468 480 Fts5Expr **ppNew, 469 481 char **pzErr 470 482 ); 471 483 472 484 /* 473 -** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bAsc); 485 +** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc); 474 486 ** rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr); 475 487 ** rc = sqlite3Fts5ExprNext(pExpr) 476 488 ** ){ 477 489 ** // The document with rowid iRowid matches the expression! 478 490 ** i64 iRowid = sqlite3Fts5ExprRowid(pExpr); 479 491 ** } 480 492 */ 481 -int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, int bAsc); 493 +int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, int bDesc); 482 494 int sqlite3Fts5ExprNext(Fts5Expr*); 483 495 int sqlite3Fts5ExprEof(Fts5Expr*); 484 496 i64 sqlite3Fts5ExprRowid(Fts5Expr*); 485 497 486 498 void sqlite3Fts5ExprFree(Fts5Expr*); 487 499 488 500 /* Called during startup to register a UDF with SQLite */
Changes to ext/fts5/fts5_expr.c.
28 28 void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(size_t)); 29 29 void sqlite3Fts5ParserFree(void*, void (*freeProc)(void*)); 30 30 void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*); 31 31 32 32 struct Fts5Expr { 33 33 Fts5Index *pIndex; 34 34 Fts5ExprNode *pRoot; 35 - int bAsc; 35 + int bDesc; /* Iterate in descending docid order */ 36 36 int nPhrase; /* Number of phrases in expression */ 37 37 Fts5ExprPhrase **apExprPhrase; /* Pointers to phrase objects */ 38 38 }; 39 39 40 40 /* 41 41 ** eType: 42 42 ** Expression node type. Always one of: ................................................................................ 596 596 } 597 597 } 598 598 599 599 return SQLITE_OK; 600 600 } 601 601 602 602 /* 603 -** Advance iterator pIter until it points to a value equal to or smaller 604 -** than the initial value of *piMin. If this means the iterator points 605 -** to a value smaller than *piMin, update *piMin to the new smallest value. 603 +** Advance iterator pIter until it points to a value equal to or laster 604 +** than the initial value of *piLast. If this means the iterator points 605 +** to a value laster than *piLast, update *piLast to the new lastest value. 606 606 ** 607 607 ** If the iterator reaches EOF, set *pbEof to true before returning. If 608 608 ** an error occurs, set *pRc to an error code. If either *pbEof or *pRc 609 609 ** are set, return a non-zero value. Otherwise, return zero. 610 610 */ 611 611 static int fts5ExprAdvanceto( 612 612 Fts5IndexIter *pIter, /* Iterator to advance */ 613 - int bAsc, /* True if iterator is "rowid ASC" */ 613 + int bDesc, /* True if iterator is "rowid DESC" */ 614 614 i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ 615 615 int *pRc, /* OUT: Error code */ 616 616 int *pbEof /* OUT: Set to true if EOF */ 617 617 ){ 618 618 i64 iLast = *piLast; 619 619 i64 iRowid; 620 620 621 621 iRowid = sqlite3Fts5IterRowid(pIter); 622 - if( (bAsc==0 && iRowid>iLast) || (bAsc && iRowid<iLast) ){ 622 + if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){ 623 623 sqlite3Fts5IterNextFrom(pIter, iLast); 624 624 if( sqlite3Fts5IterEof(pIter) ){ 625 625 *pbEof = 1; 626 626 return 1; 627 627 } 628 628 iRowid = sqlite3Fts5IterRowid(pIter); 629 - assert( (bAsc==0 && iRowid<=iLast) || (bAsc==1 && iRowid>=iLast) ); 629 + assert( (bDesc==0 && iRowid>=iLast) || (bDesc==1 && iRowid<=iLast) ); 630 630 } 631 631 *piLast = iRowid; 632 632 633 633 return 0; 634 634 } 635 635 636 636 /* ................................................................................ 652 652 ){ 653 653 Fts5ExprNearset *pNear = pNode->pNear; 654 654 int rc = SQLITE_OK; 655 655 int i, j; /* Phrase and token index, respectively */ 656 656 i64 iLast; /* Lastest rowid any iterator points to */ 657 657 int bMatch; /* True if all terms are at the same rowid */ 658 658 659 - /* Set iLast, the lastest rowid any iterator points to. If the iterator 660 - ** skips through rowids in the default descending order, this means the 661 - ** minimum rowid. Or, if the iterator is "ORDER BY rowid ASC", then it 662 - ** means the maximum rowid. */ 659 + /* Initialize iLast, the "lastest" rowid any iterator points to. If the 660 + ** iterator skips through rowids in the default ascending order, this means 661 + ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it 662 + ** means the minimum rowid. */ 663 663 iLast = sqlite3Fts5IterRowid(pNear->apPhrase[0]->aTerm[0].pIter); 664 - if( bFromValid && (iFrom>iLast)==(pExpr->bAsc!=0) ){ 664 + if( bFromValid && (iFrom>iLast)==(pExpr->bDesc==0) ){ 665 + assert( pExpr->bDesc || iFrom>=iLast ); 665 666 iLast = iFrom; 666 667 } 667 668 668 669 do { 669 670 bMatch = 1; 670 671 for(i=0; i<pNear->nPhrase; i++){ 671 672 Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; 672 673 for(j=0; j<pPhrase->nTerm; j++){ 673 674 Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; 674 675 i64 iRowid = sqlite3Fts5IterRowid(pIter); 675 676 if( iRowid!=iLast ) bMatch = 0; 676 - if( fts5ExprAdvanceto(pIter, pExpr->bAsc, &iLast, &rc, &pNode->bEof) ){ 677 + if( fts5ExprAdvanceto(pIter, pExpr->bDesc, &iLast, &rc, &pNode->bEof) ){ 677 678 return rc; 678 679 } 679 680 } 680 681 } 681 682 }while( bMatch==0 ); 682 683 683 684 pNode->iRowid = iLast; ................................................................................ 770 771 for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){ 771 772 pPhrase = pNear->apPhrase[i]; 772 773 for(j=0; j<pPhrase->nTerm; j++){ 773 774 pTerm = &pPhrase->aTerm[j]; 774 775 rc = sqlite3Fts5IndexQuery( 775 776 pExpr->pIndex, pTerm->zTerm, strlen(pTerm->zTerm), 776 777 (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | 777 - (pExpr->bAsc ? FTS5INDEX_QUERY_ASC : 0), 778 + (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), 778 779 &pTerm->pIter 779 780 ); 780 781 assert( rc==SQLITE_OK || pTerm->pIter==0 ); 781 782 if( pTerm->pIter==0 || sqlite3Fts5IterEof(pTerm->pIter) ){ 782 783 pNode->bEof = 1; 783 784 break; 784 785 } ................................................................................ 806 807 static int fts5NodeCompare( 807 808 Fts5Expr *pExpr, 808 809 Fts5ExprNode *p1, 809 810 Fts5ExprNode *p2 810 811 ){ 811 812 if( p2->bEof ) return -1; 812 813 if( p1->bEof ) return +1; 813 - if( pExpr->bAsc ){ 814 + if( pExpr->bDesc==0 ){ 814 815 if( p1->iRowid<p2->iRowid ) return -1; 815 816 return (p1->iRowid > p2->iRowid); 816 817 }else{ 817 818 if( p1->iRowid>p2->iRowid ) return -1; 818 819 return (p1->iRowid < p2->iRowid); 819 820 } 820 821 } ................................................................................ 907 908 break; 908 909 } 909 910 910 911 case FTS5_AND: { 911 912 Fts5ExprNode *p1 = pNode->pLeft; 912 913 Fts5ExprNode *p2 = pNode->pRight; 913 914 914 - 915 915 while( p1->bEof==0 && p2->bEof==0 && p2->iRowid!=p1->iRowid ){ 916 916 Fts5ExprNode *pAdv; 917 - assert( pExpr->bAsc==0 || pExpr->bAsc==1 ); 918 - if( pExpr->bAsc==(p1->iRowid < p2->iRowid) ){ 917 + assert( pExpr->bDesc==0 || pExpr->bDesc==1 ); 918 + if( pExpr->bDesc==(p1->iRowid > p2->iRowid) ){ 919 919 pAdv = p1; 920 - if( bFromValid==0 || pExpr->bAsc==(p2->iRowid > iFrom) ){ 920 + if( bFromValid==0 || pExpr->bDesc==(p2->iRowid < iFrom) ){ 921 921 iFrom = p2->iRowid; 922 922 } 923 923 }else{ 924 924 pAdv = p2; 925 - if( bFromValid==0 || pExpr->bAsc==(p1->iRowid > iFrom) ){ 925 + if( bFromValid==0 || pExpr->bDesc==(p1->iRowid < iFrom) ){ 926 926 iFrom = p1->iRowid; 927 927 } 928 928 } 929 929 rc = fts5ExprNodeNext(pExpr, pAdv, 1, iFrom); 930 930 if( rc!=SQLITE_OK ) break; 931 931 } 932 932 if( p1->bEof || p2->bEof ){ ................................................................................ 999 999 return rc; 1000 1000 } 1001 1001 1002 1002 1003 1003 1004 1004 /* 1005 1005 ** Begin iterating through the set of documents in index pIdx matched by 1006 -** the MATCH expression passed as the first argument. If the "bAsc" parameter 1007 -** is passed a non-zero value, iteration is in ascending rowid order. Or, 1008 -** if it is zero, in descending order. 1006 +** the MATCH expression passed as the first argument. If the "bDesc" parameter 1007 +** is passed a non-zero value, iteration is in descending rowid order. Or, 1008 +** if it is zero, in ascending order. 1009 1009 ** 1010 1010 ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It 1011 1011 ** is not considered an error if the query does not match any documents. 1012 1012 */ 1013 -int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, int bAsc){ 1013 +int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, int bDesc){ 1014 1014 int rc = SQLITE_OK; 1015 1015 if( p->pRoot ){ 1016 1016 p->pIndex = pIdx; 1017 - p->bAsc = bAsc; 1017 + p->bDesc = bDesc; 1018 1018 rc = fts5ExprNodeFirst(p, p->pRoot); 1019 1019 } 1020 1020 return rc; 1021 1021 } 1022 1022 1023 1023 /* 1024 1024 ** Move to the next document
Changes to ext/fts5/fts5_hash.c.
51 51 ** nData: 52 52 ** Bytes of data written since iRowidOff. 53 53 */ 54 54 struct Fts5HashEntry { 55 55 Fts5HashEntry *pNext; /* Next hash entry with same hash-key */ 56 56 57 57 int nAlloc; /* Total size of allocation */ 58 - int iRowidOff; /* Offset of last rowid written */ 58 + int iSzPoslist; /* Offset of space for 4-byte poslist size */ 59 59 int nData; /* Total bytes of data (incl. structure) */ 60 60 61 61 int iCol; /* Column of last value written */ 62 62 int iPos; /* Position of last value written */ 63 63 i64 iRowid; /* Rowid of last value written */ 64 64 char zKey[0]; /* Nul-terminated entry key */ 65 65 }; 66 66 67 +/* 68 +** Format value iVal as a 4-byte varint and write it to buffer a[]. 4 bytes 69 +** are used even if the value could fit in a smaller amount of space. 70 +*/ 71 +static void fts5Put4ByteVarint(u8 *a, int iVal){ 72 + a[0] = (0x80 | (u8)(iVal >> 21)); 73 + a[1] = (0x80 | (u8)(iVal >> 14)); 74 + a[2] = (0x80 | (u8)(iVal >> 7)); 75 + a[3] = (0x7F & (u8)(iVal)); 76 +} 67 77 68 78 /* 69 79 ** Allocate a new hash table. 70 80 */ 71 81 int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){ 72 82 int rc = SQLITE_OK; 73 83 Fts5Hash *pNew; ................................................................................ 157 167 158 168 sqlite3_free(apOld); 159 169 pHash->nSlot = nNew; 160 170 pHash->aSlot = apNew; 161 171 return SQLITE_OK; 162 172 } 163 173 164 -/* 165 -** Store the 32-bit integer passed as the second argument in buffer p. 166 -*/ 167 -static int fts5PutNativeInt(u8 *p, int i){ 168 - assert( sizeof(i)==4 ); 169 - memcpy(p, &i, sizeof(i)); 170 - return sizeof(i); 171 -} 172 - 173 -/* 174 -** Read and return the 32-bit integer stored in buffer p. 175 -*/ 176 -static int fts5GetNativeU32(u8 *p){ 177 - int i; 178 - assert( sizeof(i)==4 ); 179 - memcpy(&i, p, sizeof(i)); 180 - return i; 181 -} 182 - 183 174 int sqlite3Fts5HashWrite( 184 175 Fts5Hash *pHash, 185 176 i64 iRowid, /* Rowid for this entry */ 186 177 int iCol, /* Column token appears in (-ve -> delete) */ 187 178 int iPos, /* Position of token within column */ 188 179 const char *pToken, int nToken /* Token to add or remove to or from index */ 189 180 ){ 190 181 unsigned int iHash = fts5HashKey(pHash->nSlot, pToken, nToken); 191 182 Fts5HashEntry *p; 192 183 u8 *pPtr; 193 184 int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ 194 185 195 - /* Attempt to locate an existing hash object */ 186 + /* Attempt to locate an existing hash entry */ 196 187 for(p=pHash->aSlot[iHash]; p; p=p->pNext){ 197 188 if( memcmp(p->zKey, pToken, nToken)==0 && p->zKey[nToken]==0 ) break; 198 189 } 199 190 200 191 /* If an existing hash entry cannot be found, create a new one. */ 201 192 if( p==0 ){ 202 193 int nByte = sizeof(Fts5HashEntry) + nToken + 1 + 64; ................................................................................ 210 201 211 202 p = (Fts5HashEntry*)sqlite3_malloc(nByte); 212 203 if( !p ) return SQLITE_NOMEM; 213 204 memset(p, 0, sizeof(Fts5HashEntry)); 214 205 p->nAlloc = nByte; 215 206 memcpy(p->zKey, pToken, nToken); 216 207 p->zKey[nToken] = '\0'; 217 - p->iRowidOff = p->nData = nToken + 1 + sizeof(Fts5HashEntry); 208 + p->nData = nToken + 1 + sizeof(Fts5HashEntry); 218 209 p->nData += sqlite3PutVarint(&((u8*)p)[p->nData], iRowid); 210 + p->iSzPoslist = p->nData; 211 + p->nData += 4; 219 212 p->iRowid = iRowid; 220 213 p->pNext = pHash->aSlot[iHash]; 221 214 pHash->aSlot[iHash] = p; 222 215 pHash->nEntry++; 223 - 224 216 nIncr += p->nData; 225 217 } 226 218 227 219 /* Check there is enough space to append a new entry. Worst case scenario 228 220 ** is: 229 221 ** 230 - ** + 4 bytes for the previous entry size field, 231 222 ** + 9 bytes for a new rowid, 223 + ** + 4 bytes reserved for the "poslist size" varint. 232 224 ** + 1 byte for a "new column" byte, 233 225 ** + 3 bytes for a new column number (16-bit max) as a varint, 234 226 ** + 5 bytes for the new position offset (32-bit max). 235 227 */ 236 - if( (p->nAlloc - p->nData) < (4 + 9 + 1 + 3 + 5) ){ 228 + if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){ 237 229 int nNew = p->nAlloc * 2; 238 230 Fts5HashEntry *pNew; 239 231 Fts5HashEntry **pp; 240 232 pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew); 241 233 if( pNew==0 ) return SQLITE_NOMEM; 242 234 pNew->nAlloc = nNew; 243 235 for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pNext); ................................................................................ 246 238 } 247 239 pPtr = (u8*)p; 248 240 nIncr -= p->nData; 249 241 250 242 /* If this is a new rowid, append the 4-byte size field for the previous 251 243 ** entry, and the new rowid for this entry. */ 252 244 if( iRowid!=p->iRowid ){ 253 - p->nData += fts5PutNativeInt(&pPtr[p->nData], p->nData - p->iRowidOff); 254 - p->iRowidOff = p->nData; 255 - p->nData += sqlite3PutVarint(&pPtr[p->nData], iRowid); 245 + assert( p->iSzPoslist>0 ); 246 + fts5Put4ByteVarint(&pPtr[p->iSzPoslist], p->nData - p->iSzPoslist - 4); 247 + p->nData += sqlite3PutVarint(&pPtr[p->nData], iRowid - p->iRowid); 248 + p->iSzPoslist = p->nData; 249 + p->nData += 4; 256 250 p->iCol = 0; 257 251 p->iPos = 0; 258 252 p->iRowid = iRowid; 259 253 } 260 254 261 255 if( iCol>=0 ){ 262 256 /* Append a new column value, if necessary */ ................................................................................ 375 369 int rc; 376 370 377 371 rc = fts5HashEntrySort(pHash, &pList); 378 372 if( rc==SQLITE_OK ){ 379 373 while( pList ){ 380 374 Fts5HashEntry *pNext = pList->pNext; 381 375 if( rc==SQLITE_OK ){ 376 + const int nSz = pList->nData - pList->iSzPoslist - 4; 377 + const int nKey = strlen(pList->zKey); 378 + i64 iRowid = 0; 382 379 u8 *pPtr = (u8*)pList; 383 - int nKey = strlen(pList->zKey); 384 - int iOff = pList->iRowidOff; 385 - int iEnd = sizeof(Fts5HashEntry) + nKey + 1; 386 - int nByte = pList->nData - pList->iRowidOff; 380 + int iOff = sizeof(Fts5HashEntry) + nKey + 1; 387 381 382 + /* Fill in the final poslist size field */ 383 + fts5Put4ByteVarint(&pPtr[pList->iSzPoslist], nSz); 384 + 385 + /* Issue the new-term callback */ 388 386 rc = xTerm(pCtx, pList->zKey, nKey); 389 - while( rc==SQLITE_OK && iOff ){ 390 - int nVarint; 391 - i64 iRowid; 392 - nVarint = getVarint(&pPtr[iOff], (u64*)&iRowid); 393 - rc = xEntry(pCtx, iRowid, &pPtr[iOff+nVarint], nByte-nVarint); 394 - if( iOff==iEnd ){ 395 - iOff = 0; 396 - }else{ 397 - nByte = fts5GetNativeU32(&pPtr[iOff-sizeof(int)]); 398 - iOff = iOff - sizeof(int) - nByte; 399 - } 387 + 388 + /* Issue the xEntry callbacks */ 389 + while( rc==SQLITE_OK && iOff<pList->nData ){ 390 + i64 iDelta; /* Rowid delta value */ 391 + int nPoslist; /* Size of position list in bytes */ 392 + iOff += getVarint(&pPtr[iOff], (u64*)&iDelta); 393 + iRowid += iDelta; 394 + iOff += fts5GetVarint32(&pPtr[iOff], nPoslist); 395 + rc = xEntry(pCtx, iRowid, &pPtr[iOff], nPoslist); 396 + iOff += nPoslist; 400 397 } 401 - if( rc==SQLITE_OK ){ 402 - rc = xTermDone(pCtx); 403 - } 398 + 399 + /* Issue the term-done callback */ 400 + if( rc==SQLITE_OK ) rc = xTermDone(pCtx); 404 401 } 405 402 sqlite3_free(pList); 406 403 pList = pNext; 407 404 } 408 405 } 409 406 return rc; 410 407 } 411 - 412 - 413 408
Changes to ext/fts5/fts5_index.c.
40 40 ** incremental merge operations. 41 41 ** 42 42 */ 43 43 44 44 #define FTS5_OPT_WORK_UNIT 1000 /* Number of leaf pages per optimize step */ 45 45 #define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */ 46 46 47 -#define FTS5_MIN_DLIDX_SIZE 4 /* Add dlidx if this many empty pages */ 47 +#define FTS5_MIN_DLIDX_SIZE 4000 /* Add dlidx if this many empty pages */ 48 48 49 49 /* 50 50 ** Details: 51 51 ** 52 52 ** The %_data table managed by this module, 53 53 ** 54 54 ** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB); ................................................................................ 266 266 ** without overreading if the records are corrupt. 267 267 */ 268 268 #define FTS5_DATA_ZERO_PADDING 8 269 269 270 270 typedef struct Fts5BtreeIter Fts5BtreeIter; 271 271 typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel; 272 272 typedef struct Fts5ChunkIter Fts5ChunkIter; 273 -typedef struct Fts5Data Fts5Data; 274 273 typedef struct Fts5DlidxIter Fts5DlidxIter; 275 274 typedef struct Fts5MultiSegIter Fts5MultiSegIter; 276 275 typedef struct Fts5NodeIter Fts5NodeIter; 277 276 typedef struct Fts5PageWriter Fts5PageWriter; 278 277 typedef struct Fts5PosIter Fts5PosIter; 279 278 typedef struct Fts5SegIter Fts5SegIter; 280 279 typedef struct Fts5DoclistIter Fts5DoclistIter; ................................................................................ 307 306 sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ 308 307 sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */ 309 308 sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */ 310 309 int nRead; /* Total number of blocks read */ 311 310 }; 312 311 313 312 struct Fts5DoclistIter { 314 - int bAsc; 313 + int bDesc; /* True for DESC order, false for ASC */ 315 314 u8 *a; 316 315 int n; 317 316 int i; 318 317 319 318 /* Output variables. aPoslist==0 at EOF */ 320 319 i64 iRowid; 321 320 u8 *aPoslist; ................................................................................ 329 328 Fts5Index *pIndex; 330 329 Fts5Structure *pStruct; 331 330 Fts5MultiSegIter *pMulti; 332 331 Fts5DoclistIter *pDoclist; 333 332 Fts5Buffer poslist; /* Buffer containing current poslist */ 334 333 }; 335 334 336 -/* 337 -** A single record read from the %_data table. 338 -*/ 339 -struct Fts5Data { 340 - u8 *p; /* Pointer to buffer containing record */ 341 - int n; /* Size of record in bytes */ 342 - int nRef; /* Ref count */ 343 -}; 344 - 345 335 /* 346 336 ** The contents of the "structure" record for each index are represented 347 337 ** using an Fts5Structure record in memory. Which uses instances of the 348 338 ** other Fts5StructureXXX types as components. 349 339 */ 350 340 struct Fts5StructureSegment { 351 341 int iSegid; /* Segment id */ ................................................................................ 1478 1468 */ 1479 1469 static void fts5DlidxIterFree(Fts5DlidxIter *pIter){ 1480 1470 if( pIter ){ 1481 1471 fts5DataRelease(pIter->pData); 1482 1472 sqlite3_free(pIter); 1483 1473 } 1484 1474 } 1475 + 1476 +static void fts5LeafHeader(Fts5Data *pLeaf, int *piRowid, int *piTerm){ 1477 + *piRowid = (int)fts5GetU16(&pLeaf->p[0]); 1478 + *piTerm = (int)fts5GetU16(&pLeaf->p[2]); 1479 +} 1485 1480 1486 1481 /* 1487 1482 ** Load the next leaf page into the segment iterator. 1488 1483 */ 1489 1484 static void fts5SegIterNextPage( 1490 1485 Fts5Index *p, /* FTS5 backend object */ 1491 1486 Fts5SegIter *pIter /* Iterator to advance to next page */ ................................................................................ 1499 1494 ); 1500 1495 }else{ 1501 1496 pIter->pLeaf = 0; 1502 1497 } 1503 1498 } 1504 1499 1505 1500 /* 1506 -** Leave pIter->iLeafOffset as the offset to the size field of the first 1507 -** position list. The position list belonging to document pIter->iRowid. 1501 +** Fts5SegIter.iLeafOffset currently points to the first byte of the 1502 +** "nSuffix" field of a term. Function parameter nKeep contains the value 1503 +** of the "nPrefix" field (if there was one - it is passed 0 if this is 1504 +** the first term in the segment). 1505 +** 1506 +** This function populates (Fts5SegIter.term) and (Fts5SegIter.iRowid) 1507 +** accordingly and leaves (Fts5SegIter.iLeafOffset) set to the offset to 1508 +** the size field of the first position list. The position list belonging 1509 +** to document (Fts5SegIter.iRowid). 1508 1510 */ 1509 1511 static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ 1510 1512 u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ 1511 1513 int iOff = pIter->iLeafOffset; /* Offset to read at */ 1512 1514 int nNew; /* Bytes of new data */ 1513 1515 1514 1516 iOff += fts5GetVarint32(&a[iOff], nNew); ................................................................................ 1565 1567 if( p->rc==SQLITE_OK ){ 1566 1568 u8 *a = pIter->pLeaf->p; 1567 1569 pIter->iLeafOffset = fts5GetU16(&a[2]); 1568 1570 fts5SegIterLoadTerm(p, pIter, 0); 1569 1571 } 1570 1572 } 1571 1573 1572 -static void fts5LeafHeader(Fts5Data *pLeaf, int *piRowid, int *piTerm){ 1573 - *piRowid = (int)fts5GetU16(&pLeaf->p[0]); 1574 - *piTerm = (int)fts5GetU16(&pLeaf->p[2]); 1575 -} 1576 - 1577 1574 /* 1578 1575 ** This function is only ever called on iterators created by calls to 1579 1576 ** Fts5IndexQuery() with the FTS5INDEX_QUERY_ASC flag set. 1580 1577 ** 1581 1578 ** When this function is called, iterator pIter points to the first rowid 1582 1579 ** on the current leaf associated with the term being queried. This function 1583 1580 ** advances it to point to the last such rowid and, if necessary, initializes ................................................................................ 1594 1591 int nPos; 1595 1592 1596 1593 i += fts5GetVarint32(&a[i], nPos); 1597 1594 i += nPos; 1598 1595 if( i>=n ) break; 1599 1596 i += getVarint(&a[i], (u64*)&iDelta); 1600 1597 if( iDelta==0 ) break; 1601 - pIter->iRowid -= iDelta; 1598 + pIter->iRowid += iDelta; 1602 1599 1603 1600 if( iRowidOffset>=pIter->nRowidOffset ){ 1604 1601 int nNew = pIter->nRowidOffset + 8; 1605 1602 int *aNew = (int*)sqlite3_realloc(pIter->aRowidOffset, nNew*sizeof(int)); 1606 1603 if( aNew==0 ){ 1607 1604 p->rc = SQLITE_NOMEM; 1608 1605 break; ................................................................................ 1674 1671 int bRet = 0; 1675 1672 Fts5Data *pLeaf = pIter->pLeaf; 1676 1673 if( p->rc==SQLITE_OK && pLeaf ){ 1677 1674 if( pIter->iLeafOffset<pLeaf->n ){ 1678 1675 bRet = (pLeaf->p[pIter->iLeafOffset]==0x00); 1679 1676 }else{ 1680 1677 Fts5Data *pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID( 1681 - pIter->iIdx, pIter->pSeg->iSegid, 0, pIter->iLeafPgno 1678 + pIter->iIdx, pIter->pSeg->iSegid, 0, pIter->iLeafPgno+1 1682 1679 )); 1683 1680 if( pNew ){ 1684 1681 bRet = (pNew->p[4]==0x00); 1685 1682 fts5DataRelease(pNew); 1686 1683 } 1687 1684 } 1688 1685 } ................................................................................ 1709 1706 i64 iDelta; 1710 1707 pIter->iRowidOffset--; 1711 1708 1712 1709 pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset]; 1713 1710 iOff += fts5GetVarint32(&a[iOff], nPos); 1714 1711 iOff += nPos; 1715 1712 getVarint(&a[iOff], (u64*)&iDelta); 1716 - pIter->iRowid += iDelta; 1713 + pIter->iRowid -= iDelta; 1717 1714 }else{ 1718 1715 fts5SegIterReverseNewPage(p, pIter); 1719 1716 } 1720 1717 }else{ 1721 1718 Fts5Data *pLeaf = pIter->pLeaf; 1722 1719 int iOff; 1723 1720 int bNewTerm = 0; ................................................................................ 1744 1741 if( iOff>=n ){ 1745 1742 fts5SegIterNextPage(p, pIter); 1746 1743 pIter->iLeafOffset = 4; 1747 1744 }else if( iOff!=fts5GetU16(&a[2]) ){ 1748 1745 pIter->iLeafOffset += fts5GetVarint32(&a[iOff], nKeep); 1749 1746 } 1750 1747 }else{ 1751 - pIter->iRowid -= iDelta; 1748 + pIter->iRowid += iDelta; 1752 1749 } 1753 1750 }else{ 1754 1751 iOff = 0; 1755 1752 /* Next entry is not on the current page */ 1756 1753 while( iOff==0 ){ 1757 1754 fts5SegIterNextPage(p, pIter); 1758 1755 pLeaf = pIter->pLeaf; ................................................................................ 1918 1915 Fts5SegIter *pIter /* Object to populate */ 1919 1916 ){ 1920 1917 int iPg = 1; 1921 1918 int h; 1922 1919 int bGe = ((flags & FTS5INDEX_QUERY_PREFIX) && iIdx==0); 1923 1920 int bDlidx = 0; /* True if there is a doclist-index */ 1924 1921 1925 - assert( bGe==0 || (flags & FTS5INDEX_QUERY_ASC)==0 ); 1922 + assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); 1926 1923 assert( pTerm && nTerm ); 1927 1924 memset(pIter, 0, sizeof(*pIter)); 1928 1925 pIter->pSeg = pSeg; 1929 1926 pIter->iIdx = iIdx; 1930 1927 1931 1928 /* This block sets stack variable iPg to the leaf page number that may 1932 1929 ** contain term (pTerm/nTerm), if it is present in the segment. */ ................................................................................ 1976 1973 pIter->pLeaf = 0; 1977 1974 } 1978 1975 } 1979 1976 1980 1977 if( p->rc==SQLITE_OK && bGe==0 ){ 1981 1978 pIter->flags |= FTS5_SEGITER_ONETERM; 1982 1979 if( pIter->pLeaf ){ 1983 - if( flags & FTS5INDEX_QUERY_ASC ){ 1980 + if( flags & FTS5INDEX_QUERY_DESC ){ 1984 1981 pIter->flags |= FTS5_SEGITER_REVERSE; 1985 1982 } 1986 1983 if( bDlidx ){ 1987 1984 fts5SegIterLoadDlidx(p, iIdx, pIter); 1988 1985 } 1989 - if( flags & FTS5INDEX_QUERY_ASC ){ 1986 + if( flags & FTS5INDEX_QUERY_DESC ){ 1990 1987 fts5SegIterReverse(p, iIdx, pIter); 1991 1988 } 1992 1989 } 1993 1990 } 1994 1991 } 1995 1992 1996 1993 /* ................................................................................ 2038 2035 iRes = i1; 2039 2036 }else{ 2040 2037 int res = fts5BufferCompare(&p1->term, &p2->term); 2041 2038 if( res==0 ){ 2042 2039 assert( i2>i1 ); 2043 2040 assert( i2!=0 ); 2044 2041 if( p1->iRowid==p2->iRowid ) return i2; 2045 - res = ((p1->iRowid < p2->iRowid)==pIter->bRev) ? -1 : +1; 2042 + res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1; 2046 2043 } 2047 2044 assert( res!=0 ); 2048 2045 if( res<0 ){ 2049 2046 iRes = i1; 2050 2047 }else{ 2051 2048 iRes = i2; 2052 2049 } 2053 2050 } 2054 2051 2055 2052 pIter->aFirst[iOut] = iRes; 2056 2053 return 0; 2057 2054 } 2058 2055 2059 -/* 2060 -** Free the iterator object passed as the second argument. 2061 -*/ 2062 -static void fts5MultiIterFree(Fts5Index *p, Fts5MultiSegIter *pIter){ 2063 - if( pIter ){ 2064 - int i; 2065 - for(i=0; i<pIter->nSeg; i++){ 2066 - fts5SegIterClear(&pIter->aSeg[i]); 2067 - } 2068 - sqlite3_free(pIter); 2069 - } 2070 -} 2071 - 2072 -static void fts5MultiIterAdvanced( 2073 - Fts5Index *p, /* FTS5 backend to iterate within */ 2074 - Fts5MultiSegIter *pIter, /* Iterator to update aFirst[] array for */ 2075 - int iChanged, /* Index of sub-iterator just advanced */ 2076 - int iMinset /* Minimum entry in aFirst[] to set */ 2077 -){ 2078 - int i; 2079 - for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){ 2080 - int iEq; 2081 - if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){ 2082 - fts5SegIterNext(p, &pIter->aSeg[iEq]); 2083 - i = pIter->nSeg + iEq; 2084 - } 2085 - } 2086 -} 2087 - 2088 2056 /* 2089 2057 ** Move the seg-iter so that it points to the first rowid on page iLeafPgno. 2090 2058 ** It is an error if leaf iLeafPgno contains no rowid. 2091 2059 */ 2092 2060 static void fts5SegIterGotoPage( 2093 2061 Fts5Index *p, /* FTS5 backend object */ 2094 2062 Fts5SegIter *pIter, /* Iterator to advance */ ................................................................................ 2166 2134 if( pIter->pLeaf==0 ) break; 2167 2135 if( bRev==0 && pIter->iRowid<=iMatch ) break; 2168 2136 if( bRev!=0 && pIter->iRowid>=iMatch ) break; 2169 2137 bMove = 1; 2170 2138 } 2171 2139 } 2172 2140 2141 + 2142 +/* 2143 +** Free the iterator object passed as the second argument. 2144 +*/ 2145 +static void fts5MultiIterFree(Fts5Index *p, Fts5MultiSegIter *pIter){ 2146 + if( pIter ){ 2147 + int i; 2148 + for(i=0; i<pIter->nSeg; i++){ 2149 + fts5SegIterClear(&pIter->aSeg[i]); 2150 + } 2151 + sqlite3_free(pIter); 2152 + } 2153 +} 2154 + 2155 +static void fts5MultiIterAdvanced( 2156 + Fts5Index *p, /* FTS5 backend to iterate within */ 2157 + Fts5MultiSegIter *pIter, /* Iterator to update aFirst[] array for */ 2158 + int iChanged, /* Index of sub-iterator just advanced */ 2159 + int iMinset /* Minimum entry in aFirst[] to set */ 2160 +){ 2161 + int i; 2162 + for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){ 2163 + int iEq; 2164 + if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){ 2165 + fts5SegIterNext(p, &pIter->aSeg[iEq]); 2166 + i = pIter->nSeg + iEq; 2167 + } 2168 + } 2169 +} 2170 + 2173 2171 /* 2174 2172 ** Move the iterator to the next entry. 2175 2173 ** 2176 2174 ** If an error occurs, an error code is left in Fts5Index.rc. It is not 2177 2175 ** considered an error if the iterator reaches EOF, or if it is already at 2178 2176 ** EOF when this function is called. 2179 2177 */ ................................................................................ 2244 2242 sizeof(Fts5SegIter) * nSlot + /* pNew->aSeg[] */ 2245 2243 sizeof(u16) * nSlot /* pNew->aFirst[] */ 2246 2244 ); 2247 2245 if( pNew==0 ) return; 2248 2246 pNew->nSeg = nSlot; 2249 2247 pNew->aSeg = (Fts5SegIter*)&pNew[1]; 2250 2248 pNew->aFirst = (u16*)&pNew->aSeg[nSlot]; 2251 - pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_ASC)); 2249 + pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC)); 2252 2250 pNew->bSkipEmpty = bSkipEmpty; 2253 2251 2254 2252 /* Initialize each of the component segment iterators. */ 2255 2253 if( iLevel<0 ){ 2256 2254 Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; 2257 2255 for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){ 2258 2256 for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){ ................................................................................ 2324 2322 i64 iMatch 2325 2323 ){ 2326 2324 while( 1 ){ 2327 2325 i64 iRowid; 2328 2326 fts5MultiIterNext(p, pIter, 1, iMatch); 2329 2327 if( fts5MultiIterEof(p, pIter) ) break; 2330 2328 iRowid = fts5MultiIterRowid(pIter); 2331 - if( pIter->bRev==0 && iRowid<=iMatch ) break; 2332 - if( pIter->bRev!=0 && iRowid>=iMatch ) break; 2329 + if( pIter->bRev==0 && iRowid>=iMatch ) break; 2330 + if( pIter->bRev!=0 && iRowid<=iMatch ) break; 2333 2331 } 2334 2332 } 2335 2333 2336 2334 /* 2337 2335 ** Return a pointer to a buffer containing the term associated with the 2338 2336 ** entry that the iterator currently points to. 2339 2337 */ ................................................................................ 2790 2788 fts5WriteDlidxAppend(p, pWriter, iRowid); 2791 2789 } 2792 2790 2793 2791 /* Write the docid. */ 2794 2792 if( pWriter->bFirstRowidInDoclist || pWriter->bFirstRowidInPage ){ 2795 2793 fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid); 2796 2794 }else{ 2797 - assert( p->rc || iRowid<pWriter->iPrevRowid ); 2798 - fts5BufferAppendVarint(&p->rc, &pPage->buf, pWriter->iPrevRowid - iRowid); 2795 + assert( p->rc || iRowid>pWriter->iPrevRowid ); 2796 + fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid); 2799 2797 } 2800 2798 pWriter->iPrevRowid = iRowid; 2801 2799 pWriter->bFirstRowidInDoclist = 0; 2802 2800 pWriter->bFirstRowidInPage = 0; 2803 2801 2804 2802 if( pPage->buf.n>=p->pConfig->pgsz ){ 2805 2803 fts5WriteFlushLeaf(p, pWriter); ................................................................................ 3707 3705 } 3708 3706 3709 3707 static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ 3710 3708 if( pIter->i<pIter->n ){ 3711 3709 if( pIter->i ){ 3712 3710 i64 iDelta; 3713 3711 pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&iDelta); 3714 - if( pIter->bAsc ){ 3715 - pIter->iRowid += iDelta; 3712 + if( pIter->bDesc ){ 3713 + pIter->iRowid -= iDelta; 3716 3714 }else{ 3717 - pIter->iRowid -= iDelta; 3715 + pIter->iRowid += iDelta; 3718 3716 } 3719 3717 }else{ 3720 3718 pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid); 3721 3719 } 3722 3720 pIter->i += fts5GetVarint32(&pIter->a[pIter->i], pIter->nPoslist); 3723 3721 pIter->aPoslist = &pIter->a[pIter->i]; 3724 3722 pIter->i += pIter->nPoslist; ................................................................................ 3725 3723 }else{ 3726 3724 pIter->aPoslist = 0; 3727 3725 } 3728 3726 } 3729 3727 3730 3728 static void fts5DoclistIterInit( 3731 3729 Fts5Buffer *pBuf, 3732 - int bAsc, 3730 + int bDesc, 3733 3731 Fts5DoclistIter *pIter 3734 3732 ){ 3735 3733 memset(pIter, 0, sizeof(*pIter)); 3736 3734 pIter->a = pBuf->p; 3737 3735 pIter->n = pBuf->n; 3738 - pIter->bAsc = bAsc; 3736 + pIter->bDesc = bDesc; 3739 3737 fts5DoclistIterNext(pIter); 3740 3738 } 3741 3739 3742 3740 /* 3743 3741 ** Append a doclist to buffer pBuf. 3744 3742 */ 3745 3743 static void fts5MergeAppendDocid( 3746 3744 int *pRc, /* IN/OUT: Error code */ 3747 - int bAsc, 3745 + int bDesc, 3748 3746 Fts5Buffer *pBuf, /* Buffer to write to */ 3749 3747 i64 *piLastRowid, /* IN/OUT: Previous rowid written (if any) */ 3750 3748 i64 iRowid /* Rowid to append */ 3751 3749 ){ 3752 3750 if( pBuf->n==0 ){ 3753 3751 fts5BufferAppendVarint(pRc, pBuf, iRowid); 3754 - }else if( bAsc==0 ){ 3752 + }else if( bDesc ){ 3755 3753 fts5BufferAppendVarint(pRc, pBuf, *piLastRowid - iRowid); 3756 3754 }else{ 3757 3755 fts5BufferAppendVarint(pRc, pBuf, iRowid - *piLastRowid); 3758 3756 } 3759 3757 *piLastRowid = iRowid; 3760 3758 } 3761 3759 ................................................................................ 3765 3763 ** returning. 3766 3764 ** 3767 3765 ** If an error occurs, an error code is left in p->rc. If an error has 3768 3766 ** already occurred, this function is a no-op. 3769 3767 */ 3770 3768 static void fts5MergePrefixLists( 3771 3769 Fts5Index *p, /* FTS5 backend object */ 3772 - int bAsc, 3770 + int bDesc, 3773 3771 Fts5Buffer *p1, /* First list to merge */ 3774 3772 Fts5Buffer *p2 /* Second list to merge */ 3775 3773 ){ 3776 3774 if( p2->n ){ 3777 3775 i64 iLastRowid = 0; 3778 3776 Fts5DoclistIter i1; 3779 3777 Fts5DoclistIter i2; 3780 3778 Fts5Buffer out; 3781 3779 Fts5Buffer tmp; 3782 3780 memset(&out, 0, sizeof(out)); 3783 3781 memset(&tmp, 0, sizeof(tmp)); 3784 3782 3785 - fts5DoclistIterInit(p1, bAsc, &i1); 3786 - fts5DoclistIterInit(p2, bAsc, &i2); 3783 + fts5DoclistIterInit(p1, bDesc, &i1); 3784 + fts5DoclistIterInit(p2, bDesc, &i2); 3787 3785 while( i1.aPoslist!=0 || i2.aPoslist!=0 ){ 3788 3786 if( i2.aPoslist==0 || (i1.aPoslist && 3789 - ( (!bAsc && i1.iRowid>i2.iRowid) || (bAsc && i1.iRowid<i2.iRowid) ) 3787 + ( (bDesc && i1.iRowid>i2.iRowid) || (!bDesc && i1.iRowid<i2.iRowid) ) 3790 3788 )){ 3791 3789 /* Copy entry from i1 */ 3792 - fts5MergeAppendDocid(&p->rc, bAsc, &out, &iLastRowid, i1.iRowid); 3790 + fts5MergeAppendDocid(&p->rc, bDesc, &out, &iLastRowid, i1.iRowid); 3793 3791 fts5BufferAppendVarint(&p->rc, &out, i1.nPoslist); 3794 3792 fts5BufferAppendBlob(&p->rc, &out, i1.nPoslist, i1.aPoslist); 3795 3793 fts5DoclistIterNext(&i1); 3796 3794 } 3797 3795 else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){ 3798 3796 /* Copy entry from i2 */ 3799 - fts5MergeAppendDocid(&p->rc, bAsc, &out, &iLastRowid, i2.iRowid); 3797 + fts5MergeAppendDocid(&p->rc, bDesc, &out, &iLastRowid, i2.iRowid); 3800 3798 fts5BufferAppendVarint(&p->rc, &out, i2.nPoslist); 3801 3799 fts5BufferAppendBlob(&p->rc, &out, i2.nPoslist, i2.aPoslist); 3802 3800 fts5DoclistIterNext(&i2); 3803 3801 } 3804 3802 else{ 3805 3803 Fts5PoslistReader r1; 3806 3804 Fts5PoslistReader r2; 3807 3805 Fts5PoslistWriter writer; 3808 3806 3809 3807 memset(&writer, 0, sizeof(writer)); 3810 3808 3811 3809 /* Merge the two position lists. */ 3812 - fts5MergeAppendDocid(&p->rc, bAsc, &out, &iLastRowid, i2.iRowid); 3810 + fts5MergeAppendDocid(&p->rc, bDesc, &out, &iLastRowid, i2.iRowid); 3813 3811 fts5BufferZero(&tmp); 3814 3812 sqlite3Fts5PoslistReaderInit(-1, i1.aPoslist, i1.nPoslist, &r1); 3815 3813 sqlite3Fts5PoslistReaderInit(-1, i2.aPoslist, i2.nPoslist, &r2); 3816 3814 while( p->rc==SQLITE_OK && (r1.bEof==0 || r2.bEof==0) ){ 3817 3815 i64 iNew; 3818 3816 if( r2.bEof || (r1.bEof==0 && r1.iPos<r2.iPos) ){ 3819 3817 iNew = r1.iPos; ................................................................................ 3843 3841 Fts5Buffer tmp = *p1; 3844 3842 *p1 = *p2; 3845 3843 *p2 = tmp; 3846 3844 } 3847 3845 3848 3846 static void fts5SetupPrefixIter( 3849 3847 Fts5Index *p, /* Index to read from */ 3850 - int bAsc, /* True for "ORDER BY rowid ASC" */ 3848 + int bDesc, /* True for "ORDER BY rowid DESC" */ 3851 3849 const u8 *pToken, /* Buffer containing prefix to match */ 3852 3850 int nToken, /* Size of buffer pToken in bytes */ 3853 3851 Fts5IndexIter *pIter /* Populate this object */ 3854 3852 ){ 3855 3853 Fts5Structure *pStruct; 3856 3854 Fts5Buffer *aBuf; 3857 3855 const int nBuf = 32; ................................................................................ 3874 3872 i64 iRowid = fts5MultiIterRowid(p1); 3875 3873 int nTerm; 3876 3874 const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm); 3877 3875 assert( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); 3878 3876 if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break; 3879 3877 3880 3878 if( doclist.n>0 3881 - && ((!bAsc && iRowid>=iLastRowid) || (bAsc && iRowid<=iLastRowid)) 3879 + && ((!bDesc && iRowid<=iLastRowid) || (bDesc && iRowid>=iLastRowid)) 3882 3880 ){ 3883 3881 3884 3882 for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ 3885 3883 assert( i<nBuf ); 3886 3884 if( aBuf[i].n==0 ){ 3887 3885 fts5BufferSwap(&doclist, &aBuf[i]); 3888 3886 fts5BufferZero(&doclist); 3889 3887 }else{ 3890 - fts5MergePrefixLists(p, bAsc, &doclist, &aBuf[i]); 3888 + fts5MergePrefixLists(p, bDesc, &doclist, &aBuf[i]); 3891 3889 fts5BufferZero(&aBuf[i]); 3892 3890 } 3893 3891 } 3894 3892 } 3895 3893 if( doclist.n==0 ){ 3896 3894 fts5BufferAppendVarint(&p->rc, &doclist, iRowid); 3897 - }else if( bAsc==0 ){ 3895 + }else if( bDesc ){ 3898 3896 fts5BufferAppendVarint(&p->rc, &doclist, iLastRowid - iRowid); 3899 3897 }else{ 3900 3898 fts5BufferAppendVarint(&p->rc, &doclist, iRowid - iLastRowid); 3901 3899 } 3902 3900 iLastRowid = iRowid; 3903 3901 fts5MultiIterPoslist(p, p1, 1, &doclist); 3904 3902 } 3905 3903 3906 3904 for(i=0; i<nBuf; i++){ 3907 - fts5MergePrefixLists(p, bAsc, &doclist, &aBuf[i]); 3905 + fts5MergePrefixLists(p, bDesc, &doclist, &aBuf[i]); 3908 3906 fts5BufferFree(&aBuf[i]); 3909 3907 } 3910 3908 fts5MultiIterFree(p, p1); 3911 3909 3912 3910 pDoclist = (Fts5DoclistIter*)fts5IdxMalloc(p, sizeof(Fts5DoclistIter)); 3913 3911 if( !pDoclist ){ 3914 3912 fts5BufferFree(&doclist); 3915 3913 }else{ 3916 3914 pIter->pDoclist = pDoclist; 3917 - fts5DoclistIterInit(&doclist, bAsc, pIter->pDoclist); 3915 + fts5DoclistIterInit(&doclist, bDesc, pIter->pDoclist); 3918 3916 } 3919 3917 } 3920 3918 3921 3919 fts5StructureRelease(pStruct); 3922 3920 sqlite3_free(aBuf); 3923 3921 } 3924 3922 ................................................................................ 4269 4267 pRet->pStruct = fts5StructureRead(p, iIdx); 4270 4268 if( pRet->pStruct ){ 4271 4269 fts5MultiIterNew(p, pRet->pStruct, 4272 4270 iIdx, 1, flags, (const u8*)pToken, nToken, -1, 0, &pRet->pMulti 4273 4271 ); 4274 4272 } 4275 4273 }else{ 4276 - int bAsc = (flags & FTS5INDEX_QUERY_ASC)!=0; 4277 - fts5SetupPrefixIter(p, bAsc, (const u8*)pToken, nToken, pRet); 4274 + int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; 4275 + fts5SetupPrefixIter(p, bDesc, (const u8*)pToken, nToken, pRet); 4278 4276 } 4279 4277 } 4280 4278 4281 4279 if( p->rc ){ 4282 4280 sqlite3Fts5IterClose(pRet); 4283 4281 pRet = 0; 4284 4282 } ................................................................................ 4317 4315 ** matching rowid that occurs at or after iMatch. The definition of "at 4318 4316 ** or after" depends on whether this iterator iterates in ascending or 4319 4317 ** descending rowid order. 4320 4318 */ 4321 4319 static void fts5DoclistIterNextFrom(Fts5DoclistIter *p, i64 iMatch){ 4322 4320 do{ 4323 4321 i64 iRowid = p->iRowid; 4324 - if( p->bAsc!=0 && iRowid>=iMatch ) break; 4325 - if( p->bAsc==0 && iRowid<=iMatch ) break; 4322 + if( p->bDesc==0 && iRowid>=iMatch ) break; 4323 + if( p->bDesc!=0 && iRowid<=iMatch ) break; 4326 4324 fts5DoclistIterNext(p); 4327 4325 }while( p->aPoslist ); 4328 4326 } 4329 4327 4330 4328 /* 4331 4329 ** Move to the next matching rowid that occurs at or after iMatch. The 4332 4330 ** definition of "at or after" depends on whether this iterator iterates ................................................................................ 4598 4596 int nPos; 4599 4597 iOff += fts5GetVarint32(&a[iOff], nPos); 4600 4598 iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos)); 4601 4599 if( iOff<n ){ 4602 4600 i64 iDelta; 4603 4601 iOff += sqlite3GetVarint(&a[iOff], (u64*)&iDelta); 4604 4602 if( iDelta==0 ) return iOff; 4605 - iDocid -= iDelta; 4603 + iDocid += iDelta; 4606 4604 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid); 4607 4605 } 4608 4606 } 4609 4607 4610 4608 return iOff; 4611 4609 } 4612 4610 ................................................................................ 4687 4685 iOff = iRowidOff; 4688 4686 }else if( iTermOff ){ 4689 4687 iOff = iTermOff; 4690 4688 }else{ 4691 4689 iOff = n; 4692 4690 } 4693 4691 fts5DecodePoslist(&rc, &s, &a[4], iOff-4); 4694 - 4695 4692 4696 4693 assert( iRowidOff==0 || iOff==iRowidOff ); 4697 4694 if( iRowidOff ){ 4698 4695 iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff); 4699 4696 } 4700 4697 4701 4698 assert( iTermOff==0 || iOff==iTermOff );
Changes to ext/fts5/test/fts5aa.test.
181 181 set y [doc] 182 182 set z [doc] 183 183 set rowid [expr int(rand() * 100)] 184 184 execsql { REPLACE INTO t1(rowid,x,y,z) VALUES($rowid, $x, $y, $z) } 185 185 } 186 186 execsql { INSERT INTO t1(t1) VALUES('integrity-check'); } 187 187 } {} 188 + if {$i==2} break 188 189 } 190 +#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} 189 191 190 192 #------------------------------------------------------------------------- 191 193 # 192 194 reset_db 193 195 do_execsql_test 8.0 { 194 196 CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3"); 195 197 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); ................................................................................ 308 310 do_execsql_test 13.1 { 309 311 CREATE VIRTUAL TABLE t1 USING fts5(x); 310 312 INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o'); 311 313 } {} 312 314 313 315 do_execsql_test 13.2 { 314 316 SELECT rowid FROM t1 WHERE t1 MATCH 'o'; 315 -} {2 1} 317 +} {1 2} 316 318 317 319 do_execsql_test 13.4 { 318 320 DELETE FROM t1 WHERE rowid=2; 319 321 } {} 320 322 321 323 do_execsql_test 13.5 { 322 324 SELECT rowid FROM t1 WHERE t1 MATCH 'o';
Changes to ext/fts5/test/fts5ab.test.
26 26 CREATE VIRTUAL TABLE t1 USING fts5(a, b); 27 27 INSERT INTO t1 VALUES('hello', 'world'); 28 28 INSERT INTO t1 VALUES('one two', 'three four'); 29 29 INSERT INTO t1(rowid, a, b) VALUES(45, 'forty', 'five'); 30 30 } 31 31 32 32 do_execsql_test 1.1 { 33 - SELECT * FROM t1; 33 + SELECT * FROM t1 ORDER BY rowid DESC; 34 34 } { forty five {one two} {three four} hello world } 35 35 36 36 do_execsql_test 1.2 { 37 - SELECT rowid FROM t1; 37 + SELECT rowid FROM t1 ORDER BY rowid DESC; 38 38 } {45 2 1} 39 39 40 40 do_execsql_test 1.3 { 41 41 SELECT rowid FROM t1 ORDER BY rowid ASC; 42 42 } {1 2 45} 43 43 44 44 do_execsql_test 1.4 { ................................................................................ 86 86 5 e {5 4} 87 87 6 f {4} 88 88 7 g {4} 89 89 8 x {6} 90 90 9 y {6} 91 91 10 z {6} 92 92 } { 93 - do_execsql_test 2.7.$tn { SELECT rowid FROM t1 WHERE t1 MATCH $expr } $res 93 + do_execsql_test 2.7.$tn.1 { 94 + SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid DESC 95 + } $res 96 + do_execsql_test 2.7.$tn.2 { 97 + SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid ASC 98 + } [lsort -integer $res] 94 99 } 95 100 96 101 #------------------------------------------------------------------------- 97 102 # 98 103 reset_db 99 104 do_execsql_test 3.0 { 100 105 CREATE VIRTUAL TABLE t1 USING fts5(a,b); ................................................................................ 123 128 3 {abase + abash} {1} 124 129 4 {abash + abase} {9} 125 130 5 {abaft + abashing} {8 5} 126 131 6 {abandon + abandoning} {10} 127 132 7 {"abashing abases abasement abaft abashing"} {8} 128 133 } { 129 134 do_execsql_test 3.2.$tn { 130 - SELECT rowid FROM t1 WHERE t1 MATCH $expr 135 + SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid DESC 131 136 } $res 132 137 } 133 138 134 139 do_execsql_test 3.3 { 135 140 SELECT rowid FROM t1 WHERE t1 MATCH 'NEAR(aback abate, 2)' 136 141 } {6} 137 142 ................................................................................ 141 146 3 {abase + abash} {1} 142 147 4 {abash + abase} {9} 143 148 5 {abaft + abashing} {5 8} 144 149 6 {abandon + abandoning} {10} 145 150 7 {"abashing abases abasement abaft abashing"} {8} 146 151 } { 147 152 do_execsql_test 3.4.$tn { 148 - SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid ASC 153 + SELECT rowid FROM t1 WHERE t1 MATCH $expr 149 154 } $res 150 155 } 151 156 152 157 #------------------------------------------------------------------------- 153 158 # Documents with more than 2M tokens. 154 159 # 155 160 ................................................................................ 161 166 2 "[string repeat {a a } 1500000] x" \ 162 167 ] { 163 168 do_execsql_test 4.$tn { INSERT INTO s1 VALUES($doc) } 164 169 } 165 170 166 171 do_execsql_test 4.3 { 167 172 SELECT rowid FROM s1 WHERE s1 MATCH 'x' 168 -} {2 1} 173 +} {1 2} 169 174 170 175 do_execsql_test 4.4 { 171 176 SELECT rowid FROM s1 WHERE s1 MATCH '"a x"' 172 -} {2 1} 177 +} {1 2} 173 178 174 179 #------------------------------------------------------------------------- 175 180 # Check that a special case of segment promotion works. The case is where 176 181 # a new segment is written to level L, but the oldest segment within level 177 182 # (L-2) is larger than it. 178 183 # 179 184 do_execsql_test 5.0 { ................................................................................ 229 234 } {0 1} 230 235 231 236 do_test 5.4 { 232 237 execsql { INSERT INTO s2 VALUES(rnddoc(160)) } 233 238 fts5_level_segs s2 234 239 } {2 0} 235 240 241 +#------------------------------------------------------------------------- 242 +# 243 +do_execsql_test 6.0 { 244 + CREATE VIRTUAL TABLE s3 USING fts5(x); 245 + BEGIN; 246 + INSERT INTO s3 VALUES('a b c'); 247 + INSERT INTO s3 VALUES('A B C'); 248 +} 249 + 250 +do_execsql_test 6.1 { 251 + SELECT rowid FROM s3 WHERE s3 MATCH 'a' 252 +} {2 1} 253 + 254 +do_execsql_test 6.2 { 255 + COMMIT; 256 + SELECT rowid FROM s3 WHERE s3 MATCH 'a' 257 +} {2 1} 236 258 237 259 finish_test 238 260
Changes to ext/fts5/test/fts5ac.test.
130 130 99 {r c v w i v h a t a c v c r e} {h h u m g o f b a e o} 131 131 } 132 132 133 133 do_test 1.1 { 134 134 foreach {id x y} $data { 135 135 execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) } 136 136 } 137 + execsql { INSERT INTO xx(xx) VALUES('integrity-check') } 137 138 } {} 138 139 139 140 # Usage: 140 141 # 141 142 # poslist aCol ?-pc VARNAME? ?-near N? ?-col C? -- phrase1 phrase2... 142 143 # 143 144 proc poslist {aCol args} { ................................................................................ 249 250 # formatted as follows: 250 251 # 251 252 # <rowid> {<phrase 0 matches> <phrase 1 matches>...} 252 253 # 253 254 # where each <phrase X matches> element is a list of phrase matches in the 254 255 # same form as returned by auxiliary scalar function fts5_test(). 255 256 # 256 -proc matchdata {bPos expr {bAsc 0}} { 257 +proc matchdata {bPos expr {bAsc 1}} { 257 258 258 259 set tclexpr [db one {SELECT fts5_expr_tcl($expr, 'nearset $cols', 'x', 'y')}] 259 260 set res [list] 260 261 261 262 #puts $tclexpr 262 263 foreach {id x y} $::data { 263 264 set cols [list $x $y] ................................................................................ 303 304 304 305 sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist 305 306 306 307 #------------------------------------------------------------------------- 307 308 # Test phrase queries. 308 309 # 309 310 foreach {tn phrase} { 311 + 8 "c" 312 + 310 313 1 "o" 311 314 2 "b q" 312 315 3 "e a e" 313 316 4 "m d g q q b k b w f q q p p" 314 317 5 "l o o l v v k" 315 318 6 "a" 316 319 7 "b" ................................................................................ 396 399 #------------------------------------------------------------------------- 397 400 # 398 401 do_execsql_test 6.integrity { 399 402 INSERT INTO xx(xx) VALUES('integrity-check'); 400 403 } 401 404 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r} 402 405 foreach {bAsc sql} { 403 - 0 {SELECT rowid FROM xx WHERE xx MATCH $expr} 404 - 1 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid ASC} 406 + 1 {SELECT rowid FROM xx WHERE xx MATCH $expr} 407 + 0 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid DESC} 405 408 } { 406 409 foreach {tn expr} { 407 410 0.1 x 408 411 1 { NEAR(r c) } 409 412 2 { NEAR(r c, 5) } 410 413 3 { NEAR(r c, 3) } 411 414 4 { NEAR(r c, 2) }
Changes to ext/fts5/test/fts5ad.test.
32 32 foreach {tn match res} { 33 33 1 {c*} {1} 34 34 2 {i*} {3 2} 35 35 3 {t*} {3 1} 36 36 4 {r*} {3 1} 37 37 } { 38 38 do_execsql_test 1.$tn { 39 - SELECT rowid FROM yy WHERE yy MATCH $match 39 + SELECT rowid FROM yy WHERE yy MATCH $match ORDER BY rowid DESC 40 40 } $res 41 41 } 42 42 43 43 foreach {tn match res} { 44 44 5 {c*} {1} 45 45 6 {i*} {2 3} 46 46 7 {t*} {1 3} 47 47 8 {r*} {1 3} 48 48 } { 49 49 do_execsql_test 1.$tn { 50 - SELECT rowid FROM yy WHERE yy MATCH $match ORDER BY rowid ASC 50 + SELECT rowid FROM yy WHERE yy MATCH $match 51 51 } $res 52 52 } 53 53 54 54 foreach {T create} { 55 55 2 { 56 56 CREATE VIRTUAL TABLE t1 USING fts5(a, b); 57 57 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); ................................................................................ 190 190 } 191 191 if {$bMatch} { lappend ret $rowid } 192 192 } 193 193 return $ret 194 194 } 195 195 196 196 foreach {bAsc sql} { 197 - 0 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix} 198 - 1 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix ORDER BY rowid ASC} 197 + 1 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix} 198 + 0 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix ORDER BY rowid DESC} 199 199 } { 200 200 foreach {tn prefix} { 201 201 1 {a*} 2 {ab*} 3 {abc*} 4 {abcd*} 5 {abcde*} 202 202 6 {f*} 7 {fg*} 8 {fgh*} 9 {fghi*} 10 {fghij*} 203 203 11 {k*} 12 {kl*} 13 {klm*} 14 {klmn*} 15 {klmno*} 204 204 16 {p*} 17 {pq*} 18 {pqr*} 19 {pqrs*} 20 {pqrst*} 205 205 21 {u*} 22 {uv*} 23 {uvw*} 24 {uvwx*} 25 {uvwxy*} 26 {uvwxyz*}
Changes to ext/fts5/test/fts5ae.test.
105 105 106 106 do_execsql_test 3.3 { 107 107 INSERT INTO t3 108 108 VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o'); 109 109 SELECT rowid, fts5_test_poslist(t3) 110 110 FROM t3 WHERE t3 MATCH 'a OR b AND c'; 111 111 } { 112 - 3 0.0.5 113 112 1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15 2.1.2} 113 + 3 0.0.5 114 114 } 115 115 116 116 #------------------------------------------------------------------------- 117 117 # 118 118 do_execsql_test 4.0 { 119 119 CREATE VIRTUAL TABLE t4 USING fts5(x, y); 120 120 INSERT INTO t4 ................................................................................ 186 186 INSERT INTO t6 VALUES('There are more', 'things in heaven and earth'); 187 187 INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.'); 188 188 } 189 189 190 190 do_execsql_test 6.2 { 191 191 SELECT rowid, fts5_test_tokenize(t6) FROM t6 WHERE t6 MATCH 't*' 192 192 } { 193 - 2 {{horatio than are} {dreamt of in your philosophy}} 194 193 1 {{there are more} {things in heaven and earth}} 194 + 2 {{horatio than are} {dreamt of in your philosophy}} 195 195 } 196 196 197 197 #------------------------------------------------------------------------- 198 198 # Test the xQueryPhrase() API 199 199 # 200 200 reset_db 201 201 fts5_aux_test_functions db
Changes to ext/fts5/test/fts5ak.test.
36 36 INSERT INTO ft1 VALUES('i d i g c d c h b f'); 37 37 INSERT INTO ft1 VALUES('g d a e h a b c f j'); 38 38 } 39 39 40 40 do_execsql_test 1.2 { 41 41 SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'e'; 42 42 } { 43 - {g d a [e] h a b c f j} 44 - {i c c f a d g h j [e]} 45 - {j f c [e] d a h j d b} 43 + {[e] j a [e] f h b f h h} 44 + {d c j d c j b c g [e]} 46 45 {i a d [e] g j g d a a} 47 - {d c j d c j b c g [e]} 48 - {[e] j a [e] f h b f h h} 46 + {j f c [e] d a h j d b} 47 + {i c c f a d g h j [e]} 48 + {g d a [e] h a b c f j} 49 49 } 50 50 51 51 do_execsql_test 1.3 { 52 52 SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'h + d'; 53 53 } { 54 - {j f [h d] g h i b d f} 55 54 {[h d] b j c c g a c a} 55 + {j f [h d] g h i b d f} 56 56 } 57 57 58 58 do_execsql_test 1.4 { 59 59 SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d'; 60 60 } { 61 61 {i [d d] a g i b g [d d]} 62 62 } 63 63 64 64 do_execsql_test 1.5 { 65 65 SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'e e e' 66 66 } { 67 - {g d a [e] h a b c f j} 68 - {i c c f a d g h j [e]} 69 - {j f c [e] d a h j d b} 67 + {[e] j a [e] f h b f h h} 68 + {d c j d c j b c g [e]} 70 69 {i a d [e] g j g d a a} 71 - {d c j d c j b c g [e]} 72 - {[e] j a [e] f h b f h h} 70 + {j f c [e] d a h j d b} 71 + {i c c f a d g h j [e]} 72 + {g d a [e] h a b c f j} 73 73 } 74 74 75 75 do_execsql_test 1.6 { 76 76 SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d d + d'; 77 77 } { 78 78 {i [d d] a g i b g [d d]} 79 79 } ................................................................................ 129 129 130 130 -- The following SELECT statement returns these three rows: 131 131 -- '[a b c] x [c d e]' 132 132 -- '[a b c] [c d e]' 133 133 -- '[a b c d e]' 134 134 SELECT highlight(ft, 0, '[', ']') FROM ft WHERE ft MATCH 'a+b+c AND c+d+e'; 135 135 } { 136 - {[a b c d e]} 136 + {[a b c] x [c d e]} 137 137 {[a b c] [c d e]} 138 - {[a b c] x [c d e]} 138 + {[a b c d e]} 139 139 } 140 140 141 141 142 142 finish_test 143 143
Changes to ext/fts5/test/fts5al.test.
100 100 } {{123 456} {123 456}} 101 101 102 102 proc rowidtest {cmd} { $cmd xRowid } 103 103 sqlite3_fts5_create_function db rowidtest rowidtest 104 104 105 105 do_execsql_test 3.3.1 { 106 106 SELECT rowidtest(t1) FROM t1 WHERE t1 MATCH 'q' 107 -} {2 1} 107 +} {1 2} 108 108 109 109 proc insttest {cmd} { 110 110 set res [list] 111 111 for {set i 0} {$i < [$cmd xInstCount]} {incr i} { 112 112 lappend res [$cmd xInst $i] 113 113 } 114 114 set res 115 115 } 116 116 sqlite3_fts5_create_function db insttest insttest 117 117 118 118 do_execsql_test 3.4.1 { 119 119 SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'q' 120 120 } { 121 - {{0 0 5}} 122 121 {{0 0 0}} 122 + {{0 0 5}} 123 123 } 124 124 125 125 do_execsql_test 3.4.2 { 126 126 SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'r+e OR w' 127 127 } { 128 - {{0 0 2} {1 0 4}} 129 128 {{1 0 1}} 129 + {{0 0 2} {1 0 4}} 130 130 } 131 131 132 132 proc coltest {cmd} { 133 133 list [$cmd xColumnSize 0] [$cmd xColumnText 0] 134 134 } 135 135 sqlite3_fts5_create_function db coltest coltest 136 136 137 137 do_execsql_test 3.5.1 { 138 138 SELECT coltest(t1) FROM t1 WHERE t1 MATCH 'q' 139 139 } { 140 - {6 {y t r e w q}} {6 {q w e r t y}} 140 + {6 {q w e r t y}} 141 + {6 {y t r e w q}} 141 142 } 142 143 143 144 #------------------------------------------------------------------------- 144 145 # Tests for remapping the "rank" column. 145 146 # 146 147 # 4.1.*: Mapped to a function with no arguments. 147 148 # 4.2.*: Mapped to a function with one or more arguments. ................................................................................ 184 185 } 185 186 186 187 do_execsql_test 4.1.3 { 187 188 SELECT rowid, rank FROM t2 188 189 WHERE t2 MATCH 'a' AND rank MATCH 'firstinst()' 189 190 ORDER BY rank DESC 190 191 } { 191 - 5 103 9 102 6 9 10 8 3 6 2 4 7 0 1 0 192 + 5 103 9 102 6 9 10 8 3 6 2 4 1 0 7 0 192 193 } 193 194 194 195 do_execsql_test 4.1.4 { 195 196 INSERT INTO t2(t2, rank) VALUES('rank', 'firstinst()'); 196 197 SELECT rowid, rank FROM t2 WHERE t2 MATCH 'a' ORDER BY rowid ASC 197 198 } { 198 199 1 0 2 4 3 6 5 103 199 200 6 9 7 0 9 102 10 8 200 201 } 201 202 202 203 do_execsql_test 4.1.5 { 203 204 SELECT rowid, rank FROM t2 WHERE t2 MATCH 'a' ORDER BY rank DESC 204 205 } { 205 - 5 103 9 102 6 9 10 8 3 6 2 4 7 0 1 0 206 + 5 103 9 102 6 9 10 8 3 6 2 4 1 0 7 0 206 207 } 207 208 208 209 do_execsql_test 4.1.6 { 209 210 INSERT INTO t2(t2, rank) VALUES('rank', 'firstinst ( ) '); 210 211 SELECT rowid, rank FROM t2 WHERE t2 MATCH 'a' ORDER BY rank DESC 211 212 } { 212 - 5 103 9 102 6 9 10 8 3 6 2 4 7 0 1 0 213 + 5 103 9 102 6 9 10 8 3 6 2 4 1 0 7 0 213 214 } 214 215 215 216 proc rowidplus {cmd ival} { 216 217 expr [$cmd xRowid] + $ival 217 218 } 218 219 sqlite3_fts5_create_function db rowidplus rowidplus 219 220 ................................................................................ 253 254 breakpoint 254 255 255 256 do_execsql_test 4.3.2 { 256 257 SELECT * FROM t3 257 258 WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(4)' 258 259 ORDER BY rank ASC 259 260 } { 260 - {a four} {a five} {a one} {a two} {a three} 261 + {a four} {a one} {a five} {a two} {a three} 261 262 } 262 263 do_execsql_test 4.3.3 { 263 264 SELECT *, rank FROM t3 264 265 WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(3)' 265 266 ORDER BY rank ASC 266 267 } { 267 - {a three} 0 {a four} 1 {a one} 1 {a five} 2 {a two} 2 268 + {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 268 269 } 269 270 270 271 271 272 finish_test 272 273
Changes to ext/fts5/test/fts5content.test.
22 22 INSERT INTO f1(rowid, a, b) VALUES(1, 'one', 'o n e'); 23 23 INSERT INTO f1(rowid, a, b) VALUES(2, 'two', 't w o'); 24 24 INSERT INTO f1(rowid, a, b) VALUES(3, 'three', 't h r e e'); 25 25 } 26 26 27 27 do_execsql_test 1.2 { 28 28 SELECT rowid FROM f1 WHERE f1 MATCH 'o'; 29 -} {2 1} 29 +} {1 2} 30 30 31 31 do_execsql_test 1.3 { 32 32 INSERT INTO f1(a, b) VALUES('four', 'f o u r'); 33 33 SELECT rowid FROM f1 WHERE f1 MATCH 'o'; 34 -} {4 2 1} 34 +} {1 2 4} 35 35 36 36 do_execsql_test 1.4 { 37 37 SELECT rowid, a, b FROM f1 WHERE f1 MATCH 'o'; 38 -} {4 {} {} 2 {} {} 1 {} {}} 38 +} {1 {} {} 2 {} {} 4 {} {}} 39 39 40 40 do_execsql_test 1.5 { 41 41 SELECT rowid, highlight(f1, 0, '[', ']') FROM f1 WHERE f1 MATCH 'o'; 42 -} {4 {} 2 {} 1 {}} 42 +} {1 {} 2 {} 4 {}} 43 43 44 44 do_execsql_test 1.6 { 45 45 SELECT rowid, highlight(f1, 0, '[', ']') IS NULL FROM f1 WHERE f1 MATCH 'o'; 46 -} {4 1 2 1 1 1} 46 +} {1 1 2 1 4 1} 47 47 48 48 do_execsql_test 1.7 { 49 49 SELECT rowid, snippet(f1, -1, '[', ']', '...', 5) IS NULL 50 50 FROM f1 WHERE f1 MATCH 'o'; 51 -} {4 1 2 1 1 1} 51 +} {1 1 2 1 4 1} 52 52 53 53 do_execsql_test 1.8 { 54 54 SELECT rowid, snippet(f1, 1, '[', ']', '...', 5) IS NULL 55 55 FROM f1 WHERE f1 MATCH 'o'; 56 -} {4 1 2 1 1 1} 56 +} {1 1 2 1 4 1} 57 57 58 58 do_execsql_test 1.9 { 59 59 SELECT rowid FROM f1; 60 -} {4 3 2 1} 60 +} {1 2 3 4} 61 61 62 62 do_execsql_test 1.10 { 63 63 SELECT * FROM f1; 64 64 } {{} {} {} {} {} {} {} {}} 65 65 66 66 do_execsql_test 1.11 { 67 67 SELECT rowid, a, b FROM f1 ORDER BY rowid ASC; ................................................................................ 81 81 82 82 do_execsql_test 1.15 { 83 83 INSERT INTO f1(f1, rowid, a, b) VALUES('delete', 2, 'two', 't w o'); 84 84 } {} 85 85 86 86 do_execsql_test 1.16 { 87 87 SELECT rowid FROM f1 WHERE f1 MATCH 'o'; 88 -} {4 1} 88 +} {1 4} 89 89 90 90 do_execsql_test 1.17 { 91 91 SELECT rowid FROM f1; 92 -} {4 3 1} 92 +} {1 3 4} 93 93 94 94 #------------------------------------------------------------------------- 95 95 # External content tables 96 96 # 97 97 reset_db 98 98 do_execsql_test 2.1 { 99 99 -- Create a table. And an external content fts5 table to index it.
Changes to ext/fts5/test/fts5corrupt.test.
49 49 catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } 50 50 } {1 {database disk image is malformed}} 51 51 52 52 db_restore_and_reopen 53 53 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} 54 54 55 55 56 - 57 56 #-------------------------------------------------------------------- 57 +# 58 58 do_execsql_test 2.0 { 59 59 CREATE VIRTUAL TABLE t2 USING fts5(x); 60 - INSERT INTO t2(t2, rank) VALUES('pgsz', 32); 60 + INSERT INTO t2(t2, rank) VALUES('pgsz', 64); 61 61 } 62 +db func rnddoc fts5_rnddoc 62 63 do_test 2.1 { 63 - db transaction { 64 - for {set i 0} {$i < 20} {incr i} { 65 - execsql { INSERT INTO t2 VALUES('xxxxxxxxxx') } 66 - } 67 - for {set i 0} {$i < 20} {incr i} { 68 - execsql { INSERT INTO t2 VALUES('xxxxxxxxxzzzz') } 69 - } 64 + for {set i 0} {$i < 500} {incr i} { 65 + execsql { INSERT INTO t2 VALUES(rnddoc(50)) } 66 + execsql { INSERT INTO t2(t2) VALUES('integrity-check') } 70 67 } 71 68 } {} 72 -db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t2_data} {puts $r} 69 + 70 +#-------------------------------------------------------------------- 71 +# 73 72 74 73 finish_test 75 74
Changes to ext/fts5/test/fts5fault1.test.
86 86 INSERT INTO t2 VALUES('k fe fd rd a gi ho kk', 'ng m c r d ml rm r'); 87 87 } 88 88 faultsim_save_and_close 89 89 90 90 foreach {tn expr res} { 91 91 1 { dk } 7 92 92 2 { m f } 1 93 - 3 { f* } {10 9 8 6 5 4 3 1} 94 - 4 { m OR f } {10 9 8 5 4 1} 93 + 3 { f* } {1 3 4 5 6 8 9 10} 94 + 4 { m OR f } {1 4 5 8 9 10} 95 95 5 { sn + gh } {5} 96 96 6 { "sn gh" } {5} 97 97 7 { NEAR(r a, 5) } {9} 98 - 8 { m* f* } {10 9 8 6 4 1} 99 - 9 { m* + f* } {8 1} 98 + 8 { m* f* } {1 4 6 8 9 10} 99 + 9 { m* + f* } {1 8} 100 100 } { 101 101 do_faultsim_test 4.$tn -prep { 102 102 faultsim_restore_and_reopen 103 103 } -body " 104 104 execsql { SELECT rowid FROM t2 WHERE t2 MATCH '$expr' } 105 105 " -test " 106 106 faultsim_test_result {[list 0 $res]} ................................................................................ 298 298 299 299 reset_db 300 300 db func rnddoc rnddoc 301 301 302 302 do_test 8.0 { 303 303 execsql { CREATE VIRTUAL TABLE x1 USING fts5(a) } 304 304 set ::res [list] 305 - for {set i 100} {$i>0} {incr i -1} { 305 + for {set i 1} {$i<100} {incr i 1} { 306 306 execsql { INSERT INTO x1 VALUES( rnddoc(50) ) } 307 307 lappend ::res $i 308 308 } 309 309 } {} 310 310 311 311 do_faultsim_test 8.1 -faults oom* -prep { 312 312 } -body {
Changes to ext/fts5/test/fts5rowid.test.
168 168 } 169 169 foreach str $strlist { execsql { INSERT INTO x4 VALUES($str) } } 170 170 execsql COMMIT 171 171 } {} 172 172 173 173 do_execsql_test 5.1 { 174 174 SELECT rowid FROM x4 WHERE x4 MATCH 'a' 175 -} {4 3 2 1} 175 +} {1 2 3 4} 176 176 177 177 set res [db one {SELECT count(*) FROM x4_data}] 178 178 do_execsql_test 5.2 { 179 179 SELECT count(fts5_decode(rowid, block)) FROM x4_data; 180 180 } $res 181 181 182 182 finish_test 183 183