Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch fts4aux Excluding Merge-Ins
This is equivalent to a diff from ed759d5a to 0147d973
2011-06-01
| ||
11:26 | Toward version 3.7.5.1. This check-in updates the version number and also patches the pager so that it never spills cache while in WAL mode. The latter patch fixes ticket [2d1a5c67dfc23] and also prevents excessive WAL file sizes if the pager cache starts thrashing. A few test cases fail. (Leaf check-in: 2e5a912a user: drh tags: branch-3.7.5) | |
2011-02-10
| ||
01:49 | This is a version of the SQLite 3.7.5 release with Apple's changes for MacOS. (check-in: 55d2e55b user: drh tags: apple-osx) | |
2011-02-02
| ||
04:40 | Merge fts4aux branch. (check-in: c6d9f7d8 user: dan tags: trunk) | |
04:26 | Add missing file fts3_aux.c. (Closed-Leaf check-in: 0147d973 user: dan tags: fts4aux) | |
2011-02-01
| ||
18:59 | Remove extra instances of the text "checked out." from sqlite.h.in. Also add new file fts3_aux.c to mksqlite3c.tcl. (check-in: 9897da22 user: dan tags: fts4aux) | |
16:34 | Add virtual table module "fts4aux", used to inspect the full-text index of an fts4 table directly. Also add the "compress" and "uncompress" fts4 options. (check-in: b010ddcc user: dan tags: fts4aux) | |
00:04 | Version 3.7.5 (check-in: ed759d5a user: drh tags: trunk, release, version-3.7.5) | |
2011-01-28
| ||
16:45 | Add test script pagerfault3.test. No changes to code or existing tests. (check-in: 682fe41e user: dan tags: trunk) | |
Changes to ext/fts3/fts3.c.
︙ | ︙ | |||
444 445 446 447 448 449 450 451 452 453 454 455 456 457 | assert( p->pSegments==0 ); /* Free any prepared statements held */ for(i=0; i<SizeofArray(p->aStmt); i++){ sqlite3_finalize(p->aStmt[i]); } sqlite3_free(p->zSegmentsTbl); /* Invoke the tokenizer destructor to free the tokenizer. */ p->pTokenizer->pModule->xDestroy(p->pTokenizer); sqlite3_free(p); return SQLITE_OK; } | > > | 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 | assert( p->pSegments==0 ); /* Free any prepared statements held */ for(i=0; i<SizeofArray(p->aStmt); i++){ sqlite3_finalize(p->aStmt[i]); } sqlite3_free(p->zSegmentsTbl); sqlite3_free(p->zReadExprlist); sqlite3_free(p->zWriteExprlist); /* Invoke the tokenizer destructor to free the tokenizer. */ p->pTokenizer->pModule->xDestroy(p->pTokenizer); sqlite3_free(p); return SQLITE_OK; } |
︙ | ︙ | |||
660 661 662 663 664 665 666 667 668 669 670 671 672 673 | zValue = sqlite3_mprintf("%s", &zCsr[1]); if( zValue ){ sqlite3Fts3Dequote(zValue); } *pzValue = zValue; return 1; } /* ** This function is the implementation of both the xConnect and xCreate ** methods of the FTS3 virtual table. ** ** The argv[] array contains the following: ** | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 | zValue = sqlite3_mprintf("%s", &zCsr[1]); if( zValue ){ sqlite3Fts3Dequote(zValue); } *pzValue = zValue; return 1; } /* ** Append the output of a printf() style formatting to an existing string. */ static void fts3Appendf( int *pRc, /* IN/OUT: Error code */ char **pz, /* IN/OUT: Pointer to string buffer */ const char *zFormat, /* Printf format string to append */ ... /* Arguments for printf format string */ ){ if( *pRc==SQLITE_OK ){ va_list ap; char *z; va_start(ap, zFormat); z = sqlite3_vmprintf(zFormat, ap); if( z && *pz ){ char *z2 = sqlite3_mprintf("%s%s", *pz, z); sqlite3_free(z); z = z2; } if( z==0 ) *pRc = SQLITE_NOMEM; sqlite3_free(*pz); *pz = z; } } /* ** Return a list of comma separated SQL expressions that could be used ** in a SELECT statement such as the following: ** ** SELECT <list of expressions> FROM %_content AS x ... ** ** to return the docid, followed by each column of text data in order ** from left to write. If parameter zFunc is not NULL, then instead of ** being returned directly each column of text data is passed to an SQL ** function named zFunc first. For example, if zFunc is "unzip" and the ** table has the three user-defined columns "a", "b", and "c", the following ** string is returned: ** ** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c')" ** ** The pointer returned points to a buffer allocated by sqlite3_malloc(). It ** is the responsibility of the caller to eventually free it. ** ** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and ** a NULL pointer is returned). Otherwise, if an OOM error is encountered ** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If ** no error occurs, *pRc is left unmodified. */ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ char *zRet = 0; int i; if( !zFunc ) zFunc = ""; fts3Appendf(pRc, &zRet, "docid"); for(i=0; i<p->nColumn; i++){ fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunc, i, p->azColumn[i]); } return zRet; } /* ** Return a list of N comma separated question marks, where N is the number ** of columns in the %_content table (one for the docid plus one for each ** user-defined text column). ** ** If argument zFunc is not NULL, then all but the first question mark ** is preceded by zFunc and an open bracket, and followed by a closed ** bracket. For example, if zFunc is "zip" and the FTS3 table has three ** user-defined text columns, the following string is returned: ** ** "?, zip(?), zip(?), zip(?)" ** ** The pointer returned points to a buffer allocated by sqlite3_malloc(). It ** is the responsibility of the caller to eventually free it. ** ** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and ** a NULL pointer is returned). Otherwise, if an OOM error is encountered ** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If ** no error occurs, *pRc is left unmodified. */ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ char *zRet = 0; int i; if( !zFunc ) zFunc = ""; fts3Appendf(pRc, &zRet, "?"); for(i=0; i<p->nColumn; i++){ fts3Appendf(pRc, &zRet, ",%s(?)", zFunc); } return zRet; } /* ** This function is the implementation of both the xConnect and xCreate ** methods of the FTS3 virtual table. ** ** The argv[] array contains the following: ** |
︙ | ︙ | |||
697 698 699 700 701 702 703 704 705 706 707 708 709 710 | int nDb; /* Bytes required to hold database name */ int nName; /* Bytes required to hold table name */ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ int bNoDocsize = 0; /* True to omit %_docsize table */ const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ assert( strlen(argv[0])==4 ); assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) ); nDb = (int)strlen(argv[1]) + 1; nName = (int)strlen(argv[2]) + 1; | > > > | 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 | int nDb; /* Bytes required to hold database name */ int nName; /* Bytes required to hold table name */ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ int bNoDocsize = 0; /* True to omit %_docsize table */ const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ char *zCompress = 0; char *zUncompress = 0; assert( strlen(argv[0])==4 ); assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) ); nDb = (int)strlen(argv[1]) + 1; nName = (int)strlen(argv[2]) + 1; |
︙ | ︙ | |||
747 748 749 750 751 752 753 754 755 756 757 758 759 760 | if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){ if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){ bNoDocsize = 1; }else{ *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); rc = SQLITE_ERROR; } }else{ *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); rc = SQLITE_ERROR; } sqlite3_free(zVal); } | > > > > > > | 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 | if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){ if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){ bNoDocsize = 1; }else{ *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); rc = SQLITE_ERROR; } }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){ zCompress = zVal; zVal = 0; }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){ zUncompress = zVal; zVal = 0; }else{ *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); rc = SQLITE_ERROR; } sqlite3_free(zVal); } |
︙ | ︙ | |||
820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 | memcpy(zCsr, z, n); zCsr[n] = '\0'; sqlite3Fts3Dequote(zCsr); p->azColumn[iCol] = zCsr; zCsr += n+1; assert( zCsr <= &((char *)p)[nByte] ); } /* If this is an xCreate call, create the underlying tables in the ** database. TODO: For xConnect(), it could verify that said tables exist. */ if( isCreate ){ rc = fts3CreateTables(p); } /* Figure out the page-size for the database. This is required in order to ** estimate the cost of loading large doclists from the database (see ** function sqlite3Fts3SegReaderCost() for details). */ fts3DatabasePageSize(&rc, p); /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); fts3_init_out: | > > > > > > > > > | > | 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 | memcpy(zCsr, z, n); zCsr[n] = '\0'; sqlite3Fts3Dequote(zCsr); p->azColumn[iCol] = zCsr; zCsr += n+1; assert( zCsr <= &((char *)p)[nByte] ); } if( (zCompress==0)!=(zUncompress==0) ){ char const *zMissing = (zCompress==0 ? "compress" : "uncompress"); rc = SQLITE_ERROR; *pzErr = sqlite3_mprintf("missing %s parameter", zMissing); } p->zReadExprlist = fts3ReadExprList(p, zUncompress, &rc); p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); if( rc!=SQLITE_OK ) goto fts3_init_out; /* If this is an xCreate call, create the underlying tables in the ** database. TODO: For xConnect(), it could verify that said tables exist. */ if( isCreate ){ rc = fts3CreateTables(p); } /* Figure out the page-size for the database. This is required in order to ** estimate the cost of loading large doclists from the database (see ** function sqlite3Fts3SegReaderCost() for details). */ fts3DatabasePageSize(&rc, p); /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); fts3_init_out: sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ fts3DisconnectMethod((sqlite3_vtab *)p); }else if( pTokenizer ){ pTokenizer->pModule->xDestroy(pTokenizer); } |
︙ | ︙ | |||
1931 1932 1933 1934 1935 1936 1937 | if( !*ppOut ) return SQLITE_NOMEM; sqlite3Fts3PutVarint(*ppOut, docid); } return SQLITE_OK; } | < < < < | | > > | > | > > > > > | | > > > > > > > > < > > | > | < | | > > | | < > | > > | > > > | > | > > > > > > > | | > | | > > > > > | > > > > > > > | | < | < > | | | < > < < < | < < | > > > > < | > | | < < | < < < | < < < < | | < < < < < < | < < < | | < | < < | > < < < < < < < < < | < < < < | < < < | < < | < < < < < < < | | > > > > > | 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 | if( !*ppOut ) return SQLITE_NOMEM; sqlite3Fts3PutVarint(*ppOut, docid); } return SQLITE_OK; } int sqlite3Fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ Fts3SegReaderCursor *pCsr /* Cursor object to populate */ ){ int rc = SQLITE_OK; int rc2; int iAge = 0; sqlite3_stmt *pStmt = 0; Fts3SegReader *pPending = 0; assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel==FTS3_SEGCURSOR_PENDING || iLevel>=0 ); assert( FTS3_SEGCURSOR_PENDING<0 ); assert( FTS3_SEGCURSOR_ALL<0 ); assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) ); memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); /* If iLevel is less than 0, include a seg-reader for the pending-terms. */ if( iLevel<0 ){ rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pPending); if( rc==SQLITE_OK && pPending ){ int nByte = (sizeof(Fts3SegReader *) * 16); pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); if( pCsr->apSegment==0 ){ rc = SQLITE_NOMEM; }else{ pCsr->apSegment[0] = pPending; pCsr->nSegment = 1; pPending = 0; } } } if( iLevel!=FTS3_SEGCURSOR_PENDING ){ if( rc==SQLITE_OK ){ rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt); } while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ /* Read the values returned by the SELECT into local variables. */ sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); int nRoot = sqlite3_column_bytes(pStmt, 4); char const *zRoot = sqlite3_column_blob(pStmt, 4); /* If nSegment is a multiple of 16 the array needs to be extended. */ if( (pCsr->nSegment%16)==0 ){ Fts3SegReader **apNew; int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); if( !apNew ){ rc = SQLITE_NOMEM; goto finished; } pCsr->apSegment = apNew; } /* If zTerm is not NULL, and this segment is not stored entirely on its ** root node, the range of leaves scanned can be reduced. Do this. */ if( iStartBlock && zTerm ){ sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); if( rc!=SQLITE_OK ) goto finished; if( isPrefix==0 ) iLeavesEndBlock = iStartBlock; } rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pCsr->apSegment[pCsr->nSegment] ); if( rc!=SQLITE_OK ) goto finished; pCsr->nSegment++; iAge++; } } finished: rc2 = sqlite3_reset(pStmt); if( rc==SQLITE_DONE ) rc = rc2; sqlite3Fts3SegReaderFree(pPending); return rc; } static int fts3TermSegReaderCursor( Fts3Cursor *pCsr, /* Virtual table cursor handle */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ Fts3SegReaderCursor **ppSegcsr /* OUT: Allocated seg-reader cursor */ ){ Fts3SegReaderCursor *pSegcsr; /* Object to allocate and return */ int rc = SQLITE_NOMEM; /* Return code */ pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor)); if( pSegcsr ){ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; int i; int nCost = 0; rc = sqlite3Fts3SegReaderCursor( p, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, pSegcsr); for(i=0; rc==SQLITE_OK && i<pSegcsr->nSegment; i++){ rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); } pSegcsr->nCost = nCost; } *ppSegcsr = pSegcsr; return rc; } static void fts3SegReaderCursorFree(Fts3SegReaderCursor *pSegcsr){ sqlite3Fts3SegReaderFinish(pSegcsr); sqlite3_free(pSegcsr); } /* ** This function retreives the doclist for the specified term (or term ** prefix) from the database. ** ** The returned doclist may be in one of two formats, depending on the ** value of parameter isReqPos. If isReqPos is zero, then the doclist is |
︙ | ︙ | |||
2077 2078 2079 2080 2081 2082 2083 | Fts3PhraseToken *pTok, /* Token to query for */ int iColumn, /* Column to query (or -ve for all columns) */ int isReqPos, /* True to include position lists in output */ int *pnOut, /* OUT: Size of buffer at *ppOut */ char **ppOut /* OUT: Malloced result buffer */ ){ int rc; /* Return code */ | | | | | | > > > | > | > > < | | | 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 | Fts3PhraseToken *pTok, /* Token to query for */ int iColumn, /* Column to query (or -ve for all columns) */ int isReqPos, /* True to include position lists in output */ int *pnOut, /* OUT: Size of buffer at *ppOut */ char **ppOut /* OUT: Malloced result buffer */ ){ int rc; /* Return code */ Fts3SegReaderCursor *pSegcsr; /* Seg-reader cursor for this term */ TermSelect tsc; /* Context object for fts3TermSelectCb() */ Fts3SegFilter filter; /* Segment term filter configuration */ pSegcsr = pTok->pSegcsr; memset(&tsc, 0, sizeof(TermSelect)); tsc.isReqPos = isReqPos; filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0) | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); filter.iCol = iColumn; filter.zTerm = pTok->z; filter.nTerm = pTok->n; rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); while( SQLITE_OK==rc && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) ){ rc = fts3TermSelectCb(p, (void *)&tsc, pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist ); } if( rc==SQLITE_OK ){ rc = fts3TermSelectMerge(&tsc); } if( rc==SQLITE_OK ){ *ppOut = tsc.aaOutput[0]; *pnOut = tsc.anOutput[0]; }else{ int i; for(i=0; i<SizeofArray(tsc.aaOutput); i++){ sqlite3_free(tsc.aaOutput[i]); } } fts3SegReaderCursorFree(pSegcsr); pTok->pSegcsr = 0; return rc; } /* ** This function counts the total number of docids in the doclist stored ** in buffer aList[], size nList bytes. ** |
︙ | ︙ | |||
2234 2235 2236 2237 2238 2239 2240 | /* If this is an xFilter() evaluation, create a segment-reader for each ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo ** evaluation, only create segment-readers if there are no Fts3DeferredToken ** objects attached to the phrase-tokens. */ for(ii=0; ii<pPhrase->nToken; ii++){ Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; | | | | | 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 | /* If this is an xFilter() evaluation, create a segment-reader for each ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo ** evaluation, only create segment-readers if there are no Fts3DeferredToken ** objects attached to the phrase-tokens. */ for(ii=0; ii<pPhrase->nToken; ii++){ Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; if( pTok->pSegcsr==0 ){ if( (pCsr->eEvalmode==FTS3_EVAL_FILTER) || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) ){ rc = fts3TermSegReaderCursor( pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr ); if( rc!=SQLITE_OK ) return rc; } } } for(ii=0; ii<pPhrase->nToken; ii++){ |
︙ | ︙ | |||
2271 2272 2273 2274 2275 2276 2277 | pTok = &pPhrase->aToken[iTok]; }else{ int nMinCost = 0x7FFFFFFF; int jj; /* Find the remaining token with the lowest cost. */ for(jj=0; jj<pPhrase->nToken; jj++){ | | | | | | | 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 | pTok = &pPhrase->aToken[iTok]; }else{ int nMinCost = 0x7FFFFFFF; int jj; /* Find the remaining token with the lowest cost. */ for(jj=0; jj<pPhrase->nToken; jj++){ Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[jj].pSegcsr; if( pSegcsr && pSegcsr->nCost<nMinCost ){ iTok = jj; nMinCost = pSegcsr->nCost; } } pTok = &pPhrase->aToken[iTok]; /* This branch is taken if it is determined that loading the doclist ** for the next token would require more IO than loading all documents ** currently identified by doclist pOut/nOut. No further doclists will ** be loaded from the full-text index for this phrase. */ if( nMinCost>nDoc && ii>0 ){ rc = fts3DeferExpression(pCsr, pCsr->pExpr); break; } } if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){ rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList); }else{ if( pTok->pSegcsr ){ rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList); } pTok->bFulltext = 1; } assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pSegcsr==0 ); if( rc!=SQLITE_OK ) break; if( isFirst ){ pOut = pList; nOut = nList; if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){ nDoc = fts3DoclistCountDocids(1, pOut, nOut); |
︙ | ︙ | |||
2476 2477 2478 2479 2480 2481 2482 | if( pExpr->eType==FTSQUERY_PHRASE ){ Fts3Phrase *pPhrase = pExpr->pPhrase; int ii; for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){ Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; | | | | | 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 | if( pExpr->eType==FTSQUERY_PHRASE ){ Fts3Phrase *pPhrase = pExpr->pPhrase; int ii; for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){ Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; if( pTok->pSegcsr==0 ){ rc = fts3TermSegReaderCursor( pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr ); } } }else{ rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr); if( rc==SQLITE_OK ){ rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr); |
︙ | ︙ | |||
2502 2503 2504 2505 2506 2507 2508 | */ static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ if( pExpr ){ Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase ){ int kk; for(kk=0; kk<pPhrase->nToken; kk++){ | | | | < | < | 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 | */ static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ if( pExpr ){ Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase ){ int kk; for(kk=0; kk<pPhrase->nToken; kk++){ fts3SegReaderCursorFree(pPhrase->aToken[kk].pSegcsr); pPhrase->aToken[kk].pSegcsr = 0; } } fts3ExprFreeSegReaders(pExpr->pLeft); fts3ExprFreeSegReaders(pExpr->pRight); } } /* ** Return the sum of the costs of all tokens in the expression pExpr. This ** function must be called after Fts3SegReaderArrays have been allocated ** for all tokens using fts3ExprAllocateSegReaders(). */ static int fts3ExprCost(Fts3Expr *pExpr){ int nCost; /* Return value */ if( pExpr->eType==FTSQUERY_PHRASE ){ Fts3Phrase *pPhrase = pExpr->pPhrase; int ii; nCost = 0; for(ii=0; ii<pPhrase->nToken; ii++){ Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[ii].pSegcsr; if( pSegcsr ) nCost += pSegcsr->nCost; } }else{ nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight); } return nCost; } |
︙ | ︙ | |||
2868 2869 2870 2871 2872 2873 2874 | sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ int idxNum, /* Strategy index */ const char *idxStr, /* Unused */ int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ const char *azSql[] = { | | | | 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 | sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ int idxNum, /* Strategy index */ const char *idxStr, /* Unused */ int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ const char *azSql[] = { "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?", /* non-full-scan */ "SELECT %s FROM %Q.'%q_content' AS x ", /* full-scan */ }; int rc; /* Return code */ char *zSql; /* SQL statement used to access %_content */ Fts3Table *p = (Fts3Table *)pCursor->pVtab; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; UNUSED_PARAMETER(idxStr); |
︙ | ︙ | |||
2924 2925 2926 2927 2928 2929 2930 | } /* Compile a SELECT statement for this cursor. For a full-table-scan, the ** statement loops through all rows of the %_content table. For a ** full-text query or docid lookup, the statement retrieves a single ** row by docid. */ | | > | 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 | } /* Compile a SELECT statement for this cursor. For a full-table-scan, the ** statement loops through all rows of the %_content table. For a ** full-text query or docid lookup, the statement retrieves a single ** row by docid. */ zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH]; zSql = sqlite3_mprintf(zSql, p->zReadExprlist, p->zDb, p->zName); if( !zSql ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); sqlite3_free(zSql); } if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){ |
︙ | ︙ | |||
3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 | const sqlite3_tokenizer_module *pPorter = 0; #ifdef SQLITE_ENABLE_ICU const sqlite3_tokenizer_module *pIcu = 0; sqlite3Fts3IcuTokenizerModule(&pIcu); #endif sqlite3Fts3SimpleTokenizerModule(&pSimple); sqlite3Fts3PorterTokenizerModule(&pPorter); /* Allocate and initialise the hash-table used to store tokenizers. */ pHash = sqlite3_malloc(sizeof(Fts3Hash)); if( !pHash ){ rc = SQLITE_NOMEM; | > > > | 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 | const sqlite3_tokenizer_module *pPorter = 0; #ifdef SQLITE_ENABLE_ICU const sqlite3_tokenizer_module *pIcu = 0; sqlite3Fts3IcuTokenizerModule(&pIcu); #endif rc = sqlite3Fts3InitAux(db); if( rc!=SQLITE_OK ) return rc; sqlite3Fts3SimpleTokenizerModule(&pSimple); sqlite3Fts3PorterTokenizerModule(&pPorter); /* Allocate and initialise the hash-table used to store tokenizers. */ pHash = sqlite3_malloc(sizeof(Fts3Hash)); if( !pHash ){ rc = SQLITE_NOMEM; |
︙ | ︙ |
Changes to ext/fts3/fts3Int.h.
︙ | ︙ | |||
103 104 105 106 107 108 109 | typedef struct Fts3Expr Fts3Expr; typedef struct Fts3Phrase Fts3Phrase; typedef struct Fts3PhraseToken Fts3PhraseToken; typedef struct Fts3SegFilter Fts3SegFilter; typedef struct Fts3DeferredToken Fts3DeferredToken; typedef struct Fts3SegReader Fts3SegReader; | | | 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 | typedef struct Fts3Expr Fts3Expr; typedef struct Fts3Phrase Fts3Phrase; typedef struct Fts3PhraseToken Fts3PhraseToken; typedef struct Fts3SegFilter Fts3SegFilter; typedef struct Fts3DeferredToken Fts3DeferredToken; typedef struct Fts3SegReader Fts3SegReader; typedef struct Fts3SegReaderCursor Fts3SegReaderCursor; /* ** A connection to a fulltext index is an instance of the following ** structure. The xCreate and xConnect methods create an instance ** of this structure and xDestroy and xDisconnect free that instance. ** All other methods receive a pointer to the structure as one of their ** arguments. |
︙ | ︙ | |||
126 127 128 129 130 131 132 133 134 135 136 137 138 139 | sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ sqlite3_stmt *aStmt[24]; int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ | > > > | 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 | sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ sqlite3_stmt *aStmt[24]; char *zReadExprlist; char *zWriteExprlist; int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ |
︙ | ︙ | |||
213 214 215 216 217 218 219 | ** on the assumption that the */ struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ int bFulltext; /* True if full-text index was used */ | | | 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 | ** on the assumption that the */ struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ int bFulltext; /* True if full-text index was used */ Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ }; struct Fts3Phrase { /* Variables populated by fts3_expr.c when parsing a MATCH expression */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ |
︙ | ︙ | |||
281 282 283 284 285 286 287 | int sqlite3Fts3PendingTermsFlush(Fts3Table *); void sqlite3Fts3PendingTermsClear(Fts3Table *); int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); | < < < < | > > > > > > > | > > > > > > > > > > > > > > > > > > > > | 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 | int sqlite3Fts3PendingTermsFlush(Fts3Table *); void sqlite3Fts3PendingTermsClear(Fts3Table *); int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **); int sqlite3Fts3ReadLock(Fts3Table *); int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); void sqlite3Fts3SegmentsClose(Fts3Table *); #define FTS3_SEGCURSOR_PENDING -1 #define FTS3_SEGCURSOR_ALL -2 int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*); int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *); void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *); int sqlite3Fts3SegReaderCursor( Fts3Table *, int, const char *, int, int, Fts3SegReaderCursor *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 #define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 #define FTS3_SEGMENT_PREFIX 0x00000008 /* Type passed as 4th argument to SegmentReaderIterate() */ struct Fts3SegFilter { const char *zTerm; int nTerm; int iCol; int flags; }; struct Fts3SegReaderCursor { /* Used internally by sqlite3Fts3SegReaderXXX() calls */ Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ int nSegment; /* Size of apSegment array */ int nAdvance; /* How many seg-readers to advance */ Fts3SegFilter *pFilter; /* Pointer to filter object */ char *aBuffer; /* Buffer to merge doclists in */ int nBuffer; /* Allocated size of aBuffer[] in bytes */ /* Cost of running this iterator. Used by fts3.c only. */ int nCost; /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ char *zTerm; /* Pointer to term buffer */ int nTerm; /* Size of zTerm in bytes */ char *aDoclist; /* Pointer to doclist buffer */ int nDoclist; /* Size of aDoclist[] in bytes */ }; /* fts3.c */ int sqlite3Fts3PutVarint(char *, sqlite3_int64); int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); |
︙ | ︙ | |||
351 352 353 354 355 356 357 358 | char **, int, int, const char *, int, Fts3Expr ** ); void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); #endif #endif /* _FTSINT_H */ | > > > | 377 378 379 380 381 382 383 384 385 386 387 | char **, int, int, const char *, int, Fts3Expr ** ); void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); #endif /* fts3_aux.c */ int sqlite3Fts3InitAux(sqlite3 *db); #endif /* _FTSINT_H */ |
Added ext/fts3/fts3_aux.c.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 | /* ** 2011 Jan 27 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** ** */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include "fts3Int.h" #include <string.h> #include <assert.h> typedef struct Fts3auxTable Fts3auxTable; typedef struct Fts3auxCursor Fts3auxCursor; struct Fts3auxTable { sqlite3_vtab base; /* Base class used by SQLite core */ Fts3Table *pFts3Tab; }; struct Fts3auxCursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ Fts3SegFilter filter; Fts3SegReaderCursor csr; int isEof; sqlite3_int64 iRowid; sqlite3_int64 nDoc; sqlite3_int64 nOcc; }; /* ** Schema of the terms table. */ #define FTS3_TERMS_SCHEMA "CREATE TABLE x(term, documents, occurrences)" /* ** This function does all the work for both the xConnect and xCreate methods. ** These tables have no persistent representation of their own, so xConnect ** and xCreate are identical operations. */ static int fts3auxConnectMethod( sqlite3 *db, /* Database connection */ void *pUnused, /* Unused */ int argc, /* Number of elements in argv array */ const char * const *argv, /* xCreate/xConnect argument array */ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ char **pzErr /* OUT: sqlite3_malloc'd error message */ ){ char const *zDb; /* Name of database (e.g. "main") */ char const *zFts3; /* Name of fts3 table */ int nDb; /* Result of strlen(zDb) */ int nFts3; /* Result of strlen(zFts3) */ int nByte; /* Bytes of space to allocate here */ int rc; /* value returned by declare_vtab() */ Fts3auxTable *p; /* Virtual table object to return */ /* The user should specify a single argument - the name of an fts3 table. */ if( argc!=4 ){ *pzErr = sqlite3_mprintf("wrong number of arguments"); return SQLITE_ERROR; } zDb = argv[1]; nDb = strlen(zDb); zFts3 = argv[3]; nFts3 = strlen(zFts3); rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA); if( rc!=SQLITE_OK ) return rc; nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; p = (Fts3auxTable *)sqlite3_malloc(nByte); if( !p ) return SQLITE_NOMEM; memset(p, 0, nByte); p->pFts3Tab = (Fts3Table *)&p[1]; p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); *ppVtab = (sqlite3_vtab *)p; return SQLITE_OK; } /* ** This function does the work for both the xDisconnect and xDestroy methods. ** These tables have no persistent representation of their own, so xDisconnect ** and xDestroy are identical operations. */ static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ Fts3auxTable *p = (Fts3auxTable *)pVtab; Fts3Table *pFts3 = p->pFts3Tab; int i; /* Free any prepared statements held */ for(i=0; i<SizeofArray(pFts3->aStmt); i++){ sqlite3_finalize(pFts3->aStmt[i]); } sqlite3_free(pFts3->zSegmentsTbl); sqlite3_free(p); return SQLITE_OK; } /* ** xBestIndex - Analyze a WHERE and ORDER BY clause. */ static int fts3auxBestIndexMethod( sqlite3_vtab *pVTab, sqlite3_index_info *pInfo ){ /* This vtab delivers always results in "ORDER BY term ASC" order. */ if( pInfo->nOrderBy==1 && pInfo->aOrderBy[0].iColumn==0 && pInfo->aOrderBy[0].desc==0 ){ pInfo->orderByConsumed = 1; } pInfo->estimatedCost = 20000; return SQLITE_OK; } /* ** xOpen - Open a cursor. */ static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ Fts3auxCursor *pCsr; /* Pointer to cursor object to return */ pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor)); if( !pCsr ) return SQLITE_NOMEM; memset(pCsr, 0, sizeof(Fts3auxCursor)); *ppCsr = (sqlite3_vtab_cursor *)pCsr; return SQLITE_OK; } /* ** xClose - Close a cursor. */ static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){ Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; sqlite3Fts3SegmentsClose(pFts3); sqlite3Fts3SegReaderFinish(&pCsr->csr); sqlite3_free(pCsr); return SQLITE_OK; } /* ** xNext - Advance the cursor to the next row, if any. */ static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; int rc; rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); if( rc==SQLITE_ROW ){ int i; int isIgnore = 1; int nDoclist = pCsr->csr.nDoclist; char *aDoclist = pCsr->csr.aDoclist; /* Now count the number of documents and positions in the doclist ** in pCsr->csr.aDoclist[]. Store the number of documents in pCsr->nDoc ** and the number of occurrences in pCsr->nOcc. */ pCsr->nDoc = 0; pCsr->nOcc = 0; i = 0; while( i<nDoclist ){ sqlite3_int64 v = 0; i += sqlite3Fts3GetVarint(&aDoclist[i], &v); if( isIgnore ){ isIgnore = 0; }else if( v>1 ){ pCsr->nOcc++; }else{ if( v==0 ) pCsr->nDoc++; isIgnore = 1; } } rc = SQLITE_OK; pCsr->iRowid++; }else{ pCsr->isEof = 1; } return rc; } /* ** xFilter - Initialize a cursor to point at the start of its data. */ static int fts3auxFilterMethod( sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ int idxNum, /* Strategy index */ const char *idxStr, /* Unused */ int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; int rc; sqlite3Fts3SegReaderFinish(&pCsr->csr); memset(&pCsr->csr, 0, sizeof(Fts3SegReaderCursor)); pCsr->isEof = 0; pCsr->iRowid = 0; pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL, 0, 0,1,&pCsr->csr); if( rc==SQLITE_OK ){ rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); } if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor); return rc; } /* ** xEof - Return true if the cursor is at EOF, or false otherwise. */ static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; return pCsr->isEof; } /* ** xColumn - Return a column value. */ static int fts3auxColumnMethod( sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ Fts3auxCursor *p = (Fts3auxCursor *)pCursor; assert( p->isEof==0 ); if( iCol==0 ){ /* Column "term" */ sqlite3_result_text(pContext, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); }else if( iCol==1 ){ /* Column "documents" */ sqlite3_result_int64(pContext, p->nDoc); }else{ /* Column "occurrences" */ sqlite3_result_int64(pContext, p->nOcc); } return SQLITE_OK; } /* ** xRowid - Return the current rowid for the cursor. */ static int fts3auxRowidMethod( sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ sqlite_int64 *pRowid /* OUT: Rowid value */ ){ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; *pRowid = pCsr->iRowid; return SQLITE_OK; } /* ** Register the fts3aux module with database connection db. Return SQLITE_OK ** if successful or an error code if sqlite3_create_module() fails. */ int sqlite3Fts3InitAux(sqlite3 *db){ static const sqlite3_module fts3aux_module = { 0, /* iVersion */ fts3auxConnectMethod, /* xCreate */ fts3auxConnectMethod, /* xConnect */ fts3auxBestIndexMethod, /* xBestIndex */ fts3auxDisconnectMethod, /* xDisconnect */ fts3auxDisconnectMethod, /* xDestroy */ fts3auxOpenMethod, /* xOpen */ fts3auxCloseMethod, /* xClose */ fts3auxFilterMethod, /* xFilter */ fts3auxNextMethod, /* xNext */ fts3auxEofMethod, /* xEof */ fts3auxColumnMethod, /* xColumn */ fts3auxRowidMethod, /* xRowid */ 0, /* xUpdate */ 0, /* xBegin */ 0, /* xSync */ 0, /* xCommit */ 0, /* xRollback */ 0, /* xFindFunction */ 0 /* xRename */ }; int rc; /* Return code */ rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0); return rc; } #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ |
Changes to ext/fts3/fts3_snippet.c.
︙ | ︙ | |||
956 957 958 959 960 961 962 963 964 965 966 967 968 969 | if( rc!=SQLITE_OK ) return rc; } pStmt = *ppStmt; assert( sqlite3_data_count(pStmt)==1 ); a = sqlite3_column_blob(pStmt, 0); a += sqlite3Fts3GetVarint(a, &nDoc); *pnDoc = (u32)nDoc; if( paLen ) *paLen = a; return SQLITE_OK; } /* | > | 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 | if( rc!=SQLITE_OK ) return rc; } pStmt = *ppStmt; assert( sqlite3_data_count(pStmt)==1 ); a = sqlite3_column_blob(pStmt, 0); a += sqlite3Fts3GetVarint(a, &nDoc); if( nDoc==0 ) return SQLITE_CORRUPT; *pnDoc = (u32)nDoc; if( paLen ) *paLen = a; return SQLITE_OK; } /* |
︙ | ︙ | |||
1162 1163 1164 1165 1166 1167 1168 1169 1170 | sqlite3_int64 nDoc; /* Number of rows in table */ const char *a; /* Aggregate column length array */ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a); if( rc==SQLITE_OK ){ int iCol; for(iCol=0; iCol<pInfo->nCol; iCol++){ sqlite3_int64 nToken; a += sqlite3Fts3GetVarint(a, &nToken); | > | > | 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 | sqlite3_int64 nDoc; /* Number of rows in table */ const char *a; /* Aggregate column length array */ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a); if( rc==SQLITE_OK ){ int iCol; for(iCol=0; iCol<pInfo->nCol; iCol++){ u32 iVal; sqlite3_int64 nToken; a += sqlite3Fts3GetVarint(a, &nToken); iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); pInfo->aMatchinfo[iCol] = iVal; } } } break; case FTS3_MATCHINFO_LENGTH: { sqlite3_stmt *pSelectDocsize = 0; |
︙ | ︙ |
Changes to ext/fts3/fts3_write.c.
︙ | ︙ | |||
208 209 210 211 212 213 214 | /* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", /* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", /* 2 */ "DELETE FROM %Q.'%q_content'", /* 3 */ "DELETE FROM %Q.'%q_segments'", /* 4 */ "DELETE FROM %Q.'%q_segdir'", /* 5 */ "DELETE FROM %Q.'%q_docsize'", /* 6 */ "DELETE FROM %Q.'%q_stat'", | | | < < | | < < < < < < < < < | | 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 | /* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", /* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", /* 2 */ "DELETE FROM %Q.'%q_content'", /* 3 */ "DELETE FROM %Q.'%q_segments'", /* 4 */ "DELETE FROM %Q.'%q_segdir'", /* 5 */ "DELETE FROM %Q.'%q_docsize'", /* 6 */ "DELETE FROM %Q.'%q_stat'", /* 7 */ "SELECT %s FROM %Q.'%q_content' AS x WHERE rowid=?", /* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", /* 9 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", /* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", /* 11 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", /* Return segments in order from oldest to newest.*/ /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC", /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", /* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", /* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); assert( eStmt<SizeofArray(azSql) && eStmt>=0 ); pStmt = p->aStmt[eStmt]; if( !pStmt ){ char *zSql; if( eStmt==SQL_CONTENT_INSERT ){ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist, p->zDb, p->zName); }else{ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); } if( !zSql ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL); |
︙ | ︙ | |||
296 297 298 299 300 301 302 | rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0); if( rc==SQLITE_OK ){ if( eStmt==SQL_SELECT_DOCSIZE ){ sqlite3_bind_int64(pStmt, 1, iDocid); } rc = sqlite3_step(pStmt); | | | 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 | rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0); if( rc==SQLITE_OK ){ if( eStmt==SQL_SELECT_DOCSIZE ){ sqlite3_bind_int64(pStmt, 1, iDocid); } rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ rc = sqlite3_reset(pStmt); if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT; pStmt = 0; }else{ rc = SQLITE_OK; } } |
︙ | ︙ | |||
397 398 399 400 401 402 403 | ** ** 0: idx ** 1: start_block ** 2: leaves_end_block ** 3: end_block ** 4: root */ | | > > > | > > > > > > | 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 | ** ** 0: idx ** 1: start_block ** 2: leaves_end_block ** 3: end_block ** 4: root */ int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){ int rc; sqlite3_stmt *pStmt = 0; if( iLevel<0 ){ rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0); }else{ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel); } *ppStmt = pStmt; return rc; } /* ** Append a single varint to a PendingList buffer. SQLITE_OK is returned ** if successful, or an SQLite error code otherwise. ** |
︙ | ︙ | |||
1100 1101 1102 1103 1104 1105 1106 1107 1108 | ** the table. The following nCol varints contain the total amount of ** data stored in all rows of each column of the table, from left ** to right. */ sqlite3_stmt *pStmt; sqlite3_int64 nDoc = 0; sqlite3_int64 nByte = 0; const char *a; rc = sqlite3Fts3SelectDoctotal(p, &pStmt); | > > | | > | | | | < | 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 | ** the table. The following nCol varints contain the total amount of ** data stored in all rows of each column of the table, from left ** to right. */ sqlite3_stmt *pStmt; sqlite3_int64 nDoc = 0; sqlite3_int64 nByte = 0; const char *pEnd; const char *a; rc = sqlite3Fts3SelectDoctotal(p, &pStmt); if( rc!=SQLITE_OK ) return rc; a = sqlite3_column_blob(pStmt, 0); assert( a ); pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; a += sqlite3Fts3GetVarint(a, &nDoc); while( a<pEnd ){ a += sqlite3Fts3GetVarint(a, &nByte); } if( nDoc==0 || nByte==0 ){ sqlite3_reset(pStmt); return SQLITE_CORRUPT; } pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz); |
︙ | ︙ | |||
1299 1300 1301 1302 1303 1304 1305 | if( isPrefix ){ sqlite3_free(aElem); } *ppReader = pReader; return rc; } | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 | if( isPrefix ){ sqlite3_free(aElem); } *ppReader = pReader; return rc; } /* ** Compare the entries pointed to by two Fts3SegReader structures. ** Comparison is as follows: ** ** 1) EOF is greater than not EOF. ** ** 2) The current terms (if any) are compared using memcmp(). If one |
︙ | ︙ | |||
1939 1940 1941 1942 1943 1944 1945 | *pisEmpty = sqlite3_column_int(pStmt, 0); } rc = sqlite3_reset(pStmt); } return rc; } | < < < < < < < < < < < < < < < < < < < | 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 | *pisEmpty = sqlite3_column_int(pStmt, 0); } rc = sqlite3_reset(pStmt); } return rc; } /* ** Set *pnSegment to the total number of segments in the database. Set ** *pnMax to the largest segment level in the database (segment levels ** are stored in the 'level' column of the %_segdir table). ** ** Return SQLITE_OK if successful, or an SQLite error code if not. */ |
︙ | ︙ | |||
2016 2017 2018 2019 2020 2021 2022 | rc = sqlite3_reset(pDelete); } } if( rc!=SQLITE_OK ){ return rc; } | > > > > > | < < | 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 | rc = sqlite3_reset(pDelete); } } if( rc!=SQLITE_OK ){ return rc; } if( iLevel==FTS3_SEGCURSOR_ALL ){ fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ sqlite3Fts3PendingTermsClear(p); }else{ assert( iLevel>=0 ); rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pDelete, 1, iLevel); sqlite3_step(pDelete); rc = sqlite3_reset(pDelete); } } return rc; } /* ** When this function is called, buffer *ppList (size *pnList bytes) contains |
︙ | ︙ | |||
2073 2074 2075 2076 2077 2078 2079 | p += sqlite3Fts3GetVarint32(p, &iCurrent); } *ppList = pList; *pnList = nList; } | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | | < | < < | < < < < < < < | < < < < > | | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > > > | > | | < | | | | | > | | | | | | | 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 | p += sqlite3Fts3GetVarint32(p, &iCurrent); } *ppList = pList; *pnList = nList; } int sqlite3Fts3SegReaderStart( Fts3Table *p, /* Virtual table handle */ Fts3SegReaderCursor *pCsr, /* Cursor object */ Fts3SegFilter *pFilter /* Restrictions on range of iteration */ ){ int i; /* Initialize the cursor object */ pCsr->pFilter = pFilter; /* If the Fts3SegFilter defines a specific term (or term prefix) to search ** for, then advance each segment iterator until it points to a term of ** equal or greater value than the specified term. This prevents many ** unnecessary merge/sort operations for the case where single segment ** b-tree leaf nodes contain more than one term. */ for(i=0; i<pCsr->nSegment; i++){ int nTerm = pFilter->nTerm; const char *zTerm = pFilter->zTerm; Fts3SegReader *pSeg = pCsr->apSegment[i]; do { int rc = fts3SegReaderNext(p, pSeg); if( rc!=SQLITE_OK ) return rc; }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); } fts3SegReaderSort( pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp); return SQLITE_OK; } int sqlite3Fts3SegReaderStep( Fts3Table *p, /* Virtual table handle */ Fts3SegReaderCursor *pCsr /* Cursor object */ ){ int rc = SQLITE_OK; int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); Fts3SegReader **apSegment = pCsr->apSegment; int nSegment = pCsr->nSegment; Fts3SegFilter *pFilter = pCsr->pFilter; if( pCsr->nSegment==0 ) return SQLITE_OK; do { int nMerge; int i; /* Advance the first pCsr->nAdvance entries in the apSegment[] array ** forward. Then sort the list in order of current term again. */ for(i=0; i<pCsr->nAdvance; i++){ rc = fts3SegReaderNext(p, apSegment[i]); if( rc!=SQLITE_OK ) return rc; } fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); pCsr->nAdvance = 0; /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */ assert( rc==SQLITE_OK ); if( apSegment[0]->aNode==0 ) break; pCsr->nTerm = apSegment[0]->nTerm; pCsr->zTerm = apSegment[0]->zTerm; /* If this is a prefix-search, and if the term that apSegment[0] points ** to does not share a suffix with pFilter->zTerm/nTerm, then all ** required callbacks have been made. In this case exit early. ** ** Similarly, if this is a search for an exact match, and the first term ** of segment apSegment[0] is not a match, exit early. */ if( pFilter->zTerm ){ if( pCsr->nTerm<pFilter->nTerm || (!isPrefix && pCsr->nTerm>pFilter->nTerm) || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) ){ break; } } nMerge = 1; while( nMerge<nSegment && apSegment[nMerge]->aNode && apSegment[nMerge]->nTerm==pCsr->nTerm && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm) ){ nMerge++; } assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); if( nMerge==1 && !isIgnoreEmpty ){ pCsr->aDoclist = apSegment[0]->aDoclist; pCsr->nDoclist = apSegment[0]->nDoclist; rc = SQLITE_ROW; }else{ int nDoclist = 0; /* Size of doclist */ sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ /* The current term of the first nMerge entries in the array ** of Fts3SegReader objects is the same. The doclists must be merged ** and a single term returned with the merged doclist. */ for(i=0; i<nMerge; i++){ fts3SegReaderFirstDocid(apSegment[i]); } fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp); while( apSegment[0]->pOffsetList ){ int j; /* Number of segments that share a docid */ |
︙ | ︙ | |||
2238 2239 2240 2241 2242 2243 2244 | if( isColFilter ){ fts3ColumnFilter(pFilter->iCol, &pList, &nList); } if( !isIgnoreEmpty || nList>0 ){ nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); | | | | | < | | > > | | < > | | > > < | < < < < | > > > > > | | < | > | > > > | < < < < < | < < < > > > > | | < < < < < < | | | > > > | < < < < > | < | < | < > | < | < < < < < < | < < | | < < < | < < < > < > < < < | < < < < < | < < < < < < < | < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 | if( isColFilter ){ fts3ColumnFilter(pFilter->iCol, &pList, &nList); } if( !isIgnoreEmpty || nList>0 ){ nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); if( nDoclist+nByte>pCsr->nBuffer ){ char *aNew; pCsr->nBuffer = (nDoclist+nByte)*2; aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); if( !aNew ){ return SQLITE_NOMEM; } pCsr->aBuffer = aNew; } nDoclist += sqlite3Fts3PutVarint( &pCsr->aBuffer[nDoclist], iDocid-iPrev ); iPrev = iDocid; if( isRequirePos ){ memcpy(&pCsr->aBuffer[nDoclist], pList, nList); nDoclist += nList; pCsr->aBuffer[nDoclist++] = '\0'; } } fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp); } if( nDoclist>0 ){ pCsr->aDoclist = pCsr->aBuffer; pCsr->nDoclist = nDoclist; rc = SQLITE_ROW; } } pCsr->nAdvance = nMerge; }while( rc==SQLITE_OK ); return rc; } void sqlite3Fts3SegReaderFinish( Fts3SegReaderCursor *pCsr /* Cursor object */ ){ if( pCsr ){ int i; for(i=0; i<pCsr->nSegment; i++){ sqlite3Fts3SegReaderFree(pCsr->apSegment[i]); } sqlite3_free(pCsr->apSegment); sqlite3_free(pCsr->aBuffer); pCsr->nSegment = 0; pCsr->apSegment = 0; pCsr->aBuffer = 0; } } /* ** Merge all level iLevel segments in the database into a single ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a ** single segment with a level equal to the numerically largest level ** currently present in the database. ** ** If this function is called with iLevel<0, but there is only one ** segment in the database, SQLITE_DONE is returned immediately. ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ int rc; /* Return code */ int iIdx; /* Index of new segment */ int iNewLevel = 0; /* Level to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the the numerically ** greatest segment level currently present in the database. The index ** of the new segment is always 0. */ int nDummy; /* TODO: Remove this */ if( csr.nSegment==1 ){ rc = SQLITE_DONE; goto finished; } iIdx = 0; rc = fts3SegmentCountMax(p, &nDummy, &iNewLevel); }else{ /* This call is to merge all segments at level iLevel. Find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ iNewLevel = iLevel+1; rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx); } if( rc!=SQLITE_OK ) goto finished; assert( csr.nSegment>0 ); assert( iNewLevel>=0 ); memset(&filter, 0, sizeof(Fts3SegFilter)); filter.flags = FTS3_SEGMENT_REQUIRE_POS; filter.flags |= (iLevel==FTS3_SEGCURSOR_ALL ? FTS3_SEGMENT_IGNORE_EMPTY : 0); rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); while( SQLITE_OK==rc ){ rc = sqlite3Fts3SegReaderStep(p, &csr); if( rc!=SQLITE_ROW ) break; rc = fts3SegWriterAdd(p, &pWriter, 1, csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist); } if( rc!=SQLITE_OK ) goto finished; assert( pWriter ); rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment); if( rc!=SQLITE_OK ) goto finished; rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); finished: fts3SegWriterFree(pWriter); sqlite3Fts3SegReaderFinish(&csr); return rc; } /* ** Flush the contents of pendingTerms to a level 0 segment. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ return fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING); } /* ** Encode N integers as varints into a blob. */ static void fts3EncodeIntArray( int N, /* The number of integers to encode */ |
︙ | ︙ | |||
2615 2616 2617 2618 2619 2620 2621 | int rc; /* Return Code */ const char *zVal = (const char *)sqlite3_value_text(pVal); int nVal = sqlite3_value_bytes(pVal); if( !zVal ){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ | | | 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 | int rc; /* Return Code */ const char *zVal = (const char *)sqlite3_value_text(pVal); int nVal = sqlite3_value_bytes(pVal); if( !zVal ){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); if( rc==SQLITE_DONE ){ rc = SQLITE_OK; }else{ sqlite3Fts3PendingTermsClear(p); } #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ |
︙ | ︙ | |||
2873 2874 2875 2876 2877 2878 2879 | ** merge all segments in the database (including the new segment, if ** there was any data to flush) into a single segment. */ int sqlite3Fts3Optimize(Fts3Table *p){ int rc; rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); if( rc==SQLITE_OK ){ | | | 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 | ** merge all segments in the database (including the new segment, if ** there was any data to flush) into a single segment. */ int sqlite3Fts3Optimize(Fts3Table *p){ int rc; rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); if( rc==SQLITE_OK ){ rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); if( rc==SQLITE_OK ){ rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); if( rc==SQLITE_OK ){ sqlite3Fts3PendingTermsClear(p); } }else{ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); |
︙ | ︙ |
Changes to main.mk.
︙ | ︙ | |||
49 50 51 52 53 54 55 | TCCX += -I$(TOP)/ext/async # Object files for the SQLite library. # LIBOBJ+= alter.o analyze.o attach.o auth.o \ backup.o bitvec.o btmutex.o btree.o build.o \ callback.o complete.o ctime.o date.o delete.o expr.o fault.o fkey.o \ | | | 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | TCCX += -I$(TOP)/ext/async # Object files for the SQLite library. # LIBOBJ+= alter.o analyze.o attach.o auth.o \ backup.o bitvec.o btmutex.o btree.o build.o \ callback.o complete.o ctime.o date.o delete.o expr.o fault.o fkey.o \ fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o fts3_write.o \ func.o global.o hash.o \ icu.o insert.o journal.o legacy.o loadext.o \ main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \ memjournal.o \ mutex.o mutex_noop.o mutex_os2.o mutex_unix.o mutex_w32.o \ notify.o opcodes.o os.o os_os2.o os_unix.o os_win.o \ |
︙ | ︙ | |||
183 184 185 186 187 188 189 190 191 192 193 194 195 196 | $(TOP)/ext/fts2/fts2_tokenizer.h \ $(TOP)/ext/fts2/fts2_tokenizer.c \ $(TOP)/ext/fts2/fts2_tokenizer1.c SRC += \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3.h \ $(TOP)/ext/fts3/fts3Int.h \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_hash.c \ $(TOP)/ext/fts3/fts3_hash.h \ $(TOP)/ext/fts3/fts3_icu.c \ $(TOP)/ext/fts3/fts3_porter.c \ $(TOP)/ext/fts3/fts3_snippet.c \ $(TOP)/ext/fts3/fts3_tokenizer.h \ | > | 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 | $(TOP)/ext/fts2/fts2_tokenizer.h \ $(TOP)/ext/fts2/fts2_tokenizer.c \ $(TOP)/ext/fts2/fts2_tokenizer1.c SRC += \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3.h \ $(TOP)/ext/fts3/fts3Int.h \ $(TOP)/ext/fts3/fts3_aux.c \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_hash.c \ $(TOP)/ext/fts3/fts3_hash.h \ $(TOP)/ext/fts3/fts3_icu.c \ $(TOP)/ext/fts3/fts3_porter.c \ $(TOP)/ext/fts3/fts3_snippet.c \ $(TOP)/ext/fts3/fts3_tokenizer.h \ |
︙ | ︙ | |||
289 290 291 292 293 294 295 296 297 298 299 300 301 302 | $(TOP)/src/vdbeapi.c \ $(TOP)/src/vdbeaux.c \ $(TOP)/src/vdbe.c \ $(TOP)/src/vdbemem.c \ $(TOP)/src/where.c \ parse.c \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ $(TOP)/ext/async/sqlite3async.c # Header files used by all library source files. # | > | 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 | $(TOP)/src/vdbeapi.c \ $(TOP)/src/vdbeaux.c \ $(TOP)/src/vdbe.c \ $(TOP)/src/vdbemem.c \ $(TOP)/src/where.c \ parse.c \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3_aux.c \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ $(TOP)/ext/async/sqlite3async.c # Header files used by all library source files. # |
︙ | ︙ | |||
457 458 459 460 461 462 463 464 465 466 467 468 469 470 | $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer.c fts2_tokenizer1.o: $(TOP)/ext/fts2/fts2_tokenizer1.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer1.c fts3.o: $(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c fts3_expr.o: $(TOP)/ext/fts3/fts3_expr.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c | > > > | 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 | $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer.c fts2_tokenizer1.o: $(TOP)/ext/fts2/fts2_tokenizer1.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer1.c fts3.o: $(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c fts3_aux.o: $(TOP)/ext/fts3/fts3_aux.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_aux.c fts3_expr.o: $(TOP)/ext/fts3/fts3_expr.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c |
︙ | ︙ |
Changes to src/pager.c.
︙ | ︙ | |||
2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 | PAGER_INCR(pPager->nRead); IOTRACE(("PGIN %p %d\n", pPager, pgno)); PAGERTRACE(("FETCH %d page %d hash(%08x)\n", PAGERID(pPager), pgno, pager_pagehash(pPg))); return rc; } #ifndef SQLITE_OMIT_WAL /* ** This function is invoked once for each page that has already been ** written into the log file when a WAL transaction is rolled back. ** Parameter iPg is the page number of said page. The pCtx argument ** is actually a pointer to the Pager structure. | > > > > > > > > > > > > > > > > > > > > > > | 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 | PAGER_INCR(pPager->nRead); IOTRACE(("PGIN %p %d\n", pPager, pgno)); PAGERTRACE(("FETCH %d page %d hash(%08x)\n", PAGERID(pPager), pgno, pager_pagehash(pPg))); return rc; } /* ** Update the value of the change-counter at offsets 24 and 92 in ** the header and the sqlite version number at offset 96. ** ** This is an unconditional update. See also the pager_incr_changecounter() ** routine which only updates the change-counter if the update is actually ** needed, as determined by the pPager->changeCountDone state variable. */ static void pager_write_changecounter(PgHdr *pPg){ u32 change_counter; /* Increment the value just read and write it back to byte 24. */ change_counter = sqlite3Get4byte((u8*)pPg->pPager->dbFileVers)+1; put32bits(((char*)pPg->pData)+24, change_counter); /* Also store the SQLite version number in bytes 96..99 and in ** bytes 92..95 store the change counter for which the version number ** is valid. */ put32bits(((char*)pPg->pData)+92, change_counter); put32bits(((char*)pPg->pData)+96, SQLITE_VERSION_NUMBER); } #ifndef SQLITE_OMIT_WAL /* ** This function is invoked once for each page that has already been ** written into the log file when a WAL transaction is rolled back. ** Parameter iPg is the page number of said page. The pCtx argument ** is actually a pointer to the Pager structure. |
︙ | ︙ | |||
2917 2918 2919 2920 2921 2922 2923 | rc = pagerUndoCallback((void *)pPager, pList->pgno); pList = pNext; } return rc; } | < < < < < < < < < < < < < < < < < < < < < < < | 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 | rc = pagerUndoCallback((void *)pPager, pList->pgno); pList = pNext; } return rc; } /* ** This function is a wrapper around sqlite3WalFrames(). As well as logging ** the contents of the list of pages headed by pList (connected by pDirty), ** this function notifies any active backup processes that the pages have ** changed. ** ** The list of pages passed into this routine is always sorted by page number. |
︙ | ︙ |
Changes to src/sqlite.h.in.
︙ | ︙ | |||
5530 5531 5532 5533 5534 5535 5536 | ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_USED</dt> ** <dd>This parameter returns the number of lookaside memory slots currently ** checked out.</dd>)^ ** ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_HIT</dt> ** <dd>This parameter returns the number malloc attempts that were ** satisfied using lookaside memory. Only the high-water value is meaningful; | | < | < | < | 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 | ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_USED</dt> ** <dd>This parameter returns the number of lookaside memory slots currently ** checked out.</dd>)^ ** ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_HIT</dt> ** <dd>This parameter returns the number malloc attempts that were ** satisfied using lookaside memory. Only the high-water value is meaningful; ** the current value is always zero.)^ ** ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE</dt> ** <dd>This parameter returns the number malloc attempts that might have ** been satisfied using lookaside memory but failed due to the amount of ** memory requested being larger than the lookaside slot size. ** Only the high-water value is meaningful; ** the current value is always zero.)^ ** ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL</dt> ** <dd>This parameter returns the number malloc attempts that might have ** been satisfied using lookaside memory but failed due to all lookaside ** memory already being in use. ** Only the high-water value is meaningful; ** the current value is always zero.)^ ** ** ^(<dt>SQLITE_DBSTATUS_CACHE_USED</dt> ** <dd>This parameter returns the approximate number of of bytes of heap ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. ** ** ^(<dt>SQLITE_DBSTATUS_SCHEMA_USED</dt> |
︙ | ︙ |
Added test/fts3aux1.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | # 2011 January 27 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # This file implements regression tests for SQLite library. The # focus of this script is testing the FTS3 module. # set testdir [file dirname $argv0] source $testdir/tester.tcl ifcapable !fts3 { finish_test ; return } set ::testprefix fts3aux1 do_execsql_test 1.1 { CREATE VIRTUAL TABLE t1 USING fts4; INSERT INTO t1 VALUES('one two three four'); INSERT INTO t1 VALUES('three four five six'); INSERT INTO t1 VALUES('one three five seven'); CREATE VIRTUAL TABLE terms USING fts4aux(t1); SELECT * FROM terms; } { five 2 2 four 2 2 one 2 2 seven 1 1 six 1 1 three 3 3 two 1 1 } do_execsql_test 1.2 { INSERT INTO t1 VALUES('one one one three three three'); SELECT * FROM terms; } { five 2 2 four 2 2 one 3 5 seven 1 1 six 1 1 three 4 6 two 1 1 } do_execsql_test 1.3 { DELETE FROM t1; SELECT * FROM terms; } {} do_execsql_test 1.4 { INSERT INTO t1 VALUES('a b a b a b a'); INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; SELECT * FROM terms; } {a 256 1024 b 256 768} finish_test |
Added test/fts3comp1.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 | # 2011 January 27 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # This file implements regression tests for SQLite library. The # focus of this script is testing the FTS3 module. # set testdir [file dirname $argv0] source $testdir/tester.tcl ifcapable !fts3 { finish_test ; return } set ::testprefix fts3comp1 set next_x 0 proc zip {x} { incr ::next_x set ::strings($::next_x) $x return $::next_x } proc unzip {x} { return $::strings($x) } db func zip zip db func unzip unzip do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts4( a, b, compress=zip, uncompress=unzip ); INSERT INTO t1 VALUES('one two three', 'two four six'); } do_execsql_test 1.1 { SELECT a, b FROM t1; } {{one two three} {two four six}} do_execsql_test 1.2 { SELECT c0a, c1b FROM t1_content; } {1 2} do_execsql_test 1.3 { INSERT INTO t1 VALUES('three six nine', 'four eight twelve'); SELECT a, b FROM t1; } {{one two three} {two four six} {three six nine} {four eight twelve}} do_execsql_test 1.4 { SELECT c0a, c1b FROM t1_content; } {1 2 3 4} do_execsql_test 1.5 { CREATE VIRTUAL TABLE terms USING fts4aux(t1); SELECT * FROM terms; } { eight 1 1 four 2 2 nine 1 1 one 1 1 six 2 2 three 2 2 twelve 1 1 two 1 2 } do_execsql_test 1.6 { DELETE FROM t1 WHERE docid = 1; SELECT * FROM terms; } { eight 1 1 four 1 1 nine 1 1 six 1 1 three 1 1 twelve 1 1 } do_execsql_test 1.7 { SELECT c0a, c1b FROM t1_content; } {3 4} finish_test |
Changes to test/fts3corrupt.test.
︙ | ︙ | |||
126 127 128 129 130 131 132 133 134 135 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" ] do_catchsql_test 4.3 { UPDATE t1_segdir SET root = $blob; SELECT rowid FROM t1 WHERE t1 MATCH 'world'; } {1 {database disk image is malformed}} finish_test | > > > > > > > > > > > > > > > > > > > > > > > > > > > | 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" ] do_catchsql_test 4.3 { UPDATE t1_segdir SET root = $blob; SELECT rowid FROM t1 WHERE t1 MATCH 'world'; } {1 {database disk image is malformed}} # Test a special kind of corruption, where the %_stat table contains # an invalid entry. At one point this could lead to a division-by-zero # error in fts4. # do_execsql_test 5.0 { DROP TABLE t1; CREATE VIRTUAL TABLE t1 USING fts4; } do_test 5.1 { db func nn nn execsql BEGIN execsql { INSERT INTO t1 VALUES('one') } execsql { INSERT INTO t1 VALUES('two') } execsql { INSERT INTO t1 VALUES('three') } execsql { INSERT INTO t1 VALUES('four') } execsql COMMIT } {} do_catchsql_test 5.2 { UPDATE t1_stat SET value = X'0000'; SELECT matchinfo(t1, 'nxa') FROM t1 WHERE t1 MATCH 't*'; } {1 {database disk image is malformed}} do_catchsql_test 5.3 { UPDATE t1_stat SET value = NULL; SELECT matchinfo(t1, 'nxa') FROM t1 WHERE t1 MATCH 't*'; } {1 {database disk image is malformed}} finish_test |
Changes to test/permutations.test.
︙ | ︙ | |||
177 178 179 180 181 182 183 184 185 186 187 188 189 190 | fts3af.test fts3ag.test fts3ah.test fts3ai.test fts3aj.test fts3ak.test fts3al.test fts3am.test fts3an.test fts3ao.test fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test fts3near.test fts3query.test fts3shared.test fts3snippet.test fts3fault.test fts3malloc.test fts3matchinfo.test } lappend ::testsuitelist xxx #------------------------------------------------------------------------- # Define the coverage related test suites: # | > > | 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 | fts3af.test fts3ag.test fts3ah.test fts3ai.test fts3aj.test fts3ak.test fts3al.test fts3am.test fts3an.test fts3ao.test fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test fts3near.test fts3query.test fts3shared.test fts3snippet.test fts3fault.test fts3malloc.test fts3matchinfo.test fts3aux1.test fts3comp1.test } lappend ::testsuitelist xxx #------------------------------------------------------------------------- # Define the coverage related test suites: # |
︙ | ︙ |
Changes to test/trace2.test.
︙ | ︙ | |||
124 125 126 127 128 129 130 | INSERT INTO x1 VALUES('Wind chill values as low as -13'); } do_trace_test 2.2 { INSERT INTO x1 VALUES('North northwest wind between 8 and 14 mph'); } { "INSERT INTO x1 VALUES('North northwest wind between 8 and 14 mph');" | | < > | 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 | INSERT INTO x1 VALUES('Wind chill values as low as -13'); } do_trace_test 2.2 { INSERT INTO x1 VALUES('North northwest wind between 8 and 14 mph'); } { "INSERT INTO x1 VALUES('North northwest wind between 8 and 14 mph');" "-- INSERT INTO 'main'.'x1_content' VALUES(?,(?))" "-- REPLACE INTO 'main'.'x1_docsize' VALUES(?,?)" "-- SELECT value FROM 'main'.'x1_stat' WHERE id=0" "-- REPLACE INTO 'main'.'x1_stat' VALUES(0,?)" "-- SELECT (SELECT max(idx) FROM 'main'.'x1_segdir' WHERE level = ?) + 1" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" } do_trace_test 2.3 { INSERT INTO x1(x1) VALUES('optimize'); } { "INSERT INTO x1(x1) VALUES('optimize');" "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' ORDER BY level DESC, idx ASC" "-- SELECT count(*), max(level) FROM 'main'.'x1_segdir'" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" "-- DELETE FROM 'main'.'x1_segdir'" "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" } } finish_test |
Changes to tool/mksqlite3c.tcl.
︙ | ︙ | |||
290 291 292 293 294 295 296 297 298 299 300 301 302 303 | tokenize.c complete.c main.c notify.c fts3.c fts3_expr.c fts3_hash.c fts3_porter.c fts3_tokenizer.c fts3_tokenizer1.c fts3_write.c fts3_snippet.c | > | 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 | tokenize.c complete.c main.c notify.c fts3.c fts3_aux.c fts3_expr.c fts3_hash.c fts3_porter.c fts3_tokenizer.c fts3_tokenizer1.c fts3_write.c fts3_snippet.c |
︙ | ︙ |