Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Change fts4 so that the prefix= parameter is passes a comma-separated list of integers. For each integer N, a separate index of all prefixes of length N bytes is created. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts3-prefix-search |
| Files: | files | file ages | folders |
| SHA1: |
be59bf49402d2e2f4b95fb6668849f37 |
| User & Date: | dan 2011-05-25 18:34:53 |
Context
|
2011-05-25
| ||
| 18:47 | Merge trunk changes into experimental fts3-prefix-search branch. check-in: f0f0a03d user: dan tags: fts3-prefix-search | |
| 18:34 | Change fts4 so that the prefix= parameter is passes a comma-separated list of integers. For each integer N, a separate index of all prefixes of length N bytes is created. check-in: be59bf49 user: dan tags: fts3-prefix-search | |
|
2011-05-24
| ||
| 18:49 | If the fts4 option prefix=1 is specified, have the fts4 module maintain an index of prefixes as well as terms. check-in: b5bdc639 user: dan tags: fts3-prefix-search | |
Changes
Changes to ext/fts3/fts3.c.
827 827 fts3Appendf(pRc, &zRet, "?"); 828 828 for(i=0; i<p->nColumn; i++){ 829 829 fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); 830 830 } 831 831 sqlite3_free(zFree); 832 832 return zRet; 833 833 } 834 + 835 +static int fts3GobbleInt(const char **pp, int *pnOut){ 836 + const char *p = *pp; 837 + int nInt = 0; 838 + for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ 839 + nInt = nInt * 10 + (p[0] - '0'); 840 + } 841 + if( p==*pp ) return SQLITE_ERROR; 842 + *pnOut = nInt; 843 + *pp = p; 844 + return SQLITE_OK; 845 +} 846 + 847 + 848 +static int fts3PrefixParameter( 849 + const char *zParam, /* ABC in prefix=ABC parameter to parse */ 850 + int *pnIndex, /* OUT: size of *apIndex[] array */ 851 + struct Fts3Index **apIndex, /* OUT: Array of indexes for this table */ 852 + struct Fts3Index **apFree /* OUT: Free this with sqlite3_free() */ 853 +){ 854 + struct Fts3Index *aIndex; 855 + int nIndex = 1; 856 + 857 + if( zParam && zParam[0] ){ 858 + const char *p; 859 + nIndex++; 860 + for(p=zParam; *p; p++){ 861 + if( *p==',' ) nIndex++; 862 + } 863 + } 864 + 865 + aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); 866 + *apIndex = *apFree = aIndex; 867 + *pnIndex = nIndex; 868 + if( !aIndex ){ 869 + return SQLITE_NOMEM; 870 + } 871 + 872 + memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); 873 + if( zParam ){ 874 + const char *p = zParam; 875 + int i; 876 + for(i=1; i<nIndex; i++){ 877 + int nPrefix; 878 + if( fts3GobbleInt(&p, &nPrefix) ) return SQLITE_ERROR; 879 + aIndex[i].nPrefix = nPrefix; 880 + p++; 881 + } 882 + } 883 + 884 + return SQLITE_OK; 885 +} 834 886 835 887 /* 836 888 ** This function is the implementation of both the xConnect and xCreate 837 889 ** methods of the FTS3 virtual table. 838 890 ** 839 891 ** The argv[] array contains the following: 840 892 ** ................................................................................ 861 913 int nString = 0; /* Bytes required to hold all column names */ 862 914 int nCol = 0; /* Number of columns in the FTS table */ 863 915 char *zCsr; /* Space for holding column names */ 864 916 int nDb; /* Bytes required to hold database name */ 865 917 int nName; /* Bytes required to hold table name */ 866 918 int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ 867 919 int bNoDocsize = 0; /* True to omit %_docsize table */ 868 - int bPrefix = 0; /* True to include a prefix-search index */ 869 920 const char **aCol; /* Array of column names */ 870 921 sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ 922 + 923 + char *zPrefix = 0; /* Prefix parameter value (or NULL) */ 924 + int nIndex; /* Size of aIndex[] array */ 925 + struct Fts3Index *aIndex; /* Array of indexes for this table */ 926 + struct Fts3Index *aFree = 0; /* Free this before returning */ 871 927 872 928 char *zCompress = 0; 873 929 char *zUncompress = 0; 874 930 875 931 assert( strlen(argv[0])==4 ); 876 932 assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) 877 933 || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) ................................................................................ 925 981 }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){ 926 982 zCompress = zVal; 927 983 zVal = 0; 928 984 }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){ 929 985 zUncompress = zVal; 930 986 zVal = 0; 931 987 }else if( nKey==6 && 0==sqlite3_strnicmp(z, "prefix", 6) ){ 932 - bPrefix = 1; 988 + sqlite3_free(zPrefix); 989 + zPrefix = zVal; 990 + zVal = 0; 933 991 }else{ 934 992 *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); 935 993 rc = SQLITE_ERROR; 936 994 } 937 995 sqlite3_free(zVal); 938 996 } 939 997 ................................................................................ 954 1012 955 1013 if( pTokenizer==0 ){ 956 1014 rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr); 957 1015 if( rc!=SQLITE_OK ) goto fts3_init_out; 958 1016 } 959 1017 assert( pTokenizer ); 960 1018 1019 + rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree); 1020 + if( rc==SQLITE_ERROR ){ 1021 + assert( zPrefix ); 1022 + *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix); 1023 + } 1024 + if( rc ) goto fts3_init_out; 1025 + 961 1026 962 1027 /* Allocate and populate the Fts3Table structure. */ 963 - nByte = sizeof(Fts3Table) + /* Fts3Table */ 1028 + nByte = sizeof(Fts3Table) + /* Fts3Table */ 964 1029 nCol * sizeof(char *) + /* azColumn */ 1030 + nIndex * sizeof(struct Fts3Index) + /* aIndex */ 965 1031 nName + /* zName */ 966 1032 nDb + /* zDb */ 967 1033 nString; /* Space for azColumn strings */ 968 1034 p = (Fts3Table*)sqlite3_malloc(nByte); 969 1035 if( p==0 ){ 970 1036 rc = SQLITE_NOMEM; 971 1037 goto fts3_init_out; ................................................................................ 978 1044 p->pTokenizer = pTokenizer; 979 1045 p->nNodeSize = 1000; 980 1046 p->nMaxPendingData = FTS3_MAX_PENDING_DATA; 981 1047 p->bHasDocsize = (isFts4 && bNoDocsize==0); 982 1048 p->bHasStat = isFts4; 983 1049 TESTONLY( p->inTransaction = -1 ); 984 1050 TESTONLY( p->mxSavepoint = -1 ); 985 - p->bPrefix = bPrefix; 986 - fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); 987 - fts3HashInit(&p->pendingPrefixes, FTS3_HASH_STRING, 1); 1051 + 1052 + p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; 1053 + memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); 1054 + p->nIndex = nIndex; 1055 + for(i=0; i<nIndex; i++){ 1056 + fts3HashInit(&p->aIndex[i].hPending, FTS3_HASH_STRING, 1); 1057 + } 988 1058 989 1059 /* Fill in the zName and zDb fields of the vtab structure. */ 990 - zCsr = (char *)&p->azColumn[nCol]; 1060 + zCsr = (char *)&p->aIndex[nIndex]; 991 1061 p->zName = zCsr; 992 1062 memcpy(zCsr, argv[2], nName); 993 1063 zCsr += nName; 994 1064 p->zDb = zCsr; 995 1065 memcpy(zCsr, argv[1], nDb); 996 1066 zCsr += nDb; 997 1067 ................................................................................ 1030 1100 */ 1031 1101 fts3DatabasePageSize(&rc, p); 1032 1102 1033 1103 /* Declare the table schema to SQLite. */ 1034 1104 fts3DeclareVtab(&rc, p); 1035 1105 1036 1106 fts3_init_out: 1107 + sqlite3_free(zPrefix); 1108 + sqlite3_free(aFree); 1037 1109 sqlite3_free(zCompress); 1038 1110 sqlite3_free(zUncompress); 1039 1111 sqlite3_free((void *)aCol); 1040 1112 if( rc!=SQLITE_OK ){ 1041 1113 if( p ){ 1042 1114 fts3DisconnectMethod((sqlite3_vtab *)p); 1043 1115 }else if( pTokenizer ){ ................................................................................ 2140 2212 if( !*ppOut ) return SQLITE_NOMEM; 2141 2213 sqlite3Fts3PutVarint(*ppOut, docid); 2142 2214 } 2143 2215 2144 2216 return SQLITE_OK; 2145 2217 } 2146 2218 2219 +/* 2220 +** Append SegReader object pNew to the end of the pCsr->apSegment[] array. 2221 +*/ 2147 2222 static int fts3SegReaderCursorAppend( 2148 2223 Fts3SegReaderCursor *pCsr, 2149 2224 Fts3SegReader *pNew 2150 2225 ){ 2151 2226 if( (pCsr->nSegment%16)==0 ){ 2152 2227 Fts3SegReader **apNew; 2153 2228 int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); ................................................................................ 2159 2234 pCsr->apSegment = apNew; 2160 2235 } 2161 2236 pCsr->apSegment[pCsr->nSegment++] = pNew; 2162 2237 return SQLITE_OK; 2163 2238 } 2164 2239 2165 2240 /* 2166 -** Set up a cursor object for iterating through the full-text index or 2167 -** a single level therein. 2241 +** Set up a cursor object for iterating through a full-text index or a 2242 +** single level therein. 2168 2243 */ 2169 2244 int sqlite3Fts3SegReaderCursor( 2170 2245 Fts3Table *p, /* FTS3 table handle */ 2246 + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ 2171 2247 int iLevel, /* Level of segments to scan */ 2172 2248 const char *zTerm, /* Term to query for */ 2173 2249 int nTerm, /* Size of zTerm in bytes */ 2174 2250 int isPrefix, /* True for a prefix search */ 2175 2251 int isScan, /* True to scan from zTerm to EOF */ 2176 2252 Fts3SegReaderCursor *pCsr /* Cursor object to populate */ 2177 2253 ){ 2178 2254 int rc = SQLITE_OK; 2179 2255 int rc2; 2180 2256 int iAge = 0; 2181 2257 sqlite3_stmt *pStmt = 0; 2182 2258 2183 - assert( iLevel==FTS3_SEGCURSOR_ALL_TERM 2259 + assert( iIndex>=0 && iIndex<p->nIndex ); 2260 + assert( iLevel==FTS3_SEGCURSOR_ALL 2184 2261 || iLevel==FTS3_SEGCURSOR_PENDING 2185 - || iLevel==FTS3_SEGCURSOR_PENDING_PREFIX 2186 - || iLevel==FTS3_SEGCURSOR_ALL_PREFIX 2187 2262 || iLevel>=0 2188 2263 ); 2189 - assert( 0>FTS3_SEGCURSOR_ALL_TERM 2190 - && 0>FTS3_SEGCURSOR_PENDING 2191 - && 0>FTS3_SEGCURSOR_PENDING_PREFIX 2192 - && 0>FTS3_SEGCURSOR_ALL_PREFIX 2193 - ); 2194 - assert( iLevel==FTS3_SEGCURSOR_ALL_TERM 2195 - || iLevel==FTS3_SEGCURSOR_ALL_PREFIX 2196 - || (zTerm==0 && isPrefix==1) 2197 - ); 2264 + assert( iLevel<FTS3_SEGDIR_MAXLEVEL ); 2265 + assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 ); 2266 + assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) ); 2198 2267 assert( isPrefix==0 || isScan==0 ); 2268 + 2269 + /* "isScan" is only set to true by the ft4aux module, an ordinary 2270 + ** full-text tables. */ 2271 + assert( isScan==0 || p->aIndex==0 ); 2199 2272 2200 2273 memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); 2201 2274 2202 - /* "isScan" is only set to true by the ft4aux module, not an ordinary 2203 - ** full-text table. The pendingTerms and pendingPrefixes tables must be 2204 - ** empty in this case. */ 2205 - assert( isScan==0 || fts3HashCount(&p->pendingTerms)==0 ); 2206 - assert( isScan==0 || fts3HashCount(&p->pendingPrefixes)==0 ); 2207 - 2208 - /* If iLevel is less than 0, include a seg-reader for the pending-terms. */ 2209 - if( iLevel<0 && isScan==0 ){ 2210 - int bPrefix = ( 2211 - iLevel==FTS3_SEGCURSOR_PENDING_PREFIX 2212 - || iLevel==FTS3_SEGCURSOR_ALL_PREFIX 2213 - ); 2214 - Fts3SegReader *pPending = 0; 2215 - 2216 - rc = sqlite3Fts3SegReaderPending(p,zTerm,nTerm,isPrefix,bPrefix,&pPending); 2217 - if( rc==SQLITE_OK && pPending ){ 2218 - rc = fts3SegReaderCursorAppend(pCsr, pPending); 2275 + /* If iLevel is less than 0 and this is not a scan, include a seg-reader 2276 + ** for the pending-terms. If this is a scan, then this call must be being 2277 + ** made by an fts4aux module, not an FTS table. In this case calling 2278 + ** Fts3SegReaderPending might segfault, as the data structures used by 2279 + ** fts4aux are not completely populated. So it's easiest to filter these 2280 + ** calls out here. */ 2281 + if( iLevel<0 && p->aIndex ){ 2282 + Fts3SegReader *pSeg = 0; 2283 + rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg); 2284 + if( rc==SQLITE_OK && pSeg ){ 2285 + rc = fts3SegReaderCursorAppend(pCsr, pSeg); 2219 2286 } 2220 2287 } 2221 2288 2222 - if( iLevel!=FTS3_SEGCURSOR_PENDING && iLevel!=FTS3_SEGCURSOR_PENDING_PREFIX ){ 2289 + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ 2223 2290 if( rc==SQLITE_OK ){ 2224 - rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt); 2291 + rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt); 2225 2292 } 2293 + 2226 2294 while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ 2227 2295 Fts3SegReader *pSeg = 0; 2228 2296 2229 2297 /* Read the values returned by the SELECT into local variables. */ 2230 2298 sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); 2231 2299 sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); 2232 2300 sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); ................................................................................ 2269 2337 Fts3SegReaderCursor *pSegcsr; /* Object to allocate and return */ 2270 2338 int rc = SQLITE_NOMEM; /* Return code */ 2271 2339 2272 2340 pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor)); 2273 2341 if( pSegcsr ){ 2274 2342 int i; 2275 2343 int nCost = 0; 2344 + int bFound = 0; /* True once an index has been found */ 2276 2345 Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; 2277 2346 2278 - if( isPrefix && p->bPrefix && nTerm<=FTS3_MAX_PREFIX ){ 2279 - rc = sqlite3Fts3SegReaderCursor( 2280 - p, FTS3_SEGCURSOR_ALL_PREFIX, zTerm, nTerm, 0, 0, pSegcsr); 2347 + if( isPrefix ){ 2348 + for(i=1; i<p->nIndex; i++){ 2349 + if( p->aIndex[i].nPrefix==nTerm ){ 2350 + bFound = 1; 2351 + rc = sqlite3Fts3SegReaderCursor( 2352 + p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr); 2353 + break; 2354 + } 2355 + } 2356 + } 2281 2357 2282 - }else{ 2358 + if( bFound==0 ){ 2283 2359 rc = sqlite3Fts3SegReaderCursor( 2284 - p, FTS3_SEGCURSOR_ALL_TERM, zTerm, nTerm, isPrefix, 0, pSegcsr); 2360 + p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr 2361 + ); 2285 2362 } 2286 - 2287 2363 for(i=0; rc==SQLITE_OK && i<pSegcsr->nSegment; i++){ 2288 2364 rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); 2289 2365 } 2290 2366 pSegcsr->nCost = nCost; 2291 2367 } 2292 2368 2293 2369 *ppSegcsr = pSegcsr; ................................................................................ 3336 3412 */ 3337 3413 static int fts3RollbackMethod(sqlite3_vtab *pVtab){ 3338 3414 Fts3Table *p = (Fts3Table*)pVtab; 3339 3415 sqlite3Fts3PendingTermsClear(p); 3340 3416 assert( p->inTransaction!=0 ); 3341 3417 TESTONLY( p->inTransaction = 0 ); 3342 3418 TESTONLY( p->mxSavepoint = -1; ); 3343 - sqlite3Fts3PendingPrefixesClear((Fts3Table *)pVtab); 3344 3419 return SQLITE_OK; 3345 3420 } 3346 3421 3347 3422 /* 3348 3423 ** Load the doclist associated with expression pExpr to pExpr->aDoclist. 3349 3424 ** The loaded doclist contains positions as well as the document ids. 3350 3425 ** This is used by the matchinfo(), snippet() and offsets() auxillary ................................................................................ 3711 3786 } 3712 3787 static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ 3713 3788 Fts3Table *p = (Fts3Table*)pVtab; 3714 3789 assert( p->inTransaction ); 3715 3790 assert( p->mxSavepoint >= iSavepoint ); 3716 3791 TESTONLY( p->mxSavepoint = iSavepoint ); 3717 3792 sqlite3Fts3PendingTermsClear(p); 3718 - sqlite3Fts3PendingPrefixesClear((Fts3Table *)pVtab); 3719 3793 return SQLITE_OK; 3720 3794 } 3721 3795 3722 3796 static const sqlite3_module fts3Module = { 3723 3797 /* iVersion */ 2, 3724 3798 /* xCreate */ fts3CreateMethod, 3725 3799 /* xConnect */ fts3ConnectMethod,
Changes to ext/fts3/fts3Int.h.
19 19 # define NDEBUG 1 20 20 #endif 21 21 22 22 #include "sqlite3.h" 23 23 #include "fts3_tokenizer.h" 24 24 #include "fts3_hash.h" 25 25 26 -#define FTS3_MAX_PREFIX 8 27 - 28 26 /* 29 27 ** This constant controls how often segments are merged. Once there are 30 28 ** FTS3_MERGE_COUNT segments of level N, they are merged into a single 31 29 ** segment of level N+1. 32 30 */ 33 31 #define FTS3_MERGE_COUNT 16 34 32 ................................................................................ 52 50 /* 53 51 ** Maximum length of a varint encoded integer. The varint format is different 54 52 ** from that used by SQLite, so the maximum length is 10, not 9. 55 53 */ 56 54 #define FTS3_VARINT_MAX 10 57 55 58 56 /* 59 -** FTS4 virtual tables may maintain two separate indexes. One that indexes 60 -** all document terms (the same index FTS3 tables maintain) and another used 61 -** for prefixes. B+-trees that are part of the prefix index have values for 62 -** the %_segdir.level column that are equal to or greater than the following 63 -** value. 57 +** FTS4 virtual tables may maintain multiple indexes - one index of all terms 58 +** in the document set and zero or more prefix indexes. All indexes are stored 59 +** as one or more b+-trees in the %_segments and %_segdir tables. 60 +** 61 +** It is possible to determine which index a b+-tree belongs to based on the 62 +** value stored in the "%_segdir.level" column. Given this value L, the index 63 +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with 64 +** level values between 0 and 1023 (inclusive) belong to index 0, all levels 65 +** between 1024 and 2047 to index 1, and so on. 64 66 ** 65 -** It is considered impossible for the regular index to use levels this large. 66 -** In theory it could, but that would require that at least 2^1024 separate 67 -** write operations to be made within the lifetime of the database. 67 +** It is considered impossible for an index to use more than 1024 levels. In 68 +** theory though this may happen, but only after at least 69 +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. 68 70 */ 69 -#define FTS3_SEGDIR_PREFIXLEVEL 1024 70 -#define FTS3_SEGDIR_PREFIXLEVEL_STR "1024" 71 +#define FTS3_SEGDIR_MAXLEVEL 1024 72 +#define FTS3_SEGDIR_MAXLEVEL_STR "1024" 71 73 72 74 /* 73 75 ** The testcase() macro is only used by the amalgamation. If undefined, 74 76 ** make it a no-op. 75 77 */ 76 78 #ifndef testcase 77 79 # define testcase(X) ................................................................................ 168 170 169 171 char *zReadExprlist; 170 172 char *zWriteExprlist; 171 173 172 174 int nNodeSize; /* Soft limit for node size */ 173 175 u8 bHasStat; /* True if %_stat table exists */ 174 176 u8 bHasDocsize; /* True if %_docsize table exists */ 175 - u8 bPrefix; /* True if there is a prefix index */ 176 177 int nPgsz; /* Page size for host database */ 177 178 char *zSegmentsTbl; /* Name of %_segments table */ 178 179 sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ 179 180 180 - /* The following hash table is used to buffer pending index updates during 181 + /* TODO: Fix the first paragraph of this comment. 182 + ** 183 + ** The following hash table is used to buffer pending index updates during 181 184 ** transactions. Variable nPendingData estimates the memory size of the 182 185 ** pending data, including hash table overhead, but not malloc overhead. 183 186 ** When nPendingData exceeds nMaxPendingData, the buffer is flushed 184 187 ** automatically. Variable iPrevDocid is the docid of the most recently 185 188 ** inserted record. 189 + ** 190 + ** A single FTS4 table may have multiple full-text indexes. For each index 191 + ** there is an entry in the aIndex[] array. Index 0 is an index of all the 192 + ** terms that appear in the document set. Each subsequent index in aIndex[] 193 + ** is an index of prefixes of a specific length. 186 194 */ 187 - int nMaxPendingData; 188 - int nPendingData; 189 - sqlite_int64 iPrevDocid; 190 - Fts3Hash pendingTerms; 191 - Fts3Hash pendingPrefixes; 195 + int nIndex; /* Size of aIndex[] */ 196 + struct Fts3Index { 197 + int nPrefix; /* Prefix length (0 for main terms index) */ 198 + Fts3Hash hPending; /* Pending terms table for this index */ 199 + } *aIndex; 200 + int nMaxPendingData; /* Max pending data before flush to disk */ 201 + int nPendingData; /* Current bytes of pending data */ 202 + sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ 192 203 193 204 #if defined(SQLITE_DEBUG) 194 205 /* State variables used for validating that the transaction control 195 206 ** methods of the virtual table are called at appropriate times. These 196 207 ** values do not contribution to the FTS computation; they are used for 197 208 ** verifying the SQLite core. 198 209 */ ................................................................................ 332 343 int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); 333 344 int sqlite3Fts3PendingTermsFlush(Fts3Table *); 334 345 void sqlite3Fts3PendingTermsClear(Fts3Table *); 335 346 int sqlite3Fts3Optimize(Fts3Table *); 336 347 int sqlite3Fts3SegReaderNew(int, sqlite3_int64, 337 348 sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); 338 349 int sqlite3Fts3SegReaderPending( 339 - Fts3Table*,const char*,int,int,int,Fts3SegReader**); 350 + Fts3Table*,int,const char*,int,int,Fts3SegReader**); 340 351 void sqlite3Fts3SegReaderFree(Fts3SegReader *); 341 352 int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); 342 -int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **); 353 +int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **); 343 354 int sqlite3Fts3ReadLock(Fts3Table *); 344 355 int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); 345 356 346 357 int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); 347 358 int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); 348 359 349 360 void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); ................................................................................ 351 362 int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); 352 363 void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); 353 364 char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); 354 365 void sqlite3Fts3SegmentsClose(Fts3Table *); 355 366 356 367 /* Special values interpreted by sqlite3SegReaderCursor() */ 357 368 #define FTS3_SEGCURSOR_PENDING -1 358 -#define FTS3_SEGCURSOR_PENDING_PREFIX -2 359 -#define FTS3_SEGCURSOR_ALL_PREFIX -3 360 -#define FTS3_SEGCURSOR_ALL_TERM -4 369 +#define FTS3_SEGCURSOR_ALL -2 361 370 362 371 int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*); 363 372 int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *); 364 373 void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *); 365 374 int sqlite3Fts3SegReaderCursor( 366 - Fts3Table *, int, const char *, int, int, int, Fts3SegReaderCursor *); 375 + Fts3Table *, int, int, const char *, int, int, int, Fts3SegReaderCursor *); 367 376 368 377 /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ 369 378 #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 370 379 #define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 371 380 #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 372 381 #define FTS3_SEGMENT_PREFIX 0x00000008 373 382 #define FTS3_SEGMENT_SCAN 0x00000010
Changes to ext/fts3/fts3_aux.c.
92 92 if( !p ) return SQLITE_NOMEM; 93 93 memset(p, 0, nByte); 94 94 95 95 p->pFts3Tab = (Fts3Table *)&p[1]; 96 96 p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; 97 97 p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; 98 98 p->pFts3Tab->db = db; 99 + p->pFts3Tab->nIndex = 1; 99 100 100 101 memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); 101 102 memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); 102 103 sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); 103 104 104 105 *ppVtab = (sqlite3_vtab *)p; 105 106 return SQLITE_OK; ................................................................................ 371 372 if( idxNum&FTS4AUX_LE_CONSTRAINT ){ 372 373 int iIdx = (idxNum&FTS4AUX_GE_CONSTRAINT) ? 1 : 0; 373 374 pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx])); 374 375 pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]); 375 376 if( pCsr->zStop==0 ) return SQLITE_NOMEM; 376 377 } 377 378 378 - rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL_TERM, 379 + rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL, 379 380 pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr 380 381 ); 381 382 if( rc==SQLITE_OK ){ 382 383 rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); 383 384 } 384 385 385 386 if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor);
Changes to ext/fts3/fts3_term.c.
23 23 #include <assert.h> 24 24 25 25 typedef struct Fts3termTable Fts3termTable; 26 26 typedef struct Fts3termCursor Fts3termCursor; 27 27 28 28 struct Fts3termTable { 29 29 sqlite3_vtab base; /* Base class used by SQLite core */ 30 - int bPrefix; /* True for an fts4prefix table */ 30 + int iIndex; /* Index for Fts3Table.aIndex[] */ 31 31 Fts3Table *pFts3Tab; 32 32 }; 33 33 34 34 struct Fts3termCursor { 35 35 sqlite3_vtab_cursor base; /* Base class used by SQLite core */ 36 36 Fts3SegReaderCursor csr; /* Must be right after "base" */ 37 37 Fts3SegFilter filter; ................................................................................ 66 66 char const *zDb; /* Name of database (e.g. "main") */ 67 67 char const *zFts3; /* Name of fts3 table */ 68 68 int nDb; /* Result of strlen(zDb) */ 69 69 int nFts3; /* Result of strlen(zFts3) */ 70 70 int nByte; /* Bytes of space to allocate here */ 71 71 int rc; /* value returned by declare_vtab() */ 72 72 Fts3termTable *p; /* Virtual table object to return */ 73 + int iIndex = 0; 74 + 75 + if( argc==5 ){ 76 + iIndex = atoi(argv[4]); 77 + argc--; 78 + } 73 79 74 80 /* The user should specify a single argument - the name of an fts3 table. */ 75 81 if( argc!=4 ){ 76 82 *pzErr = sqlite3_mprintf( 77 83 "wrong number of arguments to fts4term constructor" 78 84 ); 79 85 return SQLITE_ERROR; ................................................................................ 92 98 if( !p ) return SQLITE_NOMEM; 93 99 memset(p, 0, nByte); 94 100 95 101 p->pFts3Tab = (Fts3Table *)&p[1]; 96 102 p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; 97 103 p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; 98 104 p->pFts3Tab->db = db; 99 - p->bPrefix = (int)pCtx; 105 + p->pFts3Tab->nIndex = iIndex+1; 106 + p->iIndex = iIndex; 100 107 101 108 memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); 102 109 memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); 103 110 sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); 104 111 105 112 *ppVtab = (sqlite3_vtab *)p; 106 113 return SQLITE_OK; ................................................................................ 259 266 testcase(pCsr->filter.zTerm); 260 267 sqlite3Fts3SegReaderFinish(&pCsr->csr); 261 268 memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); 262 269 263 270 pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; 264 271 pCsr->filter.flags |= FTS3_SEGMENT_SCAN; 265 272 266 - rc = sqlite3Fts3SegReaderCursor(pFts3, 267 - p->bPrefix ? FTS3_SEGCURSOR_ALL_PREFIX : FTS3_SEGCURSOR_ALL_TERM, 273 + rc = sqlite3Fts3SegReaderCursor(pFts3, p->iIndex, FTS3_SEGCURSOR_ALL, 268 274 pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr 269 275 ); 270 276 if( rc==SQLITE_OK ){ 271 277 rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); 272 278 } 273 279 if( rc==SQLITE_OK ){ 274 280 rc = fts3termNextMethod(pCursor); ................................................................................ 351 357 0, /* xRollback */ 352 358 0, /* xFindFunction */ 353 359 0 /* xRename */ 354 360 }; 355 361 int rc; /* Return code */ 356 362 357 363 rc = sqlite3_create_module(db, "fts4term", &fts3term_module, 0); 358 - if( rc==SQLITE_OK ){ 359 - rc = sqlite3_create_module(db, "fts4prefix", &fts3term_module, (void*)1); 360 - } 361 364 return rc; 362 365 } 363 366 364 367 #endif 365 368 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
Changes to ext/fts3/fts3_write.c.
179 179 #define SQL_DELETE_ALL_STAT 6 180 180 #define SQL_SELECT_CONTENT_BY_ROWID 7 181 181 #define SQL_NEXT_SEGMENT_INDEX 8 182 182 #define SQL_INSERT_SEGMENTS 9 183 183 #define SQL_NEXT_SEGMENTS_ID 10 184 184 #define SQL_INSERT_SEGDIR 11 185 185 #define SQL_SELECT_LEVEL 12 186 -#define SQL_SELECT_ALL_LEVEL 13 186 +#define SQL_SELECT_LEVEL_RANGE 13 187 187 #define SQL_SELECT_LEVEL_COUNT 14 188 188 #define SQL_SELECT_SEGDIR_MAX_LEVEL 15 189 -#define SQL_DELETE_SEGDIR_BY_LEVEL 16 189 +#define SQL_DELETE_SEGDIR_LEVEL 16 190 190 #define SQL_DELETE_SEGMENTS_RANGE 17 191 191 #define SQL_CONTENT_INSERT 18 192 192 #define SQL_DELETE_DOCSIZE 19 193 193 #define SQL_REPLACE_DOCSIZE 20 194 194 #define SQL_SELECT_DOCSIZE 21 195 195 #define SQL_SELECT_DOCTOTAL 22 196 196 #define SQL_REPLACE_DOCTOTAL 23 197 + 197 198 #define SQL_SELECT_ALL_PREFIX_LEVEL 24 198 - 199 199 #define SQL_DELETE_ALL_TERMS_SEGDIR 25 200 -#define SQL_DELETE_ALL_PREFIX_SEGDIR 26 200 + 201 +#define SQL_DELETE_SEGDIR_RANGE 26 201 202 202 203 /* 203 204 ** This function is used to obtain an SQLite prepared statement handle 204 205 ** for the statement identified by the second argument. If successful, 205 206 ** *pp is set to the requested statement handle and SQLITE_OK returned. 206 207 ** Otherwise, an SQLite error code is returned and *pp is set to 0. 207 208 ** ................................................................................ 230 231 /* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", 231 232 /* 11 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", 232 233 233 234 /* Return segments in order from oldest to newest.*/ 234 235 /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " 235 236 "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", 236 237 /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " 237 - "FROM %Q.'%q_segdir' WHERE level < " FTS3_SEGDIR_PREFIXLEVEL_STR 238 - " ORDER BY level DESC, idx ASC", 238 + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" 239 + "ORDER BY level DESC, idx ASC", 239 240 240 241 /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", 241 -/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level < (?+1)*" 242 - FTS3_SEGDIR_PREFIXLEVEL_STR, 242 +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", 243 243 244 244 /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", 245 245 /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", 246 246 /* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", 247 247 /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", 248 248 /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", 249 249 /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", 250 250 /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", 251 251 /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", 252 -/* 24 */ "SELECT idx, start_block, leaves_end_block, end_block, root " 253 - "FROM %Q.'%q_segdir' WHERE level >= " FTS3_SEGDIR_PREFIXLEVEL_STR 254 - " ORDER BY level DESC, idx ASC", 255 -/* 25 */ "DELETE FROM %Q.'%q_segdir' WHERE level<" FTS3_SEGDIR_PREFIXLEVEL_STR, 256 -/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level>=" FTS3_SEGDIR_PREFIXLEVEL_STR, 252 +/* 24 */ "", 253 +/* 25 */ "", 254 + 255 +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", 256 + 257 257 }; 258 258 int rc = SQLITE_OK; 259 259 sqlite3_stmt *pStmt; 260 260 261 261 assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); 262 262 assert( eStmt<SizeofArray(azSql) && eStmt>=0 ); 263 263 ................................................................................ 405 405 ** 406 406 ** 0: idx 407 407 ** 1: start_block 408 408 ** 2: leaves_end_block 409 409 ** 3: end_block 410 410 ** 4: root 411 411 */ 412 -int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){ 412 +int sqlite3Fts3AllSegdirs( 413 + Fts3Table *p, /* FTS3 table */ 414 + int iIndex, /* Index for p->aIndex[] */ 415 + int iLevel, /* Level to select */ 416 + sqlite3_stmt **ppStmt /* OUT: Compiled statement */ 417 +){ 413 418 int rc; 414 419 sqlite3_stmt *pStmt = 0; 415 - if( iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){ 416 - rc = fts3SqlStmt(p, SQL_SELECT_ALL_PREFIX_LEVEL, &pStmt, 0); 417 - }else if( iLevel==FTS3_SEGCURSOR_ALL_TERM ){ 418 - rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0); 420 + 421 + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); 422 + assert( iLevel<FTS3_SEGDIR_MAXLEVEL ); 423 + assert( iIndex>=0 && iIndex<p->nIndex ); 424 + 425 + if( iLevel==FTS3_SEGCURSOR_ALL ){ 426 + /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ 427 + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); 428 + if( rc==SQLITE_OK ){ 429 + sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); 430 + sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1); 431 + } 419 432 }else{ 420 - assert( iLevel>=0 ); 433 + /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ 421 434 rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); 422 - if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel); 435 + if( rc==SQLITE_OK ){ 436 + sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL); 437 + } 423 438 } 424 439 *ppStmt = pStmt; 425 440 return rc; 426 441 } 427 442 428 443 429 444 /* ................................................................................ 560 575 } 561 576 } 562 577 if( rc==SQLITE_OK ){ 563 578 p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); 564 579 } 565 580 return rc; 566 581 } 567 - 568 - 569 - 570 582 571 583 /* 572 584 ** Tokenize the nul-terminated string zText and add all tokens to the 573 585 ** pending-terms hash-table. The docid used is that currently stored in 574 586 ** p->iPrevDocid, and the column is specified by argument iCol. 575 587 ** 576 588 ** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. ................................................................................ 612 624 } 613 625 pCsr->pTokenizer = pTokenizer; 614 626 615 627 xNext = pModule->xNext; 616 628 while( SQLITE_OK==rc 617 629 && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) 618 630 ){ 631 + int i; 619 632 if( iPos>=nWord ) nWord = iPos+1; 620 633 621 634 /* Positions cannot be negative; we use -1 as a terminator internally. 622 635 ** Tokens must have a non-zero length. 623 636 */ 624 637 if( iPos<0 || !zToken || nToken<=0 ){ 625 638 rc = SQLITE_ERROR; 626 639 break; 627 640 } 628 641 629 - rc = fts3PendingTermsAddOne(p,iCol,iPos,&p->pendingTerms,zToken,nToken); 630 - if( p->bPrefix ){ 631 - int n = (nToken > FTS3_MAX_PREFIX ? FTS3_MAX_PREFIX : nToken); 632 - for(; n>0 && rc==SQLITE_OK; n--){ 633 - rc = fts3PendingTermsAddOne(p,iCol,iPos,&p->pendingPrefixes,zToken,n); 634 - } 642 + /* Add the term to the terms index */ 643 + rc = fts3PendingTermsAddOne( 644 + p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken 645 + ); 646 + 647 + /* Add the term to each of the prefix indexes that it is not too 648 + ** short for. */ 649 + for(i=1; rc==SQLITE_OK && i<p->nIndex; i++){ 650 + struct Fts3Index *pIndex = &p->aIndex[i]; 651 + if( nToken<pIndex->nPrefix ) continue; 652 + rc = fts3PendingTermsAddOne( 653 + p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix 654 + ); 635 655 } 636 656 } 637 657 638 658 pModule->xClose(pCsr); 639 659 *pnWord = nWord; 640 660 return (rc==SQLITE_DONE ? SQLITE_OK : rc); 641 661 } ................................................................................ 657 677 if( rc!=SQLITE_OK ) return rc; 658 678 } 659 679 p->iPrevDocid = iDocid; 660 680 return SQLITE_OK; 661 681 } 662 682 663 683 /* 664 -** Discard the contents of the pending-terms hash table. 684 +** Discard the contents of the pending-terms hash tables. 665 685 */ 666 686 void sqlite3Fts3PendingTermsClear(Fts3Table *p){ 667 - Fts3HashElem *pElem; 668 - for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ 669 - sqlite3_free(fts3HashData(pElem)); 687 + int i; 688 + for(i=0; i<p->nIndex; i++){ 689 + Fts3HashElem *pElem; 690 + Fts3Hash *pHash = &p->aIndex[i].hPending; 691 + for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ 692 + sqlite3_free(fts3HashData(pElem)); 693 + } 694 + fts3HashClear(pHash); 670 695 } 671 - fts3HashClear(&p->pendingTerms); 672 696 p->nPendingData = 0; 673 697 } 674 698 675 -/* 676 -** Discard the contents of the pending-prefixes hash table. 677 -*/ 678 -void sqlite3Fts3PendingPrefixesClear(Fts3Table *p){ 679 - Fts3HashElem *pElem; 680 - for(pElem=fts3HashFirst(&p->pendingPrefixes); pElem; pElem=fts3HashNext(pElem)){ 681 - sqlite3_free(fts3HashData(pElem)); 682 - } 683 - fts3HashClear(&p->pendingPrefixes); 684 -} 685 - 686 699 /* 687 700 ** This function is called by the xUpdate() method as part of an INSERT 688 701 ** operation. It adds entries for each term in the new record to the 689 702 ** pendingTerms hash table. 690 703 ** 691 704 ** Argument apVal is the same as the similarly named argument passed to 692 705 ** fts3InsertData(). Parameter iDocid is the docid of the new row. ................................................................................ 776 789 ** pending terms. 777 790 */ 778 791 static int fts3DeleteAll(Fts3Table *p){ 779 792 int rc = SQLITE_OK; /* Return code */ 780 793 781 794 /* Discard the contents of the pending-terms hash table. */ 782 795 sqlite3Fts3PendingTermsClear(p); 783 - sqlite3Fts3PendingPrefixesClear(p); 784 796 785 797 /* Delete everything from the %_content, %_segments and %_segdir tables. */ 786 798 fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); 787 799 fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); 788 800 fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); 789 801 if( p->bHasDocsize ){ 790 802 fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0); ................................................................................ 832 844 *pRC = rc; 833 845 } 834 846 835 847 /* 836 848 ** Forward declaration to account for the circular dependency between 837 849 ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). 838 850 */ 839 -static int fts3SegmentMerge(Fts3Table *, int); 851 +static int fts3SegmentMerge(Fts3Table *, int, int); 840 852 841 853 /* 842 854 ** This function allocates a new level iLevel index in the segdir table. 843 855 ** Usually, indexes are allocated within a level sequentially starting 844 856 ** with 0, so the allocated index is one greater than the value returned 845 857 ** by: 846 858 ** ................................................................................ 849 861 ** However, if there are already FTS3_MERGE_COUNT indexes at the requested 850 862 ** level, they are merged into a single level (iLevel+1) segment and the 851 863 ** allocated index is 0. 852 864 ** 853 865 ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK 854 866 ** returned. Otherwise, an SQLite error code is returned. 855 867 */ 856 -static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ 868 +static int fts3AllocateSegdirIdx( 869 + Fts3Table *p, 870 + int iIndex, /* Index for p->aIndex */ 871 + int iLevel, 872 + int *piIdx 873 +){ 857 874 int rc; /* Return Code */ 858 875 sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ 859 876 int iNext = 0; /* Result of query pNextIdx */ 860 877 861 878 /* Set variable iNext to the next available segdir index at level iLevel. */ 862 879 rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); 863 880 if( rc==SQLITE_OK ){ 864 - sqlite3_bind_int(pNextIdx, 1, iLevel); 881 + sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); 865 882 if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ 866 883 iNext = sqlite3_column_int(pNextIdx, 0); 867 884 } 868 885 rc = sqlite3_reset(pNextIdx); 869 886 } 870 887 871 888 if( rc==SQLITE_OK ){ 872 889 /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already 873 890 ** full, merge all segments in level iLevel into a single iLevel+1 874 891 ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, 875 892 ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. 876 893 */ 877 894 if( iNext>=FTS3_MERGE_COUNT ){ 878 - rc = fts3SegmentMerge(p, iLevel); 895 + rc = fts3SegmentMerge(p, iIndex, iLevel); 879 896 *piIdx = 0; 880 897 }else{ 881 898 *piIdx = iNext; 882 899 } 883 900 } 884 901 885 902 return rc; ................................................................................ 1305 1322 ** 1306 1323 ** Whereas if isPrefixIter is zero, the terms visited are: 1307 1324 ** 1308 1325 ** firebird mysql sqlite 1309 1326 */ 1310 1327 int sqlite3Fts3SegReaderPending( 1311 1328 Fts3Table *p, /* Virtual table handle */ 1329 + int iIndex, /* Index for p->aIndex */ 1312 1330 const char *zTerm, /* Term to search for */ 1313 1331 int nTerm, /* Size of buffer zTerm */ 1314 - int isMultiTerm, /* True to visit multiple terms */ 1315 - int isPrefixIter, /* 0->pendingTerms, 1->pendingPrefixes */ 1332 + int bPrefix, /* True for a prefix iterator */ 1316 1333 Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ 1317 1334 ){ 1318 1335 Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ 1319 1336 Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ 1320 1337 int nElem = 0; /* Size of array at aElem */ 1321 1338 int rc = SQLITE_OK; /* Return Code */ 1339 + Fts3Hash *pHash; 1322 1340 1323 - if( isMultiTerm ){ 1341 + pHash = &p->aIndex[iIndex].hPending; 1342 + if( bPrefix ){ 1324 1343 int nAlloc = 0; /* Size of allocated array at aElem */ 1325 1344 Fts3HashElem *pE = 0; /* Iterator variable */ 1326 - Fts3Hash *pHash; 1327 - 1328 - pHash = (isPrefixIter ? &p->pendingPrefixes : &p->pendingTerms); 1329 1345 1330 1346 for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ 1331 1347 char *zKey = (char *)fts3HashKey(pE); 1332 1348 int nKey = fts3HashKeysize(pE); 1333 1349 if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ 1334 1350 if( nElem==nAlloc ){ 1335 1351 Fts3HashElem **aElem2; ................................................................................ 1356 1372 if( nElem>1 ){ 1357 1373 qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); 1358 1374 } 1359 1375 1360 1376 }else{ 1361 1377 /* The query is a simple term lookup that matches at most one term in 1362 1378 ** the index. All that is required is a straight hash-lookup. */ 1363 - Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm); 1379 + Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm); 1364 1380 if( pE ){ 1365 1381 aElem = &pE; 1366 1382 nElem = 1; 1367 1383 } 1368 1384 } 1369 1385 1370 1386 if( nElem>0 ){ ................................................................................ 1376 1392 memset(pReader, 0, nByte); 1377 1393 pReader->iIdx = 0x7FFFFFFF; 1378 1394 pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; 1379 1395 memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); 1380 1396 } 1381 1397 } 1382 1398 1383 - if( isMultiTerm ){ 1399 + if( bPrefix ){ 1384 1400 sqlite3_free(aElem); 1385 1401 } 1386 1402 *ppReader = pReader; 1387 1403 return rc; 1388 1404 } 1389 1405 1390 1406 /* ................................................................................ 1988 2004 } 1989 2005 rc = sqlite3_reset(pStmt); 1990 2006 } 1991 2007 return rc; 1992 2008 } 1993 2009 1994 2010 /* 1995 -** Set *pnMax to the largest segment level in the database for either the 1996 -** terms index (if parameter bPrefixIndex is 0) or the prefixes index (if 1997 -** parameter bPrefixIndex is 1). 2011 +** Set *pnMax to the largest segment level in the database for the index 2012 +** iIndex. 1998 2013 ** 1999 2014 ** Segment levels are stored in the 'level' column of the %_segdir table. 2000 2015 ** 2001 2016 ** Return SQLITE_OK if successful, or an SQLite error code if not. 2002 2017 */ 2003 -static int fts3SegmentMaxLevel(Fts3Table *p, int bPrefixIndex, int *pnMax){ 2018 +static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ 2004 2019 sqlite3_stmt *pStmt; 2005 2020 int rc; 2006 - assert( bPrefixIndex==0 || bPrefixIndex==1 ); 2021 + assert( iIndex>=0 && iIndex<p->nIndex ); 2007 2022 2008 2023 /* Set pStmt to the compiled version of: 2009 2024 ** 2010 - ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level < (?+1) * 1024 2025 + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? 2011 2026 ** 2012 2027 ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). 2013 2028 */ 2014 2029 rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); 2015 2030 if( rc!=SQLITE_OK ) return rc; 2016 - sqlite3_bind_int(pStmt, 1, bPrefixIndex); 2031 + sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); 2032 + sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1); 2017 2033 if( SQLITE_ROW==sqlite3_step(pStmt) ){ 2018 2034 *pnMax = sqlite3_column_int(pStmt, 0); 2019 2035 } 2020 2036 return sqlite3_reset(pStmt); 2021 2037 } 2022 2038 2023 2039 /* ................................................................................ 2032 2048 ** 2) deletes all %_segdir entries with level iLevel, or all %_segdir 2033 2049 ** entries regardless of level if (iLevel<0). 2034 2050 ** 2035 2051 ** SQLITE_OK is returned if successful, otherwise an SQLite error code. 2036 2052 */ 2037 2053 static int fts3DeleteSegdir( 2038 2054 Fts3Table *p, /* Virtual table handle */ 2055 + int iIndex, /* Index for p->aIndex */ 2039 2056 int iLevel, /* Level of %_segdir entries to delete */ 2040 2057 Fts3SegReader **apSegment, /* Array of SegReader objects */ 2041 2058 int nReader /* Size of array apSegment */ 2042 2059 ){ 2043 2060 int rc; /* Return Code */ 2044 2061 int i; /* Iterator variable */ 2045 2062 sqlite3_stmt *pDelete; /* SQL statement to delete rows */ ................................................................................ 2054 2071 rc = sqlite3_reset(pDelete); 2055 2072 } 2056 2073 } 2057 2074 if( rc!=SQLITE_OK ){ 2058 2075 return rc; 2059 2076 } 2060 2077 2061 - assert( iLevel>=0 2062 - || iLevel==FTS3_SEGCURSOR_ALL_TERM 2063 - || iLevel==FTS3_SEGCURSOR_ALL_PREFIX 2064 - || iLevel==FTS3_SEGCURSOR_PENDING 2065 - || iLevel==FTS3_SEGCURSOR_PENDING_PREFIX 2066 - ); 2067 - if( iLevel==FTS3_SEGCURSOR_ALL_TERM ){ 2068 - fts3SqlExec(&rc, p, SQL_DELETE_ALL_TERMS_SEGDIR, 0); 2069 - }else if( iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){ 2070 - fts3SqlExec(&rc, p, SQL_DELETE_ALL_PREFIX_SEGDIR, 0); 2071 - }else if( iLevel==FTS3_SEGCURSOR_PENDING_PREFIX ){ 2072 - sqlite3Fts3PendingPrefixesClear(p); 2073 - }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ 2074 - sqlite3Fts3PendingTermsClear(p); 2075 - }else if( iLevel>=0 ){ 2076 - rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); 2078 + assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL ); 2079 + if( iLevel==FTS3_SEGCURSOR_ALL ){ 2080 + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); 2081 + if( rc==SQLITE_OK ){ 2082 + sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); 2083 + sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1); 2084 + } 2085 + }else{ 2086 + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); 2077 2087 if( rc==SQLITE_OK ){ 2078 - sqlite3_bind_int(pDelete, 1, iLevel); 2079 - sqlite3_step(pDelete); 2080 - rc = sqlite3_reset(pDelete); 2088 + sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); 2081 2089 } 2082 2090 } 2091 + 2092 + if( rc==SQLITE_OK ){ 2093 + sqlite3_step(pDelete); 2094 + rc = sqlite3_reset(pDelete); 2095 + } 2083 2096 2084 2097 return rc; 2085 2098 } 2086 2099 2087 2100 /* 2088 2101 ** When this function is called, buffer *ppList (size *pnList bytes) contains 2089 2102 ** a position list that may (or may not) feature multiple columns. This ................................................................................ 2319 2332 ** currently present in the database. 2320 2333 ** 2321 2334 ** If this function is called with iLevel<0, but there is only one 2322 2335 ** segment in the database, SQLITE_DONE is returned immediately. 2323 2336 ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, 2324 2337 ** an SQLite error code is returned. 2325 2338 */ 2326 -static int fts3SegmentMerge(Fts3Table *p, int iLevel){ 2339 +static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ 2327 2340 int rc; /* Return code */ 2328 2341 int iIdx = 0; /* Index of new segment */ 2329 - int iNewLevel = 0; /* Level to create new segment at */ 2342 + int iNewLevel = 0; /* Level/index to create new segment at */ 2330 2343 SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ 2331 2344 Fts3SegFilter filter; /* Segment term filter condition */ 2332 2345 Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ 2333 2346 int bIgnoreEmpty = 0; /* True to ignore empty segments */ 2334 2347 2335 - rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, 0, &csr); 2348 + assert( iLevel==FTS3_SEGCURSOR_ALL 2349 + || iLevel==FTS3_SEGCURSOR_PENDING 2350 + || iLevel>=0 2351 + ); 2352 + assert( iLevel<FTS3_SEGDIR_MAXLEVEL ); 2353 + assert( iIndex>=0 && iIndex<p->nIndex ); 2354 + 2355 + rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr); 2336 2356 if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; 2337 2357 2338 - if( iLevel==FTS3_SEGCURSOR_ALL_TERM || iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){ 2358 + if( iLevel==FTS3_SEGCURSOR_ALL ){ 2339 2359 /* This call is to merge all segments in the database to a single 2340 2360 ** segment. The level of the new segment is equal to the the numerically 2341 - ** greatest segment level currently present in the database. The index 2342 - ** of the new segment is always 0. */ 2361 + ** greatest segment level currently present in the database for this 2362 + ** index. The idx of the new segment is always 0. */ 2343 2363 if( csr.nSegment==1 ){ 2344 2364 rc = SQLITE_DONE; 2345 2365 goto finished; 2346 2366 } 2347 - rc = fts3SegmentMaxLevel(p, iLevel==FTS3_SEGCURSOR_ALL_PREFIX, &iNewLevel); 2367 + rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel); 2348 2368 bIgnoreEmpty = 1; 2369 + 2370 + }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ 2371 + iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL; 2372 + rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx); 2349 2373 }else{ 2350 2374 /* This call is to merge all segments at level iLevel. find the next 2351 2375 ** available segment index at level iLevel+1. The call to 2352 2376 ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to 2353 2377 ** a single iLevel+2 segment if necessary. */ 2354 - if( iLevel==FTS3_SEGCURSOR_PENDING ){ 2355 - iNewLevel = 0; 2356 - }else if( iLevel==FTS3_SEGCURSOR_PENDING_PREFIX ){ 2357 - iNewLevel = FTS3_SEGDIR_PREFIXLEVEL; 2358 - }else{ 2359 - iNewLevel = iLevel+1; 2360 - } 2361 - rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx); 2378 + rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx); 2379 + iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1; 2362 2380 } 2363 2381 if( rc!=SQLITE_OK ) goto finished; 2364 2382 assert( csr.nSegment>0 ); 2365 - assert( iNewLevel>=0 ); 2383 + assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) ); 2384 + assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) ); 2366 2385 2367 2386 memset(&filter, 0, sizeof(Fts3SegFilter)); 2368 2387 filter.flags = FTS3_SEGMENT_REQUIRE_POS; 2369 2388 filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0); 2370 2389 2371 2390 rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); 2372 2391 while( SQLITE_OK==rc ){ ................................................................................ 2374 2393 if( rc!=SQLITE_ROW ) break; 2375 2394 rc = fts3SegWriterAdd(p, &pWriter, 1, 2376 2395 csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist); 2377 2396 } 2378 2397 if( rc!=SQLITE_OK ) goto finished; 2379 2398 assert( pWriter ); 2380 2399 2381 - rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment); 2382 - if( rc!=SQLITE_OK ) goto finished; 2400 + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ 2401 + rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment); 2402 + if( rc!=SQLITE_OK ) goto finished; 2403 + } 2383 2404 rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); 2384 2405 2385 2406 finished: 2386 2407 fts3SegWriterFree(pWriter); 2387 2408 sqlite3Fts3SegReaderFinish(&csr); 2388 2409 return rc; 2389 2410 } 2390 2411 2391 2412 2392 2413 /* 2393 -** Flush the contents of pendingTerms to a level 0 segment. 2414 +** Flush the contents of pendingTerms to level 0 segments. 2394 2415 */ 2395 2416 int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ 2396 2417 int rc = SQLITE_OK; 2397 - if( p->bPrefix ){ 2398 - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING_PREFIX); 2418 + int i; 2419 + for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){ 2420 + rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING); 2421 + if( rc==SQLITE_DONE ) rc = SQLITE_OK; 2399 2422 } 2400 - if( rc==SQLITE_OK || rc==SQLITE_DONE ){ 2401 - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING); 2402 - } 2403 - if( rc==SQLITE_DONE ){ 2404 - rc = SQLITE_OK; 2405 - } 2423 + sqlite3Fts3PendingTermsClear(p); 2406 2424 return rc; 2407 2425 } 2408 2426 2409 2427 /* 2410 2428 ** Encode N integers as varints into a blob. 2411 2429 */ 2412 2430 static void fts3EncodeIntArray( ................................................................................ 2549 2567 return; 2550 2568 } 2551 2569 sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC); 2552 2570 sqlite3_step(pStmt); 2553 2571 *pRC = sqlite3_reset(pStmt); 2554 2572 sqlite3_free(a); 2555 2573 } 2574 + 2575 +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ 2576 + int i; 2577 + int bSeenDone = 0; 2578 + int rc = SQLITE_OK; 2579 + for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){ 2580 + rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL); 2581 + if( rc==SQLITE_DONE ){ 2582 + bSeenDone = 1; 2583 + rc = SQLITE_OK; 2584 + } 2585 + } 2586 + sqlite3Fts3SegmentsClose(p); 2587 + sqlite3Fts3PendingTermsClear(p); 2588 + 2589 + return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; 2590 +} 2556 2591 2557 2592 /* 2558 2593 ** Handle a 'special' INSERT of the form: 2559 2594 ** 2560 2595 ** "INSERT INTO tbl(tbl) VALUES(<expr>)" 2561 2596 ** 2562 2597 ** Argument pVal contains the result of <expr>. Currently the only ................................................................................ 2566 2601 int rc; /* Return Code */ 2567 2602 const char *zVal = (const char *)sqlite3_value_text(pVal); 2568 2603 int nVal = sqlite3_value_bytes(pVal); 2569 2604 2570 2605 if( !zVal ){ 2571 2606 return SQLITE_NOMEM; 2572 2607 }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ 2573 - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_PREFIX); 2574 - if( rc==SQLITE_OK ){ 2575 - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_TERM); 2576 - } 2608 + rc = fts3DoOptimize(p, 0); 2577 2609 #ifdef SQLITE_TEST 2578 2610 }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ 2579 2611 p->nNodeSize = atoi(&zVal[9]); 2580 2612 rc = SQLITE_OK; 2581 2613 }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ 2582 2614 p->nMaxPendingData = atoi(&zVal[11]); 2583 2615 rc = SQLITE_OK; 2584 2616 #endif 2585 2617 }else{ 2586 2618 rc = SQLITE_ERROR; 2587 2619 } 2588 2620 2589 - sqlite3Fts3SegmentsClose(p); 2590 - sqlite3Fts3PendingTermsClear(p); 2591 - sqlite3Fts3PendingPrefixesClear(p); 2592 2621 return rc; 2593 2622 } 2594 2623 2595 2624 /* 2596 2625 ** Return the deferred doclist associated with deferred token pDeferred. 2597 2626 ** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already 2598 2627 ** been called to allocate and populate the doclist. ................................................................................ 2901 2930 /* 2902 2931 ** Flush any data in the pending-terms hash table to disk. If successful, 2903 2932 ** merge all segments in the database (including the new segment, if 2904 2933 ** there was any data to flush) into a single segment. 2905 2934 */ 2906 2935 int sqlite3Fts3Optimize(Fts3Table *p){ 2907 2936 int rc; 2908 - int bReturnDone = 0; 2909 2937 rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); 2910 2938 if( rc==SQLITE_OK ){ 2911 - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_PREFIX); 2912 - if( rc==SQLITE_OK ){ 2913 - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_TERM); 2914 - } 2915 - if( rc==SQLITE_DONE ){ 2916 - bReturnDone = 1; 2917 - rc = SQLITE_OK; 2918 - } 2919 - if( rc==SQLITE_OK ){ 2920 - rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); 2921 - if( rc==SQLITE_OK ){ 2922 - sqlite3Fts3PendingTermsClear(p); 2923 - sqlite3Fts3PendingPrefixesClear(p); 2924 - } 2939 + rc = fts3DoOptimize(p, 1); 2940 + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ 2941 + int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); 2942 + if( rc2!=SQLITE_OK ) rc = rc2; 2925 2943 }else{ 2926 2944 sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); 2927 2945 sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); 2928 2946 } 2929 2947 } 2930 2948 sqlite3Fts3SegmentsClose(p); 2931 - return ((rc==SQLITE_OK && bReturnDone) ? SQLITE_DONE : rc); 2949 + return rc; 2932 2950 } 2933 2951 2934 2952 #endif
Changes to test/fts3prefix.test.
11 11 # This file implements regression tests for SQLite library. The 12 12 # focus of this script is testing the FTS3 module. 13 13 # 14 14 15 15 set testdir [file dirname $argv0] 16 16 source $testdir/tester.tcl 17 17 set testprefix fts3prefix 18 + 19 +ifcapable !fts3 { 20 + finish_test 21 + return 22 +} 18 23 19 24 # This proc tests that the prefixes index appears to represent the same content 20 25 # as the terms index. 21 26 # 22 -proc fts3_terms_and_prefixes {db tbl} { 23 - $db eval "CREATE VIRTUAL TABLE fts3check1 USING fts4term($tbl);" 24 - $db eval "CREATE VIRTUAL TABLE fts3check2 USING fts4prefix($tbl);" 25 - 26 - $db eval { 27 - CREATE TEMP TABLE terms AS SELECT * FROM fts3check1; 28 - CREATE TEMP TABLE prefixes AS SELECT * FROM fts3check2; 29 - CREATE INDEX temp.idx ON prefixes(term); 30 - DROP TABLE fts3check1; 31 - DROP TABLE fts3check2; 32 - } 33 - 34 - $db eval { SELECT term, docid, col, pos FROM temp.terms } a { 35 - set nMax [expr [string length $a(term)] - 1] 36 - if {$nMax>8} {set nMax 8} 37 - for {set n 0} {$n < $nMax} {incr n} { 38 - set t [string range $a(term) 0 $n] 27 +proc fts3_terms_and_prefixes {db tbl prefixlengths} { 28 + 29 + set iIndex 0 30 + foreach len $prefixlengths { 31 + incr iIndex 32 + $db eval { 33 + DROP TABLE IF EXISTS fts3check1; 34 + DROP TABLE IF EXISTS fts3check2; 35 + } 36 + $db eval "CREATE VIRTUAL TABLE fts3check1 USING fts4term($tbl, 0);" 37 + $db eval "CREATE VIRTUAL TABLE fts3check2 USING fts4term($tbl, $iIndex);" 38 + 39 + $db eval { 40 + DROP TABLE IF EXISTS temp.terms; 41 + DROP TABLE IF EXISTS temp.prefixes; 42 + CREATE TEMP TABLE terms AS SELECT * FROM fts3check1; 43 + CREATE TEMP TABLE prefixes AS SELECT * FROM fts3check2; 44 + CREATE INDEX temp.idx ON prefixes(term); 45 + DROP TABLE fts3check1; 46 + DROP TABLE fts3check2; 47 + } 48 + 49 + set nExpect 0 50 + $db eval { SELECT term, docid, col, pos FROM temp.terms } a { 51 + if {[string length $a(term)]<$len} continue 52 + incr nExpect 53 + set prefix [string range $a(term) 0 [expr $len-1]] 39 54 set r [$db one { 40 55 SELECT count(*) FROM temp.prefixes WHERE 41 - term = $t AND docid = $a(docid) AND col = $a(col) AND pos = $a(pos) 56 + term = $prefix AND docid = $a(docid) AND col = $a(col) AND pos = $a(pos) 42 57 }] 43 58 if {$r != 1} { 44 59 error "$t, $a(docid), $a(col), $a(pos)" 45 60 } 46 61 } 47 - } 48 62 49 - execsql { DROP TABLE temp.prefixes } 50 - execsql { DROP TABLE temp.terms } 63 + set nCount [$db one {SELECT count(*) FROM temp.prefixes}] 64 + if {$nCount != $nExpect} { 65 + error "prefixes.count(*) is $nCount expected $nExpect" 66 + } 67 + 68 + execsql { DROP TABLE temp.prefixes } 69 + execsql { DROP TABLE temp.terms } 51 70 52 - set terms_layout [$db eval " 53 - SELECT level, idx FROM ${tbl}_segdir WHERE level < 1024 ORDER by 1, 2 54 - "] 55 - set prefixes_layout [$db eval " 56 - SELECT level-1024, idx FROM ${tbl}_segdir WHERE level >= 1024 ORDER by 1, 2 57 - "] 71 + set list [list] 72 + $db eval " 73 + SELECT sum( 1 << (16*(level%1024)) ) AS total, (level/1024) AS tree 74 + FROM ${tbl}_segdir GROUP BY tree 75 + " { 76 + lappend list [list $total $tree] 77 + } 58 78 59 - if {$terms_layout != $prefixes_layout} { 60 - puts "TERMS LAYOUT: $terms_layout" 61 - puts "PREFIX LAYOUT: $prefixes_layout" 62 - error "Terms and prefixes are comprised of different b-trees" 79 + if { [lsort -integer -index 0 $list] != [lsort -integer -index 1 $list] } { 80 + error "inconsistent tree structures: $list" 81 + } 63 82 } 64 83 65 84 return "" 66 85 } 67 -proc fts3_tap_test {tn db tbl} { 68 - uplevel [list do_test $tn [list fts3_terms_and_prefixes $db $tbl] ""] 86 +proc fts3_tap_test {tn db tbl lens} { 87 + uplevel [list do_test $tn [list fts3_terms_and_prefixes $db $tbl $lens] ""] 69 88 } 70 89 71 90 #------------------------------------------------------------------------- 72 91 # Test cases 1.* are a sanity check. They test that the prefixes index is 73 92 # being constructed correctly for the simplest possible case. 74 93 # 75 94 do_execsql_test 1.1 { 76 - CREATE VIRTUAL TABLE t1 USING fts4(prefix=1); 77 - CREATE VIRTUAL TABLE prefixes USING fts4prefix(t1); 95 + CREATE VIRTUAL TABLE t1 USING fts4(prefix='1,3,6'); 96 + 97 + CREATE VIRTUAL TABLE p1 USING fts4term(t1, 1); 98 + CREATE VIRTUAL TABLE p2 USING fts4term(t1, 2); 99 + CREATE VIRTUAL TABLE p3 USING fts4term(t1, 3); 78 100 CREATE VIRTUAL TABLE terms USING fts4term(t1); 79 101 } 80 102 do_execsql_test 1.2 { 81 103 INSERT INTO t1 VALUES('sqlite mysql firebird'); 82 104 } 83 -do_execsql_test 1.3 { 84 - SELECT term FROM prefixes; 85 -} {f fi fir fire fireb firebi firebir firebird m my mys mysq mysql s sq sql sqli sqlit sqlite} 105 +do_execsql_test 1.3.1 { SELECT term FROM p1 } {f m s} 106 +do_execsql_test 1.3.2 { SELECT term FROM p2 } {fir mys sql} 107 +do_execsql_test 1.3.3 { SELECT term FROM p3 } {firebi sqlite} 86 108 do_execsql_test 1.4 { 87 109 SELECT term FROM terms; 88 110 } {firebird mysql sqlite} 89 111 90 -fts3_tap_test 1.5 db t1 112 +fts3_tap_test 1.5 db t1 {1 3 6} 91 113 92 114 #------------------------------------------------------------------------- 93 115 # A slightly more complicated dataset. This test also verifies that DELETE 94 116 # operations do not corrupt the prefixes index. 95 117 # 96 118 do_execsql_test 2.1 { 97 119 INSERT INTO t1 VALUES('FTS3 and FTS4 are an SQLite virtual table modules'); ................................................................................ 111 133 INSERT INTO t1 VALUES('modules for SQLite. There are known issues with'); 112 134 INSERT INTO t1 VALUES('these older modules and their use should be'); 113 135 INSERT INTO t1 VALUES('avoided. Portions of the original FTS3 code were'); 114 136 INSERT INTO t1 VALUES('contributed to the SQLite project by Scott Hess of'); 115 137 INSERT INTO t1 VALUES('Google. It is now developed and maintained as part'); 116 138 INSERT INTO t1 VALUES('of SQLite. '); 117 139 } 118 -fts3_tap_test 2.2 db t1 140 +fts3_tap_test 2.2 db t1 {1 3 6} 119 141 do_execsql_test 2.3 { DELETE FROM t1 WHERE docid%2; } 120 -fts3_tap_test 2.4 db t1 142 +fts3_tap_test 2.4 db t1 {1 3 6} 121 143 122 144 do_execsql_test 2.5 { INSERT INTO t1(t1) VALUES('optimize') } 123 -fts3_tap_test 2.6 db t1 145 +fts3_tap_test 2.6 db t1 {1 3 6} 124 146 125 147 do_execsql_test 3.1 { 126 - CREATE VIRTUAL TABLE t2 USING fts4(prefix=1); 148 + CREATE VIRTUAL TABLE t2 USING fts4(prefix='1,2,3'); 127 149 INSERT INTO t2 VALUES('On 12 September the wind direction turned and'); 128 150 INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the'); 129 151 INSERT INTO t2 VALUES('fleet was forced to take shelter at'); 130 152 INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind'); 131 153 INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet'); 132 154 INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey'); 133 155 INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved'); ................................................................................ 144 166 INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved'); 145 167 INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he'); 146 168 INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of'); 147 169 INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland'); 148 170 INSERT INTO t2 VALUES('and waited for Harold''s return from the north.'); 149 171 } 150 172 151 -fts3_tap_test 3.2 db t2 173 +fts3_tap_test 3.2 db t2 {1 2 3} 152 174 do_execsql_test 3.3 { SELECT optimize(t2) FROM t2 LIMIT 1 } {{Index optimized}} 153 -fts3_tap_test 3.4 db t2 175 +fts3_tap_test 3.4 db t2 {1 2 3} 154 176 155 177 156 178 #------------------------------------------------------------------------- 157 179 # Simple tests for reading the prefix-index. 158 180 # 159 181 do_execsql_test 4.1 { 160 - CREATE VIRTUAL TABLE t3 USING fts4(prefix=1); 182 + CREATE VIRTUAL TABLE t3 USING fts4(prefix="1,4"); 161 183 INSERT INTO t3 VALUES('one two three'); 162 184 INSERT INTO t3 VALUES('four five six'); 163 185 INSERT INTO t3 VALUES('seven eight nine'); 164 186 } 165 187 do_execsql_test 4.2 { 166 188 SELECT * FROM t3 WHERE t3 MATCH 'f*' 167 189 } {{four five six}}