Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Experimental code to prevent FTS indexes from growing indefinitely as the table is updated. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts4-experimental |
Files: | files | file ages | folders |
SHA1: |
b3b505a4dd0c679437a4272109f11881 |
User & Date: | dan 2014-05-12 20:04:48.571 |
Context
2014-05-13
| ||
20:11 | Fix a problem preventing delete markers from ever being removed from the FTS index. (check-in: 7f47ae5c5d user: dan tags: fts4-experimental) | |
2014-05-12
| ||
20:04 | Experimental code to prevent FTS indexes from growing indefinitely as the table is updated. (check-in: b3b505a4dd user: dan tags: fts4-experimental) | |
2014-05-10
| ||
17:28 | When using Visual Studio 2013, add the appropriate MaxPlatformVersion attribute to the VSIX SDK manifest. (check-in: 0a4f59676b user: mistachkin tags: trunk) | |
Changes
Changes to ext/fts3/fts3Int.h.
︙ | ︙ | |||
212 213 214 215 216 217 218 | char *zLanguageid; /* languageid=xxx option, or NULL */ u8 bAutoincrmerge; /* True if automerge=1 */ u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ | | | 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 | char *zLanguageid; /* languageid=xxx option, or NULL */ u8 bAutoincrmerge; /* True if automerge=1 */ u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ sqlite3_stmt *aStmt[40]; char *zReadExprlist; char *zWriteExprlist; int nNodeSize; /* Soft limit for node size */ u8 bFts4; /* True for FTS4, false for FTS3 */ u8 bHasStat; /* True if %_stat table exists (2==unknown) */ |
︙ | ︙ |
Changes to ext/fts3/fts3_write.c.
︙ | ︙ | |||
189 190 191 192 193 194 195 196 197 198 199 200 201 202 | char *zTerm; /* Pointer to previous term buffer */ int nTerm; /* Number of bytes in zTerm */ int nMalloc; /* Size of malloc'd buffer at zMalloc */ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ int nSize; /* Size of allocation at aData */ int nData; /* Bytes of data in aData */ char *aData; /* Pointer to block from malloc() */ }; /* ** Type SegmentNode is used by the following three functions to create ** the interior part of the segment b+-tree structures (everything except ** the leaf nodes). These functions and type are only ever used by code ** within the fts3SegWriterXXX() family of functions described above. | > | 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 | char *zTerm; /* Pointer to previous term buffer */ int nTerm; /* Number of bytes in zTerm */ int nMalloc; /* Size of malloc'd buffer at zMalloc */ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ int nSize; /* Size of allocation at aData */ int nData; /* Bytes of data in aData */ char *aData; /* Pointer to block from malloc() */ i64 nLeafData; /* Number of bytes of leaf data written */ }; /* ** Type SegmentNode is used by the following three functions to create ** the interior part of the segment b+-tree structures (everything except ** the leaf nodes). These functions and type are only ever used by code ** within the fts3SegWriterXXX() family of functions described above. |
︙ | ︙ | |||
263 264 265 266 267 268 269 270 271 272 273 274 275 276 | #define SQL_DELETE_SEGDIR_ENTRY 30 #define SQL_SHIFT_SEGDIR_ENTRY 31 #define SQL_SELECT_SEGDIR 32 #define SQL_CHOMP_SEGDIR 33 #define SQL_SEGMENT_IS_APPENDABLE 34 #define SQL_SELECT_INDEXES 35 #define SQL_SELECT_MXLEVEL 36 /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, ** *pp is set to the requested statement handle and SQLITE_OK returned. ** Otherwise, an SQLite error code is returned and *pp is set to 0. ** | > > > > | 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 | #define SQL_DELETE_SEGDIR_ENTRY 30 #define SQL_SHIFT_SEGDIR_ENTRY 31 #define SQL_SELECT_SEGDIR 32 #define SQL_CHOMP_SEGDIR 33 #define SQL_SEGMENT_IS_APPENDABLE 34 #define SQL_SELECT_INDEXES 35 #define SQL_SELECT_MXLEVEL 36 #define SQL_SELECT_LEVEL_RANGE2 37 #define SQL_UPDATE_LEVEL_IDX 38 #define SQL_UPDATE_LEVEL 39 /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, ** *pp is set to the requested statement handle and SQLITE_OK returned. ** Otherwise, an SQLite error code is returned and *pp is set to 0. ** |
︙ | ︙ | |||
365 366 367 368 369 370 371 | /* SQL_SELECT_INDEXES ** Return the list of valid segment indexes for absolute level ? */ /* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", /* SQL_SELECT_MXLEVEL ** Return the largest relative level in the FTS index or indexes. */ | | > > > > > > > > > > | 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 | /* SQL_SELECT_INDEXES ** Return the list of valid segment indexes for absolute level ? */ /* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", /* SQL_SELECT_MXLEVEL ** Return the largest relative level in the FTS index or indexes. */ /* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", /* Return segments in order from oldest to newest.*/ /* 37 */ "SELECT level, idx, end_block " "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" "ORDER BY level DESC, idx ASC", /* Update statements used while promoting segments */ /* 38 */ "UPDATE %Q.'%q_segdir' SET level=-1,idx=? WHERE level=? AND idx=?", /* 39 */ "UPDATE %Q.'%q_segdir' SET level=? WHERE level=-1" }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); assert( eStmt<SizeofArray(azSql) && eStmt>=0 ); |
︙ | ︙ | |||
1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 | static int fts3WriteSegdir( Fts3Table *p, /* Virtual table handle */ sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ int iIdx, /* Value for "idx" field */ sqlite3_int64 iStartBlock, /* Value for "start_block" field */ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ sqlite3_int64 iEndBlock, /* Value for "end_block" field */ char *zRoot, /* Blob value for "root" field */ int nRoot /* Number of bytes in buffer zRoot */ ){ sqlite3_stmt *pStmt; int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, iLevel); sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); | > > | > > > > > | 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 | static int fts3WriteSegdir( Fts3Table *p, /* Virtual table handle */ sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ int iIdx, /* Value for "idx" field */ sqlite3_int64 iStartBlock, /* Value for "start_block" field */ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ sqlite3_int64 iEndBlock, /* Value for "end_block" field */ sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ char *zRoot, /* Blob value for "root" field */ int nRoot /* Number of bytes in buffer zRoot */ ){ sqlite3_stmt *pStmt; int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, iLevel); sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); if( nLeafData==0 ){ sqlite3_bind_int64(pStmt, 5, iEndBlock); }else{ char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); if( !zEnd ) return SQLITE_NOMEM; sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); } sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); } return rc; } |
︙ | ︙ | |||
2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 | nSuffix = nTerm; nReq = 1 + /* varint containing prefix size */ sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ nTerm + /* Term suffix */ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ nDoclist; /* Doclist data */ } /* If the buffer currently allocated is too small for this entry, realloc ** the buffer to make it large enough. */ if( nReq>pWriter->nSize ){ char *aNew = sqlite3_realloc(pWriter->aData, nReq); if( !aNew ) return SQLITE_NOMEM; | > > > | 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 | nSuffix = nTerm; nReq = 1 + /* varint containing prefix size */ sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ nTerm + /* Term suffix */ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ nDoclist; /* Doclist data */ } /* Increase the total number of bytes written to account for the new entry. */ pWriter->nLeafData += nReq; /* If the buffer currently allocated is too small for this entry, realloc ** the buffer to make it large enough. */ if( nReq>pWriter->nSize ){ char *aNew = sqlite3_realloc(pWriter->aData, nReq); if( !aNew ) return SQLITE_NOMEM; |
︙ | ︙ | |||
2313 2314 2315 2316 2317 2318 2319 | iLastLeaf = pWriter->iFree; rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); if( rc==SQLITE_OK ){ rc = fts3NodeWrite(p, pWriter->pTree, 1, pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); } if( rc==SQLITE_OK ){ | | | | | | 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 | iLastLeaf = pWriter->iFree; rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); if( rc==SQLITE_OK ){ rc = fts3NodeWrite(p, pWriter->pTree, 1, pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); } if( rc==SQLITE_OK ){ rc = fts3WriteSegdir(p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); } }else{ /* The entire tree fits on the root node. Write it to the segdir table. */ rc = fts3WriteSegdir(p, iLevel, iIdx, 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); } p->nLeafAdd++; return rc; } /* ** Release all memory held by the SegmentWriter object passed as the |
︙ | ︙ | |||
2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 | sqlite3_free(pCsr->aBuffer); pCsr->nSegment = 0; pCsr->apSegment = 0; pCsr->aBuffer = 0; } } /* ** Merge all level iLevel segments in the database into a single ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a ** single segment with a level equal to the numerically largest level ** currently present in the database. ** | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 | sqlite3_free(pCsr->aBuffer); pCsr->nSegment = 0; pCsr->apSegment = 0; pCsr->aBuffer = 0; } } /* ** Decode the "end_block" field, selected by column iCol of the SELECT ** statement passed as the first argument. */ static void fts3ReadEndBlockField( sqlite3_stmt *pStmt, int iCol, i64 *piEndBlock, i64 *pnByte ){ const unsigned char *zText = sqlite3_column_text(pStmt, iCol); if( zText ){ int i; i64 iVal = 0; for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ iVal = iVal*10 + (zText[i] - '0'); } *piEndBlock = iVal; while( zText[i]==' ' ) i++; iVal = 0; for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ iVal = iVal*10 + (zText[i] - '0'); } *pnByte = iVal; } } /* ** A segment of size nByte bytes has just been written to absolute level ** iAbsLevel. Promote any segments that should be promoted as a result. */ static int fts3PromoteSegments( Fts3Table *p, /* FTS table handle */ int iAbsLevel, /* Absolute level just updated */ sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ ){ int rc = SQLITE_OK; sqlite3_stmt *pRange; rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); if( rc==SQLITE_OK ){ int bOk = 1; int iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; sqlite3_bind_int(pRange, 1, iAbsLevel+1); sqlite3_bind_int(pRange, 2, iLast); while( SQLITE_ROW==sqlite3_step(pRange) ){ i64 nSize, dummy; fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); if( nSize>nByte ){ bOk = 0; break; } } rc = sqlite3_reset(pRange); if( bOk ){ int iIdx = 0; sqlite3_stmt *pUpdate1; sqlite3_stmt *pUpdate2; if( rc==SQLITE_OK ){ rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); } if( rc==SQLITE_OK ){ rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); } if( rc==SQLITE_OK ){ sqlite3_bind_int(pRange, 1, iAbsLevel); while( SQLITE_ROW==sqlite3_step(pRange) ){ sqlite3_bind_int(pUpdate1, 1, iIdx++); sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); sqlite3_step(pUpdate1); rc = sqlite3_reset(pUpdate1); if( rc!=SQLITE_OK ){ sqlite3_reset(pRange); break; } } } if( rc==SQLITE_OK ){ rc = sqlite3_reset(pRange); } if( rc==SQLITE_OK ){ sqlite3_bind_int(pUpdate2, 1, iAbsLevel); sqlite3_step(pUpdate2); rc = sqlite3_reset(pUpdate2); } } } return rc; } /* ** Merge all level iLevel segments in the database into a single ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a ** single segment with a level equal to the numerically largest level ** currently present in the database. ** |
︙ | ︙ | |||
3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 | if( iLevel!=FTS3_SEGCURSOR_PENDING ){ rc = fts3DeleteSegdir( p, iLangid, iIndex, iLevel, csr.apSegment, csr.nSegment ); if( rc!=SQLITE_OK ) goto finished; } rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); finished: fts3SegWriterFree(pWriter); sqlite3Fts3SegReaderFinish(&csr); return rc; } /* | > > > | | 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 | if( iLevel!=FTS3_SEGCURSOR_PENDING ){ rc = fts3DeleteSegdir( p, iLangid, iIndex, iLevel, csr.apSegment, csr.nSegment ); if( rc!=SQLITE_OK ) goto finished; } rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); if( rc==SQLITE_OK ){ rc = fts3PromoteSegments(p, iNewLevel, pWriter->nLeafData); } finished: fts3SegWriterFree(pWriter); sqlite3Fts3SegReaderFinish(&csr); return rc; } /* ** Flush the contents of pendingTerms to level 0 segments. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; int i; for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){ rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); |
︙ | ︙ | |||
3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 | struct IncrmergeWriter { int nLeafEst; /* Space allocated for leaf blocks */ int nWork; /* Number of leaf pages flushed */ sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ int iIdx; /* Index of *output* segment in iAbsLevel+1 */ sqlite3_int64 iStart; /* Block number of first allocated block */ sqlite3_int64 iEnd; /* Block number of last allocated block */ NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; }; /* ** An object of the following type is used to read data from a single ** FTS segment node. See the following functions: ** | > | 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 | struct IncrmergeWriter { int nLeafEst; /* Space allocated for leaf blocks */ int nWork; /* Number of leaf pages flushed */ sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ int iIdx; /* Index of *output* segment in iAbsLevel+1 */ sqlite3_int64 iStart; /* Block number of first allocated block */ sqlite3_int64 iEnd; /* Block number of last allocated block */ sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; }; /* ** An object of the following type is used to read data from a single ** FTS segment node. See the following functions: ** |
︙ | ︙ | |||
3760 3761 3762 3763 3764 3765 3766 3767 | nSuffix = nTerm; nSpace = 1; nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; } blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); | > < | 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 | nSuffix = nTerm; nSpace = 1; nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; } pWriter->nLeafData += nSpace; blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); if( rc==SQLITE_OK ){ if( pLeaf->block.n==0 ){ pLeaf->block.n = 1; pLeaf->block.a[0] = '\0'; } rc = fts3AppendToNode( &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist |
︙ | ︙ | |||
3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 | if( rc==SQLITE_OK ){ rc = fts3WriteSegdir(p, pWriter->iAbsLevel+1, /* level */ pWriter->iIdx, /* idx */ pWriter->iStart, /* start_block */ pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ pWriter->iEnd, /* end_block */ pRoot->block.a, pRoot->block.n /* root */ ); } sqlite3_free(pRoot->block.a); sqlite3_free(pRoot->key.a); *pRc = rc; | > | 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 | if( rc==SQLITE_OK ){ rc = fts3WriteSegdir(p, pWriter->iAbsLevel+1, /* level */ pWriter->iIdx, /* idx */ pWriter->iStart, /* start_block */ pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ pWriter->iEnd, /* end_block */ pWriter->nLeafData, /* end_block */ pRoot->block.a, pRoot->block.n /* root */ ); } sqlite3_free(pRoot->block.a); sqlite3_free(pRoot->key.a); *pRc = rc; |
︙ | ︙ | |||
3961 3962 3963 3964 3965 3966 3967 | /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); sqlite3_bind_int(pSelect, 2, iIdx); if( sqlite3_step(pSelect)==SQLITE_ROW ){ iStart = sqlite3_column_int64(pSelect, 1); iLeafEnd = sqlite3_column_int64(pSelect, 2); | | | 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 | /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); sqlite3_bind_int(pSelect, 2, iIdx); if( sqlite3_step(pSelect)==SQLITE_ROW ){ iStart = sqlite3_column_int64(pSelect, 1); iLeafEnd = sqlite3_column_int64(pSelect, 2); fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); nRoot = sqlite3_column_bytes(pSelect, 4); aRoot = sqlite3_column_blob(pSelect, 4); }else{ return sqlite3_reset(pSelect); } /* Check for the zero-length marker in the %_segments table */ |
︙ | ︙ | |||
4562 4563 4564 4565 4566 4567 4568 | return SQLITE_OK; } /* ** Attempt an incremental merge that writes nMerge leaf blocks. ** | | | | | | | 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 | return SQLITE_OK; } /* ** Attempt an incremental merge that writes nMerge leaf blocks. ** ** Incremental merges happen nMin segments at a time. The segments ** to be merged are the nMin oldest segments (the ones with the smallest ** values for the _segdir.idx field) in the highest level that contains ** at least nMin segments. Multiple merges might occur in an attempt to ** write the quota of nMerge leaf blocks. */ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc; /* Return code */ int nRem = nMerge; /* Number of leaf pages yet to be written */ Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ IncrmergeWriter *pWriter; /* Writer object */ |
︙ | ︙ | |||
4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 | bDirtyHint = 1; fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); } } } fts3IncrmergeRelease(p, pWriter, &rc); } sqlite3Fts3SegReaderFinish(pCsr); } /* Write the hint values into the %_stat table for the next incr-merger */ if( bDirtyHint && rc==SQLITE_OK ){ | > > > | 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 | bDirtyHint = 1; fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); } } } fts3IncrmergeRelease(p, pWriter, &rc); if( nSeg==0 ){ fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); } } sqlite3Fts3SegReaderFinish(pCsr); } /* Write the hint values into the %_stat table for the next incr-merger */ if( bDirtyHint && rc==SQLITE_OK ){ |
︙ | ︙ |
Added test/fts4growth.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 | # 2014 May 12 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # This file implements regression tests for SQLite library. The # focus of this script is testing the FTS4 module. # # set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts4growth # If SQLITE_ENABLE_FTS3 is defined, omit this file. ifcapable !fts3 { finish_test return } source $testdir/genesis.tcl do_execsql_test 1.1 { CREATE VIRTUAL TABLE x1 USING fts3; } do_test 1.2 { foreach L { {"See here, young man," said Mulga Bill, "from Walgett to the sea,} {From Conroy's Gap to Castlereagh, there's none can ride like me.} {I'm good all round at everything as everybody knows,} {Although I'm not the one to talk -- I hate a man that blows.} } { execsql { INSERT INTO x1 VALUES($L) } } execsql { SELECT end_block, length(root) FROM x1_segdir } } {{0 114} 114 {0 118} 118 {0 95} 95 {0 115} 115} do_execsql_test 1.3 { INSERT INTO x1(x1) VALUES('optimize'); SELECT level, end_block, length(root) FROM x1_segdir; } {0 {0 394} 394} do_test 1.4 { foreach L { {But riding is my special gift, my chiefest, sole delight;} {Just ask a wild duck can it swim, a wildcat can it fight.} {There's nothing clothed in hair or hide, or built of flesh or steel,} {There's nothing walks or jumps, or runs, on axle, hoof, or wheel,} {But what I'll sit, while hide will hold and girths and straps are tight:} {I'll ride this here two-wheeled concern right straight away at sight."} } { execsql { INSERT INTO x1 VALUES($L) } } execsql { INSERT INTO x1(x1) VALUES('merge=4,4'); SELECT level, end_block, length(root) FROM x1_segdir; } } {0 {0 110} 110 0 {0 132} 132 0 {0 129} 129 1 {128 658} 2} do_execsql_test 1.5 { SELECT length(block) FROM x1_segments; } {658 {}} do_test 1.6 { foreach L { {'Twas Mulga Bill, from Eaglehawk, that sought his own abode,} {That perched above Dead Man's Creek, beside the mountain road.} {He turned the cycle down the hill and mounted for the fray,} {But 'ere he'd gone a dozen yards it bolted clean away.} {It left the track, and through the trees, just like a silver steak,} {It whistled down the awful slope towards the Dead Man's Creek.} {It shaved a stump by half an inch, it dodged a big white-box:} {The very wallaroos in fright went scrambling up the rocks,} {The wombats hiding in their caves dug deeper underground,} {As Mulga Bill, as white as chalk, sat tight to every bound.} {It struck a stone and gave a spring that cleared a fallen tree,} {It raced beside a precipice as close as close could be;} {And then as Mulga Bill let out one last despairing shriek} {It made a leap of twenty feet into the Dead Man's Creek.} } { execsql { INSERT INTO x1 VALUES($L) } } execsql { SELECT level, end_block, length(root) FROM x1_segdir; } } {1 {128 658} 2 1 {130 1377} 6 0 {0 117} 117} do_execsql_test 1.7 { SELECT sum(length(block)) FROM x1_segments WHERE blockid IN (129, 130); } {1377} #------------------------------------------------------------------------- # do_execsql_test 2.1 { CREATE TABLE t1(docid, words); CREATE VIRTUAL TABLE x2 USING fts4; } fts_kjv_genesis do_test 2.2 { foreach id [db eval {SELECT docid FROM t1}] { execsql { INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id } } foreach id [db eval {SELECT docid FROM t1}] { execsql { INSERT INTO x2(docid, content) SELECT NULL, words FROM t1 WHERE docid=$id } if {[db one {SELECT count(*) FROM x2_segdir WHERE level<2}]==2} break } } {} do_execsql_test 2.3 { SELECT count(*) FROM x2_segdir WHERE level=2; SELECT count(*) FROM x2_segdir WHERE level=3; } {6 0} do_execsql_test 2.4 { INSERT INTO x2(x2) VALUES('merge=4,4'); SELECT count(*) FROM x2_segdir WHERE level=2; SELECT count(*) FROM x2_segdir WHERE level=3; } {6 1} do_execsql_test 2.5 { SELECT end_block FROM x2_segdir WHERE level=3; INSERT INTO x2(x2) VALUES('merge=4,4'); SELECT end_block FROM x2_segdir WHERE level=3; INSERT INTO x2(x2) VALUES('merge=4,4'); SELECT end_block FROM x2_segdir WHERE level=3; } {{3828 3430} {3828 10191} {3828 14109}} do_execsql_test 2.6 { SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE blockid BETWEEN start_block AND leaves_end_block AND level=3 } {14109} do_execsql_test 2.7 { INSERT INTO x2(x2) VALUES('merge=1000,4'); SELECT end_block FROM x2_segdir WHERE level=3; } {{3828 86120}} do_execsql_test 2.8 { SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE blockid BETWEEN start_block AND leaves_end_block AND level=3 } {86120} #-------------------------------------------------------------------------- do_execsql_test 3.1 { DROP TABLE IF EXISTS x2; DROP TABLE IF EXISTS t1; CREATE TABLE t1(docid, words); CREATE VIRTUAL TABLE x2 USING fts4; } fts_kjv_genesis proc t1_to_x2 {} { foreach id [db eval {SELECT docid FROM t1 LIMIT 2}] { execsql { DELETE FROM x2 WHERE docid=$id; INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id; } } } #do_test 3.2 { #t1_to_x2 #execsql {SELECT level, count(*) FROM x2_segdir GROUP BY level} #} {0 13 1 15 2 5} #proc second {x} { lindex $x 1 } #db func second second #for {set i 0} {$i <1000} {incr i} { # t1_to_x2 # db eval { # SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level; # } { # puts "$i.$level: $c" # } #} finish_test |