Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Improve test coverage of fts5_index.c. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts5 |
Files: | files | file ages | folders |
SHA1: |
7aea8c6d99737c6c72078e0b4b9c5f81 |
User & Date: | dan 2015-05-15 18:13:14.380 |
Context
2015-05-16
| ||
20:04 | Further test coverage improvements for fts5. (check-in: 927d9a64e1 user: dan tags: fts5) | |
2015-05-15
| ||
18:13 | Improve test coverage of fts5_index.c. (check-in: 7aea8c6d99 user: dan tags: fts5) | |
12:18 | Add test cases. And some fixes. (check-in: adee788586 user: dan tags: fts5) | |
Changes
Changes to ext/fts5/fts5_index.c.
︙ | ︙ | |||
3458 3459 3460 3461 3462 3463 3464 | fts5NodeIterInit(pPg->buf.p, pPg->buf.n, &ss); while( ss.aData ) fts5NodeIterNext(&p->rc, &ss); fts5BufferSet(&p->rc, &pPg->term, ss.term.n, ss.term.p); pgno = ss.iChild; fts5NodeIterFree(&ss); } } | < < < | 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 | fts5NodeIterInit(pPg->buf.p, pPg->buf.n, &ss); while( ss.aData ) fts5NodeIterNext(&p->rc, &ss); fts5BufferSet(&p->rc, &pPg->term, ss.term.n, ss.term.p); pgno = ss.iChild; fts5NodeIterFree(&ss); } } assert( p->rc!=SQLITE_OK || (pgno+pWriter->nEmpty)==pSeg->pgnoLast ); pWriter->bFirstTermInPage = 1; assert( pWriter->aWriter[0].term.n==0 ); } } /* |
︙ | ︙ | |||
4047 4048 4049 4050 4051 4052 4053 | return fts5IndexReturn(p); } int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ Fts5Structure *pStruct; pStruct = fts5StructureRead(p); | > | | > | 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 | return fts5IndexReturn(p); } int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ Fts5Structure *pStruct; pStruct = fts5StructureRead(p); if( pStruct && pStruct->nLevel ){ fts5IndexMerge(p, &pStruct, nMerge); fts5StructureWrite(p, pStruct); } fts5StructureRelease(pStruct); return fts5IndexReturn(p); } /* |
︙ | ︙ | |||
4529 4530 4531 4532 4533 4534 4535 | || (flags & FTS5INDEX_QUERY_SCAN)==FTS5INDEX_QUERY_SCAN ); if( sqlite3Fts5BufferGrow(&p->rc, &buf, nToken+1)==0 ){ memcpy(&buf.p[1], pToken, nToken); } | > | | | | > > | | | < < < | 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 | || (flags & FTS5INDEX_QUERY_SCAN)==FTS5INDEX_QUERY_SCAN ); if( sqlite3Fts5BufferGrow(&p->rc, &buf, nToken+1)==0 ){ memcpy(&buf.p[1], pToken, nToken); } #ifdef SQLITE_DEBUG if( flags & FTS5INDEX_QUERY_TEST_NOIDX ){ assert( flags & FTS5INDEX_QUERY_PREFIX ); iIdx = 1+pConfig->nPrefix; }else #endif if( flags & FTS5INDEX_QUERY_PREFIX ){ int nChar = fts5IndexCharlen(pToken, nToken); for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){ if( pConfig->aPrefix[iIdx-1]==nChar ) break; } } pRet = (Fts5IndexIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5IndexIter)); if( pRet ){ pRet->pIndex = p; if( iIdx<=pConfig->nPrefix ){ buf.p[0] = FTS5_MAIN_PREFIX + iIdx; pRet->pStruct = fts5StructureRead(p); if( pRet->pStruct ){ fts5MultiIterNew( p, pRet->pStruct, 1, flags, buf.p, nToken+1, -1, 0, &pRet->pMulti |
︙ | ︙ | |||
4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 | pLvl->pData = 0; } } sqlite3_free(pIter->aLvl); fts5BufferFree(&pIter->term); } /* ** This function is purely an internal test. It does not contribute to ** FTS functionality, or even the integrity-check, in any way. ** ** Instead, it tests that the same set of pgno/rowid combinations are ** visited regardless of whether the doclist-index identified by parameters ** iSegid/iLeaf is iterated in forwards or reverse order. */ | > < | | 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 | pLvl->pData = 0; } } sqlite3_free(pIter->aLvl); fts5BufferFree(&pIter->term); } #ifdef SQLITE_DEBUG /* ** This function is purely an internal test. It does not contribute to ** FTS functionality, or even the integrity-check, in any way. ** ** Instead, it tests that the same set of pgno/rowid combinations are ** visited regardless of whether the doclist-index identified by parameters ** iSegid/iLeaf is iterated in forwards or reverse order. */ static void fts5TestDlidxReverse( Fts5Index *p, int iSegid, /* Segment id to load from */ int iLeaf /* Load doclist-index for this leaf */ ){ Fts5DlidxIter *pDlidx = 0; u64 cksum1 = 13; u64 cksum2 = 13; |
︙ | ︙ | |||
4930 4931 4932 4933 4934 4935 4936 4937 | cksum2 += iRowid + ((i64)pgno<<32); } fts5DlidxIterFree(pDlidx); pDlidx = 0; if( p->rc==SQLITE_OK && cksum1!=cksum2 ) p->rc = FTS5_CORRUPT; } #else | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 | cksum2 += iRowid + ((i64)pgno<<32); } fts5DlidxIterFree(pDlidx); pDlidx = 0; if( p->rc==SQLITE_OK && cksum1!=cksum2 ) p->rc = FTS5_CORRUPT; } static int fts5QueryCksum( Fts5Index *p, /* Fts5 index object */ int iIdx, const char *z, /* Index key to query for */ int n, /* Size of index key in bytes */ int flags, /* Flags for Fts5IndexQuery */ u64 *pCksum /* IN/OUT: Checksum value */ ){ u64 cksum = *pCksum; Fts5IndexIter *pIdxIter = 0; int rc = sqlite3Fts5IndexQuery(p, z, n, flags, &pIdxIter); while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){ const u8 *pPos; int nPos; i64 rowid = sqlite3Fts5IterRowid(pIdxIter); rc = sqlite3Fts5IterPoslist(pIdxIter, &pPos, &nPos); if( rc==SQLITE_OK ){ Fts5PoslistReader sReader; for(sqlite3Fts5PoslistReaderInit(-1, pPos, nPos, &sReader); sReader.bEof==0; sqlite3Fts5PoslistReaderNext(&sReader) ){ int iCol = FTS5_POS2COLUMN(sReader.iPos); int iOff = FTS5_POS2OFFSET(sReader.iPos); cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); } rc = sqlite3Fts5IterNext(pIdxIter); } } sqlite3Fts5IterClose(pIdxIter); *pCksum = cksum; return rc; } /* ** This function is also purely an internal test. It does not contribute to ** FTS functionality, or even the integrity-check, in any way. */ static void fts5TestTerm( Fts5Index *p, Fts5Buffer *pPrev, /* Previous term */ const char *z, int n, /* Possibly new term to test */ u64 expected, u64 *pCksum ){ int rc = p->rc; if( pPrev->n==0 ){ fts5BufferSet(&rc, pPrev, n, (const u8*)z); }else if( rc==SQLITE_OK && (pPrev->n!=n || memcmp(pPrev->p, z, n)) ){ u32 cksum3 = *pCksum; const char *zTerm = &pPrev->p[1]; /* The term without the prefix-byte */ int nTerm = pPrev->n-1; /* Size of zTerm in bytes */ int iIdx = (pPrev->p[0] - FTS5_MAIN_PREFIX); int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX); int rc; u64 ck1 = 0; u64 ck2 = 0; /* Check that the results returned for ASC and DESC queries are ** the same. If not, call this corruption. */ rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, flags, &ck1); if( rc==SQLITE_OK ){ int f = flags|FTS5INDEX_QUERY_DESC; rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); } if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; /* If this is a prefix query, check that the results returned if the ** the index is disabled are the same. In both ASC and DESC order. */ if( iIdx>0 && rc==SQLITE_OK ){ int f = flags|FTS5INDEX_QUERY_TEST_NOIDX; ck2 = 0; rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; } if( iIdx>0 && rc==SQLITE_OK ){ int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC; ck2 = 0; rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; } cksum3 ^= ck1; fts5BufferSet(&rc, pPrev, n, (const u8*)z); if( rc==SQLITE_OK && cksum3!=expected ){ rc = FTS5_CORRUPT; } *pCksum = cksum3; } p->rc = rc; } #else # define fts5TestDlidxReverse(x,y,z) # define fts5TestTerm(u,v,w,x,y,z) #endif static void fts5IndexIntegrityCheckSegment( Fts5Index *p, /* FTS5 backend object */ Fts5StructureSegment *pSeg /* Segment to check internal consistency */ ){ Fts5BtreeIter iter; /* Used to iterate through b-tree hierarchy */ |
︙ | ︙ | |||
5042 5043 5044 5045 5046 5047 5048 | if( pLeaf ){ if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT; fts5DataRelease(pLeaf); } } fts5DlidxIterFree(pDlidx); | | | < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < > > > > | 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 | if( pLeaf ){ if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT; fts5DataRelease(pLeaf); } } fts5DlidxIterFree(pDlidx); fts5TestDlidxReverse(p, iSegid, iter.iLeaf); } } /* Page iter.iLeaf must now be the rightmost leaf-page in the segment */ if( p->rc==SQLITE_OK && iter.iLeaf!=pSeg->pgnoLast ){ p->rc = FTS5_CORRUPT; } fts5BtreeIterFree(&iter); } /* ** Run internal checks to ensure that the FTS index (a) is internally ** consistent and (b) contains entries for which the XOR of the checksums ** as calculated by fts5IndexEntryCksum() is cksum. ** ** Return SQLITE_CORRUPT if any of the internal checks fail, or if the ** checksum does not match. Return SQLITE_OK if all checks pass without ** error, or some other SQLite error code if another error (e.g. OOM) ** occurs. */ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ u64 cksum2 = 0; /* Checksum based on contents of indexes */ Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ Fts5MultiSegIter *pIter; /* Used to iterate through entire index */ Fts5Structure *pStruct; /* Index structure */ /* Used by extra internal tests only run if NDEBUG is not defined */ u64 cksum3 = 0; /* Checksum based on contents of indexes */ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ /* Load the FTS index structure */ pStruct = fts5StructureRead(p); /* Check that the internal nodes of each segment match the leaves */ if( pStruct ){ int iLvl, iSeg; |
︙ | ︙ | |||
5160 5161 5162 5163 5164 5165 5166 | while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ int iCol = FTS5_POS2COLUMN(iPos); int iTokOff = FTS5_POS2OFFSET(iPos); cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); } /* If this is a new term, query for it. Update cksum3 with the results. */ | < < < < < < < < | < < < < < < | < | < < < < < < < | < < < < < < < < < < < < < < | 5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 | while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ int iCol = FTS5_POS2COLUMN(iPos); int iTokOff = FTS5_POS2OFFSET(iPos); cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); } /* If this is a new term, query for it. Update cksum3 with the results. */ fts5TestTerm(p, &term, z, n, cksum2, &cksum3); } fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); fts5MultiIterFree(p, pIter); if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; fts5StructureRelease(pStruct); fts5BufferFree(&term); fts5BufferFree(&poslist); return fts5IndexReturn(p); } |
︙ | ︙ |
Changes to ext/fts5/test/fts5corrupt2.test.
︙ | ︙ | |||
111 112 113 114 115 116 117 | do_execsql_test 2.$i.4 { ROLLBACK; INSERT INTO t1(t1) VALUES('integrity-check'); } {} } | < < | 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | do_execsql_test 2.$i.4 { ROLLBACK; INSERT INTO t1(t1) VALUES('integrity-check'); } {} } #------------------------------------------------------------------------- # Test that corruption in leaf page headers is detected by queries that use # doclist-indexes. # set doc "A B C D E F G H I J " do_execsql_test 3.0 { CREATE VIRTUAL TABLE x3 USING fts5(tt); |
︙ | ︙ | |||
203 204 205 206 207 208 209 210 211 212 213 214 | } {1} execsql ROLLBACK } do_test 4.$tn.x { expr $nCorrupt>0 } 1 } sqlite3_fts5_may_be_corrupt 0 finish_test | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 | } {1} execsql ROLLBACK } do_test 4.$tn.x { expr $nCorrupt>0 } 1 } } set doc [string repeat "A B C " 1000] do_execsql_test 4.0 { CREATE VIRTUAL TABLE x5 USING fts5(tt); INSERT INTO x5(x5, rank) VALUES('pgsz', 32); WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10) INSERT INTO x5 SELECT $doc FROM ii; } foreach {tn hdr} { 1 "\x00\x01" } { set tn2 0 set nCorrupt 0 foreach rowid [db eval {SELECT rowid FROM x5_data WHERE rowid>10}] { if {$rowid & $mask} continue incr tn2 do_test 4.$tn.$tn2 { execsql BEGIN set fd [db incrblob main x5_data block $rowid] fconfigure $fd -encoding binary -translation binary puts -nonewline $fd $hdr close $fd catchsql { INSERT INTO x5(x5) VALUES('integrity-check') } set {} {} } {} execsql ROLLBACK } } sqlite3_fts5_may_be_corrupt 0 finish_test |
Added ext/fts5/test/fts5corrupt3.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | # 2015 Apr 24 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # This file tests that FTS5 handles corrupt databases (i.e. internal # inconsistencies in the backing tables) correctly. In this case # "correctly" means without crashing. # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5corrupt3 sqlite3_fts5_may_be_corrupt 1 # Create a simple FTS5 table containing 100 documents. Each document # contains 10 terms, each of which start with the character "x". # expr srand(0) db func rnddoc fts5_rnddoc do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x); INSERT INTO t1(t1, rank) VALUES('pgsz', 64); WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100) INSERT INTO t1 SELECT rnddoc(10) FROM ii; } set mask [expr 31 << 31] do_test 1.1 { # Pick out the rowid of the right-most b-tree leaf in the new segment. set rowid [db one { SELECT max(rowid) FROM t1_data WHERE ((rowid>>31) & 0x0F)==1 }] set L [db one {SELECT length(block) FROM t1_data WHERE rowid = $rowid}] set {} {} } {} for {set i 0} {$i < $L} {incr i} { do_test 1.2.$i { catchsql { BEGIN; UPDATE t1_data SET block = substr(block, 1, $i) WHERE id = $rowid; INSERT INTO t1(t1) VALUES('integrity-check'); } } {1 {database disk image is malformed}} catchsql ROLLBACK } sqlite3_fts5_may_be_corrupt 0 finish_test |
Changes to ext/fts5/test/fts5merge.test.
︙ | ︙ | |||
130 131 132 133 134 135 136 137 138 139 | do_test 3.4 { execsql { INSERT INTO x8(x8, rank) VALUES('automerge', 4) } while {[not_merged x8]} { execsql { INSERT INTO x8(x8, rank) VALUES('merge', 1) } } fts5_level_segs x8 } {0 1} finish_test | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 | do_test 3.4 { execsql { INSERT INTO x8(x8, rank) VALUES('automerge', 4) } while {[not_merged x8]} { execsql { INSERT INTO x8(x8, rank) VALUES('merge', 1) } } fts5_level_segs x8 } {0 1} #------------------------------------------------------------------------- # proc mydoc {} { set x [lindex {a b c d e f g h i j} [expr int(rand()*10)]] return [string repeat "$x " 30] } db func mydoc mydoc proc mycount {} { set res [list] foreach x {a b c d e f g h i j} { lappend res [db one {SELECT count(*) FROM x8 WHERE x8 MATCH $x}] } set res } #1 32 foreach {tn pgsz} { 2 1000 } { do_execsql_test 4.$tn.1 { DROP TABLE IF EXISTS x8; CREATE VIRTUAL TABLE x8 USING fts5(i); INSERT INTO x8(x8, rank) VALUES('pgsz', $pgsz); } do_execsql_test 4.$tn.2 { INSERT INTO x8(x8, rank) VALUES('merge', 1); } do_execsql_test 4.$tn.3 { WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100) INSERT INTO x8 SELECT mydoc() FROM ii; WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100) INSERT INTO x8 SELECT mydoc() FROM ii; INSERT INTO x8(x8, rank) VALUES('automerge', 2); } set expect [mycount] for {set i 0} {$i < 20} {incr i} { do_test 4.$tn.4.$i { execsql { INSERT INTO x8(x8, rank) VALUES('merge', 1); } mycount } $expect break } db eval {SELECT fts5_decode(rowid, block) AS r FROM x8_data} { puts $r } } finish_test |