Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Further modifications and test cases to improve test coverage of fts3. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA3-256: |
ea8a0d2ce0cb1ca3f4f18c72fb780d1c |
User & Date: | dan 2017-04-19 07:33:52.121 |
Context
2017-04-19
| ||
13:25 | Further improvements to coverage of fts3 module. (check-in: 6b21d0fdeb user: dan tags: trunk) | |
07:33 | Further modifications and test cases to improve test coverage of fts3. (check-in: ea8a0d2ce0 user: dan tags: trunk) | |
2017-04-18
| ||
13:50 | Use sqlite3_table_column_metadata() instead of a SELECT statement to check for the existence of a %_stat table in fts3. This leads to smaller and easier to test code. (check-in: dc2a48020a user: dan tags: trunk) | |
Changes
Changes to ext/fts3/fts3.c.
︙ | ︙ | |||
3956 3957 3958 3959 3960 3961 3962 | #ifdef SQLITE_TEST if( rc==SQLITE_OK ){ rc = sqlite3Fts3ExprInitTestInterface(db); } #endif /* Create the virtual table wrapper around the hash-table and overload | | | 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 | #ifdef SQLITE_TEST if( rc==SQLITE_OK ){ rc = sqlite3Fts3ExprInitTestInterface(db); } #endif /* Create the virtual table wrapper around the hash-table and overload ** the four scalar functions. If this is successful, register the ** module with sqlite. */ if( SQLITE_OK==rc && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) |
︙ | ︙ | |||
4539 4540 4541 4542 4543 4544 4545 | Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; u8 bEof = 0; /* This is only called if it is guaranteed that the phrase has at least ** one incremental token. In which case the bIncr flag is set. */ assert( p->bIncr==1 ); | | | 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 | Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; u8 bEof = 0; /* This is only called if it is guaranteed that the phrase has at least ** one incremental token. In which case the bIncr flag is set. */ assert( p->bIncr==1 ); if( p->nToken==1 ){ rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, &pDL->iDocid, &pDL->pList, &pDL->nList ); if( pDL->pList==0 ) bEof = 1; }else{ int bDescDoclist = pCsr->bDesc; struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS]; |
︙ | ︙ | |||
4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 | ** The average document size in pages is calculated by first calculating ** determining the average size in bytes, B. If B is less than the amount ** of data that will fit on a single leaf page of an intkey table in ** this database, then the average docsize is 1. Otherwise, it is 1 plus ** the number of overflow pages consumed by a record B bytes in size. */ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ if( pCsr->nRowAvg==0 ){ /* The average document size, which is required to calculate the cost ** of each doclist, has not yet been determined. Read the required ** data from the %_stat table to calculate it. ** ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 ** varints, where nCol is the number of columns in the FTS3 table. | > | 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 | ** The average document size in pages is calculated by first calculating ** determining the average size in bytes, B. If B is less than the amount ** of data that will fit on a single leaf page of an intkey table in ** this database, then the average docsize is 1. Otherwise, it is 1 plus ** the number of overflow pages consumed by a record B bytes in size. */ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ int rc = SQLITE_OK; if( pCsr->nRowAvg==0 ){ /* The average document size, which is required to calculate the cost ** of each doclist, has not yet been determined. Read the required ** data from the %_stat table to calculate it. ** ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 ** varints, where nCol is the number of columns in the FTS3 table. |
︙ | ︙ | |||
4811 4812 4813 4814 4815 4816 4817 | return FTS_CORRUPT_VTAB; } pCsr->nDoc = nDoc; pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); assert( pCsr->nRowAvg>0 ); rc = sqlite3_reset(pStmt); | < | | 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 | return FTS_CORRUPT_VTAB; } pCsr->nDoc = nDoc; pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); assert( pCsr->nRowAvg>0 ); rc = sqlite3_reset(pStmt); } *pnPage = pCsr->nRowAvg; return rc; } /* ** This function is called to select the tokens (if any) that will be ** deferred. The array aTC[] has already been populated when this is ** called. ** |
︙ | ︙ | |||
5165 5166 5167 5168 5169 5170 5171 | }else{ fts3EvalNextRow(pCsr, pRight, pRc); } } pExpr->iDocid = pLeft->iDocid; pExpr->bEof = (pLeft->bEof || pRight->bEof); if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){ | > | | 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 | }else{ fts3EvalNextRow(pCsr, pRight, pRc); } } pExpr->iDocid = pLeft->iDocid; pExpr->bEof = (pLeft->bEof || pRight->bEof); if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){ assert( pRight->eType==FTSQUERY_PHRASE ); if( pRight->pPhrase->doclist.aAll ){ Fts3Doclist *pDl = &pRight->pPhrase->doclist; while( *pRc==SQLITE_OK && pRight->bEof==0 ){ memset(pDl->pList, 0, pDl->nList); fts3EvalNextRow(pCsr, pRight, pRc); } } if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){ |
︙ | ︙ | |||
5194 5195 5196 5197 5198 5199 5200 | sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ fts3EvalNextRow(pCsr, pLeft, pRc); | | | 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 | sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ fts3EvalNextRow(pCsr, pLeft, pRc); }else if( pLeft->bEof || iCmp>0 ){ fts3EvalNextRow(pCsr, pRight, pRc); }else{ fts3EvalNextRow(pCsr, pLeft, pRc); fts3EvalNextRow(pCsr, pRight, pRc); } pExpr->bEof = (pLeft->bEof && pRight->bEof); |
︙ | ︙ |
Changes to test/fts3fault2.test.
︙ | ︙ | |||
185 186 187 188 189 190 191 192 193 194 195 | INSERT INTO t7 VALUES('lazy dog'); } do_faultsim_test 7.1 -faults oom* -body { execsql { SELECT docid FROM t7 WHERE t7 MATCH 't*' } } -test { faultsim_test_result {0 {1 2}} } finish_test | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 | INSERT INTO t7 VALUES('lazy dog'); } do_faultsim_test 7.1 -faults oom* -body { execsql { SELECT docid FROM t7 WHERE t7 MATCH 't*' } } -test { faultsim_test_result {0 {1 2}} } #------------------------------------------------------------------------- # Inject faults into a opening an existing fts3 table that has been # upgraded to add an %_stat table. # reset_db do_execsql_test 8.0 { CREATE VIRTUAL TABLE t8 USING fts3; INSERT INTO t8 VALUES('the quick brown fox'); INSERT INTO t8 VALUES('jumped over the'); INSERT INTO t8 VALUES('lazy dog'); INSERT INTO t8(t8) VALUES('automerge=8'); SELECT name FROM sqlite_master WHERE name LIKE 't8%'; } { t8 t8_content t8_segments t8_segdir t8_stat } faultsim_save_and_close do_faultsim_test 8.1 -faults oom* -prep { faultsim_restore_and_reopen } -body { execsql { INSERT INTO t8 VALUES('one two three') } } -test { faultsim_test_result {0 {}} } do_faultsim_test 8.2 -faults oom* -prep { faultsim_restore_and_reopen } -body { execsql { ALTER TABLE t8 RENAME TO t8ii } } -test { faultsim_test_result {0 {}} } #------------------------------------------------------------------------- reset_db set chunkconfig [fts3_configure_incr_load 1 1] do_execsql_test 9.0 { PRAGMA page_size = 512; CREATE VIRTUAL TABLE t9 USING fts3; WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50 ) INSERT INTO t9 SELECT 'one two three' FROM s; } do_faultsim_test 8.2 -faults io* -body { execsql { SELECT count(*) FROM t9 WHERE t9 MATCH '"one two three"' } } -test { faultsim_test_result {0 50} } eval fts3_configure_incr_load $chunkconfig finish_test |
Changes to test/fts3misc.test.
︙ | ︙ | |||
98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | 6 7 14 15 22 23 30 31 38 39 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 70 71 78 79 86 87 94 95 102 103 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 134 135 142 143 150 151 158 159 166 167 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 198 199 } # Range constraints on the docid using non-integer values. # do_execsql_test 2.6 { SELECT rowid FROM t2 WHERE t2 MATCH 'e' AND rowid BETWEEN NULL AND 45; } {} do_execsql_test 2.7 { SELECT rowid FROM t2 WHERE t2 MATCH 'e' AND rowid BETWEEN 11.5 AND 48.2; } { 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 34 35 38 39 42 43 46 47 48 } do_execsql_test 2.8 { SELECT rowid FROM t2 WHERE t2 MATCH 'e' AND rowid BETWEEN '11.5' AND '48.2'; } { 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 34 35 38 39 42 43 46 47 48 } | > > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 | 6 7 14 15 22 23 30 31 38 39 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 70 71 78 79 86 87 94 95 102 103 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 134 135 142 143 150 151 158 159 166 167 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 198 199 } #------------------------------------------------------------------------- # Range constraints on the docid using non-integer values. # do_execsql_test 2.6 { SELECT rowid FROM t2 WHERE t2 MATCH 'e' AND rowid BETWEEN NULL AND 45; } {} do_execsql_test 2.7 { SELECT rowid FROM t2 WHERE t2 MATCH 'e' AND rowid BETWEEN 11.5 AND 48.2; } { 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 34 35 38 39 42 43 46 47 48 } do_execsql_test 2.8 { SELECT rowid FROM t2 WHERE t2 MATCH 'e' AND rowid BETWEEN '11.5' AND '48.2'; } { 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 34 35 38 39 42 43 46 47 48 } #------------------------------------------------------------------------- # Phrase query tests. # do_execsql_test 3.1.1 { CREATE VIRTUAL TABLE t3 USING fts3; INSERT INTO t3 VALUES('a b c'); INSERT INTO t3 VALUES('d e f'); INSERT INTO t3 VALUES('a b d'); INSERT INTO t3 VALUES('1 2 3 4 5 6 7 8 9 10 11'); } do_execsql_test 3.1.2 { SELECT * FROM t3 WHERE t3 MATCH '"a b x y"' ORDER BY docid DESC } do_execsql_test 3.1.3 { SELECT * FROM t3 WHERE t3 MATCH '"a b c" OR "a b x y"' ORDER BY docid DESC } {{a b c}} do_execsql_test 3.1.4 { SELECT * FROM t3 WHERE t3 MATCH '"a* b* x* a*"' } do_execsql_test 3.1.5 { SELECT rowid FROM t3 WHERE t3 MATCH '"2 3 4 5 6 7 8 9"' } {4} #------------------------------------------------------------------------- # reset_db do_execsql_test 4.0 { PRAGMA page_size = 512; CREATE VIRTUAL TABLE t4 USING fts4; WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<8000 ) INSERT INTO t4 SELECT 'a b c a b c a b c' FROM s; } do_execsql_test 4.1 { SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"' } {8000} do_execsql_test 4.2 { SELECT quote(value) from t4_stat where id=0 } {X'C03EC0B204C0A608'} do_execsql_test 4.3 { UPDATE t4_stat SET value = X'C03EC0B204C0A60800' WHERE id=0; } do_catchsql_test 4.4 { SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"' } {1 {database disk image is malformed}} do_execsql_test 4.5 { UPDATE t4_stat SET value = X'00C03EC0B204C0A608' WHERE id=0; } do_catchsql_test 4.6 { SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"' } {1 {database disk image is malformed}} finish_test |