Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add the 'rebuild' and 'delete-all' commands. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts5 |
Files: | files | file ages | folders |
SHA1: |
0cb2fed525778d96237b5b0943047665 |
User & Date: | dan 2015-01-07 17:11:11.301 |
Context
2015-01-07
| ||
19:33 | Add the fts5 'optimize' command. (check-in: e749be563d user: dan tags: fts5) | |
17:11 | Add the 'rebuild' and 'delete-all' commands. (check-in: 0cb2fed525 user: dan tags: fts5) | |
2015-01-06
| ||
19:08 | Remove the iPos parameter from the tokenizer callback. Fix the "tokenchars" and "separators" options on the simple tokenizer. (check-in: 65f0262fb8 user: dan tags: fts5) | |
Changes
Changes to ext/fts5/fts5.c.
︙ | ︙ | |||
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 | if( rc==SQLITE_OK ){ rc = SQLITE_CORRUPT_VTAB; } } } return rc; } /* ** This function is called to handle an FTS INSERT command. In other words, ** an INSERT statement of the form: ** ** INSERT INTO fts(fts) VALUES($pCmd) ** INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal) | > > > > > > > > > | 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 | if( rc==SQLITE_OK ){ rc = SQLITE_CORRUPT_VTAB; } } } return rc; } static void fts5SetVtabError(Fts5Table *p, const char *zFormat, ...){ int rc; va_list ap; /* ... printf arguments */ va_start(ap, zFormat); assert( p->base.zErrMsg==0 ); p->base.zErrMsg = sqlite3_vmprintf(zFormat, ap); va_end(ap); } /* ** This function is called to handle an FTS INSERT command. In other words, ** an INSERT statement of the form: ** ** INSERT INTO fts(fts) VALUES($pCmd) ** INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal) |
︙ | ︙ | |||
1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 | ** more commands are added to this function. */ static int fts5SpecialInsert( Fts5Table *pTab, /* Fts5 table object */ sqlite3_value *pCmd, /* Value inserted into special column */ sqlite3_value *pVal /* Value inserted into rowid column */ ){ const char *z = (const char*)sqlite3_value_text(pCmd); int rc = SQLITE_OK; int bError = 0; | > > > > > > > > > > > > > > > > > > > > | | 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 | ** more commands are added to this function. */ static int fts5SpecialInsert( Fts5Table *pTab, /* Fts5 table object */ sqlite3_value *pCmd, /* Value inserted into special column */ sqlite3_value *pVal /* Value inserted into rowid column */ ){ Fts5Config *pConfig = pTab->pConfig; const char *z = (const char*)sqlite3_value_text(pCmd); int rc = SQLITE_OK; int bError = 0; if( 0==sqlite3_stricmp("delete-all", z) ){ if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ fts5SetVtabError(pTab, "'delete-all' may only be used with a " "contentless or external content fts5 table" ); rc = SQLITE_ERROR; }else{ rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage); } }else if( 0==sqlite3_stricmp("rebuild", z) ){ if( pConfig->eContent==FTS5_CONTENT_NONE ){ fts5SetVtabError(pTab, "'rebuild' may not be used with a contentless fts5 table" ); rc = SQLITE_ERROR; }else{ rc = sqlite3Fts5StorageRebuild(pTab->pStorage); } }else if( 0==sqlite3_stricmp("integrity-check", z) ){ rc = sqlite3Fts5StorageIntegrity(pTab->pStorage); }else{ rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, z, pVal, &bError); if( rc==SQLITE_OK ){ if( bError ){ rc = SQLITE_ERROR; }else{ |
︙ | ︙ |
Changes to ext/fts5/fts5.h.
︙ | ︙ | |||
88 89 90 91 92 93 94 | ** output by xInstCount(). ** ** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) ** if an error occurs. ** ** xRowid: ** Returns the rowid of the current row. | < < < < < < < < < < < < < < < < < < < < < | 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | ** output by xInstCount(). ** ** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) ** if an error occurs. ** ** xRowid: ** Returns the rowid of the current row. ** ** xTokenize: ** Tokenize text using the tokenizer belonging to the FTS5 table. ** ** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): ** This API function is used to query the FTS table for phrase iPhrase ** of the current query. Specifically, a query equivalent to: ** ** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY DESC ** |
︙ | ︙ |
Changes to ext/fts5/fts5Int.h.
︙ | ︙ | |||
336 337 338 339 340 341 342 343 344 345 346 347 348 349 | /* ** Return the total number of entries read from the %_data table by ** this connection since it was created. */ int sqlite3Fts5IndexReads(Fts5Index *p); /* ** End of interface to code in fts5_index.c. **************************************************************************/ /************************************************************************** ** Interface to code in fts5_hash.c. */ | > > | 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 | /* ** Return the total number of entries read from the %_data table by ** this connection since it was created. */ int sqlite3Fts5IndexReads(Fts5Index *p); int sqlite3Fts5IndexReinit(Fts5Index *p); /* ** End of interface to code in fts5_index.c. **************************************************************************/ /************************************************************************** ** Interface to code in fts5_hash.c. */ |
︙ | ︙ | |||
416 417 418 419 420 421 422 423 424 425 426 427 428 429 | int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit); int sqlite3Fts5StorageRollback(Fts5Storage *p); int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*); int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**); /* ** End of interface to code in fts5_storage.c. **************************************************************************/ /************************************************************************** | > > > | 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 | int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit); int sqlite3Fts5StorageRollback(Fts5Storage *p); int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*); int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**); int sqlite3Fts5StorageDeleteAll(Fts5Storage *p); int sqlite3Fts5StorageRebuild(Fts5Storage *p); /* ** End of interface to code in fts5_storage.c. **************************************************************************/ /************************************************************************** |
︙ | ︙ |
Changes to ext/fts5/fts5_index.c.
︙ | ︙ | |||
3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 | */ int sqlite3Fts5IndexRollback(Fts5Index *p){ fts5CloseReader(p); fts5IndexDiscardData(p); assert( p->rc==SQLITE_OK ); return SQLITE_OK; } /* ** Open a new Fts5Index handle. If the bCreate argument is true, create ** and initialize the underlying %_data table. ** ** If successful, set *pp to point to the new object and return SQLITE_OK. ** Otherwise, set *pp to NULL and return an SQLite error code. | > > > > > > > > > > > > > > > > > > > > | 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 | */ int sqlite3Fts5IndexRollback(Fts5Index *p){ fts5CloseReader(p); fts5IndexDiscardData(p); assert( p->rc==SQLITE_OK ); return SQLITE_OK; } /* ** The %_data table is completely empty when this function is called. This ** function populates it with the initial structure objects for each index, ** and the initial version of the "averages" record (a zero-byte blob). */ int sqlite3Fts5IndexReinit(Fts5Index *p){ int i; Fts5Structure s; memset(&s, 0, sizeof(Fts5Structure)); for(i=0; i<p->pConfig->nPrefix+1; i++){ fts5StructureWrite(p, i, &s); } if( p->rc==SQLITE_OK ){ p->rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0); } return fts5IndexReturn(p); } /* ** Open a new Fts5Index handle. If the bCreate argument is true, create ** and initialize the underlying %_data table. ** ** If successful, set *pp to point to the new object and return SQLITE_OK. ** Otherwise, set *pp to NULL and return an SQLite error code. |
︙ | ︙ | |||
3923 3924 3925 3926 3927 3928 3929 | p->nCrisisMerge = FTS5_CRISIS_MERGE; p->nWorkUnit = FTS5_WORK_UNIT; p->nMaxPendingData = 1024*1024; p->zDataTbl = sqlite3_mprintf("%s_data", pConfig->zName); if( p->zDataTbl==0 ){ rc = SQLITE_NOMEM; }else if( bCreate ){ | < < < < < < < < < | | 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 | p->nCrisisMerge = FTS5_CRISIS_MERGE; p->nWorkUnit = FTS5_WORK_UNIT; p->nMaxPendingData = 1024*1024; p->zDataTbl = sqlite3_mprintf("%s_data", pConfig->zName); if( p->zDataTbl==0 ){ rc = SQLITE_NOMEM; }else if( bCreate ){ rc = sqlite3Fts5CreateTable( pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr ); if( rc==SQLITE_OK ){ rc = sqlite3Fts5IndexReinit(p); } } assert( p->rc==SQLITE_OK || rc!=SQLITE_OK ); if( rc ){ sqlite3Fts5IndexClose(p, 0); *pp = 0; |
︙ | ︙ |
Changes to ext/fts5/fts5_storage.c.
︙ | ︙ | |||
121 122 123 124 125 126 127 128 129 130 131 132 | } } *ppStmt = p->aStmt[eStmt]; return rc; } /* ** Drop the shadow table with the postfix zPost (e.g. "content"). Return ** SQLITE_OK if successful or an SQLite error code otherwise. */ int sqlite3Fts5DropTable(Fts5Config *pConfig, const char *zPost){ | > > > > > > > > > > > > > > > > > > > > > > > < | < < < < < < < > > | | < < < < < < < | | | | | | | < | | 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 | } } *ppStmt = p->aStmt[eStmt]; return rc; } static int fts5ExecPrintf( sqlite3 *db, char **pzErr, const char *zFormat, ... ){ int rc; va_list ap; /* ... printf arguments */ va_start(ap, zFormat); char *zSql = sqlite3_vmprintf(zFormat, ap); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_exec(db, zSql, 0, 0, pzErr); sqlite3_free(zSql); } va_end(ap); return rc; } /* ** Drop the shadow table with the postfix zPost (e.g. "content"). Return ** SQLITE_OK if successful or an SQLite error code otherwise. */ int sqlite3Fts5DropTable(Fts5Config *pConfig, const char *zPost){ return fts5ExecPrintf(pConfig->db, 0, "DROP TABLE IF EXISTS %Q.'%q_%q'", pConfig->zDb, pConfig->zName, zPost ); } /* ** Create the shadow table named zPost, with definition zDefn. Return ** SQLITE_OK if successful, or an SQLite error code otherwise. */ int sqlite3Fts5CreateTable( Fts5Config *pConfig, /* FTS5 configuration */ const char *zPost, /* Shadow table to create (e.g. "content") */ const char *zDefn, /* Columns etc. for shadow table */ int bWithout, /* True for without rowid */ char **pzErr /* OUT: Error message */ ){ int rc; char *zErr = 0; rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s", pConfig->zDb, pConfig->zName, zPost, zDefn, bWithout?" WITHOUT ROWID":"" ); if( zErr ){ *pzErr = sqlite3_mprintf( "fts5: error creating shadow table %q_%s: %s", pConfig->zName, zPost, zErr ); sqlite3_free(zErr); } return rc; } /* ** Open a new Fts5Index handle. If the bCreate argument is true, create ** and initialize the underlying tables ** |
︙ | ︙ | |||
458 459 460 461 462 463 464 | i64 iDel, sqlite3_value **apVal ){ Fts5Config *pConfig = p->pConfig; int rc; sqlite3_stmt *pDel; | | | 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 | i64 iDel, sqlite3_value **apVal ){ Fts5Config *pConfig = p->pConfig; int rc; sqlite3_stmt *pDel; assert( pConfig->eContent!=FTS5_CONTENT_NORMAL ); rc = fts5StorageLoadTotals(p, 1); /* Delete the index records */ if( rc==SQLITE_OK ){ int iCol; Fts5InsertCtx ctx; ctx.pStorage = p; |
︙ | ︙ | |||
498 499 500 501 502 503 504 | /* Write the averages record */ if( rc==SQLITE_OK ){ rc = fts5StorageSaveTotals(p); } return rc; | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 | /* Write the averages record */ if( rc==SQLITE_OK ){ rc = fts5StorageSaveTotals(p); } return rc; } /* ** Delete all entries in the FTS5 index. */ int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){ Fts5Config *pConfig = p->pConfig; int rc; /* Delete the contents of the %_data and %_docsize tables. */ rc = fts5ExecPrintf(pConfig->db, 0, "DELETE FROM %Q.'%q_data';" "DELETE FROM %Q.'%q_docsize';", pConfig->zDb, pConfig->zName, pConfig->zDb, pConfig->zName ); /* Reinitialize the %_data table. This call creates the initial structure ** and averages records. */ if( rc==SQLITE_OK ){ rc = sqlite3Fts5IndexReinit(p->pIndex); } return rc; } int sqlite3Fts5StorageRebuild(Fts5Storage *p){ Fts5Buffer buf = {0,0,0}; Fts5Config *pConfig = p->pConfig; sqlite3_stmt *pScan = 0; Fts5InsertCtx ctx; int rc; memset(&ctx, 0, sizeof(Fts5InsertCtx)); ctx.pStorage = p; rc = sqlite3Fts5StorageDeleteAll(p); if( rc==SQLITE_OK ){ rc = fts5StorageLoadTotals(p, 1); } if( rc==SQLITE_OK ){ rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN_ASC, &pScan, 0); } while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){ i64 iRowid = sqlite3_column_int64(pScan, 0); sqlite3Fts5BufferZero(&buf); rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iRowid); for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){ ctx.szCol = 0; rc = sqlite3Fts5Tokenize(pConfig, (const char*)sqlite3_column_text(pScan, ctx.iCol+1), sqlite3_column_bytes(pScan, ctx.iCol+1), (void*)&ctx, fts5StorageInsertCallback ); sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; } p->nTotalRow++; if( rc==SQLITE_OK ){ rc = fts5StorageInsertDocsize(p, iRowid, &buf); } } sqlite3_free(buf.p); /* Write the averages record */ if( rc==SQLITE_OK ){ rc = fts5StorageSaveTotals(p); } return rc; } /* ** Allocate a new rowid. This is used for "external content" tables when ** a NULL value is inserted into the rowid column. The new rowid is allocated ** by inserting a dummy row into the %_docsize table. The dummy will be ** overwritten later. |
︙ | ︙ |
Changes to ext/fts5/test/fts5content.test.
︙ | ︙ | |||
140 141 142 143 144 145 146 147 148 149 | } {three {t h r e e x y z}} do_execsql_test 2.7 { SELECT highlight(fts_idx, 1, '[', ']') FROM fts_idx WHERE fts_idx MATCH 't AND x'; } {{[t] h r e e [x] y z}} finish_test | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 | } {three {t h r e e x y z}} do_execsql_test 2.7 { SELECT highlight(fts_idx, 1, '[', ']') FROM fts_idx WHERE fts_idx MATCH 't AND x'; } {{[t] h r e e [x] y z}} #------------------------------------------------------------------------- # Quick tests of the 'delete-all' command. # do_execsql_test 3.1 { CREATE VIRTUAL TABLE t3 USING fts5(x, content=''); INSERT INTO t3 VALUES('a b c'); INSERT INTO t3 VALUES('d e f'); } do_execsql_test 3.2 { SELECT count(*) FROM t3_docsize; SELECT count(*) FROM t3_data; } {2 4} do_execsql_test 3.3 { INSERT INTO t3(t3) VALUES('delete-all'); SELECT count(*) FROM t3_docsize; SELECT count(*) FROM t3_data; } {0 2} do_execsql_test 3.4 { INSERT INTO t3 VALUES('a b c'); INSERT INTO t3 VALUES('d e f'); SELECT rowid FROM t3 WHERE t3 MATCH 'e'; } {2} do_execsql_test 3.5 { SELECT rowid FROM t3 WHERE t3 MATCH 'c'; } {1} do_execsql_test 3.6 { SELECT count(*) FROM t3_docsize; SELECT count(*) FROM t3_data; } {2 4} do_execsql_test 3.7 { CREATE VIRTUAL TABLE t4 USING fts5(x); } {} do_catchsql_test 3.8 { INSERT INTO t4(t4) VALUES('delete-all'); } {1 {'delete-all' may only be used with a contentless or external content fts5 table}} finish_test |
Added ext/fts5/test/fts5rebuild.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | # 2014 Dec 20 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5rebuild do_execsql_test 1.1 { CREATE VIRTUAL TABLE f1 USING fts5(a, b); INSERT INTO f1(a, b) VALUES('one', 'o n e'); INSERT INTO f1(a, b) VALUES('two', 't w o'); INSERT INTO f1(a, b) VALUES('three', 't h r e e'); } do_execsql_test 1.2 { INSERT INTO f1(f1) VALUES('integrity-check'); } {} do_execsql_test 1.3 { INSERT INTO f1(f1) VALUES('rebuild'); } {} do_execsql_test 1.4 { INSERT INTO f1(f1) VALUES('integrity-check'); } {} do_execsql_test 1.5 { DELETE FROM f1_data; } {} do_catchsql_test 1.6 { INSERT INTO f1(f1) VALUES('integrity-check'); } {1 {SQL logic error or missing database}} do_execsql_test 1.7 { INSERT INTO f1(f1) VALUES('rebuild'); INSERT INTO f1(f1) VALUES('integrity-check'); } {} finish_test |