Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Tests and fixes for fts5 external content tables. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts5 |
| Files: | files | file ages | folders |
| SHA1: |
047aaf830d1e72f0fdad3832a0b617e7 |
| User & Date: | dan 2015-01-05 20:41:39 |
Context
|
2015-01-06
| ||
| 14:38 | Further fixes and test cases related to external content tables. check-in: ce6a899b user: dan tags: fts5 | |
|
2015-01-05
| ||
| 20:41 | Tests and fixes for fts5 external content tables. check-in: 047aaf83 user: dan tags: fts5 | |
|
2015-01-03
| ||
| 20:44 | Add support for external content tables to fts5. check-in: 17ef5b59 user: dan tags: fts5 | |
Changes
Changes to ext/fts5/fts5.c.
252 252 break; 253 253 } 254 254 } 255 255 #else 256 256 # define fts5CheckTransactionState(x,y,z) 257 257 #endif 258 258 259 +/* 260 +** Return true if pTab is a contentless table. 261 +*/ 262 +static int fts5IsContentless(Fts5Table *pTab){ 263 + return pTab->pConfig->eContent==FTS5_CONTENT_NONE; 264 +} 259 265 260 266 /* 261 267 ** Close a virtual table handle opened by fts5InitVtab(). If the bDestroy 262 268 ** argument is non-zero, attempt delete the shadow tables from teh database 263 269 */ 264 270 static int fts5FreeVtab(Fts5Table *pTab, int bDestroy){ 265 271 int rc = SQLITE_OK; ................................................................................ 913 919 } 914 920 } 915 921 } 916 922 }else{ 917 923 /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup 918 924 ** by rowid (ePlan==FTS5_PLAN_ROWID). */ 919 925 int eStmt = fts5StmtType(idxNum); 920 - rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt); 926 + rc = sqlite3Fts5StorageStmt( 927 + pTab->pStorage, eStmt, &pCsr->pStmt, &pTab->base.zErrMsg 928 + ); 921 929 if( rc==SQLITE_OK ){ 922 930 if( ePlan==FTS5_PLAN_ROWID ){ 923 931 sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); 924 932 } 925 933 rc = fts5NextMethod(pCursor); 926 934 } 927 935 } ................................................................................ 991 999 static int fts5SeekCursor(Fts5Cursor *pCsr){ 992 1000 int rc = SQLITE_OK; 993 1001 994 1002 /* If the cursor does not yet have a statement handle, obtain one now. */ 995 1003 if( pCsr->pStmt==0 ){ 996 1004 Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); 997 1005 int eStmt = fts5StmtType(pCsr->idxNum); 998 - rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt); 1006 + rc = sqlite3Fts5StorageStmt( 1007 + pTab->pStorage, eStmt, &pCsr->pStmt, &pTab->base.zErrMsg 1008 + ); 999 1009 assert( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ); 1000 1010 } 1001 1011 1002 1012 if( rc==SQLITE_OK && CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){ 1003 1013 assert( pCsr->pExpr ); 1004 1014 sqlite3_reset(pCsr->pStmt); 1005 1015 sqlite3_bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr)); ................................................................................ 1096 1106 ** 1. The "old" rowid, or NULL. 1097 1107 ** 2. The "new" rowid. 1098 1108 ** 3. Values for each of the nCol matchable columns. 1099 1109 ** 4. Values for the two hidden columns (<tablename> and "rank"). 1100 1110 */ 1101 1111 assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) ); 1102 1112 1103 - if( nArg>1 ){ 1104 - sqlite3_value *pCmd = sqlite3_value_type(apVal[2 + pConfig->nCol]); 1105 - if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){ 1106 - const char *z = sqlite3_value_text(pCmd); 1107 - if( pConfig->bExternalContent && sqlite3_stricmp("delete", z) ){ 1108 - return fts5SpecialDelete(pTab, apVal, pRowid); 1109 - }else{ 1110 - return fts5SpecialInsert(pTab, pCmd, apVal[2 + pConfig->nCol + 1]); 1111 - } 1112 - } 1113 - } 1114 - 1115 1113 eType0 = sqlite3_value_type(apVal[0]); 1116 1114 eConflict = sqlite3_vtab_on_conflict(pConfig->db); 1117 1115 1118 1116 assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL ); 1119 1117 assert( pVtab->zErrMsg==0 ); 1120 1118 1121 1119 if( rc==SQLITE_OK && eType0==SQLITE_INTEGER ){ 1122 - i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ 1123 - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel); 1120 + if( fts5IsContentless(pTab) ){ 1121 + pTab->base.zErrMsg = sqlite3_mprintf( 1122 + "cannot %s contentless fts5 table: %s", 1123 + (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName 1124 + ); 1125 + rc = SQLITE_ERROR; 1126 + }else{ 1127 + i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ 1128 + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel); 1129 + } 1130 + }else if( nArg>1 ){ 1131 + sqlite3_value *pCmd = apVal[2 + pConfig->nCol]; 1132 + if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){ 1133 + const char *z = sqlite3_value_text(pCmd); 1134 + if( pConfig->eContent!=FTS5_CONTENT_NORMAL 1135 + && 0==sqlite3_stricmp("delete", z) 1136 + ){ 1137 + return fts5SpecialDelete(pTab, apVal, pRowid); 1138 + }else{ 1139 + return fts5SpecialInsert(pTab, pCmd, apVal[2 + pConfig->nCol + 1]); 1140 + } 1141 + } 1124 1142 } 1143 + 1125 1144 1126 1145 if( rc==SQLITE_OK && nArg>1 ){ 1127 1146 rc = sqlite3Fts5StorageInsert(pTab->pStorage, apVal, eConflict, pRowid); 1128 1147 } 1129 1148 1130 1149 return rc; 1131 1150 } ................................................................................ 1324 1343 1325 1344 static int fts5ApiColumnText( 1326 1345 Fts5Context *pCtx, 1327 1346 int iCol, 1328 1347 const char **pz, 1329 1348 int *pn 1330 1349 ){ 1350 + int rc = SQLITE_OK; 1331 1351 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 1332 - int rc = fts5SeekCursor(pCsr); 1333 - if( rc==SQLITE_OK ){ 1334 - *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); 1335 - *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); 1352 + if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ 1353 + *pz = 0; 1354 + *pn = 0; 1355 + }else{ 1356 + rc = fts5SeekCursor(pCsr); 1357 + if( rc==SQLITE_OK ){ 1358 + *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); 1359 + *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); 1360 + } 1336 1361 } 1337 1362 return rc; 1338 1363 } 1339 1364 1340 1365 static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ 1341 1366 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 1342 1367 Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); ................................................................................ 1562 1587 ** the row that the supplied cursor currently points to. 1563 1588 */ 1564 1589 static int fts5ColumnMethod( 1565 1590 sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ 1566 1591 sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ 1567 1592 int iCol /* Index of column to read value from */ 1568 1593 ){ 1569 - Fts5Config *pConfig = ((Fts5Table*)(pCursor->pVtab))->pConfig; 1594 + Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); 1595 + Fts5Config *pConfig = pTab->pConfig; 1570 1596 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; 1571 1597 int rc = SQLITE_OK; 1572 1598 1573 1599 assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); 1574 1600 1575 1601 if( pCsr->idxNum==FTS5_PLAN_SPECIAL ){ 1576 1602 if( iCol==pConfig->nCol ){ ................................................................................ 1593 1619 FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_MATCH 1594 1620 || FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SORTED_MATCH 1595 1621 ){ 1596 1622 if( pCsr->pRank || SQLITE_OK==(rc = fts5FindRankFunction(pCsr)) ){ 1597 1623 fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, pCsr->nRankArg, pCsr->apRankArg); 1598 1624 } 1599 1625 } 1600 - }else{ 1626 + }else if( !fts5IsContentless(pTab) ){ 1601 1627 rc = fts5SeekCursor(pCsr); 1602 1628 if( rc==SQLITE_OK ){ 1603 1629 sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); 1604 1630 } 1605 1631 } 1606 1632 return rc; 1607 1633 }
Changes to ext/fts5/fts5Int.h.
72 72 sqlite3 *db; /* Database handle */ 73 73 char *zDb; /* Database holding FTS index (e.g. "main") */ 74 74 char *zName; /* Name of FTS index */ 75 75 int nCol; /* Number of columns */ 76 76 char **azCol; /* Column names */ 77 77 int nPrefix; /* Number of prefix indexes */ 78 78 int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ 79 - int bExternalContent; /* Content is external */ 80 - char *zContent; /* "content=" option value (or NULL) */ 81 - char *zContentRowid; /* "content_rowid=" option value (or NULL) */ 79 + int eContent; /* An FTS5_CONTENT value */ 80 + char *zContent; /* content table */ 81 + char *zContentRowid; /* "content_rowid=" option value */ 82 82 Fts5Tokenizer *pTok; 83 83 fts5_tokenizer *pTokApi; 84 84 85 85 /* Values loaded from the %_config table */ 86 86 int iCookie; /* Incremented when %_config is modified */ 87 87 int pgsz; /* Approximate page size used in %_data */ 88 88 int nAutomerge; /* 'automerge' setting */ 89 89 char *zRank; /* Name of rank function */ 90 90 char *zRankArgs; /* Arguments to rank function */ 91 91 }; 92 + 93 +#define FTS5_CONTENT_NORMAL 0 94 +#define FTS5_CONTENT_NONE 1 95 +#define FTS5_CONTENT_EXTERNAL 2 96 + 97 + 92 98 93 99 int sqlite3Fts5ConfigParse( 94 100 Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** 95 101 ); 96 102 void sqlite3Fts5ConfigFree(Fts5Config*); 97 103 98 104 int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig); ................................................................................ 397 403 int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **); 398 404 399 405 int sqlite3Fts5StorageDelete(Fts5Storage *p, i64); 400 406 int sqlite3Fts5StorageInsert(Fts5Storage *p, sqlite3_value **apVal, int, i64*); 401 407 402 408 int sqlite3Fts5StorageIntegrity(Fts5Storage *p); 403 409 404 -int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **); 410 +int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**); 405 411 void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*); 406 412 407 413 int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol); 408 414 int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg); 409 415 int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow); 410 416 411 417 int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
Changes to ext/fts5/fts5_aux.c.
218 218 219 219 iCol = sqlite3_value_int(apVal[0]); 220 220 memset(&ctx, 0, sizeof(HighlightContext)); 221 221 ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); 222 222 ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); 223 223 rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn); 224 224 225 - if( rc==SQLITE_OK ){ 226 - rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter); 225 + if( ctx.zIn ){ 226 + if( rc==SQLITE_OK ){ 227 + rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter); 228 + } 229 + 230 + if( rc==SQLITE_OK ){ 231 + rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); 232 + } 233 + fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); 234 + 235 + if( rc==SQLITE_OK ){ 236 + sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); 237 + }else{ 238 + sqlite3_result_error_code(pCtx, rc); 239 + } 240 + sqlite3_free(ctx.zOut); 227 241 } 228 - 229 - if( rc==SQLITE_OK ){ 230 - rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx, fts5HighlightCb); 231 - } 232 - fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); 233 - 234 - if( rc==SQLITE_OK ){ 235 - sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); 236 - }else{ 237 - sqlite3_result_error_code(pCtx, rc); 238 - } 239 - sqlite3_free(ctx.zOut); 240 242 } 241 243 /* 242 244 ** End of highlight() implementation. 243 245 **************************************************************************/ 244 246 245 247 /* 246 248 ** Implementation of snippet() function. ................................................................................ 271 273 const char *zErr = "wrong number of arguments to function snippet()"; 272 274 sqlite3_result_error(pCtx, zErr, -1); 273 275 return; 274 276 } 275 277 276 278 memset(&ctx, 0, sizeof(HighlightContext)); 277 279 iCol = sqlite3_value_int(apVal[0]); 278 - rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn); 279 280 ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); 280 281 ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); 281 282 zEllips = (const char*)sqlite3_value_text(apVal[3]); 282 283 nToken = sqlite3_value_int(apVal[4]); 283 284 iBestLast = nToken-1; 284 285 285 286 iBestCol = (iCol>=0 ? iCol : 0); ................................................................................ 324 325 325 326 if( rc==SQLITE_OK ){ 326 327 rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); 327 328 } 328 329 if( rc==SQLITE_OK ){ 329 330 rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn); 330 331 } 331 - if( rc==SQLITE_OK ){ 332 - rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); 333 - } 334 - 335 - if( (iBestStart+nToken-1)>iBestLast ){ 336 - iBestStart -= (iBestStart+nToken-1-iBestLast) / 2; 337 - } 338 - if( iBestStart+nToken>nColSize ){ 339 - iBestStart = nColSize - nToken; 340 - } 341 - if( iBestStart<0 ) iBestStart = 0; 342 - 343 - ctx.iRangeStart = iBestStart; 344 - ctx.iRangeEnd = iBestStart + nToken - 1; 345 - 346 - if( iBestStart>0 ){ 347 - fts5HighlightAppend(&rc, &ctx, zEllips, -1); 348 - } 349 - if( rc==SQLITE_OK ){ 350 - rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx, fts5HighlightCb); 351 - } 352 - if( ctx.iRangeEnd>=(nColSize-1) ){ 353 - fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); 354 - }else{ 355 - fts5HighlightAppend(&rc, &ctx, zEllips, -1); 356 - } 357 - 358 - if( rc==SQLITE_OK ){ 359 - sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); 360 - }else{ 361 - sqlite3_result_error_code(pCtx, rc); 362 - } 363 - sqlite3_free(ctx.zOut); 332 + if( ctx.zIn ){ 333 + if( rc==SQLITE_OK ){ 334 + rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); 335 + } 336 + 337 + if( (iBestStart+nToken-1)>iBestLast ){ 338 + iBestStart -= (iBestStart+nToken-1-iBestLast) / 2; 339 + } 340 + if( iBestStart+nToken>nColSize ){ 341 + iBestStart = nColSize - nToken; 342 + } 343 + if( iBestStart<0 ) iBestStart = 0; 344 + 345 + ctx.iRangeStart = iBestStart; 346 + ctx.iRangeEnd = iBestStart + nToken - 1; 347 + 348 + if( iBestStart>0 ){ 349 + fts5HighlightAppend(&rc, &ctx, zEllips, -1); 350 + } 351 + if( rc==SQLITE_OK ){ 352 + rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); 353 + } 354 + if( ctx.iRangeEnd>=(nColSize-1) ){ 355 + fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); 356 + }else{ 357 + fts5HighlightAppend(&rc, &ctx, zEllips, -1); 358 + } 359 + 360 + if( rc==SQLITE_OK ){ 361 + sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); 362 + }else{ 363 + sqlite3_result_error_code(pCtx, rc); 364 + } 365 + sqlite3_free(ctx.zOut); 366 + } 364 367 sqlite3_free(aSeen); 365 368 } 366 369 367 370 /************************************************************************/ 368 371 369 372 /* 370 373 ** The first time the bm25() function is called for a query, an instance
Changes to ext/fts5/fts5_config.c.
330 330 sqlite3_free(azArg); 331 331 sqlite3_free(pDel); 332 332 return rc; 333 333 } 334 334 335 335 if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){ 336 336 int rc = SQLITE_OK; 337 - if( pConfig->zContent ){ 337 + if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){ 338 338 *pzErr = sqlite3_mprintf("multiple content=... directives"); 339 339 rc = SQLITE_ERROR; 340 340 }else{ 341 - pConfig->zContent = sqlite3_mprintf("%Q.%Q", pConfig->zDb, zArg); 342 - pConfig->bExternalContent = 1; 341 + if( zArg[0] ){ 342 + pConfig->eContent = FTS5_CONTENT_EXTERNAL; 343 + pConfig->zContent = sqlite3_mprintf("%Q.%Q", pConfig->zDb, zArg); 344 + }else{ 345 + pConfig->eContent = FTS5_CONTENT_NONE; 346 + pConfig->zContent = sqlite3_mprintf( 347 + "%Q.'%q_docsize'", pConfig->zDb, pConfig->zName 348 + ); 349 + } 343 350 if( pConfig->zContent==0 ) rc = SQLITE_NOMEM; 344 351 } 345 352 return rc; 346 353 } 347 354 348 355 if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){ 349 356 int rc = SQLITE_OK; ................................................................................ 469 476 ** already been allocated. Otherwise, allocate an instance of the default 470 477 ** tokenizer (simple) now. */ 471 478 if( rc==SQLITE_OK && pRet->pTok==0 ){ 472 479 rc = fts5ConfigDefaultTokenizer(pGlobal, pRet); 473 480 } 474 481 475 482 /* If no zContent option was specified, fill in the default values. */ 476 - if( rc==SQLITE_OK && pRet->zContent==0 ){ 483 + if( rc==SQLITE_OK && pRet->eContent==FTS5_CONTENT_NORMAL ){ 477 484 pRet->zContent = sqlite3_mprintf("%Q.'%q_content'", pRet->zDb, pRet->zName); 478 485 if( pRet->zContent==0 ){ 479 486 rc = SQLITE_NOMEM; 480 487 }else{ 481 488 sqlite3_free(pRet->zContentRowid); 482 489 pRet->zContentRowid = 0; 483 490 }
Changes to ext/fts5/fts5_storage.c.
50 50 ** Fts5Storage.pInsertDocsize - if they have not already been prepared. 51 51 ** Return SQLITE_OK if successful, or an SQLite error code if an error 52 52 ** occurs. 53 53 */ 54 54 static int fts5StorageGetStmt( 55 55 Fts5Storage *p, /* Storage handle */ 56 56 int eStmt, /* FTS5_STMT_XXX constant */ 57 - sqlite3_stmt **ppStmt /* OUT: Prepared statement handle */ 57 + sqlite3_stmt **ppStmt, /* OUT: Prepared statement handle */ 58 + char **pzErrMsg /* OUT: Error message (if any) */ 58 59 ){ 59 60 int rc = SQLITE_OK; 60 61 61 62 assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) ); 62 63 if( p->aStmt[eStmt]==0 ){ 63 64 const char *azStmt[] = { 64 65 "SELECT * FROM %s ORDER BY id ASC", /* SCAN_ASC */ ................................................................................ 113 114 } 114 115 115 116 if( zSql==0 ){ 116 117 rc = SQLITE_NOMEM; 117 118 }else{ 118 119 rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0); 119 120 sqlite3_free(zSql); 121 + if( rc!=SQLITE_OK && pzErrMsg ){ 122 + *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db)); 123 + } 120 124 } 121 125 } 122 126 123 127 *ppStmt = p->aStmt[eStmt]; 124 128 return rc; 125 129 } 126 130 ................................................................................ 201 205 202 206 memset(p, 0, nByte); 203 207 p->aTotalSize = (i64*)&p[1]; 204 208 p->pConfig = pConfig; 205 209 p->pIndex = pIndex; 206 210 207 211 if( bCreate ){ 208 - if( pConfig->bExternalContent==0 ){ 212 + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ 209 213 char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10); 210 214 if( zDefn==0 ){ 211 215 rc = SQLITE_NOMEM; 212 216 }else{ 213 217 int i; 214 218 int iOff = sprintf(zDefn, "id INTEGER PRIMARY KEY"); 215 219 for(i=0; i<pConfig->nCol; i++){ ................................................................................ 250 254 /* Finalize all SQL statements */ 251 255 for(i=0; i<ArraySize(p->aStmt); i++){ 252 256 sqlite3_finalize(p->aStmt[i]); 253 257 } 254 258 255 259 /* If required, remove the shadow tables from the database */ 256 260 if( bDestroy ){ 257 - rc = sqlite3Fts5DropTable(p->pConfig, "content"); 261 + if( p->pConfig->eContent==FTS5_CONTENT_NORMAL ){ 262 + rc = sqlite3Fts5DropTable(p->pConfig, "content"); 263 + } 258 264 if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "docsize"); 259 265 if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "config"); 260 266 } 261 267 262 268 sqlite3_free(p); 263 269 } 264 270 return rc; ................................................................................ 294 300 ** remove the %_content row at this time though. 295 301 */ 296 302 static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){ 297 303 Fts5Config *pConfig = p->pConfig; 298 304 sqlite3_stmt *pSeek; /* SELECT to read row iDel from %_data */ 299 305 int rc; /* Return code */ 300 306 301 - rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek); 307 + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0); 302 308 if( rc==SQLITE_OK ){ 303 309 int rc2; 304 310 sqlite3_bind_int64(pSeek, 1, iDel); 305 311 if( sqlite3_step(pSeek)==SQLITE_ROW ){ 306 312 int iCol; 307 313 Fts5InsertCtx ctx; 308 314 ctx.pStorage = p; ................................................................................ 334 340 */ 335 341 static int fts5StorageInsertDocsize( 336 342 Fts5Storage *p, /* Storage module to write to */ 337 343 i64 iRowid, /* id value */ 338 344 Fts5Buffer *pBuf /* sz value */ 339 345 ){ 340 346 sqlite3_stmt *pReplace = 0; 341 - int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace); 347 + int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); 342 348 if( rc==SQLITE_OK ){ 343 349 sqlite3_bind_int64(pReplace, 1, iRowid); 344 350 sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); 345 351 sqlite3_step(pReplace); 346 352 rc = sqlite3_reset(pReplace); 347 353 } 348 354 return rc; ................................................................................ 420 426 /* Delete the index records */ 421 427 if( rc==SQLITE_OK ){ 422 428 rc = fts5StorageDeleteFromIndex(p, iDel); 423 429 } 424 430 425 431 /* Delete the %_docsize record */ 426 432 if( rc==SQLITE_OK ){ 427 - rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel); 433 + rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0); 428 434 } 429 435 if( rc==SQLITE_OK ){ 430 436 sqlite3_bind_int64(pDel, 1, iDel); 431 437 sqlite3_step(pDel); 432 438 rc = sqlite3_reset(pDel); 433 439 } 434 440 435 441 /* Delete the %_content record */ 436 442 if( rc==SQLITE_OK ){ 437 - rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel); 443 + rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0); 438 444 } 439 445 if( rc==SQLITE_OK ){ 440 446 sqlite3_bind_int64(pDel, 1, iDel); 441 447 sqlite3_step(pDel); 442 448 rc = sqlite3_reset(pDel); 443 449 } 444 450 ................................................................................ 455 461 i64 iDel, 456 462 sqlite3_value **apVal 457 463 ){ 458 464 Fts5Config *pConfig = p->pConfig; 459 465 int rc; 460 466 sqlite3_stmt *pDel; 461 467 462 - assert( p->pConfig->bExternalContent ); 468 + assert( p->pConfig->eContent!=FTS5_CONTENT_NORMAL ); 463 469 rc = fts5StorageLoadTotals(p, 1); 464 470 465 471 /* Delete the index records */ 466 472 if( rc==SQLITE_OK ){ 467 473 int iCol; 468 474 Fts5InsertCtx ctx; 469 475 ctx.pStorage = p; ................................................................................ 473 479 for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){ 474 480 rc = sqlite3Fts5Tokenize(pConfig, 475 481 (const char*)sqlite3_value_text(apVal[iCol]), 476 482 sqlite3_value_bytes(apVal[iCol]), 477 483 (void*)&ctx, 478 484 fts5StorageInsertCallback 479 485 ); 480 - p->aTotalSize[iCol-1] -= (i64)ctx.szCol; 486 + p->aTotalSize[iCol] -= (i64)ctx.szCol; 481 487 } 482 488 p->nTotalRow--; 483 489 } 484 490 485 491 /* Delete the %_docsize record */ 486 492 if( rc==SQLITE_OK ){ 487 - rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel); 493 + rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0); 488 494 } 489 495 if( rc==SQLITE_OK ){ 490 496 sqlite3_bind_int64(pDel, 1, iDel); 491 497 sqlite3_step(pDel); 492 498 rc = sqlite3_reset(pDel); 493 499 } 494 500 ................................................................................ 505 511 ** Allocate a new rowid. This is used for "external content" tables when 506 512 ** a NULL value is inserted into the rowid column. The new rowid is allocated 507 513 ** by inserting a dummy row into the %_docsize table. The dummy will be 508 514 ** overwritten later. 509 515 */ 510 516 static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){ 511 517 sqlite3_stmt *pReplace = 0; 512 - int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace); 518 + int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); 513 519 if( rc==SQLITE_OK ){ 514 520 sqlite3_bind_null(pReplace, 1); 515 521 sqlite3_bind_null(pReplace, 2); 516 522 sqlite3_step(pReplace); 517 523 rc = sqlite3_reset(pReplace); 518 524 } 519 525 if( rc==SQLITE_OK ){ ................................................................................ 539 545 Fts5InsertCtx ctx; /* Tokenization callback context object */ 540 546 Fts5Buffer buf; /* Buffer used to build up %_docsize blob */ 541 547 542 548 memset(&buf, 0, sizeof(Fts5Buffer)); 543 549 rc = fts5StorageLoadTotals(p, 1); 544 550 545 551 /* Insert the new row into the %_content table. */ 546 - if( rc==SQLITE_OK && pConfig->bExternalContent==0 ){ 547 - if( pConfig->bExternalContent ){ 552 + if( rc==SQLITE_OK ){ 553 + if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){ 548 554 if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ 549 555 *piRowid = sqlite3_value_int64(apVal[1]); 550 556 }else{ 551 557 rc = fts5StorageNewRowid(p, piRowid); 552 558 } 553 559 }else{ 554 560 if( eConflict==SQLITE_REPLACE ){ ................................................................................ 556 562 if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ 557 563 rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1])); 558 564 } 559 565 }else{ 560 566 eStmt = FTS5_STMT_INSERT_CONTENT; 561 567 } 562 568 if( rc==SQLITE_OK ){ 563 - rc = fts5StorageGetStmt(p, eStmt, &pInsert); 569 + rc = fts5StorageGetStmt(p, eStmt, &pInsert, 0); 564 570 } 565 571 for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ 566 572 rc = sqlite3_bind_value(pInsert, i, apVal[i]); 567 573 } 568 574 if( rc==SQLITE_OK ){ 569 575 sqlite3_step(pInsert); 570 576 rc = sqlite3_reset(pInsert); ................................................................................ 678 684 aTotalSize = (i64*)sqlite3_malloc(pConfig->nCol * (sizeof(int)+sizeof(i64))); 679 685 if( !aTotalSize ) return SQLITE_NOMEM; 680 686 aColSize = (int*)&aTotalSize[pConfig->nCol]; 681 687 memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol); 682 688 683 689 /* Generate the expected index checksum based on the contents of the 684 690 ** %_content table. This block stores the checksum in ctx.cksum. */ 685 - rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN_ASC, &pScan); 691 + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN_ASC, &pScan, 0); 686 692 if( rc==SQLITE_OK ){ 687 693 int rc2; 688 694 while( SQLITE_ROW==sqlite3_step(pScan) ){ 689 695 int i; 690 696 ctx.iRowid = sqlite3_column_int64(pScan, 0); 691 697 ctx.szCol = 0; 692 698 rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); ................................................................................ 741 747 return rc; 742 748 } 743 749 744 750 /* 745 751 ** Obtain an SQLite statement handle that may be used to read data from the 746 752 ** %_content table. 747 753 */ 748 -int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **pp){ 754 +int sqlite3Fts5StorageStmt( 755 + Fts5Storage *p, 756 + int eStmt, 757 + sqlite3_stmt **pp, 758 + char **pzErrMsg 759 +){ 749 760 int rc; 750 761 assert( eStmt==FTS5_STMT_SCAN_ASC 751 762 || eStmt==FTS5_STMT_SCAN_DESC 752 763 || eStmt==FTS5_STMT_LOOKUP 753 764 ); 754 - rc = fts5StorageGetStmt(p, eStmt, pp); 765 + rc = fts5StorageGetStmt(p, eStmt, pp, pzErrMsg); 755 766 if( rc==SQLITE_OK ){ 756 767 assert( p->aStmt[eStmt]==*pp ); 757 768 p->aStmt[eStmt] = 0; 758 769 } 759 770 return rc; 760 771 } 761 772 ................................................................................ 801 812 ** 802 813 ** An SQLite error code is returned if an error occurs, or SQLITE_OK 803 814 ** otherwise. 804 815 */ 805 816 int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){ 806 817 int nCol = p->pConfig->nCol; 807 818 sqlite3_stmt *pLookup = 0; 808 - int rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup); 819 + int rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); 809 820 if( rc==SQLITE_OK ){ 810 821 int bCorrupt = 1; 811 822 sqlite3_bind_int64(pLookup, 1, iRowid); 812 823 if( SQLITE_ROW==sqlite3_step(pLookup) ){ 813 824 const u8 *aBlob = sqlite3_column_blob(pLookup, 0); 814 825 int nBlob = sqlite3_column_bytes(pLookup, 0); 815 826 if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){ ................................................................................ 869 880 870 881 int sqlite3Fts5StorageConfigValue( 871 882 Fts5Storage *p, 872 883 const char *z, 873 884 sqlite3_value *pVal 874 885 ){ 875 886 sqlite3_stmt *pReplace = 0; 876 - int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace); 887 + int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0); 877 888 if( rc==SQLITE_OK ){ 878 889 sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_TRANSIENT); 879 890 sqlite3_bind_value(pReplace, 2, pVal); 880 891 sqlite3_step(pReplace); 881 892 rc = sqlite3_reset(pReplace); 882 893 } 883 894 if( rc==SQLITE_OK ){
Added ext/fts5/test/fts5content.test.
1 +# 2014 Dec 20 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#*********************************************************************** 11 +# 12 +# 13 + 14 +if {![info exists testdir]} { 15 + set testdir [file join [file dirname [info script]] .. .. .. test] 16 +} 17 +source $testdir/tester.tcl 18 +set testprefix fts5content 19 + 20 +do_execsql_test 1.1 { 21 + CREATE VIRTUAL TABLE f1 USING fts5(a, b, content=''); 22 + INSERT INTO f1(rowid, a, b) VALUES(1, 'one', 'o n e'); 23 + INSERT INTO f1(rowid, a, b) VALUES(2, 'two', 't w o'); 24 + INSERT INTO f1(rowid, a, b) VALUES(3, 'three', 't h r e e'); 25 +} 26 + 27 +do_execsql_test 1.2 { 28 + SELECT rowid FROM f1 WHERE f1 MATCH 'o'; 29 +} {2 1} 30 + 31 +do_execsql_test 1.3 { 32 + INSERT INTO f1(a, b) VALUES('four', 'f o u r'); 33 + SELECT rowid FROM f1 WHERE f1 MATCH 'o'; 34 +} {4 2 1} 35 + 36 +do_execsql_test 1.4 { 37 + SELECT rowid, a, b FROM f1 WHERE f1 MATCH 'o'; 38 +} {4 {} {} 2 {} {} 1 {} {}} 39 + 40 +do_execsql_test 1.5 { 41 + SELECT rowid, highlight(f1, 0, '[', ']') FROM f1 WHERE f1 MATCH 'o'; 42 +} {4 {} 2 {} 1 {}} 43 + 44 +do_execsql_test 1.6 { 45 + SELECT rowid, highlight(f1, 0, '[', ']') IS NULL FROM f1 WHERE f1 MATCH 'o'; 46 +} {4 1 2 1 1 1} 47 + 48 +do_execsql_test 1.7 { 49 + SELECT rowid, snippet(f1, -1, '[', ']', '...', 5) IS NULL 50 + FROM f1 WHERE f1 MATCH 'o'; 51 +} {4 1 2 1 1 1} 52 + 53 +do_execsql_test 1.8 { 54 + SELECT rowid, snippet(f1, 1, '[', ']', '...', 5) IS NULL 55 + FROM f1 WHERE f1 MATCH 'o'; 56 +} {4 1 2 1 1 1} 57 + 58 +do_execsql_test 1.9 { 59 + SELECT rowid FROM f1; 60 +} {4 3 2 1} 61 + 62 +do_execsql_test 1.10 { 63 + SELECT * FROM f1; 64 +} {{} {} {} {} {} {} {} {}} 65 + 66 +do_execsql_test 1.11 { 67 + SELECT rowid, a, b FROM f1 ORDER BY rowid ASC; 68 +} {1 {} {} 2 {} {} 3 {} {} 4 {} {}} 69 + 70 +do_execsql_test 1.12 { 71 + SELECT a IS NULL FROM f1; 72 +} {1 1 1 1} 73 + 74 +do_catchsql_test 1.13 { 75 + DELETE FROM f1 WHERE rowid = 2; 76 +} {1 {cannot DELETE from contentless fts5 table: f1}} 77 + 78 +do_catchsql_test 1.14 { 79 + UPDATE f1 SET a = 'a b c' WHERE rowid = 2; 80 +} {1 {cannot UPDATE contentless fts5 table: f1}} 81 + 82 +do_execsql_test 1.15 { 83 + INSERT INTO f1(f1, rowid, a, b) VALUES('delete', 2, 'two', 't w o'); 84 +} {} 85 + 86 +db eval { SELECT fts5_decode(id, block) AS d FROM f1_data } { puts $d } 87 + 88 +breakpoint 89 +do_execsql_test 1.16 { 90 + SELECT rowid FROM f1 WHERE f1 MATCH 'o'; 91 +} {4 1} 92 +do_execsql_test 1.17 { 93 + SELECT rowid FROM f1; 94 +} {4 3 1} 95 + 96 + 97 + 98 + 99 +finish_test