Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | fts3 functions for testing scripts. These are a first step towards
being able to write test script which verify that fts3 is internally
building indices in the expected way. Both new functions are only
defined if fts3.c is compiled with SQLITE_TEST defined, as when
building testfixture. These functions are not intended to be part of
the exposed fts3 API.
dump_terms() generates a TEXT result of all the terms in the index (or a specified segment), sorted and joined with spaces. dump_doclist() generates a TEXT representation of the doclist associated with a given term in the index (or a specified segment). (CVS 5340) |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
a48e3d95f7a656285e959cef595cbe6d |
User & Date: | shess 2008-07-03 19:53:22.000 |
Context
2008-07-04
| ||
09:15 | Remove references to temporary registers from the compiler column-cache when such registers are released. Fix for #3201. (CVS 5341) (check-in: f099d6773a user: danielk1977 tags: trunk) | |
2008-07-03
| ||
19:53 |
fts3 functions for testing scripts. These are a first step towards
being able to write test script which verify that fts3 is internally
building indices in the expected way. Both new functions are only
defined if fts3.c is compiled with SQLITE_TEST defined, as when
building testfixture. These functions are not intended to be part of
the exposed fts3 API.
dump_terms() generates a TEXT result of all the terms in the index (or a specified segment), sorted and joined with spaces. dump_doclist() generates a TEXT representation of the doclist associated with a given term in the index (or a specified segment). (CVS 5340) (check-in: a48e3d95f7 user: shess tags: trunk) | |
2008-07-02
| ||
16:10 | Fix a memory leak that can occur following a malloc failure. (CVS 5339) (check-in: cec4eba1a1 user: danielk1977 tags: trunk) | |
Changes
Changes to ext/fts3/fts3.c.
︙ | ︙ | |||
1908 1909 1910 1911 1912 1913 1914 | BLOCK_INSERT_STMT, BLOCK_SELECT_STMT, BLOCK_DELETE_STMT, SEGDIR_MAX_INDEX_STMT, SEGDIR_SET_STMT, | | > | 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 | BLOCK_INSERT_STMT, BLOCK_SELECT_STMT, BLOCK_DELETE_STMT, SEGDIR_MAX_INDEX_STMT, SEGDIR_SET_STMT, SEGDIR_SELECT_LEVEL_STMT, SEGDIR_SPAN_STMT, SEGDIR_DELETE_STMT, SEGDIR_SELECT_SEGMENT_STMT, SEGDIR_SELECT_ALL_STMT, MAX_STMT /* Always at end! */ } fulltext_statement; /* These must exactly match the enum above. */ /* TODO(shess): Is there some risk that a statement will be used in two |
︙ | ︙ | |||
1934 1935 1936 1937 1938 1939 1940 | /* BLOCK_INSERT */ "insert into %_segments (blockid, block) values (null, ?)", /* BLOCK_SELECT */ "select block from %_segments where blockid = ?", /* BLOCK_DELETE */ "delete from %_segments where blockid between ? and ?", /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?", /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)", | | > > > > > > > > | > | 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 | /* BLOCK_INSERT */ "insert into %_segments (blockid, block) values (null, ?)", /* BLOCK_SELECT */ "select block from %_segments where blockid = ?", /* BLOCK_DELETE */ "delete from %_segments where blockid between ? and ?", /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?", /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)", /* SEGDIR_SELECT_LEVEL */ "select start_block, leaves_end_block, root from %_segdir " " where level = ? order by idx", /* SEGDIR_SPAN */ "select min(start_block), max(end_block) from %_segdir " " where level = ? and start_block <> 0", /* SEGDIR_DELETE */ "delete from %_segdir where level = ?", /* NOTE(shess): The first three results of the following two ** statements must match. */ /* SEGDIR_SELECT_SEGMENT */ "select start_block, leaves_end_block, root from %_segdir " " where level = ? and idx = ?", /* SEGDIR_SELECT_ALL */ "select start_block, leaves_end_block, root from %_segdir " " order by level desc, idx asc", }; /* ** A connection to a fulltext index is an instance of the following ** structure. The xCreate and xConnect methods create an instance ** of this structure and xDestroy and xDisconnect free that instance. ** All other methods receive a pointer to the structure as one of their |
︙ | ︙ | |||
5405 5406 5407 5408 5409 5410 5411 | /* Initializes pReaders with the segments from level iLevel, returning ** the number of segments in *piReaders. Leaves pReaders in sorted ** order. */ static int leavesReadersInit(fulltext_vtab *v, int iLevel, LeavesReader *pReaders, int *piReaders){ sqlite3_stmt *s; | | | 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 | /* Initializes pReaders with the segments from level iLevel, returning ** the number of segments in *piReaders. Leaves pReaders in sorted ** order. */ static int leavesReadersInit(fulltext_vtab *v, int iLevel, LeavesReader *pReaders, int *piReaders){ sqlite3_stmt *s; int i, rc = sql_get_statement(v, SEGDIR_SELECT_LEVEL_STMT, &s); if( rc!=SQLITE_OK ) return rc; rc = sqlite3_bind_int(s, 1, iLevel); if( rc!=SQLITE_OK ) return rc; i = 0; while( (rc = sqlite3_step(s))==SQLITE_ROW ){ |
︙ | ︙ | |||
5943 5944 5945 5946 5947 5948 5949 | dataBufferInit(&doclist, 0); /* Traverse the segments from oldest to newest so that newer doclist ** elements for given docids overwrite older elements. */ while( (rc = sqlite3_step(s))==SQLITE_ROW ){ | | | | 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 | dataBufferInit(&doclist, 0); /* Traverse the segments from oldest to newest so that newer doclist ** elements for given docids overwrite older elements. */ while( (rc = sqlite3_step(s))==SQLITE_ROW ){ const char *pData = sqlite3_column_blob(s, 2); const int nData = sqlite3_column_bytes(s, 2); const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1); rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix, &doclist); if( rc!=SQLITE_OK ) goto err; } if( rc==SQLITE_DONE ){ if( doclist.nData!=0 ){ |
︙ | ︙ | |||
6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 | snippetOffsetText(&pCursor->snippet); sqlite3_result_text(pContext, pCursor->snippet.zOffset, pCursor->snippet.nOffset, SQLITE_STATIC); } } /* ** This routine implements the xFindFunction method for the FTS3 ** virtual table. */ static int fulltextFindFunction( sqlite3_vtab *pVtab, int nArg, const char *zName, void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), void **ppArg ){ if( strcmp(zName,"snippet")==0 ){ *pxFunc = snippetFunc; return 1; }else if( strcmp(zName,"offsets")==0 ){ *pxFunc = snippetOffsetsFunc; return 1; } return 0; } /* ** Rename an fts3 table. */ | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327 6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364 6365 6366 6367 6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400 6401 6402 6403 6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489 6490 6491 6492 6493 6494 6495 6496 6497 6498 6499 6500 6501 6502 6503 6504 6505 6506 6507 6508 6509 6510 6511 6512 6513 6514 6515 6516 6517 6518 6519 6520 6521 6522 6523 6524 6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542 6543 6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622 6623 6624 6625 6626 6627 6628 6629 6630 6631 6632 6633 6634 6635 6636 6637 6638 6639 6640 6641 6642 6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 | snippetOffsetText(&pCursor->snippet); sqlite3_result_text(pContext, pCursor->snippet.zOffset, pCursor->snippet.nOffset, SQLITE_STATIC); } } #ifdef SQLITE_TEST /* Generate an error of the form "<prefix>: <msg>". If msg is NULL, ** pull the error from the context's db handle. */ static void generateError(sqlite3_context *pContext, const char *prefix, const char *msg){ char buf[512]; if( msg==NULL ) msg = sqlite3_errmsg(sqlite3_context_db_handle(pContext)); sqlite3_snprintf(sizeof(buf), buf, "%s: %s", prefix, msg); sqlite3_result_error(pContext, buf, -1); } /* Helper function to collect the set of terms in the segment into ** pTerms. The segment is defined by the leaf nodes between ** iStartBlockid and iEndBlockid, inclusive, or by the contents of ** pRootData if iStartBlockid is 0 (in which case the entire segment ** fit in a leaf). */ static int collectSegmentTerms(fulltext_vtab *v, sqlite3_stmt *s, fts3Hash *pTerms){ const sqlite_int64 iStartBlockid = sqlite3_column_int64(s, 0); const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1); const char *pRootData = sqlite3_column_blob(s, 2); const int nRootData = sqlite3_column_bytes(s, 2); LeavesReader reader; int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid, pRootData, nRootData, &reader); if( rc!=SQLITE_OK ) return rc; while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){ const char *pTerm = leavesReaderTerm(&reader); const int nTerm = leavesReaderTermBytes(&reader); void *oldValue = sqlite3Fts3HashFind(pTerms, pTerm, nTerm); void *newValue = (void *)((char *)oldValue+1); /* From the comment before sqlite3Fts3HashInsert in fts3_hash.c, ** the data value passed is returned in case of malloc failure. */ if( newValue==sqlite3Fts3HashInsert(pTerms, pTerm, nTerm, newValue) ){ rc = SQLITE_NOMEM; }else{ rc = leavesReaderStep(v, &reader); } } leavesReaderDestroy(&reader); return rc; } /* Helper function to build the result string for dump_terms(). */ static int generateTermsResult(sqlite3_context *pContext, fts3Hash *pTerms){ int iTerm, nTerms, nResultBytes, iByte; char *result; TermData *pData; fts3HashElem *e; /* Iterate pTerms to generate an array of terms in pData for ** sorting. */ nTerms = fts3HashCount(pTerms); assert( nTerms>0 ); pData = sqlite3_malloc(nTerms*sizeof(TermData)); if( pData==NULL ) return SQLITE_NOMEM; nResultBytes = 0; for(iTerm = 0, e = fts3HashFirst(pTerms); e; iTerm++, e = fts3HashNext(e)){ nResultBytes += fts3HashKeysize(e)+1; /* Term plus trailing space */ assert( iTerm<nTerms ); pData[iTerm].pTerm = fts3HashKey(e); pData[iTerm].nTerm = fts3HashKeysize(e); pData[iTerm].pCollector = fts3HashData(e); /* unused */ } assert( iTerm==nTerms ); assert( nResultBytes>0 ); /* nTerms>0, nResultsBytes must be, too. */ result = sqlite3_malloc(nResultBytes); if( result==NULL ){ sqlite3_free(pData); return SQLITE_NOMEM; } if( nTerms>1 ) qsort(pData, nTerms, sizeof(*pData), termDataCmp); /* Read the terms in order to build the result. */ iByte = 0; for(iTerm=0; iTerm<nTerms; ++iTerm){ memcpy(result+iByte, pData[iTerm].pTerm, pData[iTerm].nTerm); iByte += pData[iTerm].nTerm; result[iByte++] = ' '; } assert( iByte==nResultBytes ); assert( result[nResultBytes-1]==' ' ); result[nResultBytes-1] = '\0'; /* Passes away ownership of result. */ sqlite3_result_text(pContext, result, nResultBytes-1, sqlite3_free); sqlite3_free(pData); return SQLITE_OK; } /* Implements dump_terms() for use in inspecting the fts3 index from ** tests. TEXT result containing the ordered list of terms joined by ** spaces. dump_terms(t, level, idx) dumps the terms for the segment ** specified by level, idx (in %_segdir), while dump_terms(t) dumps ** all terms in the index. In both cases t is the fts table's magic ** table-named column. */ static void dumpTermsFunc( sqlite3_context *pContext, int argc, sqlite3_value **argv ){ fulltext_cursor *pCursor; if( argc!=3 && argc!=1 ){ generateError(pContext, "dump_terms", "incorrect arguments"); }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB || sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){ generateError(pContext, "dump_terms", "illegal first argument"); }else{ fulltext_vtab *v; fts3Hash terms; sqlite3_stmt *s = NULL; int rc; memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor)); v = cursor_vtab(pCursor); /* If passed only the cursor column, get all segments. Otherwise ** get the segment described by the following two arguments. */ if( argc==1 ){ rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s); }else{ rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s); if( rc==SQLITE_OK ){ rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[1])); if( rc==SQLITE_OK ){ rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[2])); } } } if( rc!=SQLITE_OK ){ generateError(pContext, "dump_terms", NULL); return; } /* Collect the terms for each segment. */ sqlite3Fts3HashInit(&terms, FTS3_HASH_STRING, 1); while( (rc = sqlite3_step(s))==SQLITE_ROW ){ rc = collectSegmentTerms(v, s, &terms); if( rc!=SQLITE_OK ) break; } if( rc!=SQLITE_DONE ){ sqlite3_reset(s); generateError(pContext, "dump_terms", NULL); }else{ const int nTerms = fts3HashCount(&terms); if( nTerms>0 ){ rc = generateTermsResult(pContext, &terms); if( rc==SQLITE_NOMEM ){ generateError(pContext, "dump_terms", "out of memory"); }else{ assert( rc==SQLITE_OK ); } }else if( argc==3 ){ /* The specific segment asked for could not be found. */ generateError(pContext, "dump_terms", "segment not found"); }else{ /* No segments found. */ /* TODO(shess): It should be impossible to reach this. This ** case can only happen for an empty table, in which case ** SQLite has no rows to call this function on. */ sqlite3_result_null(pContext); } } sqlite3Fts3HashClear(&terms); } } /* Expand the DL_DEFAULT doclist in pData into a text result in ** pContext. */ static void createDoclistResult(sqlite3_context *pContext, const char *pData, int nData){ DataBuffer dump; DLReader dlReader; assert( pData!=NULL && nData>0 ); dataBufferInit(&dump, 0); dlrInit(&dlReader, DL_DEFAULT, pData, nData); for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){ char buf[256]; PLReader plReader; plrInit(&plReader, &dlReader); if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){ sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader)); dataBufferAppend(&dump, buf, strlen(buf)); }else{ int iColumn = plrColumn(&plReader); sqlite3_snprintf(sizeof(buf), buf, "[%lld %d[", dlrDocid(&dlReader), iColumn); dataBufferAppend(&dump, buf, strlen(buf)); for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){ if( plrColumn(&plReader)!=iColumn ){ iColumn = plrColumn(&plReader); sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn); assert( dump.nData>0 ); dump.nData--; /* Overwrite trailing space. */ assert( dump.pData[dump.nData]==' '); dataBufferAppend(&dump, buf, strlen(buf)); } if( DL_DEFAULT==DL_POSITIONS_OFFSETS ){ sqlite3_snprintf(sizeof(buf), buf, "%d,%d,%d ", plrPosition(&plReader), plrStartOffset(&plReader), plrEndOffset(&plReader)); }else if( DL_DEFAULT==DL_POSITIONS ){ sqlite3_snprintf(sizeof(buf), buf, "%d ", plrPosition(&plReader)); }else{ assert( NULL=="Unhandled DL_DEFAULT value"); } dataBufferAppend(&dump, buf, strlen(buf)); } plrDestroy(&plReader); assert( dump.nData>0 ); dump.nData--; /* Overwrite trailing space. */ assert( dump.pData[dump.nData]==' '); dataBufferAppend(&dump, "]] ", 3); } } dlrDestroy(&dlReader); assert( dump.nData>0 ); dump.nData--; /* Overwrite trailing space. */ assert( dump.pData[dump.nData]==' '); dump.pData[dump.nData] = '\0'; assert( dump.nData>0 ); /* Passes ownership of dump's buffer to pContext. */ sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free); dump.pData = NULL; dump.nData = dump.nCapacity = 0; } /* Implements dump_doclist() for use in inspecting the fts3 index from ** tests. TEXT result containing a string representation of the ** doclist for the indicated term. dump_doclist(t, term, level, idx) ** dumps the doclist for term from the segment specified by level, idx ** (in %_segdir), while dump_doclist(t, term) dumps the logical ** doclist for the term across all segments. The per-segment doclist ** can contain deletions, while the full-index doclist will not ** (deletions are omitted). ** ** Result formats differ with the setting of DL_DEFAULTS. Examples: ** ** DL_DOCIDS: [1] [3] [7] ** DL_POSITIONS: [1 0[0 4] 1[17]] [3 1[5]] ** DL_POSITIONS_OFFSETS: [1 0[0,0,3 4,23,26] 1[17,102,105]] [3 1[5,20,23]] ** ** In each case the number after the outer '[' is the docid. In the ** latter two cases, the number before the inner '[' is the column ** associated with the values within. For DL_POSITIONS the numbers ** within are the positions, for DL_POSITIONS_OFFSETS they are the ** position, the start offset, and the end offset. */ static void dumpDoclistFunc( sqlite3_context *pContext, int argc, sqlite3_value **argv ){ fulltext_cursor *pCursor; if( argc!=2 && argc!=4 ){ generateError(pContext, "dump_doclist", "incorrect arguments"); }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB || sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){ generateError(pContext, "dump_doclist", "illegal first argument"); }else if( sqlite3_value_text(argv[1])==NULL || sqlite3_value_text(argv[1])[0]=='\0' ){ generateError(pContext, "dump_doclist", "empty second argument"); }else{ const char *pTerm = (const char *)sqlite3_value_text(argv[1]); const int nTerm = strlen(pTerm); fulltext_vtab *v; int rc; DataBuffer doclist; memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor)); v = cursor_vtab(pCursor); dataBufferInit(&doclist, 0); /* termSelect() yields the same logical doclist that queries are ** run against. */ if( argc==2 ){ rc = termSelect(v, v->nColumn, pTerm, nTerm, 0, DL_DEFAULT, &doclist); }else{ sqlite3_stmt *s = NULL; /* Get our specific segment's information. */ rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s); if( rc==SQLITE_OK ){ rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[2])); if( rc==SQLITE_OK ){ rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[3])); } } if( rc==SQLITE_OK ){ rc = sqlite3_step(s); if( rc==SQLITE_DONE ){ dataBufferDestroy(&doclist); generateError(pContext, "dump_doclist", "segment not found"); return; } /* Found a segment, load it into doclist. */ if( rc==SQLITE_ROW ){ const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1); const char *pData = sqlite3_column_blob(s, 2); const int nData = sqlite3_column_bytes(s, 2); /* loadSegment() is used by termSelect() to load each ** segment's data. */ rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, 0, &doclist); if( rc==SQLITE_OK ){ rc = sqlite3_step(s); /* Should not have more than one matching segment. */ if( rc!=SQLITE_DONE ){ sqlite3_reset(s); dataBufferDestroy(&doclist); generateError(pContext, "dump_doclist", "invalid segdir"); return; } rc = SQLITE_OK; } } } sqlite3_reset(s); } if( rc==SQLITE_OK ){ if( doclist.nData>0 ){ createDoclistResult(pContext, doclist.pData, doclist.nData); }else{ /* TODO(shess): This can happen if the term is not present, or ** if all instances of the term have been deleted and this is ** an all-index dump. It may be interesting to distinguish ** these cases. */ sqlite3_result_text(pContext, "", 0, SQLITE_STATIC); } }else if( rc==SQLITE_NOMEM ){ /* Handle out-of-memory cases specially because if they are ** generated in fts3 code they may not be reflected in the db ** handle. */ /* TODO(shess): Handle this more comprehensively. ** sqlite3ErrStr() has what I need, but is internal. */ generateError(pContext, "dump_doclist", "out of memory"); }else{ generateError(pContext, "dump_doclist", NULL); } dataBufferDestroy(&doclist); } } #endif /* ** This routine implements the xFindFunction method for the FTS3 ** virtual table. */ static int fulltextFindFunction( sqlite3_vtab *pVtab, int nArg, const char *zName, void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), void **ppArg ){ if( strcmp(zName,"snippet")==0 ){ *pxFunc = snippetFunc; return 1; }else if( strcmp(zName,"offsets")==0 ){ *pxFunc = snippetOffsetsFunc; return 1; #ifdef SQLITE_TEST /* NOTE(shess): These functions are present only for testing ** purposes. No particular effort is made to optimize their ** execution or how they build their results. */ }else if( strcmp(zName,"dump_terms")==0 ){ /* fprintf(stderr, "Found dump_terms\n"); */ *pxFunc = dumpTermsFunc; return 1; }else if( strcmp(zName,"dump_doclist")==0 ){ /* fprintf(stderr, "Found dump_doclist\n"); */ *pxFunc = dumpDoclistFunc; return 1; #endif } return 0; } /* ** Rename an fts3 table. */ |
︙ | ︙ | |||
6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 | ** the two scalar functions. If this is successful, register the ** module with sqlite. */ if( SQLITE_OK==rc && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1)) ){ return sqlite3_create_module_v2( db, "fts3", &fts3Module, (void *)pHash, hashDestroy ); } /* An error has occured. Delete the hash table and return the error code. */ | > > > > | 6774 6775 6776 6777 6778 6779 6780 6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 | ** the two scalar functions. If this is successful, register the ** module with sqlite. */ if( SQLITE_OK==rc && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1)) #ifdef SQLITE_TEST && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_terms", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_doclist", -1)) #endif ){ return sqlite3_create_module_v2( db, "fts3", &fts3Module, (void *)pHash, hashDestroy ); } /* An error has occured. Delete the hash table and return the error code. */ |
︙ | ︙ |
Added test/fts3c.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 | # 2008 June 26 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # This file exercises some new testing functions in the FTS3 module, # and then uses them to do some basic tests that FTS3 is internally # working as expected. # # $Id: fts3c.test,v 1.1 2008/07/03 19:53:22 shess Exp $ # set testdir [file dirname $argv0] source $testdir/tester.tcl # If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { finish_test return } #************************************************************************* # Probe to see if support for these functions is compiled in. # TODO(shess): Change main.mk to do the right thing and remove this test. db eval { DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING fts3(c); INSERT INTO t1 (docid, c) VALUES (1, 'x'); } set s {SELECT dump_terms(t1, 1) FROM t1 LIMIT 1} set r {1 {unable to use function dump_terms in the requested context}} if {[catchsql $s]==$r} { finish_test return } #************************************************************************* # Test that the new functions give appropriate errors. do_test fts3c-0.0 { catchsql { SELECT dump_terms(t1, 1) FROM t1 LIMIT 1; } } {1 {dump_terms: incorrect arguments}} do_test fts3c-0.1 { catchsql { SELECT dump_terms(t1, 0, 0, 0) FROM t1 LIMIT 1; } } {1 {dump_terms: incorrect arguments}} do_test fts3c-0.2 { catchsql { SELECT dump_terms(1, t1) FROM t1 LIMIT 1; } } {1 {unable to use function dump_terms in the requested context}} do_test fts3c-0.3 { catchsql { SELECT dump_terms(t1, 16, 16) FROM t1 LIMIT 1; } } {1 {dump_terms: segment not found}} do_test fts3c-0.4 { catchsql { SELECT dump_doclist(t1) FROM t1 LIMIT 1; } } {1 {dump_doclist: incorrect arguments}} do_test fts3c-0.5 { catchsql { SELECT dump_doclist(t1, NULL) FROM t1 LIMIT 1; } } {1 {dump_doclist: empty second argument}} do_test fts3c-0.6 { catchsql { SELECT dump_doclist(t1, '') FROM t1 LIMIT 1; } } {1 {dump_doclist: empty second argument}} do_test fts3c-0.7 { catchsql { SELECT dump_doclist(t1, 'a', 0) FROM t1 LIMIT 1; } } {1 {dump_doclist: incorrect arguments}} do_test fts3c-0.8 { catchsql { SELECT dump_doclist(t1, 'a', 0, 0, 0) FROM t1 LIMIT 1; } } {1 {dump_doclist: incorrect arguments}} do_test fts3c-0.9 { catchsql { SELECT dump_doclist(t1, 'a', 16, 16) FROM t1 LIMIT 1; } } {1 {dump_doclist: segment not found}} #************************************************************************* # Utility function to check for the expected terms in the segment # level/index. _all version does same but for entire index. proc check_terms {test level index terms} { # TODO(shess): Figure out why uplevel in do_test can't catch # $level and $index directly. set ::level $level set ::index $index do_test $test.terms { execsql { SELECT dump_terms(t1, $::level, $::index) FROM t1 LIMIT 1; } } [list $terms] } proc check_terms_all {test terms} { do_test $test.terms { execsql { SELECT dump_terms(t1) FROM t1 LIMIT 1; } } [list $terms] } # Utility function to check for the expected doclist for the term in # segment level/index. _all version does same for entire index. proc check_doclist {test level index term doclist} { # TODO(shess): Again, why can't the non-:: versions work? set ::term $term set ::level $level set ::index $index do_test $test { execsql { SELECT dump_doclist(t1, $::term, $::level, $::index) FROM t1 LIMIT 1; } } [list $doclist] } proc check_doclist_all {test term doclist} { set ::term $term do_test $test { execsql { SELECT dump_doclist(t1, $::term) FROM t1 LIMIT 1; } } [list $doclist] } #************************************************************************* # Test the segments resulting from straight-forward inserts. db eval { DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING fts3(c); INSERT INTO t1 (docid, c) VALUES (1, 'This is a test'); INSERT INTO t1 (docid, c) VALUES (2, 'That was a test'); INSERT INTO t1 (docid, c) VALUES (3, 'This is a test'); } # Check for expected segments and expected matches. do_test fts3c-1.0.segments { execsql { SELECT level, idx FROM t1_segdir ORDER BY level, idx; } } {0 0 0 1 0 2} do_test fts3c-1.0.matches { execsql { SELECT OFFSETS(t1) FROM t1 WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY docid; } } [list {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4} \ {0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4} \ {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}] # Check the specifics of the segments constructed. # Logical view of entire index. check_terms_all fts3c-1.0.1 {a is test that this was} check_doclist_all fts3c-1.0.1.1 a {[1 0[2]] [2 0[2]] [3 0[2]]} check_doclist_all fts3c-1.0.1.2 is {[1 0[1]] [3 0[1]]} check_doclist_all fts3c-1.0.1.3 test {[1 0[3]] [2 0[3]] [3 0[3]]} check_doclist_all fts3c-1.0.1.4 that {[2 0[0]]} check_doclist_all fts3c-1.0.1.5 this {[1 0[0]] [3 0[0]]} check_doclist_all fts3c-1.0.1.6 was {[2 0[1]]} # Segment 0,0 check_terms fts3c-1.0.2 0 0 {a is test this} check_doclist fts3c-1.0.2.1 0 0 a {[1 0[2]]} check_doclist fts3c-1.0.2.2 0 0 is {[1 0[1]]} check_doclist fts3c-1.0.2.3 0 0 test {[1 0[3]]} check_doclist fts3c-1.0.2.4 0 0 this {[1 0[0]]} # Segment 0,1 check_terms fts3c-1.0.3 0 1 {a test that was} check_doclist fts3c-1.0.3.1 0 1 a {[2 0[2]]} check_doclist fts3c-1.0.3.2 0 1 test {[2 0[3]]} check_doclist fts3c-1.0.3.3 0 1 that {[2 0[0]]} check_doclist fts3c-1.0.3.4 0 1 was {[2 0[1]]} # Segment 0,2 check_terms fts3c-1.0.4 0 2 {a is test this} check_doclist fts3c-1.0.4.1 0 2 a {[3 0[2]]} check_doclist fts3c-1.0.4.2 0 2 is {[3 0[1]]} check_doclist fts3c-1.0.4.3 0 2 test {[3 0[3]]} check_doclist fts3c-1.0.4.4 0 2 this {[3 0[0]]} #************************************************************************* # Test the segments resulting from inserts followed by a delete. db eval { DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING fts3(c); INSERT INTO t1 (docid, c) VALUES (1, 'This is a test'); INSERT INTO t1 (docid, c) VALUES (2, 'That was a test'); INSERT INTO t1 (docid, c) VALUES (3, 'This is a test'); DELETE FROM t1 WHERE docid = 1; } do_test fts3c-1.1.segments { execsql { SELECT level, idx FROM t1_segdir ORDER BY level, idx; } } {0 0 0 1 0 2 0 3} do_test fts3c-1.1.matches { execsql { SELECT OFFSETS(t1) FROM t1 WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY docid; } } {{0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4} {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}} check_terms_all fts3c-1.1.1 {a is test that this was} check_doclist_all fts3c-1.1.1.1 a {[2 0[2]] [3 0[2]]} check_doclist_all fts3c-1.1.1.2 is {[3 0[1]]} check_doclist_all fts3c-1.1.1.3 test {[2 0[3]] [3 0[3]]} check_doclist_all fts3c-1.1.1.4 that {[2 0[0]]} check_doclist_all fts3c-1.1.1.5 this {[3 0[0]]} check_doclist_all fts3c-1.1.1.6 was {[2 0[1]]} check_terms fts3c-1.1.2 0 0 {a is test this} check_doclist fts3c-1.1.2.1 0 0 a {[1 0[2]]} check_doclist fts3c-1.1.2.2 0 0 is {[1 0[1]]} check_doclist fts3c-1.1.2.3 0 0 test {[1 0[3]]} check_doclist fts3c-1.1.2.4 0 0 this {[1 0[0]]} check_terms fts3c-1.1.3 0 1 {a test that was} check_doclist fts3c-1.1.3.1 0 1 a {[2 0[2]]} check_doclist fts3c-1.1.3.2 0 1 test {[2 0[3]]} check_doclist fts3c-1.1.3.3 0 1 that {[2 0[0]]} check_doclist fts3c-1.1.3.4 0 1 was {[2 0[1]]} check_terms fts3c-1.1.4 0 2 {a is test this} check_doclist fts3c-1.1.4.1 0 2 a {[3 0[2]]} check_doclist fts3c-1.1.4.2 0 2 is {[3 0[1]]} check_doclist fts3c-1.1.4.3 0 2 test {[3 0[3]]} check_doclist fts3c-1.1.4.4 0 2 this {[3 0[0]]} check_terms fts3c-1.1.5 0 3 {a is test this} check_doclist fts3c-1.1.5.1 0 3 a {[1]} check_doclist fts3c-1.1.5.2 0 3 is {[1]} check_doclist fts3c-1.1.5.3 0 3 test {[1]} check_doclist fts3c-1.1.5.4 0 3 this {[1]} #************************************************************************* # Test results when all references to certain tokens are deleted. db eval { DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING fts3(c); INSERT INTO t1 (docid, c) VALUES (1, 'This is a test'); INSERT INTO t1 (docid, c) VALUES (2, 'That was a test'); INSERT INTO t1 (docid, c) VALUES (3, 'This is a test'); DELETE FROM t1 WHERE docid IN (1,3); } # Still 4 segments because 0,3 will contain deletes for docid 1 and 3. do_test fts3c-1.2.segments { execsql { SELECT level, idx FROM t1_segdir ORDER BY level, idx; } } {0 0 0 1 0 2 0 3} do_test fts3c-1.2.matches { execsql { SELECT OFFSETS(t1) FROM t1 WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY docid; } } {{0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4}} check_terms_all fts3c-1.2.1 {a is test that this was} check_doclist_all fts3c-1.2.1.1 a {[2 0[2]]} check_doclist_all fts3c-1.2.1.2 is {} check_doclist_all fts3c-1.2.1.3 test {[2 0[3]]} check_doclist_all fts3c-1.2.1.4 that {[2 0[0]]} check_doclist_all fts3c-1.2.1.5 this {} check_doclist_all fts3c-1.2.1.6 was {[2 0[1]]} check_terms fts3c-1.2.2 0 0 {a is test this} check_doclist fts3c-1.2.2.1 0 0 a {[1 0[2]]} check_doclist fts3c-1.2.2.2 0 0 is {[1 0[1]]} check_doclist fts3c-1.2.2.3 0 0 test {[1 0[3]]} check_doclist fts3c-1.2.2.4 0 0 this {[1 0[0]]} check_terms fts3c-1.2.3 0 1 {a test that was} check_doclist fts3c-1.2.3.1 0 1 a {[2 0[2]]} check_doclist fts3c-1.2.3.2 0 1 test {[2 0[3]]} check_doclist fts3c-1.2.3.3 0 1 that {[2 0[0]]} check_doclist fts3c-1.2.3.4 0 1 was {[2 0[1]]} check_terms fts3c-1.2.4 0 2 {a is test this} check_doclist fts3c-1.2.4.1 0 2 a {[3 0[2]]} check_doclist fts3c-1.2.4.2 0 2 is {[3 0[1]]} check_doclist fts3c-1.2.4.3 0 2 test {[3 0[3]]} check_doclist fts3c-1.2.4.4 0 2 this {[3 0[0]]} check_terms fts3c-1.2.5 0 3 {a is test this} check_doclist fts3c-1.2.5.1 0 3 a {[1] [3]} check_doclist fts3c-1.2.5.2 0 3 is {[1] [3]} check_doclist fts3c-1.2.5.3 0 3 test {[1] [3]} check_doclist fts3c-1.2.5.4 0 3 this {[1] [3]} #************************************************************************* # Test results when everything is optimized manually. db eval { DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING fts3(c); INSERT INTO t1 (docid, c) VALUES (1, 'This is a test'); INSERT INTO t1 (docid, c) VALUES (2, 'That was a test'); INSERT INTO t1 (docid, c) VALUES (3, 'This is a test'); DELETE FROM t1 WHERE docid IN (1,3); DROP TABLE IF EXISTS t1old; ALTER TABLE t1 RENAME TO t1old; CREATE VIRTUAL TABLE t1 USING fts3(c); INSERT INTO t1 (docid, c) SELECT docid, c FROM t1old; DROP TABLE t1old; } # Should be a single optimal segment with the same logical results. do_test fts3c-1.3.segments { execsql { SELECT level, idx FROM t1_segdir ORDER BY level, idx; } } {0 0} do_test fts3c-1.3.matches { execsql { SELECT OFFSETS(t1) FROM t1 WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY docid; } } {{0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4}} check_terms_all fts3c-1.3.1 {a test that was} check_doclist_all fts3c-1.3.1.1 a {[2 0[2]]} check_doclist_all fts3c-1.3.1.2 test {[2 0[3]]} check_doclist_all fts3c-1.3.1.3 that {[2 0[0]]} check_doclist_all fts3c-1.3.1.4 was {[2 0[1]]} check_terms fts3c-1.3.2 0 0 {a test that was} check_doclist fts3c-1.3.2.1 0 0 a {[2 0[2]]} check_doclist fts3c-1.3.2.2 0 0 test {[2 0[3]]} check_doclist fts3c-1.3.2.3 0 0 that {[2 0[0]]} check_doclist fts3c-1.3.2.4 0 0 was {[2 0[1]]} finish_test |