Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | When gathering a doclist for querying, don't discard empty position lists until the end; this allows empty position lists to override non-empty lists encountered later in the gathering process. This fixes #1982, which was caused by the fact that for all-column queries we weren't discarding empty position lists at all. (CVS 3433) |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
111ca616713dd89b5d1e114de29c8325 |
User & Date: | adamd 2006-09-21 20:56:52.000 |
Context
2006-09-22
| ||
00:06 |
Implemented UPDATE for full-text tables.
We handle an UPDATE to a row by performing an UPDATE on the content table and by building new position lists for each term which appears in either the old or new versions of the row. We write these position lists all at once; this is presumably more efficient than a delete followed by an insert (which would first write empty position lists, then new position lists). (CVS 3434) (check-in: 757fa22400 user: adamd tags: trunk) | |
2006-09-21
| ||
20:56 | When gathering a doclist for querying, don't discard empty position lists until the end; this allows empty position lists to override non-empty lists encountered later in the gathering process. This fixes #1982, which was caused by the fact that for all-column queries we weren't discarding empty position lists at all. (CVS 3433) (check-in: 111ca61671 user: adamd tags: trunk) | |
11:02 | Be more aggressive with the SQLITE_OMIT_VACUUM macro. Saves about 150 bytes of code space. (CVS 3432) (check-in: 7e618db457 user: drh tags: trunk) | |
Changes
Changes to ext/fts1/fts1.c.
︙ | ︙ | |||
335 336 337 338 339 340 341 | sqlite_int64 ret; assert( !atEnd(pReader) ); assert( pReader->iLastPos==-1 ); getVarint(pReader->p, &ret); return ret; } | | | 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 | sqlite_int64 ret; assert( !atEnd(pReader) ); assert( pReader->iLastPos==-1 ); getVarint(pReader->p, &ret); return ret; } /* Read the next docid. See also nextDocid(). */ static sqlite_int64 readDocid(DocListReader *pReader){ sqlite_int64 ret; assert( !atEnd(pReader) ); assert( pReader->iLastPos==-1 ); pReader->p += getVarint(pReader->p, &ret); if( pReader->pDoclist->iType>=DL_POSITIONS ){ |
︙ | ︙ | |||
455 456 457 458 459 460 461 | } } printf("\n"); fflush(stdout); } #endif /* SQLITE_DEBUG */ | | | < > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | | | < | 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 | } } printf("\n"); fflush(stdout); } #endif /* SQLITE_DEBUG */ /* Trim the given doclist to contain only positions in column * [iRestrictColumn]. */ static void docListRestrictColumn(DocList *in, int iRestrictColumn){ DocListReader r; DocList out; assert( in->iType>=DL_POSITIONS ); readerInit(&r, in); docListInit(&out, DL_POSITIONS, NULL, 0); while( !atEnd(&r) ){ sqlite_int64 iDocid = readDocid(&r); int iPos, iColumn; docListAddDocid(&out, iDocid); while( (iPos = readPosition(&r, &iColumn)) != -1 ){ if( iColumn==iRestrictColumn ){ docListAddPos(&out, iColumn, iPos); } } } docListDestroy(in); *in = out; } /* Trim the given doclist by discarding any docids without any remaining * positions. */ static void docListDiscardEmpty(DocList *in) { DocListReader r; DocList out; /* TODO: It would be nice to implement this operation in place; that * could save a significant amount of memory in queries with long doclists. */ assert( in->iType>=DL_POSITIONS ); readerInit(&r, in); docListInit(&out, DL_POSITIONS, NULL, 0); while( !atEnd(&r) ){ sqlite_int64 iDocid = readDocid(&r); int match = 0; int iPos, iColumn; while( (iPos = readPosition(&r, &iColumn)) != -1 ){ if( !match ){ docListAddDocid(&out, iDocid); match = 1; } docListAddPos(&out, iColumn, iPos); } } docListDestroy(in); *in = out; } |
︙ | ︙ | |||
567 568 569 570 571 572 573 | sqlite_int64 iDocid = readDocid(&updateReader); skipPositionList(&updateReader); docListSpliceElement(&accReader, iDocid, pSource, updateReader.p-pSource); } } /* | < | > > > | < < < < | | 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 | sqlite_int64 iDocid = readDocid(&updateReader); skipPositionList(&updateReader); docListSpliceElement(&accReader, iDocid, pSource, updateReader.p-pSource); } } /* ** Read the next docid off of pIn. Return 0 if we reach the end. * * TODO: This assumes that docids are never 0, but they may actually be 0 since * users can choose docids when inserting into a full-text table. Fix this. */ static sqlite_int64 nextDocid(DocListReader *pIn){ skipPositionList(pIn); return atEnd(pIn) ? 0 : readDocid(pIn); } /* ** pLeft and pRight are two DocListReaders that are pointing to ** positions lists of the same document: iDocid. ** ** If there are no instances in pLeft or pRight where the position |
︙ | ︙ | |||
647 648 649 650 651 652 653 | DocList *pOut /* Write the combined doclist here */ ){ DocListReader left, right; sqlite_int64 docidLeft, docidRight; readerInit(&left, pLeft); readerInit(&right, pRight); | | | | | | | | 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 | DocList *pOut /* Write the combined doclist here */ ){ DocListReader left, right; sqlite_int64 docidLeft, docidRight; readerInit(&left, pLeft); readerInit(&right, pRight); docidLeft = nextDocid(&left); docidRight = nextDocid(&right); while( docidLeft>0 && docidRight>0 ){ if( docidLeft<docidRight ){ docidLeft = nextDocid(&left); }else if( docidRight<docidLeft ){ docidRight = nextDocid(&right); }else{ mergePosList(&left, &right, docidLeft, pOut); docidLeft = nextDocid(&left); docidRight = nextDocid(&right); } } } /* We have two doclists: pLeft and pRight. ** Write the intersection of these two doclists into pOut. ** Only docids are matched. Position information is ignored. |
︙ | ︙ | |||
681 682 683 684 685 686 687 | DocListReader left, right; sqlite_int64 docidLeft, docidRight; assert( pOut->iType<DL_POSITIONS ); readerInit(&left, pLeft); readerInit(&right, pRight); | | | | | | | | | | | | | | 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 | DocListReader left, right; sqlite_int64 docidLeft, docidRight; assert( pOut->iType<DL_POSITIONS ); readerInit(&left, pLeft); readerInit(&right, pRight); docidLeft = nextDocid(&left); docidRight = nextDocid(&right); while( docidLeft>0 && docidRight>0 ){ if( docidLeft<docidRight ){ docidLeft = nextDocid(&left); }else if( docidRight<docidLeft ){ docidRight = nextDocid(&right); }else{ docListAddDocid(pOut, docidLeft); docidLeft = nextDocid(&left); docidRight = nextDocid(&right); } } } /* We have two doclists: pLeft and pRight. ** Write the union of these two doclists into pOut. ** Only docids are matched. Position information is ignored. ** ** The output pOut never holds positions. */ static void docListOrMerge( DocList *pLeft, /* Doclist resulting from the words on the left */ DocList *pRight, /* Doclist for the next word to the right */ DocList *pOut /* Write the combined doclist here */ ){ DocListReader left, right; sqlite_int64 docidLeft, docidRight, priorLeft; readerInit(&left, pLeft); readerInit(&right, pRight); docidLeft = nextDocid(&left); docidRight = nextDocid(&right); while( docidLeft>0 && docidRight>0 ){ if( docidLeft<=docidRight ){ docListAddDocid(pOut, docidLeft); }else{ docListAddDocid(pOut, docidRight); } priorLeft = docidLeft; if( docidLeft<=docidRight ){ docidLeft = nextDocid(&left); } if( docidRight>0 && docidRight<=priorLeft ){ docidRight = nextDocid(&right); } } while( docidLeft>0 ){ docListAddDocid(pOut, docidLeft); docidLeft = nextDocid(&left); } while( docidRight>0 ){ docListAddDocid(pOut, docidRight); docidRight = nextDocid(&right); } } /* We have two doclists: pLeft and pRight. ** Write into pOut all documents that occur in pLeft but not ** in pRight. ** |
︙ | ︙ | |||
758 759 760 761 762 763 764 | DocList *pOut /* Write the combined doclist here */ ){ DocListReader left, right; sqlite_int64 docidLeft, docidRight, priorLeft; readerInit(&left, pLeft); readerInit(&right, pRight); | | | | | | | 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 | DocList *pOut /* Write the combined doclist here */ ){ DocListReader left, right; sqlite_int64 docidLeft, docidRight, priorLeft; readerInit(&left, pLeft); readerInit(&right, pRight); docidLeft = nextDocid(&left); docidRight = nextDocid(&right); while( docidLeft>0 && docidRight>0 ){ priorLeft = docidLeft; if( docidLeft<docidRight ){ docListAddDocid(pOut, docidLeft); } if( docidLeft<=docidRight ){ docidLeft = nextDocid(&left); } if( docidRight>0 && docidRight<=priorLeft ){ docidRight = nextDocid(&right); } } while( docidLeft>0 ){ docListAddDocid(pOut, docidLeft); docidLeft = nextDocid(&left); } } static char *string_dup_n(const char *s, int n){ char *str = malloc(n + 1); memcpy(str, s, n); str[n] = '\0'; |
︙ | ︙ | |||
1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 | doclist = old; } if( rc!=SQLITE_DONE ){ docListDestroy(&doclist); return rc; } *out = doclist; return SQLITE_OK; } /* insert into %_term (rowid, term, segment, doclist) values ([piRowid], [pTerm], [iSegment], [doclist]) ** Lets sqlite select rowid if piRowid is NULL, else uses *piRowid. | > | 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 | doclist = old; } if( rc!=SQLITE_DONE ){ docListDestroy(&doclist); return rc; } docListDiscardEmpty(&doclist); *out = doclist; return SQLITE_OK; } /* insert into %_term (rowid, term, segment, doclist) values ([piRowid], [pTerm], [iSegment], [doclist]) ** Lets sqlite select rowid if piRowid is NULL, else uses *piRowid. |
︙ | ︙ | |||
2366 2367 2368 2369 2370 2371 2372 | c->eof = 1; return rc; } } else { /* full-text query */ rc = sqlite3_reset(c->pStmt); if( rc!=SQLITE_OK ) return rc; | | | 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 | c->eof = 1; return rc; } } else { /* full-text query */ rc = sqlite3_reset(c->pStmt); if( rc!=SQLITE_OK ) return rc; iDocid = nextDocid(&c->result); if( iDocid==0 ){ c->eof = 1; return SQLITE_OK; } rc = sqlite3_bind_int64(c->pStmt, 1, iDocid); if( rc!=SQLITE_OK ) return rc; /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */ |
︙ | ︙ |