Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Fix an fts5 problem with large deletes. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts5 |
Files: | files | file ages | folders |
SHA1: |
e50e8031d6f804ebe50e0eec9a8b6e7f |
User & Date: | dan 2015-04-21 19:07:39.210 |
Context
2015-04-21
| ||
20:13 | Fix a problem in fts5fault1.test. (check-in: a21d60cb2a user: dan tags: fts5) | |
19:07 | Fix an fts5 problem with large deletes. (check-in: e50e8031d6 user: dan tags: fts5) | |
2015-04-20
| ||
18:48 | Fix some fts5 problems with very large position lists. (check-in: 2ea8f9cbe6 user: dan tags: fts5) | |
Changes
Changes to ext/fts5/fts5_index.c.
︙ | ︙ | |||
1422 1423 1424 1425 1426 1427 1428 | pIter->bEof = 1; } return pIter->bEof; } static int fts5DlidxIterEof(Fts5Index *p, Fts5DlidxIter *pIter){ | | | 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 | pIter->bEof = 1; } return pIter->bEof; } static int fts5DlidxIterEof(Fts5Index *p, Fts5DlidxIter *pIter){ return pIter->bEof; } static void fts5DlidxIterLast(Fts5DlidxIter *pIter){ if( fts5DlidxIterFirst(pIter)==0 ){ while( 0==fts5DlidxIterNext(pIter) ); pIter->bEof = 0; } |
︙ | ︙ | |||
1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 | if( (a[iOff-1] & 0x80)==0 ) break; } getVarint(&a[iOff], (u64*)&iVal); pIter->iRowid -= iVal; pIter->iLeafPgno--; while( iOff>pIter->iFirstOff && a[iOff-1]==0x00 && (a[iOff-2] & 0x80)==0 ){ iOff--; pIter->iLeafPgno--; } pIter->iOff = iOff; | > | 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 | if( (a[iOff-1] & 0x80)==0 ) break; } getVarint(&a[iOff], (u64*)&iVal); pIter->iRowid -= iVal; pIter->iLeafPgno--; /* Skip backwards passed any 0x00 bytes. */ while( iOff>pIter->iFirstOff && a[iOff-1]==0x00 && (a[iOff-2] & 0x80)==0 ){ iOff--; pIter->iLeafPgno--; } pIter->iOff = iOff; |
︙ | ︙ | |||
3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 | nCopy += nPos; iRowid += iDelta; if( bFirstDocid ){ fts5PutU16(&pBuf->p[0], pBuf->n); /* first docid on page */ pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iRowid); bFirstDocid = 0; }else{ pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iDelta); } assert( pBuf->n<=pBuf->nSpace ); if( (pBuf->n + nCopy) <= pgsz ){ /* The entire poslist will fit on the current leaf. So copy | > | 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 | nCopy += nPos; iRowid += iDelta; if( bFirstDocid ){ fts5PutU16(&pBuf->p[0], pBuf->n); /* first docid on page */ pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iRowid); bFirstDocid = 0; fts5WriteDlidxAppend(p, &writer, iRowid); }else{ pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iDelta); } assert( pBuf->n<=pBuf->nSpace ); if( (pBuf->n + nCopy) <= pgsz ){ /* The entire poslist will fit on the current leaf. So copy |
︙ | ︙ | |||
3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 | while( 1 ){ int nSpace = pgsz - pBuf->n; int n = 0; if( (nCopy - iPos)<=nSpace ){ n = nCopy - iPos; }else{ n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); } assert( n>0 ); fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); iPos += n; | > | | | > > | 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 | while( 1 ){ int nSpace = pgsz - pBuf->n; int n = 0; if( (nCopy - iPos)<=nSpace ){ n = nCopy - iPos; }else{ n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); assert( n>=nSpace ); } assert( n>0 ); fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); iPos += n; if( pBuf->n>=pgsz ){ fts5WriteFlushLeaf(p, &writer); pBuf = &writer.aWriter[0].buf; } if( iPos>=nCopy ) break; } bFirstDocid = 1; } iOff += nCopy; } } |
︙ | ︙ |
Added ext/fts5/test/fts5dlidx.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | # 2015 April 21 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # This test is focused on uses of doclist-index records. # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5dlidx if { $tcl_platform(wordSize)<8 } { finish_test return } proc do_fb_test {tn sql res} { set res2 [lsort -integer -decr $res] uplevel [list do_execsql_test $tn.1 $sql $res] uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2] } do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } foreach {tn spc1 spc2 mul} { 1 10 100 1000 2 1 1 128 } { set xdoc [list] set ydoc [list] execsql { DELETE FROM t1 } do_test 1.$tn.1 { execsql BEGIN for {set i 0} {$i < 10000} {incr i} { set rowid [expr $i * $mul] set doc "a b c a b c a b c a b c a b c" if {($i % $spc1)==0} { lappend xdoc $rowid append doc " x" if {($i % $spc2)==0} { lappend ydoc $rowid append doc " y" } } execsql { INSERT INTO t1(rowid, x) VALUES($rowid, $doc) } } execsql COMMIT execsql { INSERT INTO t1(t1) VALUES('integrity-check') } } {} do_execsql_test 1.$tn.2 { INSERT INTO t1(t1) VALUES('integrity-check') } do_fb_test 1.$tn.3.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND x' } $xdoc do_fb_test 1.$tn.3.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND a' } $xdoc do_fb_test 1.$tn.4.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND y' } $ydoc do_fb_test 1.$tn.4.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND a' } $ydoc do_fb_test 1.$tn.5.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc do_fb_test 1.$tn.5.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc } finish_test |