SQLite

Check-in [e50e8031d6]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix an fts5 problem with large deletes.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: e50e8031d6f804ebe50e0eec9a8b6e7f0152ecc3
User & Date: dan 2015-04-21 19:07:39.210
Context
2015-04-21
20:13
Fix a problem in fts5fault1.test. (check-in: a21d60cb2a user: dan tags: fts5)
19:07
Fix an fts5 problem with large deletes. (check-in: e50e8031d6 user: dan tags: fts5)
2015-04-20
18:48
Fix some fts5 problems with very large position lists. (check-in: 2ea8f9cbe6 user: dan tags: fts5)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5_index.c.
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
    pIter->bEof = 1;
  }

  return pIter->bEof;
}

static int fts5DlidxIterEof(Fts5Index *p, Fts5DlidxIter *pIter){
  return (p->rc!=SQLITE_OK || pIter->bEof);
}

static void fts5DlidxIterLast(Fts5DlidxIter *pIter){
  if( fts5DlidxIterFirst(pIter)==0 ){
    while( 0==fts5DlidxIterNext(pIter) );
    pIter->bEof = 0;
  }







|







1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
    pIter->bEof = 1;
  }

  return pIter->bEof;
}

static int fts5DlidxIterEof(Fts5Index *p, Fts5DlidxIter *pIter){
  return pIter->bEof;
}

static void fts5DlidxIterLast(Fts5DlidxIter *pIter){
  if( fts5DlidxIterFirst(pIter)==0 ){
    while( 0==fts5DlidxIterNext(pIter) );
    pIter->bEof = 0;
  }
1456
1457
1458
1459
1460
1461
1462

1463
1464
1465
1466
1467
1468
1469
      if( (a[iOff-1] & 0x80)==0 ) break;
    }

    getVarint(&a[iOff], (u64*)&iVal);
    pIter->iRowid -= iVal;
    pIter->iLeafPgno--;


    while( iOff>pIter->iFirstOff 
        && a[iOff-1]==0x00 && (a[iOff-2] & 0x80)==0 
    ){
      iOff--;
      pIter->iLeafPgno--;
    }
    pIter->iOff = iOff;







>







1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
      if( (a[iOff-1] & 0x80)==0 ) break;
    }

    getVarint(&a[iOff], (u64*)&iVal);
    pIter->iRowid -= iVal;
    pIter->iLeafPgno--;

    /* Skip backwards passed any 0x00 bytes. */
    while( iOff>pIter->iFirstOff 
        && a[iOff-1]==0x00 && (a[iOff-2] & 0x80)==0 
    ){
      iOff--;
      pIter->iLeafPgno--;
    }
    pIter->iOff = iOff;
3694
3695
3696
3697
3698
3699
3700

3701
3702
3703
3704
3705
3706
3707
          nCopy += nPos;
          iRowid += iDelta;
          
          if( bFirstDocid ){
            fts5PutU16(&pBuf->p[0], pBuf->n);   /* first docid on page */
            pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iRowid);
            bFirstDocid = 0;

          }else{
            pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iDelta);
          }
          assert( pBuf->n<=pBuf->nSpace );

          if( (pBuf->n + nCopy) <= pgsz ){
            /* The entire poslist will fit on the current leaf. So copy







>







3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
          nCopy += nPos;
          iRowid += iDelta;
          
          if( bFirstDocid ){
            fts5PutU16(&pBuf->p[0], pBuf->n);   /* first docid on page */
            pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iRowid);
            bFirstDocid = 0;
            fts5WriteDlidxAppend(p, &writer, iRowid);
          }else{
            pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iDelta);
          }
          assert( pBuf->n<=pBuf->nSpace );

          if( (pBuf->n + nCopy) <= pgsz ){
            /* The entire poslist will fit on the current leaf. So copy
3716
3717
3718
3719
3720
3721
3722

3723
3724
3725
3726
3727
3728
3729


3730
3731
3732
3733
3734
3735
3736
            while( 1 ){
              int nSpace = pgsz - pBuf->n;
              int n = 0;
              if( (nCopy - iPos)<=nSpace ){
                n = nCopy - iPos;
              }else{
                n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);

              }
              assert( n>0 );
              fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n);
              iPos += n;
              if( iPos>=nCopy ) break;
              fts5WriteFlushLeaf(p, &writer);
              pBuf = &writer.aWriter[0].buf;


            }
            bFirstDocid = 1;
          }
          iOff += nCopy;
        }
      }








>




|
|
|
>
>







3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
            while( 1 ){
              int nSpace = pgsz - pBuf->n;
              int n = 0;
              if( (nCopy - iPos)<=nSpace ){
                n = nCopy - iPos;
              }else{
                n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);
                assert( n>=nSpace );
              }
              assert( n>0 );
              fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n);
              iPos += n;
              if( pBuf->n>=pgsz ){
                fts5WriteFlushLeaf(p, &writer);
                pBuf = &writer.aWriter[0].buf;
              }
              if( iPos>=nCopy ) break;
            }
            bFirstDocid = 1;
          }
          iOff += nCopy;
        }
      }

Added ext/fts5/test/fts5dlidx.test.
































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
# 2015 April 21
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This test is focused on uses of doclist-index records.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5dlidx

if { $tcl_platform(wordSize)<8 } {
  finish_test
  return
}

proc do_fb_test {tn sql res} {
  set res2 [lsort -integer -decr $res]
  uplevel [list do_execsql_test $tn.1 $sql $res]
  uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2]
}

do_execsql_test 1.0 { 
  CREATE VIRTUAL TABLE t1 USING fts5(x);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}

foreach {tn spc1 spc2 mul} {
  1 10  100 1000
  2 1   1   128
} {
  set xdoc [list]
  set ydoc [list]
  
  execsql { DELETE FROM t1 }

  do_test 1.$tn.1 {
  
    execsql BEGIN
    for {set i 0} {$i < 10000} {incr i} {
      set rowid [expr $i * $mul]
      set doc "a b c a b c a b c a b c a b c"
      if {($i % $spc1)==0} { 
        lappend xdoc $rowid
        append doc " x" 
        if {($i % $spc2)==0} { 
          lappend ydoc $rowid
          append doc " y" 
        }
      }
      execsql { INSERT INTO t1(rowid, x) VALUES($rowid, $doc) }
    }
    execsql COMMIT
    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
  } {}
  
  do_execsql_test 1.$tn.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
  
  do_fb_test 1.$tn.3.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND x' } $xdoc
  do_fb_test 1.$tn.3.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND a' } $xdoc
  
  do_fb_test 1.$tn.4.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND y' } $ydoc
  do_fb_test 1.$tn.4.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND a' } $ydoc
  
  do_fb_test 1.$tn.5.1 { 
    SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc
  do_fb_test 1.$tn.5.2 { 
    SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc

}


finish_test