/ Check-in [3291b2a6]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Avoid use-after-free and double-free errors that could occur if an fts5 table is modified in certain ways while there are active cursors.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 3291b2a6fe6f38ae91b933e5cd2bf7d97432374b4fb1fccd92b4bd759b02ee06
User & Date: dan 2019-01-10 17:08:20
Context
2019-01-10
18:35
Fix a memory leak in fts5. check-in: ff3b011f user: dan tags: trunk
17:08
Avoid use-after-free and double-free errors that could occur if an fts5 table is modified in certain ways while there are active cursors. check-in: 3291b2a6 user: dan tags: trunk
15:17
Fix further problems with fts5 handling corrupt databases. check-in: 83c467d7 user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_index.c.

508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
....
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
....
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
**   Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered.
**   There is no way to tell if this is populated or not.
*/
struct Fts5Iter {
  Fts5IndexIter base;             /* Base class containing output vars */

  Fts5Index *pIndex;              /* Index that owns this iterator */
  Fts5Structure *pStruct;         /* Database structure for this iterator */
  Fts5Buffer poslist;             /* Buffer containing current poslist */
  Fts5Colset *pColset;            /* Restrict matches to these columns */

  /* Invoked to set output variables. */
  void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*);

  int nSeg;                       /* Size of aSeg[] array */
................................................................................
*/
static void fts5MultiIterFree(Fts5Iter *pIter){
  if( pIter ){
    int i;
    for(i=0; i<pIter->nSeg; i++){
      fts5SegIterClear(&pIter->aSeg[i]);
    }
    fts5StructureRelease(pIter->pStruct);
    fts5BufferFree(&pIter->poslist);
    sqlite3_free(pIter);
  }
}

static void fts5MultiIterAdvanced(
  Fts5Index *p,                   /* FTS5 backend to iterate within */
................................................................................
      nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
    }
  }
  *ppOut = pNew = fts5MultiIterAlloc(p, nSeg);
  if( pNew==0 ) return;
  pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC));
  pNew->bSkipEmpty = (0!=(flags & FTS5INDEX_QUERY_SKIPEMPTY));
  pNew->pStruct = pStruct;
  pNew->pColset = pColset;
  fts5StructureRef(pStruct);
  if( (flags & FTS5INDEX_QUERY_NOOUTPUT)==0 ){
    fts5IterSetOutputCb(&p->rc, pNew);
  }

  /* Initialize each of the component segment iterators. */







<







 







<







 







<







508
509
510
511
512
513
514

515
516
517
518
519
520
521
....
2753
2754
2755
2756
2757
2758
2759

2760
2761
2762
2763
2764
2765
2766
....
3398
3399
3400
3401
3402
3403
3404

3405
3406
3407
3408
3409
3410
3411
**   Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered.
**   There is no way to tell if this is populated or not.
*/
struct Fts5Iter {
  Fts5IndexIter base;             /* Base class containing output vars */

  Fts5Index *pIndex;              /* Index that owns this iterator */

  Fts5Buffer poslist;             /* Buffer containing current poslist */
  Fts5Colset *pColset;            /* Restrict matches to these columns */

  /* Invoked to set output variables. */
  void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*);

  int nSeg;                       /* Size of aSeg[] array */
................................................................................
*/
static void fts5MultiIterFree(Fts5Iter *pIter){
  if( pIter ){
    int i;
    for(i=0; i<pIter->nSeg; i++){
      fts5SegIterClear(&pIter->aSeg[i]);
    }

    fts5BufferFree(&pIter->poslist);
    sqlite3_free(pIter);
  }
}

static void fts5MultiIterAdvanced(
  Fts5Index *p,                   /* FTS5 backend to iterate within */
................................................................................
      nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
    }
  }
  *ppOut = pNew = fts5MultiIterAlloc(p, nSeg);
  if( pNew==0 ) return;
  pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC));
  pNew->bSkipEmpty = (0!=(flags & FTS5INDEX_QUERY_SKIPEMPTY));

  pNew->pColset = pColset;
  fts5StructureRef(pStruct);
  if( (flags & FTS5INDEX_QUERY_NOOUTPUT)==0 ){
    fts5IterSetOutputCb(&p->rc, pNew);
  }

  /* Initialize each of the component segment iterators. */

Changes to ext/fts5/test/fts5corrupt3.test.

3507
3508
3509
3510
3511
3512
3513
3514


































3515
3516
3517
|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
| end crash-eef41e30b388a0.db
}]} {}

do_catchsql_test 30.1 {
  SELECT fts5_decode(id, block) FROM t1_data;
} {1 {database disk image is malformed}}



































sqlite3_fts5_may_be_corrupt 0
finish_test









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
| end crash-eef41e30b388a0.db
}]} {}

do_catchsql_test 30.1 {
  SELECT fts5_decode(id, block) FROM t1_data;
} {1 {database disk image is malformed}}

#-------------------------------------------------------------------------
reset_db
do_test 31.0 {
  sqlite3 db {}
  db deserialize [decode_hexdb {
| size 8192 pagesize 4096 filename crash-7629f35f11d48e.db
| page 1 offset 0
|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 02   .....@  ........
|     32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04   ................
|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
|     96: 00 00 00 00 0d 00 00 00 01 0f c7 00 0f c7 00 00   ................
|   4032: 00 00 00 00 00 00 00 37 01 06 17 15 15 01 53 74   .......7......St
|   4048: 61 62 6c 65 64 75 61 6c 64 75 61 6c 02 43 52 45   abledualdual.CRE
|   4064: 41 54 45 20 54 41 42 4c 45 20 64 75 61 6c 28 64   ATE TABLE dual(d
|   4080: 75 6d 6d 79 20 76 61 72 28 31 29 29 0d 00 00 00   ummy var(1))....
| page 2 offset 4096
|      0: 01 0f fb 00 0f fb 00 00 00 00 00 00 00 00 00 00   ................
|   4080: 00 00 00 00 00 00 00 00 00 00 00 03 01 02 0f 58   ...............X
| end crash-7629f35f11d48e.db
}]} {}

do_execsql_test 31.1 {
  CREATE VIRTUAL TABLE t1 USING fts5(a,b,c);
  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<72)
    INSERT INTO t1(a) SELECT randomblob(2829) FROM c;
  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<10)
    INSERT INTO t1(a) SELECT randomblob(3000) FROM c;
}

do_catchsql_test 31.2 {
  DELETE FROM t1 WHERE a MATCH X'6620e574f32a';
} {0 {}}

sqlite3_fts5_may_be_corrupt 0
finish_test

Changes to ext/fts5/test/fts5update.test.

110
111
112
113
114
115
116



















117
118
119
  for {set i 0} {$i < 1000} {incr i} {
    execsql { UPDATE x2 SET x=x WHERE rowid=2 }
  }
} {}
do_execsql_test 2.2.integrity {
  INSERT INTO x2(x2) VALUES('integrity-check');
}




















}
finish_test







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
  for {set i 0} {$i < 1000} {incr i} {
    execsql { UPDATE x2 SET x=x WHERE rowid=2 }
  }
} {}
do_execsql_test 2.2.integrity {
  INSERT INTO x2(x2) VALUES('integrity-check');
}

#-------------------------------------------------------------------------
#
do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE x3 USING fts5(x, detail=%DETAIL%);
  INSERT INTO x3 VALUES('one');
  INSERT INTO x3 VALUES('two');
  INSERT INTO x3 VALUES('one');
  INSERT INTO x3 VALUES('two');
  INSERT INTO x3 VALUES('one');
}

do_test 3.1 {
  db eval { SELECT * FROM x3('one') } {
    db eval {
      INSERT INTO x3(x3) VALUES('optimize');
    }
  }
} {}

}
finish_test