/ Check-in [4bb21d82]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add extra defenses against strategically corrupt databases to fts3/4.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | branch-3.18
Files: files | file ages | folders
SHA3-256: 4bb21d8205b3c72b94442018a0544ecc55e3320ef2593f0e3350142b7f2a7663
User & Date: drh 2018-12-19 01:38:18
Context
2019-09-03
17:46
Disable the undocumented rtreenode() SQL function that is only used for testing, except when doing a build that is specifically intended for testing. check-in: 0a1cce49 user: drh tags: branch-3.18
2018-12-19
01:57
Add extra defenses against strategically corrupt databases to fts3/4. check-in: 882ef4e3 user: drh tags: branch-3.9
01:38
Add extra defenses against strategically corrupt databases to fts3/4. check-in: 4bb21d82 user: drh tags: branch-3.18
2018-11-03
16:51
Add extra defenses against strategically corrupt databases to fts3/4. check-in: d44318f5 user: dan tags: trunk
2017-07-21
07:56
Add new interfaces sqlite3_result_pointer(), and sqlite3_value_pointer() and use them to transfer the eponymous FTS3 column pointer to the snippet() and offsets() routines. This changes is a cherry-pick of [f0f49224] via [69906880cee] and is ultimately inspired by check-in [72de49f2]. check-in: 8201f4e1 user: drh tags: branch-3.18
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
....
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
  sqlite3_int64 *piFirst,         /* OUT: Selected child node */
  sqlite3_int64 *piLast           /* OUT: Selected child node */
){
  int rc = SQLITE_OK;             /* Return code */
  const char *zCsr = zNode;       /* Cursor to iterate through node */
  const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
  char *zBuffer = 0;              /* Buffer to load terms into */
  int nAlloc = 0;                 /* Size of allocated buffer */
  int isFirstTerm = 1;            /* True when processing first term on page */
  sqlite3_int64 iChild;           /* Block id of child node to descend to */

  /* Skip over the 'height' varint that occurs at the start of every 
  ** interior node. Then load the blockid of the left-child of the b-tree
  ** node into variable iChild.  
  **
................................................................................
    ** the size of zBuffer if required.  */
    if( !isFirstTerm ){
      zCsr += fts3GetVarint32(zCsr, &nPrefix);
    }
    isFirstTerm = 0;
    zCsr += fts3GetVarint32(zCsr, &nSuffix);
    
    if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){
      rc = FTS_CORRUPT_VTAB;
      goto finish_scan;
    }
    if( nPrefix+nSuffix>nAlloc ){
      char *zNew;
      nAlloc = (nPrefix+nSuffix) * 2;
      zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
      if( !zNew ){
        rc = SQLITE_NOMEM;
        goto finish_scan;
      }
      zBuffer = zNew;
    }
    assert( zBuffer );







|







 







|



|

|
|







1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
....
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
  sqlite3_int64 *piFirst,         /* OUT: Selected child node */
  sqlite3_int64 *piLast           /* OUT: Selected child node */
){
  int rc = SQLITE_OK;             /* Return code */
  const char *zCsr = zNode;       /* Cursor to iterate through node */
  const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
  char *zBuffer = 0;              /* Buffer to load terms into */
  i64 nAlloc = 0;                 /* Size of allocated buffer */
  int isFirstTerm = 1;            /* True when processing first term on page */
  sqlite3_int64 iChild;           /* Block id of child node to descend to */

  /* Skip over the 'height' varint that occurs at the start of every 
  ** interior node. Then load the blockid of the left-child of the b-tree
  ** node into variable iChild.  
  **
................................................................................
    ** the size of zBuffer if required.  */
    if( !isFirstTerm ){
      zCsr += fts3GetVarint32(zCsr, &nPrefix);
    }
    isFirstTerm = 0;
    zCsr += fts3GetVarint32(zCsr, &nSuffix);
    
    if( nPrefix<0 || nSuffix<0 || nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr ){
      rc = FTS_CORRUPT_VTAB;
      goto finish_scan;
    }
    if( (i64)nPrefix+nSuffix>nAlloc ){
      char *zNew;
      nAlloc = ((i64)nPrefix+nSuffix) * 2;
      zNew = (char *)sqlite3_realloc64(zBuffer, nAlloc);
      if( !zNew ){
        rc = SQLITE_NOMEM;
        goto finish_scan;
      }
      zBuffer = zNew;
    }
    assert( zBuffer );

Changes to ext/fts3/fts3_write.c.

1369
1370
1371
1372
1373
1374
1375
1376
1377

1378
1379
1380
1381



1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
....
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
....
3722
3723
3724
3725
3726
3727
3728



3729
3730
3731
3732
3733
3734
3735



3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
  rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
  if( rc!=SQLITE_OK ) return rc;
  
  /* Because of the FTS3_NODE_PADDING bytes of padding, the following is 
  ** safe (no risk of overread) even if the node data is corrupted. */
  pNext += fts3GetVarint32(pNext, &nPrefix);
  pNext += fts3GetVarint32(pNext, &nSuffix);
  if( nPrefix<0 || nSuffix<=0 
   || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 

  ){
    return FTS_CORRUPT_VTAB;
  }




  if( nPrefix+nSuffix>pReader->nTermAlloc ){
    int nNew = (nPrefix+nSuffix)*2;
    char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
    if( !zNew ){
      return SQLITE_NOMEM;
    }
    pReader->zTerm = zNew;
    pReader->nTermAlloc = nNew;
  }

................................................................................
  pReader->aDoclist = pNext;
  pReader->pOffsetList = 0;

  /* Check that the doclist does not appear to extend past the end of the
  ** b-tree node. And that the final byte of the doclist is 0x00. If either 
  ** of these statements is untrue, then the data structure is corrupt.
  */
  if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 
   || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
  ){
    return FTS_CORRUPT_VTAB;
  }
  return SQLITE_OK;
}

................................................................................
    p->aNode = 0;
  }else{
    if( bFirst==0 ){
      p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix);
    }
    p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);




    blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
    if( rc==SQLITE_OK ){
      memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
      p->term.n = nPrefix+nSuffix;
      p->iOff += nSuffix;
      if( p->iChild==0 ){
        p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);



        p->aDoclist = &p->aNode[p->iOff];
        p->iOff += p->nDoclist;
      }
    }
  }

  assert( p->iOff<=p->nNode );

  return rc;
}

/*
** Release all dynamic resources held by node-reader object *p.
*/
static void nodeReaderRelease(NodeReader *p){







|
|
>




>
>
>
|
|
|







 







|







 







>
>
>







>
>
>







<







1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
....
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
....
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752

3753
3754
3755
3756
3757
3758
3759
  rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
  if( rc!=SQLITE_OK ) return rc;
  
  /* Because of the FTS3_NODE_PADDING bytes of padding, the following is 
  ** safe (no risk of overread) even if the node data is corrupted. */
  pNext += fts3GetVarint32(pNext, &nPrefix);
  pNext += fts3GetVarint32(pNext, &nSuffix);
  if( nSuffix<=0 
   || (&pReader->aNode[pReader->nNode] - pNext)<nSuffix
   || nPrefix>pReader->nTermAlloc
  ){
    return FTS_CORRUPT_VTAB;
  }

  /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are
  ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer
  ** overflow - hence the (i64) casts.  */
  if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){
    i64 nNew = ((i64)nPrefix+nSuffix)*2;
    char *zNew = sqlite3_realloc64(pReader->zTerm, nNew);
    if( !zNew ){
      return SQLITE_NOMEM;
    }
    pReader->zTerm = zNew;
    pReader->nTermAlloc = nNew;
  }

................................................................................
  pReader->aDoclist = pNext;
  pReader->pOffsetList = 0;

  /* Check that the doclist does not appear to extend past the end of the
  ** b-tree node. And that the final byte of the doclist is 0x00. If either 
  ** of these statements is untrue, then the data structure is corrupt.
  */
  if( (&pReader->aNode[pReader->nNode] - pReader->aDoclist)<pReader->nDoclist
   || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
  ){
    return FTS_CORRUPT_VTAB;
  }
  return SQLITE_OK;
}

................................................................................
    p->aNode = 0;
  }else{
    if( bFirst==0 ){
      p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix);
    }
    p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);

    if( nPrefix>p->iOff || nSuffix>p->nNode-p->iOff ){
      return SQLITE_CORRUPT_VTAB;
    }
    blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
    if( rc==SQLITE_OK ){
      memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
      p->term.n = nPrefix+nSuffix;
      p->iOff += nSuffix;
      if( p->iChild==0 ){
        p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
        if( (p->nNode-p->iOff)<p->nDoclist ){
          return SQLITE_CORRUPT_VTAB;
        }
        p->aDoclist = &p->aNode[p->iOff];
        p->iOff += p->nDoclist;
      }
    }
  }

  assert( p->iOff<=p->nNode );

  return rc;
}

/*
** Release all dynamic resources held by node-reader object *p.
*/
static void nodeReaderRelease(NodeReader *p){

Added test/fts3corrupt4.test.







































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
# 2006 September 9
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the FTS3 module.
#
# $Id: fts3aa.test,v 1.1 2007/08/20 17:38:42 shess Exp $
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts3corrupt4

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}

do_execsql_test 1.0 {
  BEGIN;
    CREATE VIRTUAL TABLE ft USING fts3;
    INSERT INTO ft VALUES('aback');
    INSERT INTO ft VALUES('abaft');
    INSERT INTO ft VALUES('abandon');
  COMMIT;
}

proc blob {a} { binary decode hex $a }
db func blob blob

do_execsql_test 1.1 {
  SELECT quote(root) FROM ft_segdir;
} {X'0005616261636B03010200030266740302020003046E646F6E03030200'}

do_execsql_test 1.2 {
  UPDATE ft_segdir SET root = blob(
    '0005616261636B03010200 FFFFFFFF0702 66740302020003046E646F6E03030200'
  );
}

do_catchsql_test 1.3 {
  SELECT * FROM ft WHERE ft MATCH 'abandon';
} {1 {database disk image is malformed}}

#-------------------------------------------------------------------------
reset_db
do_execsql_test 2.0.0 {
  CREATE VIRTUAL TABLE ft USING fts3;
  INSERT INTO ft(ft) VALUES('nodesize=32');
}
do_test 2.0.1 {
  for {set i 0} {$i < 12} {incr i} {
    execsql {
      BEGIN;
        INSERT INTO ft VALUES('abc' || $i);
        INSERT INTO ft VALUES('abc' || $i || 'x' );
        INSERT INTO ft VALUES('abc' || $i || 'xx' );
      COMMIT
    }
  }
  execsql {
    SELECT count(*) FROM ft_segdir;
    SELECT count(*) FROM ft_segments;
  }
} {12 0}

do_execsql_test 2.1 {
  INSERT INTO ft(ft) VALUES('merge=1,4');
  SELECT count(*) FROM ft_segdir;
  SELECT count(*) FROM ft_segments;
} {12 3}

do_execsql_test 2.2 {
  SELECT quote(block) FROM ft_segments WHERE blockid=2
} {X'00056162633130031F0200'}

db func blob blob
do_execsql_test 2.3.1 {
  UPDATE ft_segments SET block = 
    blob('00056162633130031F0200 FFFFFFFF07FF55 66740302020003046E646F6E03030200')
    WHERE blockid=2;
} {}
do_catchsql_test 2.3.2 {
  INSERT INTO ft(ft) VALUES('merge=1,4');
} {1 {database disk image is malformed}}

do_execsql_test 2.4.1 {
  UPDATE ft_segments SET block = 
    blob('00056162633130031F0200 02FFFFFFFF07 66740302020003046E646F6E03030200')
    WHERE blockid=2;
} {}
do_catchsql_test 2.4.2 {
  INSERT INTO ft(ft) VALUES('merge=1,4');
} {1 {database disk image is malformed}}

do_execsql_test 2.5.1 {
  UPDATE ft_segments SET block = 
    blob('00056162633130031F0200 0202 6674 FFFFFF070302020003046E646F6E030200')
    WHERE blockid=2;
} {}
do_catchsql_test 2.5.2 {
  INSERT INTO ft(ft) VALUES('merge=1,4');
} {1 {database disk image is malformed}}

#-------------------------------------------------------------------------
reset_db
do_execsql_test 3.0.0 {
  CREATE VIRTUAL TABLE ft USING fts3;
  INSERT INTO ft(ft) VALUES('nodesize=32');
}
do_test 3.0.1 {
  execsql BEGIN
  for {set i 0} {$i < 20} {incr i} {
    execsql { INSERT INTO ft VALUES('abc' || $i) }
  }
  execsql {
    COMMIT;
    SELECT count(*) FROM ft_segdir;
    SELECT count(*) FROM ft_segments;
  }
} {1 5}

do_execsql_test 3.1 {
  SELECT quote(root) FROM ft_segdir
} {X'0101056162633132040136030132030136'}

db func blob blob
do_execsql_test 3.2 {
  UPDATE ft_segdir 
  SET root = blob('0101056162633132FFFFFFFF070236030132030136');
}

do_catchsql_test 3.1 {
  SELECT * FROM ft WHERE ft MATCH 'abc20'
} {1 {database disk image is malformed}}

finish_test


Changes to test/permutations.test.

249
250
251
252
253
254
255

256
257
258
259
260
261
262
} -files {
  fts3aa.test fts3ab.test fts3ac.test fts3ad.test
  fts3ae.test fts3af.test fts3ag.test fts3ah.test
  fts3ai.test fts3aj.test fts3ak.test fts3al.test
  fts3am.test fts3an.test fts3ao.test fts3atoken.test
  fts3auto.test fts3aux1.test fts3aux2.test fts3b.test
  fts3comp1.test fts3conf.test fts3corrupt2.test fts3corrupt.test

  fts3cov.test fts3c.test fts3defer2.test fts3defer3.test
  fts3defer.test fts3drop.test fts3d.test fts3e.test
  fts3expr2.test fts3expr3.test fts3expr4.test fts3expr5.test
  fts3expr.test fts3fault2.test fts3fault.test fts3first.test
  fts3join.test fts3malloc.test fts3matchinfo.test fts3near.test
  fts3offsets.test fts3prefix2.test fts3prefix.test fts3query.test
  fts3shared.test fts3snippet.test fts3sort.test fts3tok1.test







>







249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
} -files {
  fts3aa.test fts3ab.test fts3ac.test fts3ad.test
  fts3ae.test fts3af.test fts3ag.test fts3ah.test
  fts3ai.test fts3aj.test fts3ak.test fts3al.test
  fts3am.test fts3an.test fts3ao.test fts3atoken.test
  fts3auto.test fts3aux1.test fts3aux2.test fts3b.test
  fts3comp1.test fts3conf.test fts3corrupt2.test fts3corrupt.test
  fts3corrupt4.test
  fts3cov.test fts3c.test fts3defer2.test fts3defer3.test
  fts3defer.test fts3drop.test fts3d.test fts3e.test
  fts3expr2.test fts3expr3.test fts3expr4.test fts3expr5.test
  fts3expr.test fts3fault2.test fts3fault.test fts3first.test
  fts3join.test fts3malloc.test fts3matchinfo.test fts3near.test
  fts3offsets.test fts3prefix2.test fts3prefix.test fts3query.test
  fts3shared.test fts3snippet.test fts3sort.test fts3tok1.test