/ Check-in [4f315e4a]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add extra defenses against strategically corrupt databases to fts3/4.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | branch-3.19
Files: files | file ages | folders
SHA3-256: 4f315e4a3d15b9196924db10dee9c1444964fc4590a153950bab9b1ec3ca4f0e
User & Date: drh 2018-12-19 01:44:50
Context
2018-12-19
14:44
Fix a crash that can follow an OOM in fts3 on this branch. check-in: de078148 user: dan tags: branch-3.19
01:44
Add extra defenses against strategically corrupt databases to fts3/4. check-in: 4f315e4a user: drh tags: branch-3.19
2018-11-03
16:51
Add extra defenses against strategically corrupt databases to fts3/4. check-in: d44318f5 user: dan tags: trunk
2018-03-16
07:49
Fix a problem in test script thread001.test causing a spurious "-1 files were left open" error when run separately. Cherrypick of [1774f1c3b]. check-in: 6cf8172d user: dan tags: branch-3.19
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

  1808   1808     sqlite3_int64 *piFirst,         /* OUT: Selected child node */
  1809   1809     sqlite3_int64 *piLast           /* OUT: Selected child node */
  1810   1810   ){
  1811   1811     int rc = SQLITE_OK;             /* Return code */
  1812   1812     const char *zCsr = zNode;       /* Cursor to iterate through node */
  1813   1813     const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
  1814   1814     char *zBuffer = 0;              /* Buffer to load terms into */
  1815         -  int nAlloc = 0;                 /* Size of allocated buffer */
         1815  +  i64 nAlloc = 0;                 /* Size of allocated buffer */
  1816   1816     int isFirstTerm = 1;            /* True when processing first term on page */
  1817   1817     sqlite3_int64 iChild;           /* Block id of child node to descend to */
  1818   1818   
  1819   1819     /* Skip over the 'height' varint that occurs at the start of every 
  1820   1820     ** interior node. Then load the blockid of the left-child of the b-tree
  1821   1821     ** node into variable iChild.  
  1822   1822     **
................................................................................
  1846   1846       if( !isFirstTerm ){
  1847   1847         zCsr += fts3GetVarint32(zCsr, &nPrefix);
  1848   1848       }
  1849   1849       isFirstTerm = 0;
  1850   1850       zCsr += fts3GetVarint32(zCsr, &nSuffix);
  1851   1851       
  1852   1852       assert( nPrefix>=0 && nSuffix>=0 );
  1853         -    if( &zCsr[nSuffix]>zEnd ){
         1853  +    if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr ){
  1854   1854         rc = FTS_CORRUPT_VTAB;
  1855   1855         goto finish_scan;
  1856   1856       }
  1857         -    if( nPrefix+nSuffix>nAlloc ){
         1857  +    if( (i64)nPrefix+nSuffix>nAlloc ){
  1858   1858         char *zNew;
  1859         -      nAlloc = (nPrefix+nSuffix) * 2;
  1860         -      zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
         1859  +      nAlloc = ((i64)nPrefix+nSuffix) * 2;
         1860  +      zNew = (char *)sqlite3_realloc64(zBuffer, nAlloc);
  1861   1861         if( !zNew ){
  1862   1862           rc = SQLITE_NOMEM;
  1863   1863           goto finish_scan;
  1864   1864         }
  1865   1865         zBuffer = zNew;
  1866   1866       }
  1867   1867       assert( zBuffer );

Changes to ext/fts3/fts3_write.c.

  1369   1369     rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
  1370   1370     if( rc!=SQLITE_OK ) return rc;
  1371   1371     
  1372   1372     /* Because of the FTS3_NODE_PADDING bytes of padding, the following is 
  1373   1373     ** safe (no risk of overread) even if the node data is corrupted. */
  1374   1374     pNext += fts3GetVarint32(pNext, &nPrefix);
  1375   1375     pNext += fts3GetVarint32(pNext, &nSuffix);
  1376         -  if( nPrefix<0 || nSuffix<=0 
  1377         -   || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 
         1376  +  if( nSuffix<=0 
         1377  +   || (&pReader->aNode[pReader->nNode] - pNext)<nSuffix
         1378  +   || nPrefix>pReader->nTermAlloc
  1378   1379     ){
  1379   1380       return FTS_CORRUPT_VTAB;
  1380   1381     }
  1381   1382   
  1382         -  if( nPrefix+nSuffix>pReader->nTermAlloc ){
  1383         -    int nNew = (nPrefix+nSuffix)*2;
  1384         -    char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
         1383  +  /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are
         1384  +  ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer
         1385  +  ** overflow - hence the (i64) casts.  */
         1386  +  if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){
         1387  +    i64 nNew = ((i64)nPrefix+nSuffix)*2;
         1388  +    char *zNew = sqlite3_realloc64(pReader->zTerm, nNew);
  1385   1389       if( !zNew ){
  1386   1390         return SQLITE_NOMEM;
  1387   1391       }
  1388   1392       pReader->zTerm = zNew;
  1389   1393       pReader->nTermAlloc = nNew;
  1390   1394     }
  1391   1395   
................................................................................
  1399   1403     pReader->aDoclist = pNext;
  1400   1404     pReader->pOffsetList = 0;
  1401   1405   
  1402   1406     /* Check that the doclist does not appear to extend past the end of the
  1403   1407     ** b-tree node. And that the final byte of the doclist is 0x00. If either 
  1404   1408     ** of these statements is untrue, then the data structure is corrupt.
  1405   1409     */
  1406         -  if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 
         1410  +  if( (&pReader->aNode[pReader->nNode] - pReader->aDoclist)<pReader->nDoclist
  1407   1411      || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
  1408   1412     ){
  1409   1413       return FTS_CORRUPT_VTAB;
  1410   1414     }
  1411   1415     return SQLITE_OK;
  1412   1416   }
  1413   1417   
................................................................................
  3722   3726       p->aNode = 0;
  3723   3727     }else{
  3724   3728       if( bFirst==0 ){
  3725   3729         p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix);
  3726   3730       }
  3727   3731       p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);
  3728   3732   
         3733  +    if( nPrefix>p->iOff || nSuffix>p->nNode-p->iOff ){
         3734  +      return SQLITE_CORRUPT_VTAB;
         3735  +    }
  3729   3736       blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
  3730   3737       if( rc==SQLITE_OK ){
  3731   3738         memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
  3732   3739         p->term.n = nPrefix+nSuffix;
  3733   3740         p->iOff += nSuffix;
  3734   3741         if( p->iChild==0 ){
  3735   3742           p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
         3743  +        if( (p->nNode-p->iOff)<p->nDoclist ){
         3744  +          return SQLITE_CORRUPT_VTAB;
         3745  +        }
  3736   3746           p->aDoclist = &p->aNode[p->iOff];
  3737   3747           p->iOff += p->nDoclist;
  3738   3748         }
  3739   3749       }
  3740   3750     }
  3741   3751   
  3742   3752     assert( p->iOff<=p->nNode );
  3743         -
  3744   3753     return rc;
  3745   3754   }
  3746   3755   
  3747   3756   /*
  3748   3757   ** Release all dynamic resources held by node-reader object *p.
  3749   3758   */
  3750   3759   static void nodeReaderRelease(NodeReader *p){

Added test/fts3corrupt4.test.

            1  +# 2006 September 9
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS3 module.
           13  +#
           14  +# $Id: fts3aa.test,v 1.1 2007/08/20 17:38:42 shess Exp $
           15  +#
           16  +
           17  +set testdir [file dirname $argv0]
           18  +source $testdir/tester.tcl
           19  +set testprefix fts3corrupt4
           20  +
           21  +# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           22  +ifcapable !fts3 {
           23  +  finish_test
           24  +  return
           25  +}
           26  +
           27  +do_execsql_test 1.0 {
           28  +  BEGIN;
           29  +    CREATE VIRTUAL TABLE ft USING fts3;
           30  +    INSERT INTO ft VALUES('aback');
           31  +    INSERT INTO ft VALUES('abaft');
           32  +    INSERT INTO ft VALUES('abandon');
           33  +  COMMIT;
           34  +}
           35  +
           36  +proc blob {a} { binary decode hex $a }
           37  +db func blob blob
           38  +
           39  +do_execsql_test 1.1 {
           40  +  SELECT quote(root) FROM ft_segdir;
           41  +} {X'0005616261636B03010200030266740302020003046E646F6E03030200'}
           42  +
           43  +do_execsql_test 1.2 {
           44  +  UPDATE ft_segdir SET root = blob(
           45  +    '0005616261636B03010200 FFFFFFFF0702 66740302020003046E646F6E03030200'
           46  +  );
           47  +}
           48  +
           49  +do_catchsql_test 1.3 {
           50  +  SELECT * FROM ft WHERE ft MATCH 'abandon';
           51  +} {1 {database disk image is malformed}}
           52  +
           53  +#-------------------------------------------------------------------------
           54  +reset_db
           55  +do_execsql_test 2.0.0 {
           56  +  CREATE VIRTUAL TABLE ft USING fts3;
           57  +  INSERT INTO ft(ft) VALUES('nodesize=32');
           58  +}
           59  +do_test 2.0.1 {
           60  +  for {set i 0} {$i < 12} {incr i} {
           61  +    execsql {
           62  +      BEGIN;
           63  +        INSERT INTO ft VALUES('abc' || $i);
           64  +        INSERT INTO ft VALUES('abc' || $i || 'x' );
           65  +        INSERT INTO ft VALUES('abc' || $i || 'xx' );
           66  +      COMMIT
           67  +    }
           68  +  }
           69  +  execsql {
           70  +    SELECT count(*) FROM ft_segdir;
           71  +    SELECT count(*) FROM ft_segments;
           72  +  }
           73  +} {12 0}
           74  +
           75  +do_execsql_test 2.1 {
           76  +  INSERT INTO ft(ft) VALUES('merge=1,4');
           77  +  SELECT count(*) FROM ft_segdir;
           78  +  SELECT count(*) FROM ft_segments;
           79  +} {12 3}
           80  +
           81  +do_execsql_test 2.2 {
           82  +  SELECT quote(block) FROM ft_segments WHERE blockid=2
           83  +} {X'00056162633130031F0200'}
           84  +
           85  +db func blob blob
           86  +do_execsql_test 2.3.1 {
           87  +  UPDATE ft_segments SET block = 
           88  +    blob('00056162633130031F0200 FFFFFFFF07FF55 66740302020003046E646F6E03030200')
           89  +    WHERE blockid=2;
           90  +} {}
           91  +do_catchsql_test 2.3.2 {
           92  +  INSERT INTO ft(ft) VALUES('merge=1,4');
           93  +} {1 {database disk image is malformed}}
           94  +
           95  +do_execsql_test 2.4.1 {
           96  +  UPDATE ft_segments SET block = 
           97  +    blob('00056162633130031F0200 02FFFFFFFF07 66740302020003046E646F6E03030200')
           98  +    WHERE blockid=2;
           99  +} {}
          100  +do_catchsql_test 2.4.2 {
          101  +  INSERT INTO ft(ft) VALUES('merge=1,4');
          102  +} {1 {database disk image is malformed}}
          103  +
          104  +do_execsql_test 2.5.1 {
          105  +  UPDATE ft_segments SET block = 
          106  +    blob('00056162633130031F0200 0202 6674 FFFFFF070302020003046E646F6E030200')
          107  +    WHERE blockid=2;
          108  +} {}
          109  +do_catchsql_test 2.5.2 {
          110  +  INSERT INTO ft(ft) VALUES('merge=1,4');
          111  +} {1 {database disk image is malformed}}
          112  +
          113  +#-------------------------------------------------------------------------
          114  +reset_db
          115  +do_execsql_test 3.0.0 {
          116  +  CREATE VIRTUAL TABLE ft USING fts3;
          117  +  INSERT INTO ft(ft) VALUES('nodesize=32');
          118  +}
          119  +do_test 3.0.1 {
          120  +  execsql BEGIN
          121  +  for {set i 0} {$i < 20} {incr i} {
          122  +    execsql { INSERT INTO ft VALUES('abc' || $i) }
          123  +  }
          124  +  execsql {
          125  +    COMMIT;
          126  +    SELECT count(*) FROM ft_segdir;
          127  +    SELECT count(*) FROM ft_segments;
          128  +  }
          129  +} {1 5}
          130  +
          131  +do_execsql_test 3.1 {
          132  +  SELECT quote(root) FROM ft_segdir
          133  +} {X'0101056162633132040136030132030136'}
          134  +
          135  +db func blob blob
          136  +do_execsql_test 3.2 {
          137  +  UPDATE ft_segdir 
          138  +  SET root = blob('0101056162633132FFFFFFFF070236030132030136');
          139  +}
          140  +
          141  +do_catchsql_test 3.1 {
          142  +  SELECT * FROM ft WHERE ft MATCH 'abc20'
          143  +} {1 {database disk image is malformed}}
          144  +
          145  +finish_test
          146  +
          147  +

Changes to test/permutations.test.

   249    249   } -files {
   250    250     fts3aa.test fts3ab.test fts3ac.test fts3ad.test
   251    251     fts3ae.test fts3af.test fts3ag.test fts3ah.test
   252    252     fts3ai.test fts3aj.test fts3ak.test fts3al.test
   253    253     fts3am.test fts3an.test fts3ao.test fts3atoken.test
   254    254     fts3auto.test fts3aux1.test fts3aux2.test fts3b.test
   255    255     fts3comp1.test fts3conf.test fts3corrupt2.test fts3corrupt.test
          256  +  fts3corrupt4.test
   256    257     fts3cov.test fts3c.test fts3defer2.test fts3defer3.test
   257    258     fts3defer.test fts3drop.test fts3d.test fts3e.test
   258    259     fts3expr2.test fts3expr3.test fts3expr4.test fts3expr5.test
   259    260     fts3expr.test fts3fault2.test fts3fault.test fts3first.test
   260    261     fts3join.test fts3malloc.test fts3matchinfo.test fts3near.test
   261    262     fts3offsets.test fts3prefix2.test fts3prefix.test fts3query.test
   262    263     fts3shared.test fts3snippet.test fts3sort.test fts3tok1.test