/ Check-in [2ead43f0]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the "PRAGMA cell_size_check=ON" command.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | cell-size-check-pragma
Files: files | file ages | folders
SHA1: 2ead43f074d01312c7642e1df9abccc95547f019
User & Date: drh 2015-05-27 03:46:18
Context
2015-05-27
15:10
Disallow the use of "rowid" in CTEs - it has never worked correctly and it makes no sense, so we might as well make it an explicit error. Also: add the PRAGMA cell_size_check=ON command. check-in: 19e2cebc user: drh tags: trunk
03:46
Add the "PRAGMA cell_size_check=ON" command. Closed-Leaf check-in: 2ead43f0 user: drh tags: cell-size-check-pragma
2015-05-26
20:31
Avoid a buffer overread when comparing against a corrupt record that spans at least one overflow page. check-in: 62a5b363 user: dan tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   998    998   test:	$(TESTPROGS) fuzztest
   999    999   	./testfixture$(TEXE) $(TOP)/test/veryquick.test
  1000   1000   
  1001   1001   # Run a test using valgrind.  This can take a really long time
  1002   1002   # because valgrind is so much slower than a native machine.
  1003   1003   #
  1004   1004   valgrindtest:	$(TESTPROGS) fuzzcheck$(TEXE)
  1005         -	valgrind -v ./fuzzcheck$(TEXE) $(FUZZDATA)
         1005  +	valgrind -v ./fuzzcheck$(TEXE) --cell-size-check $(FUZZDATA)
  1006   1006   	OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind
  1007   1007   
  1008   1008   # A very fast test that checks basic sanity.  The name comes from
  1009   1009   # the 60s-era electronics testing:  "Turn it on and see if smoke
  1010   1010   # comes out."
  1011   1011   #
  1012   1012   smoketest:	$(TESTPROGS) fuzzcheck$(TEXE)

Changes to main.mk.

   679    679   test:	$(TESTPROGS) fuzztest
   680    680   	./testfixture$(EXE) $(TOP)/test/veryquick.test
   681    681   
   682    682   # Run a test using valgrind.  This can take a really long time
   683    683   # because valgrind is so much slower than a native machine.
   684    684   #
   685    685   valgrindtest:	$(TESTPROGS) fuzzcheck$(EXE) $(FUZZDATA)
   686         -	valgrind -v ./fuzzcheck$(EXE) $(FUZZDATA)
          686  +	valgrind -v ./fuzzcheck$(EXE) --cell-size-check $(FUZZDATA)
   687    687   	OMIT_MISUSE=1 valgrind -v ./testfixture$(EXE) $(TOP)/test/permutations.test valgrind
   688    688   
   689    689   # A very fast test that checks basic sanity.  The name comes from
   690    690   # the 60s-era electronics testing:  "Turn it on and see if smoke
   691    691   # comes out."
   692    692   #
   693    693   smoketest:	$(TESTPROGS) fuzzcheck$(EXE)

Changes to src/btree.c.

  1192   1192     iCellLast = usableSize - 4;
  1193   1193     for(i=0; i<nCell; i++){
  1194   1194       u8 *pAddr;     /* The i-th cell pointer */
  1195   1195       pAddr = &data[cellOffset + i*2];
  1196   1196       pc = get2byte(pAddr);
  1197   1197       testcase( pc==iCellFirst );
  1198   1198       testcase( pc==iCellLast );
  1199         -#if !defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK)
  1200   1199       /* These conditions have already been verified in btreeInitPage()
  1201         -    ** if SQLITE_ENABLE_OVERSIZE_CELL_CHECK is defined 
         1200  +    ** if PRAGMA cell_size_check=ON.
  1202   1201       */
  1203   1202       if( pc<iCellFirst || pc>iCellLast ){
  1204   1203         return SQLITE_CORRUPT_BKPT;
  1205   1204       }
  1206         -#endif
  1207   1205       assert( pc>=iCellFirst && pc<=iCellLast );
  1208   1206       size = cellSizePtr(pPage, &src[pc]);
  1209   1207       cbrk -= size;
  1210         -#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK)
  1211         -    if( cbrk<iCellFirst ){
  1212         -      return SQLITE_CORRUPT_BKPT;
  1213         -    }
  1214         -#else
  1215   1208       if( cbrk<iCellFirst || pc+size>usableSize ){
  1216   1209         return SQLITE_CORRUPT_BKPT;
  1217   1210       }
  1218         -#endif
  1219   1211       assert( cbrk+size<=usableSize && cbrk>=iCellFirst );
  1220   1212       testcase( cbrk+size==usableSize );
  1221   1213       testcase( pc+size==usableSize );
  1222   1214       put2byte(pAddr, cbrk);
  1223   1215       if( temp==0 ){
  1224   1216         int x;
  1225   1217         if( cbrk==pc ) continue;
................................................................................
  1552   1544   ** SQLITE_CORRUPT.  Note that a return of SQLITE_OK does not
  1553   1545   ** guarantee that the page is well-formed.  It only shows that
  1554   1546   ** we failed to detect any corruption.
  1555   1547   */
  1556   1548   static int btreeInitPage(MemPage *pPage){
  1557   1549   
  1558   1550     assert( pPage->pBt!=0 );
         1551  +  assert( pPage->pBt->db!=0 );
  1559   1552     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  1560   1553     assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) );
  1561   1554     assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) );
  1562   1555     assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) );
  1563   1556   
  1564   1557     if( !pPage->isInit ){
  1565   1558       u16 pc;            /* Address of a freeblock within pPage->aData[] */
................................................................................
  1610   1603       **
  1611   1604       ** The following block of code checks early to see if a cell extends
  1612   1605       ** past the end of a page boundary and causes SQLITE_CORRUPT to be 
  1613   1606       ** returned if it does.
  1614   1607       */
  1615   1608       iCellFirst = cellOffset + 2*pPage->nCell;
  1616   1609       iCellLast = usableSize - 4;
  1617         -#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK)
  1618         -    {
         1610  +    if( pBt->db->flags & SQLITE_CellSizeCk ){
  1619   1611         int i;            /* Index into the cell pointer array */
  1620   1612         int sz;           /* Size of a cell */
  1621   1613   
  1622   1614         if( !pPage->leaf ) iCellLast--;
  1623   1615         for(i=0; i<pPage->nCell; i++){
  1624   1616           pc = get2byte(&data[cellOffset+i*2]);
  1625   1617           testcase( pc==iCellFirst );
................................................................................
  1631   1623           testcase( pc+sz==usableSize );
  1632   1624           if( pc+sz>usableSize ){
  1633   1625             return SQLITE_CORRUPT_BKPT;
  1634   1626           }
  1635   1627         }
  1636   1628         if( !pPage->leaf ) iCellLast++;
  1637   1629       }  
  1638         -#endif
  1639   1630   
  1640   1631       /* Compute the total free space on the page
  1641   1632       ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the
  1642   1633       ** start of the first freeblock on the page, or is zero if there are no
  1643   1634       ** freeblocks. */
  1644   1635       pc = get2byte(&data[hdr+1]);
  1645   1636       nFree = data[hdr+7] + top;  /* Init nFree to non-freeblock free space */
................................................................................
  4947   4938             ** fits entirely on the main b-tree page.  */
  4948   4939             testcase( pCell+nCell+2==pPage->aDataEnd );
  4949   4940             c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey);
  4950   4941           }else{
  4951   4942             /* The record flows over onto one or more overflow pages. In
  4952   4943             ** this case the whole cell needs to be parsed, a buffer allocated
  4953   4944             ** and accessPayload() used to retrieve the record into the
  4954         -          ** buffer before VdbeRecordCompare() can be called. An extra
  4955         -          ** byte of zeroed padding is allocated at the end of the buffer,
  4956         -          ** as this stops the record-compare routines from reading past
  4957         -          ** the end of the buffer if the record is corrupt.  */
         4945  +          ** buffer before VdbeRecordCompare() can be called. */
  4958   4946             void *pCellKey;
  4959   4947             u8 * const pCellBody = pCell - pPage->childPtrSize;
  4960   4948             btreeParseCellPtr(pPage, pCellBody, &pCur->info);
  4961   4949             nCell = (int)pCur->info.nKey;
  4962         -          pCellKey = sqlite3Malloc( nCell+1 );
         4950  +          pCellKey = sqlite3Malloc( nCell );
  4963   4951             if( pCellKey==0 ){
  4964   4952               rc = SQLITE_NOMEM;
  4965   4953               goto moveto_finish;
  4966   4954             }
  4967   4955             pCur->aiIdx[pCur->iPage] = (u16)idx;
  4968   4956             rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 2);
  4969         -          ((unsigned char *)pCellKey)[nCell] = 0;
  4970   4957             if( rc ){
  4971   4958               sqlite3_free(pCellKey);
  4972   4959               goto moveto_finish;
  4973   4960             }
  4974   4961             c = xRecordCompare(nCell, pCellKey, pIdxKey);
  4975   4962             sqlite3_free(pCellKey);
  4976   4963           }

Changes to src/main.c.

  2754   2754                    | SQLITE_RecTriggers
  2755   2755   #endif
  2756   2756   #if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS
  2757   2757                    | SQLITE_ForeignKeys
  2758   2758   #endif
  2759   2759   #if defined(SQLITE_REVERSE_UNORDERED_SELECTS)
  2760   2760                    | SQLITE_ReverseOrder
         2761  +#endif
         2762  +#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK)
         2763  +                 | SQLITE_CellSizeCk
  2761   2764   #endif
  2762   2765         ;
  2763   2766     sqlite3HashInit(&db->aCollSeq);
  2764   2767   #ifndef SQLITE_OMIT_VIRTUALTABLE
  2765   2768     sqlite3HashInit(&db->aModule);
  2766   2769   #endif
  2767   2770   

Changes to src/pragma.h.

    95     95       /* ePragFlag: */ 0,
    96     96       /* iArg:      */ SQLITE_CacheSpill },
    97     97   #endif
    98     98     { /* zName:     */ "case_sensitive_like",
    99     99       /* ePragTyp:  */ PragTyp_CASE_SENSITIVE_LIKE,
   100    100       /* ePragFlag: */ 0,
   101    101       /* iArg:      */ 0 },
          102  +  { /* zName:     */ "cell_size_check",
          103  +    /* ePragTyp:  */ PragTyp_FLAG,
          104  +    /* ePragFlag: */ 0,
          105  +    /* iArg:      */ SQLITE_CellSizeCk },
   102    106   #if !defined(SQLITE_OMIT_FLAG_PRAGMAS)
   103    107     { /* zName:     */ "checkpoint_fullfsync",
   104    108       /* ePragTyp:  */ PragTyp_FLAG,
   105    109       /* ePragFlag: */ 0,
   106    110       /* iArg:      */ SQLITE_CkptFullFSync },
   107    111   #endif
   108    112   #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS)
................................................................................
   452    456   #if !defined(SQLITE_OMIT_FLAG_PRAGMAS)
   453    457     { /* zName:     */ "writable_schema",
   454    458       /* ePragTyp:  */ PragTyp_FLAG,
   455    459       /* ePragFlag: */ 0,
   456    460       /* iArg:      */ SQLITE_WriteSchema|SQLITE_RecoveryMode },
   457    461   #endif
   458    462   };
   459         -/* Number of pragmas: 59 on by default, 72 total. */
          463  +/* Number of pragmas: 60 on by default, 73 total. */

Changes to src/sqliteInt.h.

  1249   1249   #define SQLITE_PreferBuiltin  0x00200000  /* Preference to built-in funcs */
  1250   1250   #define SQLITE_LoadExtension  0x00400000  /* Enable load_extension */
  1251   1251   #define SQLITE_EnableTrigger  0x00800000  /* True to enable triggers */
  1252   1252   #define SQLITE_DeferFKs       0x01000000  /* Defer all FK constraints */
  1253   1253   #define SQLITE_QueryOnly      0x02000000  /* Disable database changes */
  1254   1254   #define SQLITE_VdbeEQP        0x04000000  /* Debug EXPLAIN QUERY PLAN */
  1255   1255   #define SQLITE_Vacuum         0x08000000  /* Currently in a VACUUM */
         1256  +#define SQLITE_CellSizeCk     0x10000000  /* Check btree cell sizes on load */
  1256   1257   
  1257   1258   
  1258   1259   /*
  1259   1260   ** Bits of the sqlite3.dbOptFlags field that are used by the
  1260   1261   ** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to
  1261   1262   ** selectively disable various optimizations.
  1262   1263   */

Changes to test/fuzzcheck.c.

   592    592   */
   593    593   static void showHelp(void){
   594    594     printf("Usage: %s [options] SOURCE-DB ?ARGS...?\n", g.zArgv0);
   595    595     printf(
   596    596   "Read databases and SQL scripts from SOURCE-DB and execute each script against\n"
   597    597   "each database, checking for crashes and memory leaks.\n"
   598    598   "Options:\n"
          599  +"  --cell-size-check     Set the PRAGMA cell_size_check=ON\n"
   599    600   "  --dbid N              Use only the database where dbid=N\n"
   600    601   "  --help                Show this help text\n"    
   601    602   "  -q                    Reduced output\n"
   602    603   "  --quiet               Reduced output\n"
   603    604   "  --load-sql ARGS...    Load SQL scripts fro files into SOURCE-DB\n"
   604    605   "  --load-db ARGS...     Load template databases from files into SOURCE_DB\n"
   605    606   "  -m TEXT               Add a description to the database\n"
................................................................................
   630    631     char *zMsg = 0;              /* Add this message */
   631    632     int nSrcDb = 0;              /* Number of source databases */
   632    633     char **azSrcDb = 0;          /* Array of source database names */
   633    634     int iSrcDb;                  /* Loop over all source databases */
   634    635     int nTest = 0;               /* Total number of tests performed */
   635    636     char *zDbName = "";          /* Appreviated name of a source database */
   636    637     const char *zFailCode = 0;   /* Value of the TEST_FAILURE environment variable */
          638  +  int cellSzCkFlag = 0;        /* --cell-size-check */
   637    639   
   638    640     iBegin = timeOfDay();
   639    641     g.zArgv0 = argv[0];
   640    642     zFailCode = getenv("TEST_FAILURE");
   641    643     for(i=1; i<argc; i++){
   642    644       const char *z = argv[i];
   643    645       if( z[0]=='-' ){
   644    646         z++;
   645    647         if( z[0]=='-' ) z++;
          648  +      if( strcmp(z,"cell-size-check")==0 ){
          649  +        cellSzCkFlag = 1;
          650  +      }else
   646    651         if( strcmp(z,"dbid")==0 ){
   647    652           if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
   648    653           onlyDbid = atoi(argv[++i]);
   649    654         }else
   650    655         if( strcmp(z,"help")==0 ){
   651    656           showHelp();
   652    657           return 0;
................................................................................
   821    826           openFlags = SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE;
   822    827           if( nativeFlag && pDb->sz==0 ){
   823    828             openFlags |= SQLITE_OPEN_MEMORY;
   824    829             zVfs = 0;
   825    830           }
   826    831           rc = sqlite3_open_v2("main.db", &db, openFlags, zVfs);
   827    832           if( rc ) fatalError("cannot open inmem database");
          833  +        if( cellSzCkFlag ) runSql(db, "PRAGMA cell_size_check=ON", runFlags);
   828    834           runSql(db, (char*)pSql->a, runFlags);
   829    835           sqlite3_close(db);
   830    836           if( sqlite3_memory_used()>0 ) fatalError("memory leak");
   831    837           reformatVfs();
   832    838           nTest++;
   833    839           g.zTestName[0] = 0;
   834    840   

Changes to tool/mkpragmatab.tcl.

   132    132   
   133    133     NAME: defer_foreign_keys
   134    134     TYPE: FLAG
   135    135     ARG:  SQLITE_DeferFKs
   136    136     IF:   !defined(SQLITE_OMIT_FLAG_PRAGMAS)
   137    137     IF:   !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER)
   138    138   
          139  +  NAME: cell_size_check
          140  +  TYPE: FLAG
          141  +  ARG:  SQLITE_CellSizeCk
          142  +
   139    143     NAME: default_cache_size
   140    144     FLAG: NeedSchema
   141    145     IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED)
   142    146   
   143    147     NAME: page_size
   144    148     IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS)
   145    149