/ Check-in [2ab14a84]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the SQLITE_DIRECT_OVERFLOW_READ compile time option.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 2ab14a846727857175eac7961b7338c4d3b417ca
User & Date: dan 2011-10-08 14:57:07
References
2014-10-01
11:54 New ticket [e3a29096] out-of-bounds write when using non-default malloc implementation and SQLITE_DIRECT_OVERFLOW_READ. artifact: 72fdf842 user: dan
2011-12-21
16:25 New ticket [ac0ff496] Database corruption due to integer overflow with SQLITE_DIRECT_OVERFLOW_READ. artifact: 09f179ff user: drh
Context
2011-10-08
21:39
All the soft_heap_limit to be exceeded by 10% in test cases. check-in: 4be9dccc user: drh tags: trunk
14:57
Add the SQLITE_DIRECT_OVERFLOW_READ compile time option. check-in: 2ab14a84 user: dan tags: trunk
2011-10-07
23:52
Change the ANALYZE command so that it no longer tried to delete unused sqlite_stat2 and sqlite_stat3 tables. Change the DROP TABLE command so that it is able to drop those tables. check-in: 589f3f56 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/btree.c.

  3934   3934   #endif
  3935   3935             rc = getOverflowPage(pBt, nextPage, 0, &nextPage);
  3936   3936           offset -= ovflSize;
  3937   3937         }else{
  3938   3938           /* Need to read this page properly. It contains some of the
  3939   3939           ** range of data that is being read (eOp==0) or written (eOp!=0).
  3940   3940           */
  3941         -        DbPage *pDbPage;
         3941  +#ifdef SQLITE_DIRECT_OVERFLOW_READ
         3942  +        sqlite3_file *fd;
         3943  +#endif
  3942   3944           int a = amt;
  3943         -        rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage);
  3944         -        if( rc==SQLITE_OK ){
  3945         -          aPayload = sqlite3PagerGetData(pDbPage);
  3946         -          nextPage = get4byte(aPayload);
  3947         -          if( a + offset > ovflSize ){
  3948         -            a = ovflSize - offset;
  3949         -          }
  3950         -          rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage);
  3951         -          sqlite3PagerUnref(pDbPage);
  3952         -          offset = 0;
  3953         -          amt -= a;
  3954         -          pBuf += a;
  3955         -        }
         3945  +        if( a + offset > ovflSize ){
         3946  +          a = ovflSize - offset;
         3947  +        }
         3948  +
         3949  +#ifdef SQLITE_DIRECT_OVERFLOW_READ
         3950  +        /* If all the following are true:
         3951  +        **
         3952  +        **   1) this is a read operation, and 
         3953  +        **   2) data is required from the start of this overflow page, and
         3954  +        **   3) the database is file-backed, and
         3955  +        **   4) there is no open write-transaction, and
         3956  +        **   5) the database is not a WAL database,
         3957  +        **
         3958  +        ** then data can be read directly from the database file into the
         3959  +        ** output buffer, bypassing the page-cache altogether. This speeds
         3960  +        ** up loading large records that span many overflow pages.
         3961  +        */
         3962  +        if( eOp==0                                             /* (1) */
         3963  +         && offset==0                                          /* (2) */
         3964  +         && pBt->inTransaction==TRANS_READ                     /* (4) */
         3965  +         && (fd = sqlite3PagerFile(pBt->pPager))->pMethods     /* (3) */
         3966  +         && pBt->pPage1->aData[19]==0x01                       /* (5) */
         3967  +        ){
         3968  +          u8 aSave[4];
         3969  +          u8 *aWrite = &pBuf[-4];
         3970  +          memcpy(aSave, aWrite, 4);
         3971  +          rc = sqlite3OsRead(fd, aWrite, a+4, pBt->pageSize * (nextPage-1));
         3972  +          nextPage = get4byte(aWrite);
         3973  +          memcpy(aWrite, aSave, 4);
         3974  +        }else
         3975  +#endif
         3976  +
         3977  +        {
         3978  +          DbPage *pDbPage;
         3979  +          rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage);
         3980  +          if( rc==SQLITE_OK ){
         3981  +            aPayload = sqlite3PagerGetData(pDbPage);
         3982  +            nextPage = get4byte(aPayload);
         3983  +            rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage);
         3984  +            sqlite3PagerUnref(pDbPage);
         3985  +            offset = 0;
         3986  +          }
         3987  +        }
         3988  +        amt -= a;
         3989  +        pBuf += a;
  3956   3990         }
  3957   3991       }
  3958   3992     }
  3959   3993   
  3960   3994     if( rc==SQLITE_OK && amt>0 ){
  3961   3995       return SQLITE_CORRUPT_BKPT;
  3962   3996     }

Changes to src/test_config.c.

    50     50   #endif
    51     51   
    52     52   #ifdef SQLITE_DEBUG
    53     53     Tcl_SetVar2(interp, "sqlite_options", "debug", "1", TCL_GLOBAL_ONLY);
    54     54   #else
    55     55     Tcl_SetVar2(interp, "sqlite_options", "debug", "0", TCL_GLOBAL_ONLY);
    56     56   #endif
           57  +
           58  +#ifdef SQLITE_DIRECT_OVERFLOW_READ
           59  +  Tcl_SetVar2(interp, "sqlite_options", "direct_read", "1", TCL_GLOBAL_ONLY);
           60  +#else
           61  +  Tcl_SetVar2(interp, "sqlite_options", "direct_read", "0", TCL_GLOBAL_ONLY);
           62  +#endif
    57     63   
    58     64   #ifdef SQLITE_DISABLE_DIRSYNC
    59     65     Tcl_SetVar2(interp, "sqlite_options", "dirsync", "0", TCL_GLOBAL_ONLY);
    60     66   #else
    61     67     Tcl_SetVar2(interp, "sqlite_options", "dirsync", "1", TCL_GLOBAL_ONLY);
    62     68   #endif
    63     69   

Changes to test/corrupt3.test.

    21     21   # Do not use a codec for tests in this file, as the database file is
    22     22   # manipulated directly using tcl scripts (using the [hexio_write] command).
    23     23   #
    24     24   do_not_use_codec
    25     25   
    26     26   # We must have the page_size pragma for these tests to work.
    27     27   #
    28         -ifcapable !pager_pragmas {
           28  +ifcapable !pager_pragmas||direct_read {
    29     29     finish_test
    30     30     return
    31     31   }
    32     32   
    33     33   # Create a database with an overflow page.
    34     34   #
    35     35   do_test corrupt3-1.1 {

Changes to test/pager1.test.

  1762   1762   # Test the pagers response to the b-tree layer requesting illegal page 
  1763   1763   # numbers:
  1764   1764   #
  1765   1765   #   + The locking page,
  1766   1766   #   + Page 0,
  1767   1767   #   + A page with a page number greater than (2^31-1).
  1768   1768   #
         1769  +# These tests will not work if SQLITE_DIRECT_OVERFLOW_READ is defined. In
         1770  +# that case IO errors are sometimes reported instead of SQLITE_CORRUPT.
         1771  +#
         1772  +ifcapable !direct_read {
  1769   1773   do_test pager1-18.1 {
  1770   1774     faultsim_delete_and_reopen
  1771   1775     db func a_string a_string
  1772   1776     execsql { 
  1773   1777       PRAGMA page_size = 1024;
  1774   1778       CREATE TABLE t1(a, b);
  1775   1779       INSERT INTO t1 VALUES(a_string(500), a_string(200));
................................................................................
  1837   1841     set root [db one "SELECT rootpage FROM sqlite_master"]
  1838   1842     db close
  1839   1843   
  1840   1844     hexio_write test.db [expr ($root-1)*1024 + 8] 00000000
  1841   1845     sqlite3 db test.db
  1842   1846     catchsql { SELECT length(x) FROM t1 }
  1843   1847   } {1 {database disk image is malformed}}
         1848  +}
  1844   1849   
  1845   1850   do_test pager1-19.1 {
  1846   1851     sqlite3 db ""
  1847   1852     db func a_string a_string
  1848   1853     execsql {
  1849   1854       PRAGMA page_size = 512;
  1850   1855       PRAGMA auto_vacuum = 1;

Changes to test/pageropt.test.

    13     13   # pager optimizations implemented in version 3.3.14 work.
    14     14   #
    15     15   # $Id: pageropt.test,v 1.5 2008/08/20 14:49:25 danielk1977 Exp $
    16     16   
    17     17   set testdir [file dirname $argv0]
    18     18   source $testdir/tester.tcl
    19     19   
    20         -ifcapable {!pager_pragmas||secure_delete} {
           20  +ifcapable {!pager_pragmas||secure_delete||direct_read} {
    21     21     finish_test
    22     22     return
    23     23   }
    24     24   
    25     25   # Run the SQL statement supplied by the argument and return
    26     26   # the results.  Prepend four integers to the beginning of the
    27     27   # result which are