SQLite4
Check-in [b942b91a3d]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add some human readable text to the bt file header. Refuse to open a database (SQLITE_NOTADB) if a valid header cannot be located in the database or wal files.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:b942b91a3deb3e19b17cbc9145bb575e8201bcc6
User & Date: dan 2014-02-15 17:04:56
Context
2014-02-15
20:29
Defer opening a bt database until it is first read. check-in: d9560b73de user: dan tags: trunk
17:04
Add some human readable text to the bt file header. Refuse to open a database (SQLITE_NOTADB) if a valid header cannot be located in the database or wal files. check-in: b942b91a3d user: dan tags: trunk
2014-02-14
18:53
Fix memory leaks in test suite. check-in: f4d0f55571 user: dan tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/btInt.h.

    48     48   /* By default pages are 1024 bytes in size. */
    49     49   #define BT_DEFAULT_PGSZ 1024
    50     50   
    51     51   /* By default blocks are 512K bytes in size. */
    52     52   #define BT_DEFAULT_BLKSZ (512*1024)
    53     53   
    54     54   /*
           55  +** This structure is the in-memory representation of all data stored in
           56  +** the database header at the start of the db file.
    55     57   **
    56     58   ** pgsz, blksz:
    57     59   **   Byte offset 0 of the database file is the first byte of both page 1
    58     60   **   and block 1. Each page is pHdr->pgsz bytes in size. Each block is
    59     61   **   pHdr->blksz bytes in size. It is guaranteed that the block-size is
    60     62   **   an integer multiple of the page-size.
    61     63   **
................................................................................
    69     71   **   sub-tree, not including any overflow pages. Pages (except overflow 
    70     72   **   pages) are always allocated contiguously within the block.
    71     73   **
    72     74   **   The reason these are likely a stop-gap is that the write-magnification
    73     75   **   caused by using a b-tree for to populate level-0 sub-trees is too 
    74     76   **   expensive.
    75     77   */
           78  +#define BT_DBHDR_STRING "SQLite4 bt database 0001"
    76     79   struct BtDbHdr {
           80  +  char azStr[24];                 /* Copy of BT_DBHDR_STRING */
    77     81     u32 pgsz;                       /* Page size in bytes */
    78     82     u32 blksz;                      /* Block size in bytes */
    79     83     u32 nPg;                        /* Size of database file in pages */
    80     84   
    81     85     u32 iRoot;                      /* B-tree root page */
    82     86     u32 iMRoot;                     /* Root page of meta-tree */
    83     87     u32 iSRoot;                     /* Root page of schedule-tree */

Changes to src/bt_log.c.

   872    872       }
   873    873     }
   874    874   
   875    875     aLog[5] = iLast;
   876    876     return btLogHashRollback(pLog, btLogFrameHash(pLog, iLast), iLast);
   877    877   }
   878    878   
   879         -static void btLogDecodeDbhdr(BtLog *pLog, u8 *aData, BtDbHdr *pHdr){
          879  +static int btLogDecodeDbhdr(BtLog *pLog, u8 *aData, BtDbHdr *pHdr){
   880    880     BtDbHdrCksum hdr;
   881    881     u32 aCksum[2] = {0,0};
   882    882   
   883    883     if( aData ){
   884    884       memcpy(&hdr, aData, sizeof(BtDbHdrCksum));
   885    885       btLogChecksum32(1, (u8*)&hdr, offsetof(BtDbHdrCksum, aCksum), 0, aCksum);
   886    886     }
   887    887   
   888    888     if( aData==0 || aCksum[0]!=hdr.aCksum[0] || aCksum[1]!=hdr.aCksum[1] ){
   889         -    memset(&hdr, 0, sizeof(BtDbHdrCksum));
   890         -    hdr.hdr.pgsz = pLog->pLock->nPgsz;
   891         -    hdr.hdr.blksz = pLog->pLock->nBlksz;
   892         -    hdr.hdr.nPg = 2;
   893         -    hdr.hdr.iRoot = 2;
          889  +    return SQLITE4_NOTFOUND;
   894    890     }
   895    891   
   896    892     memcpy(pHdr, &hdr, sizeof(BtDbHdr));
          893  +  return SQLITE4_OK;
          894  +}
          895  +
          896  +static void btLogZeroDbhdr(BtLog *pLog, BtDbHdr *pHdr){
          897  +  assert( sizeof(pHdr->azStr)==strlen(BT_DBHDR_STRING) );
          898  +
          899  +  memset(pHdr, 0, sizeof(BtDbHdr));
          900  +  memcpy(pHdr->azStr, BT_DBHDR_STRING, strlen(BT_DBHDR_STRING));
          901  +  pHdr->pgsz = pLog->pLock->nPgsz;
          902  +  pHdr->blksz = pLog->pLock->nBlksz;
          903  +  pHdr->nPg = 2;
          904  +  pHdr->iRoot = 2;
   897    905   }
   898    906   
   899    907   static int btLogReadDbhdr(BtLog *pLog, BtDbHdr *pHdr, u32 iFrame){
   900    908     BtLock *p = pLog->pLock;        /* BtLock handle */
   901    909     int rc;                         /* Return code */
   902    910     i64 nByte;                      /* Size of database file in byte */
   903         -  u8 *aBuffer[sizeof(BtDbHdrCksum)];
          911  +  u8 aBuffer[sizeof(BtDbHdrCksum)];
   904    912     u8 *aData = 0;
   905    913   
   906    914     if( iFrame==0 ){
   907    915       rc = p->pVfs->xSize(p->pFd, &nByte);
   908    916       if( rc==SQLITE4_OK && nByte>0 ){
   909    917         rc = p->pVfs->xRead(p->pFd, 0, aBuffer, sizeof(BtDbHdrCksum));
   910    918         aData = aBuffer;
................................................................................
   913    921       i64 iOff = btLogFrameOffset(pLog, pLog->snapshot.dbhdr.pgsz, iFrame);
   914    922       iOff += sizeof(BtFrameHdr);
   915    923       rc = p->pVfs->xRead(pLog->pFd, iOff, aBuffer, sizeof(BtDbHdrCksum));
   916    924       aData = aBuffer;
   917    925     }
   918    926   
   919    927     if( rc==SQLITE4_OK ){
   920         -    btLogDecodeDbhdr(pLog, aData, pHdr);
          928  +    rc = btLogDecodeDbhdr(pLog, aData, pHdr);
   921    929     }
   922    930     return rc;
   923    931   }
   924    932   
   925    933   static int btLogUpdateDbhdr(BtLog *pLog, u8 *aData){
   926    934     BtDbHdrCksum hdr;
   927    935   
................................................................................
  1008   1016         pShm->ckpt.iFirstRead = pHdr->iFirstFrame;
  1009   1017         pShm->ckpt.iFirstRecover = pHdr->iFirstFrame;
  1010   1018         rc = btLogRollbackRecovery(pLog, &ctx);
  1011   1019         pLog->snapshot.iNextFrame = ctx.iNextFrame;
  1012   1020         pLog->snapshot.dbhdr.pgsz = pHdr->nPgsz;
  1013   1021         assert( pShm->ckpt.iFirstRead>0 );
  1014   1022       }
         1023  +    assert( rc!=SQLITE4_NOTFOUND );
  1015   1024   
  1016   1025       /* Based on the wal-header, the page-size and number of pages in the
  1017   1026       ** database are now known and stored in snapshot.dbhdr. But the other
  1018   1027       ** header field values (iCookie, iRoot etc.) are still unknown. Read
  1019   1028       ** them from page 1 of the database file now.  */
  1020         -    rc = btLogReadDbhdr(pLog, &pLog->snapshot.dbhdr, ctx.iPageOneFrame);
         1029  +    if( rc==SQLITE4_OK ){
         1030  +      rc = btLogReadDbhdr(pLog, &pLog->snapshot.dbhdr, ctx.iPageOneFrame);
         1031  +    }
  1021   1032   
  1022   1033     }else if( rc==SQLITE4_OK ){
  1023   1034       /* There is no data in the log file. Read the database header directly
  1024   1035       ** from offset 0 of the database file.  */
  1025   1036       btLogZeroSnapshot(pLog);
  1026   1037       rc = btLogReadDbhdr(pLog, &pLog->snapshot.dbhdr, 0);
  1027   1038     }
         1039  +
         1040  +  if( rc==SQLITE4_NOTFOUND ){
         1041  +    /* Check the size of the db file. If it is greater than zero bytes in
         1042  +    ** size, refuse to open the file (as it is probably not a database
         1043  +    ** file). Or, if it is exactly zero bytes in size, this is a brand
         1044  +    ** new database.  */
         1045  +    rc = pVfs->xSize(pLog->pLock->pFd, &nByte);
         1046  +    if( rc==SQLITE4_OK ){
         1047  +      if( nByte==0 ){
         1048  +        btLogZeroDbhdr(pLog, &pLog->snapshot.dbhdr);
         1049  +      }else{
         1050  +        rc = btErrorBkpt(SQLITE4_NOTADB);
         1051  +      }
         1052  +    }
         1053  +  }
  1028   1054   
  1029   1055     if( rc==SQLITE4_OK ){
  1030   1056       btDebugTopology(
  1031   1057           pLog->pLock, "recovered", pLog->snapshot.iHashSide, pLog->snapshot.aLog
  1032   1058       );
  1033   1059   
  1034   1060       btDebugDbhdr(pLog->pLock, "read", &pLog->snapshot.dbhdr);
................................................................................
  1076   1102       if( rc==SQLITE4_OK ){
  1077   1103         rc = btLogUpdateSharedHdr(pLog);
  1078   1104       }
  1079   1105     }
  1080   1106   
  1081   1107    open_out:
  1082   1108     if( rc!=SQLITE4_OK ){
  1083         -    sqlite4_free(pEnv, pLog);
         1109  +    sqlite4BtLogClose(pLog, 0);
  1084   1110       pLog = 0;
  1085   1111     }
  1086   1112     *ppLog = pLog;
  1087   1113     return rc;
  1088   1114   }
  1089   1115   
  1090   1116   /*
................................................................................
  1092   1118   */
  1093   1119   int sqlite4BtLogClose(BtLog *pLog, int bCleanup){
  1094   1120     int rc = SQLITE4_OK;
  1095   1121     if( pLog ){
  1096   1122       sqlite4_env *pEnv = pLog->pLock->pEnv;
  1097   1123       bt_env *pVfs = pLog->pLock->pVfs;
  1098   1124   
  1099         -    pVfs->xClose(pLog->pFd);
         1125  +    if( pLog->pFd ) pVfs->xClose(pLog->pFd);
  1100   1126       if( bCleanup ){
  1101   1127         BtPager *pPager = (BtPager*)pLog->pLock;
  1102   1128         const char *zWal = sqlite4BtPagerFilename(pPager, BT_PAGERFILE_LOG);
  1103   1129         rc = pVfs->xUnlink(pEnv, pVfs, zWal);
  1104   1130       }
  1105   1131   
  1106   1132       sqlite4_free(pEnv, pLog->apShm);
................................................................................
  1823   1849   }
  1824   1850   
  1825   1851   static int btLogMerge(BtLog *pLog, u8 *aBuf){
  1826   1852     bt_db *db = (bt_db*)sqlite4BtPagerExtra((BtPager*)pLog->pLock);
  1827   1853     return sqlite4BtMerge(db, &pLog->snapshot.dbhdr, aBuf);
  1828   1854   }
  1829   1855   
  1830         -
  1831   1856   int sqlite4BtLogCheckpoint(BtLog *pLog, int nFrameBuffer){
  1832   1857     BtLock *pLock = pLog->pLock;
  1833   1858     int rc;
  1834   1859   
  1835   1860     /* Take the CHECKPOINTER lock. */
  1836   1861     rc = sqlite4BtLockCkpt(pLock);
  1837   1862     if( rc==SQLITE4_OK ){
................................................................................
  2022   2047       pLog->snapshot.dbhdr.iCookie = iCookie;
  2023   2048       btLogUpdateDbhdr(pLog, sqlite4BtPageData(pOne));
  2024   2049     }
  2025   2050     sqlite4BtPageRelease(pOne);
  2026   2051   
  2027   2052     return rc;
  2028   2053   }
  2029         -
  2030   2054   

Changes to src/bt_pager.c.

   251    251       }
   252    252     }
   253    253     btHashClear(p);
   254    254   }
   255    255   
   256    256   static int btCheckpoint(BtLock *pLock){
   257    257     BtPager *p = (BtPager*)pLock;
          258  +  if( p->pLog==0 ) return SQLITE4_BUSY;
   258    259     return sqlite4BtLogCheckpoint(p->pLog, 0);
   259    260   }
   260    261   
   261    262   static int btCleanup(BtLock *pLock){
   262    263     BtPager *p = (BtPager*)pLock;
   263    264     int rc = sqlite4BtLogClose(p->pLog, 1);
   264    265     p->pLog = 0;

Changes to test/attach.test.

   780    780       }
   781    781     } {1 {no such table: AAAAAA}}
   782    782   }
   783    783   
   784    784   # Create a malformed file (a file that is not a valid database)
   785    785   # and try to attach it
   786    786   #
   787         -# UPDATE: This does not fail with LSM. 
   788         -#
   789    787   do_test attach-8.1 {
   790    788     set fd [open test2.db w]
   791    789     puts $fd "This file is not a valid SQLite database"
   792    790     close $fd
   793    791     catchsql {
   794    792       ATTACH 'test2.db' AS t2;
   795    793     }
   796         -} {0 {}}
          794  +} {1 {unable to open database: test2.db}}
   797    795   do_test attach-8.2 {
   798    796     db errorcode
   799         -} {0}
          797  +} {26}
   800    798   forcedelete test2.db
   801    799   do_test attach-8.3 {
   802    800     sqlite4 db2 test2.db
   803    801     db2 eval {CREATE TABLE t1(x); BEGIN EXCLUSIVE}
   804    802     catchsql {
   805    803       ATTACH 'test2.db' AS t3;
   806    804     }

Added test/bt1.test.

            1  +# 2012 November 02
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +set testdir [file dirname $argv0]
           13  +source $testdir/tester.tcl
           14  +set testprefix bt1
           15  +
           16  +# Set tcl variable $str to the hex representation of the 24 byte string 
           17  +# located at the start of every bt database file.
           18  +set str ""
           19  +binary scan "SQLite4 bt database 0001" c24 I
           20  +foreach i $I {
           21  +  append str [format %02X $i ]
           22  +}
           23  +
           24  +#-------------------------------------------------------------------------
           25  +# Test that, assuming there is no *-wal file, the bt module will refuse
           26  +# to open an existing database that does not contain a valid header.
           27  +#
           28  +do_test 1.0 {
           29  +  execsql {
           30  +    CREATE TABLE t1(x);
           31  +    INSERT INTO t1 VALUES('abcd');
           32  +  }
           33  +
           34  +  db close
           35  +  list [file exists test.db] [file exists test.db-wal]
           36  +} {1 0}
           37  +
           38  +do_test 1.1 { 
           39  +  hexio_read test.db 0 24 
           40  +} $str
           41  +
           42  +do_test 1.2 { 
           43  +  sqlite4 db test.db
           44  +  execsql { SELECT * FROM t1 }
           45  +} {abcd}
           46  +
           47  +do_test 1.3 { 
           48  +  db close
           49  +  hexio_write test.db  8 55555555
           50  +
           51  +  list [catch {sqlite4 db test.db} msg] $msg
           52  +} {1 {file is encrypted or is not a database}}
           53  +
           54  +#-------------------------------------------------------------------------
           55  +# Test that, if there is a *-wal file that contains a valid copy of page
           56  +# 1 (with the db header), it is possible to open the database even if
           57  +# the header at byte offset 0 is damaged.
           58  +#
           59  +reset_db
           60  +do_test 2.0 {
           61  +  execsql {
           62  +    CREATE TABLE t1(x);
           63  +    INSERT INTO t1 VALUES(randomblob(20000));
           64  +  }
           65  +  db close
           66  +  list [file exists test.db] [file exists test.db-wal]
           67  +} {1 0}
           68  +
           69  +do_test 2.1 {
           70  +  sqlite4 db test.db
           71  +  # This moves pages to the free-list. Meaning page 1 must be 
           72  +  # updated to set the pointer to the first free-list page.
           73  +  execsql { UPDATE t1 SET x = 'abcd' }
           74  +  db_save
           75  +  db close
           76  +  db_restore
           77  +  list [file exists test.db] [file exists test.db-wal]
           78  +} {1 1}
           79  +
           80  +do_test 2.2 { 
           81  +  hexio_write test.db  8 55555555
           82  +  sqlite4 db test.db
           83  +  execsql { SELECT * FROM t1 }
           84  +} {abcd}
           85  +
           86  +finish_test
           87  +

Changes to test/permutations.test.

   130    130   #   quick
   131    131   #   full
   132    132   #
   133    133   lappend ::testsuitelist xxx
   134    134   
   135    135   test_suite "bt" -prefix "bt-" -description {
   136    136   } -files {
          137  +bt1.test
          138  +
   137    139   aggerror.test
   138    140   alter.test
   139    141   alter3.test
   140    142   alter4.test
   141    143   analyze.test
   142    144   analyze3.test
   143    145   analyze4.test