SQLite4
Check-in [5d57889261]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Begin adding tests to check that bt database transactions are robust in the face of system failure.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 5d57889261f0d0fa454d3efa90f6474b57a09066
User & Date: dan 2013-11-07 20:22:29
Context
2013-11-08
17:50
Add missing calls to xSync(). Fix a problem with recovering wrapped logs. check-in: 93af0d7d05 user: dan tags: trunk
2013-11-07
20:22
Begin adding tests to check that bt database transactions are robust in the face of system failure. check-in: 5d57889261 user: dan tags: trunk
2013-11-04
18:21
Fix various multi-client bugs preventing the multi-threaded tests from passing. check-in: 3c32332c59 user: dan tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to lsm-test/lsmtest.h.

    91     91   int test_lsm_zip_open(const char *zFilename, int bClear, TestDb **ppDb);
    92     92   int test_lsm_small_open(const char *zFilename, int bClear, TestDb **ppDb);
    93     93   int test_lsm_mt2(const char *zFilename, int bClear, TestDb **ppDb);
    94     94   int test_lsm_mt3(const char *zFilename, int bClear, TestDb **ppDb);
    95     95   
    96     96   int tdb_lsm_configure(lsm_db *, const char *);
    97     97   
           98  +/* Functions in lsmtest_tdb4.c */
           99  +int test_bt_open(const char *zFilename, int bClear, TestDb **ppDb);
          100  +
    98    101   
    99    102   /* Functions in testutil.c. */
   100    103   int  testPrngInit(void);
   101    104   u32  testPrngValue(u32 iVal);
   102    105   void testPrngArray(u32 iVal, u32 *aOut, int nOut);
   103    106   void testPrngString(u32 iVal, char *aOut, int nOut);
   104    107   
................................................................................
   167    170   /* test5.c */
   168    171   void test_mt(const char *zSystem, const char *zPattern, int *pRc);
   169    172   
   170    173   /* lsmtest6.c */
   171    174   void test_oom(const char *zPattern, int *pRc);
   172    175   void testDeleteLsmdb(const char *zFile);
   173    176   
   174         -void testSaveLsmdb(const char *zFile);
   175         -void testRestoreLsmdb(const char *zFile);
          177  +void testSaveDb(const char *zFile, const char *zAuxExt);
          178  +void testRestoreDb(const char *zFile, const char *zAuxExt);
   176    179   void testCopyLsmdb(const char *zFrom, const char *zTo);
   177    180   
   178    181   /* lsmtest7.c */
   179    182   void test_api(const char *zPattern, int *pRc);
   180    183   
   181    184   /* lsmtest8.c */
   182    185   void do_writer_crash_test(const char *zPattern, int *pRc);

Changes to lsm-test/lsmtest2.c.

   186    186     if( *pRc==0 ){
   187    187       int rc;
   188    188       TestDb *pDb;
   189    189       rc = tdb_lsm_open(zCfg, zFile, 1, &pDb);
   190    190       if( rc==0 ){
   191    191         testWriteDatasourceRange(pDb, pData, 0, nRow, &rc);
   192    192         testClose(&pDb);
   193         -      if( rc==0 ) testSaveLsmdb(zFile);
          193  +      if( rc==0 ) testSaveDb(zFile, "log");
          194  +    }
          195  +    *pRc = rc;
          196  +  }
          197  +}
          198  +
          199  +void testSetupSavedBtdb(
          200  +  const char *zFile,
          201  +  Datasource *pData,
          202  +  int nRow,
          203  +  int *pRc
          204  +){
          205  +  if( *pRc==0 ){
          206  +    int rc;
          207  +    TestDb *pDb;
          208  +    rc = tdb_open("bt", zFile, 1, &pDb);
          209  +    if( rc==0 ){
          210  +      testWriteDatasourceRange(pDb, pData, 0, nRow, &rc);
          211  +      testClose(&pDb);
          212  +      if( rc==0 ) testSaveDb(zFile, "wal");
   194    213       }
   195    214       *pRc = rc;
   196    215     }
   197    216   }
   198    217   
   199    218   /*
   200    219   ** This function is a no-op if *pRc is non-zero when it is called.
................................................................................
   215    234       char zCksum[TEST_CKSUM_BYTES];
   216    235       TestDb *pDb;
   217    236   
   218    237       *pRc = tdb_lsm_open((bCompress?"compression=1 mmap=0":""), zFile, 0, &pDb);
   219    238       testCksumDatabase(pDb, zCksum);
   220    239       testClose(&pDb);
   221    240   
          241  +    if( *pRc==0 ){
          242  +      int r1 = 0;
          243  +      int r2 = -1;
          244  +
          245  +      r1 = strcmp(zCksum, zExpect1);
          246  +      if( zExpect2 ) r2 = strcmp(zCksum, zExpect2);
          247  +      if( r1 && r2 ){
          248  +        if( zExpect2 ){
          249  +          testPrintError("testCompareCksumLsmdb: \"%s\" != (\"%s\" OR \"%s\")",
          250  +              zCksum, zExpect1, zExpect2
          251  +          );
          252  +        }else{
          253  +          testPrintError("testCompareCksumLsmdb: \"%s\" != \"%s\"",
          254  +              zCksum, zExpect1
          255  +          );
          256  +        }
          257  +        *pRc = 1;
          258  +        test_failed();
          259  +      }
          260  +    }
          261  +  }
          262  +}
          263  +
          264  +static void testCompareCksumBtdb(
          265  +  const char *zFile,              /* Path to LSM database */
          266  +  const char *zExpect1,           /* Expected checksum 1 */
          267  +  const char *zExpect2,           /* Expected checksum 2 (or NULL) */
          268  +  int *pRc                        /* IN/OUT: Test case error code */
          269  +){
          270  +  if( *pRc==0 ){
          271  +    char zCksum[TEST_CKSUM_BYTES];
          272  +    TestDb *pDb;
          273  +
          274  +    *pRc = tdb_open("bt", zFile, 0, &pDb);
          275  +    testCksumDatabase(pDb, zCksum);
          276  +    testClose(&pDb);
          277  +
   222    278       if( *pRc==0 ){
   223    279         int r1 = 0;
   224    280         int r2 = -1;
   225    281   
   226    282         r1 = strcmp(zCksum, zExpect1);
   227    283         if( zExpect2 ) r2 = strcmp(zCksum, zExpect2);
   228    284         if( r1 && r2 ){
................................................................................
   286    342     for(i=0; i<nIter && *pRc==0; i++){
   287    343       int iWork;
   288    344       int testrc = 0;
   289    345   
   290    346       testCaseProgress(i, nIter, testCaseNDot(), &iDot);
   291    347   
   292    348       /* Restore and open the database. */
   293         -    testRestoreLsmdb(DBNAME);
          349  +    testRestoreDb(DBNAME, "log");
   294    350       testrc = tdb_lsm_open(azConfig[bCompress], DBNAME, 0, &pDb);
   295    351       assert( testrc==0 );
   296    352   
   297    353       /* Call lsm_work() on the db */
   298    354       tdb_lsm_prepare_sync_crash(pDb, 1 + (i%(nWork*2)));
   299    355       for(iWork=0; testrc==0 && iWork<nWork; iWork++){
   300    356         int nWrite = 0;
................................................................................
   341    397     for(i=0; i<nIter && *pRc==0; i++){
   342    398       int iIns;
   343    399       int testrc = 0;
   344    400   
   345    401       testCaseProgress(i, nIter, testCaseNDot(), &iDot);
   346    402   
   347    403       /* Restore and open the database. */
   348         -    testRestoreLsmdb(DBNAME);
          404  +    testRestoreDb(DBNAME, "log");
   349    405       testrc = tdb_lsm_open("safety=2", DBNAME, 0, &pDb);
   350    406       assert( testrc==0 );
   351    407   
   352    408       /* Insert nInsert records into the database. Crash midway through. */
   353    409       tdb_lsm_prepare_sync_crash(pDb, 1 + (i%(nInsert+2)));
   354    410       for(iIns=0; iIns<nInsert; iIns++){
   355    411         void *pKey; int nKey;
................................................................................
   359    415         testrc = tdb_write(pDb, pKey, nKey, pVal, nVal);
   360    416         if( testrc ) break;
   361    417       }
   362    418       tdb_close(pDb);
   363    419   
   364    420       /* Check that no data was lost when the system crashed. */
   365    421       testCompareCksumLsmdb(DBNAME, bCompress,
          422  +      testCksumArrayGet(pCksumDb, 100 + iIns),
          423  +      testCksumArrayGet(pCksumDb, 100 + iIns + 1),
          424  +      pRc
          425  +    );
          426  +  }
          427  +
          428  +  testDatasourceFree(pData);
          429  +  testCksumArrayFree(pCksumDb);
          430  +}
          431  +
          432  +
          433  +/*
          434  +** This test verifies that if a system crash occurs while committing a
          435  +** transaction to the log file, no earlier transactions are lost or damaged.
          436  +*/
          437  +static void crash_test2b(int bCompress, int *pRc){
          438  +  const char *DBNAME = "testdb.bt";
          439  +  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 1000, 1000};
          440  +
          441  +  const int nIter = 200;
          442  +  const int nInsert = 20;
          443  +
          444  +  int i;
          445  +  int iDot = 0;
          446  +  Datasource *pData;
          447  +  CksumDb *pCksumDb;
          448  +  TestDb *pDb;
          449  +
          450  +  /* Allocate datasource. And calculate the expected checksums. */
          451  +  pData = testDatasourceNew(&defn);
          452  +  pCksumDb = testCksumArrayNew(pData, 100, 100+nInsert+1, 1);
          453  +
          454  +  /* Setup and save the initial database. */
          455  +  testSetupSavedBtdb(DBNAME, pData, 100, pRc);
          456  +
          457  +  for(i=0; i<nIter && *pRc==0; i++){
          458  +    int iIns;
          459  +    int testrc = 0;
          460  +
          461  +    testCaseProgress(i, nIter, testCaseNDot(), &iDot);
          462  +
          463  +    /* Restore and open the database. */
          464  +    testRestoreDb(DBNAME, "wal");
          465  +    testrc = tdb_open("bt", DBNAME, 0, &pDb);
          466  +    assert( testrc==0 );
          467  +
          468  +    /* Insert nInsert records into the database. Crash midway through. */
          469  +    tdb_bt_prepare_sync_crash(pDb, 1 + (i%(nInsert+2)));
          470  +    for(iIns=0; iIns<nInsert; iIns++){
          471  +      void *pKey; int nKey;
          472  +      void *pVal; int nVal;
          473  +
          474  +      testDatasourceEntry(pData, 100+iIns, &pKey, &nKey, &pVal, &nVal);
          475  +      testrc = tdb_write(pDb, pKey, nKey, pVal, nVal);
          476  +      if( testrc ) break;
          477  +    }
          478  +    tdb_close(pDb);
          479  +
          480  +    /* Check that no data was lost when the system crashed. */
          481  +    testCompareCksumBtdb(DBNAME, 
   366    482         testCksumArrayGet(pCksumDb, 100 + iIns),
   367    483         testCksumArrayGet(pCksumDb, 100 + iIns + 1),
   368    484         pRc
   369    485       );
   370    486     }
   371    487   
   372    488     testDatasourceFree(pData);
................................................................................
   395    511   
   396    512     /* Setup and save the initial database. */
   397    513     testSetupSavedLsmdb("", DBNAME, pData, 100, pRc);
   398    514   
   399    515     for(i=0; i<nIter && *pRc==0; i++){
   400    516       int iOpen;
   401    517       testCaseProgress(i, nIter, testCaseNDot(), &iDot);
   402         -    testRestoreLsmdb(DBNAME);
          518  +    testRestoreDb(DBNAME, "log");
   403    519   
   404    520       for(iOpen=0; iOpen<5; iOpen++){
   405    521         /* Open the database. Insert 10 more records. */
   406    522         pDb = testOpen("lsm", 0, pRc);
   407    523         testWriteDatasourceRange(pDb, pData, 100+iOpen*10, 10, pRc);
   408    524   
   409    525         /* Schedule a crash simulation then close the db. */
................................................................................
   425    541   
   426    542   void do_crash_test(const char *zPattern, int *pRc){
   427    543     struct Test {
   428    544       const char *zTest;
   429    545       void (*x)(int, int *);
   430    546       int bCompress;
   431    547     } aTest [] = {
          548  +    { "crash.bt.2",     crash_test2b, 0 },
          549  +
   432    550       { "crash.lsm.1",     crash_test1, 0 },
   433    551       { "crash.lsm_zip.1", crash_test1, 1 },
   434    552       { "crash.lsm.2",     crash_test2, 0 },
   435    553       { "crash.lsm.3",     crash_test3, 0 },
   436    554     };
   437    555     int i;
   438    556   

Changes to lsm-test/lsmtest6.c.

   300    300     copy_file(zLog1, zLog2);
   301    301     copy_file(zShm1, zShm2);
   302    302   
   303    303     testFree(zLog1); testFree(zLog2); testFree(zShm1); testFree(zShm2);
   304    304   }
   305    305   
   306    306   /*
   307         -** File zFile is the path to an LSM database. This function makes backups
          307  +** File zFile is the path to a database. This function makes backups
   308    308   ** of the database file and its log as follows:
   309    309   **
   310         -**     cp $(zFile)     $(zFile)-save
   311         -**     cp $(zFile)-log $(zFile)-save-log
          310  +**     cp $(zFile)         $(zFile)-save
          311  +**     cp $(zFile)-$(zAux) $(zFile)-save-$(zAux)
   312    312   **
   313         -** Function testRestoreLsmdb() can be used to copy the files back in the
          313  +** Function testRestoreDb() can be used to copy the files back in the
   314    314   ** other direction.
   315    315   */
   316         -void testSaveLsmdb(const char *zFile){
   317         -  char *zLog = testMallocPrintf("%s-log", zFile);
          316  +void testSaveDb(const char *zFile, const char *zAux){
          317  +  char *zLog = testMallocPrintf("%s-%s", zFile, zAux);
   318    318     char *zFileSave = testMallocPrintf("%s-save", zFile);
   319         -  char *zLogSave = testMallocPrintf("%s-log-save", zFile);
          319  +  char *zLogSave = testMallocPrintf("%s-%s-save", zFile, zAux);
   320    320   
   321    321     unlink(zFileSave);
   322    322     unlink(zLogSave);
   323    323     copy_file(zFile, zFileSave);
   324    324     copy_file(zLog, zLogSave);
   325    325   
   326    326     testFree(zLog); testFree(zFileSave); testFree(zLogSave);
   327    327   }
   328    328   
   329    329   /*
   330         -** File zFile is the path to an LSM database. This function restores
   331         -** a backup of the database made by a previous call to testSaveLsmdb().
          330  +** File zFile is the path to a database. This function restores
          331  +** a backup of the database made by a previous call to testSaveDb().
   332    332   ** Specifically, it does the equivalent of:
   333    333   **
   334         -**     cp $(zFile)-save     $(zFile)
   335         -**     cp $(zFile)-save-log $(zFile)-log
          334  +**     cp $(zFile)-save         $(zFile)
          335  +**     cp $(zFile)-save-$(zAux) $(zFile)-$(zAux)
   336    336   */
   337         -void testRestoreLsmdb(const char *zFile){
   338         -  char *zLog = testMallocPrintf("%s-log", zFile);
          337  +void testRestoreDb(const char *zFile, const char *zAux){
          338  +  char *zLog = testMallocPrintf("%s-%s", zFile, zAux);
   339    339     char *zFileSave = testMallocPrintf("%s-save", zFile);
   340         -  char *zLogSave = testMallocPrintf("%s-log-save", zFile);
          340  +  char *zLogSave = testMallocPrintf("%s-%s-save", zFile, zAux);
   341    341   
   342    342     copy_file(zFileSave, zFile);
   343    343     copy_file(zLogSave, zLog);
   344    344   
   345    345     testFree(zLog); testFree(zFileSave); testFree(zLogSave);
   346    346   }
   347    347   
................................................................................
   389    389     if( rc==LSM_OK ) rc = lsm_open(pDb, LSMTEST6_TESTDB);
   390    390   
   391    391     for(ii=0; rc==LSM_OK && ii<ArraySize(azStr); ii+=2){
   392    392       rc = lsmWriteStr(pDb, azStr[ii], azStr[ii+1]);
   393    393     }
   394    394     lsm_close(pDb);
   395    395   
   396         -  testSaveLsmdb(LSMTEST6_TESTDB);
          396  +  testSaveDb(LSMTEST6_TESTDB, "log");
   397    397     assert( rc==LSM_OK );
   398    398   }
   399    399   
   400    400   static Datasource *getDatasource(void){
   401    401     const DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 200, 250 };
   402    402     return testDatasourceNew(&defn);
   403    403   }
................................................................................
   433    433       void *pVal; int nVal;
   434    434       testDatasourceEntry(pData, ii, &pKey, &nKey, &pVal, &nVal);
   435    435       lsm_insert(pDb, pKey, nKey, pVal, nVal);
   436    436     }
   437    437     testDatasourceFree(pData);
   438    438     lsm_close(pDb);
   439    439   
   440         -  testSaveLsmdb(LSMTEST6_TESTDB);
          440  +  testSaveDb(LSMTEST6_TESTDB, "log");
   441    441     assert( rc==LSM_OK );
   442    442   }
   443    443   
   444    444   /*
   445    445   ** Test the results of OOM conditions in lsm_new().
   446    446   */
   447    447   static void simple_oom_1(OomTest *pOom){
................................................................................
   507    507   }
   508    508   
   509    509   static void simple_oom_5(OomTest *pOom){
   510    510     Datasource *pData = getDatasource();
   511    511     int rc = LSM_OK;
   512    512     lsm_db *pDb;
   513    513   
   514         -  testRestoreLsmdb(LSMTEST6_TESTDB);
          514  +  testRestoreDb(LSMTEST6_TESTDB, "log");
   515    515     testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
   516    516   
   517    517     testOomFetchData(pOom, pDb, pData, 3333, &rc);
   518    518     testOomFetchData(pOom, pDb, pData, 0, &rc);
   519    519     testOomFetchData(pOom, pDb, pData, 4999, &rc);
   520    520   
   521    521     lsm_close(pDb);
................................................................................
   523    523   }
   524    524   
   525    525   static void simple_oom_6(OomTest *pOom){
   526    526     Datasource *pData = getDatasource();
   527    527     int rc = LSM_OK;
   528    528     lsm_db *pDb;
   529    529   
   530         -  testRestoreLsmdb(LSMTEST6_TESTDB);
          530  +  testRestoreDb(LSMTEST6_TESTDB, "log");
   531    531     testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
   532    532   
   533    533     testOomWriteData(pOom, pDb, pData, 5000, &rc);
   534    534     testOomWriteData(pOom, pDb, pData, 5001, &rc);
   535    535     testOomWriteData(pOom, pDb, pData, 5002, &rc);
   536    536     testOomFetchData(pOom, pDb, pData, 5001, &rc);
   537    537     testOomFetchData(pOom, pDb, pData, 1234, &rc);
................................................................................
   541    541   }
   542    542   
   543    543   static void simple_oom_7(OomTest *pOom){
   544    544     Datasource *pData = getDatasource();
   545    545     int rc = LSM_OK;
   546    546     lsm_db *pDb;
   547    547   
   548         -  testRestoreLsmdb(LSMTEST6_TESTDB);
          548  +  testRestoreDb(LSMTEST6_TESTDB, "log");
   549    549     testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
   550    550     testOomScan(pOom, pDb, 0, "abc", 3, 20, &rc);
   551    551     lsm_close(pDb);
   552    552     testDatasourceFree(pData);
   553    553   }
   554    554   
   555    555   static void simple_oom_8(OomTest *pOom){
   556    556     Datasource *pData = getDatasource();
   557    557     int rc = LSM_OK;
   558    558     lsm_db *pDb;
   559         -  testRestoreLsmdb(LSMTEST6_TESTDB);
          559  +  testRestoreDb(LSMTEST6_TESTDB, "log");
   560    560     testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
   561    561     testOomScan(pOom, pDb, 1, "xyz", 3, 20, &rc);
   562    562     lsm_close(pDb);
   563    563     testDatasourceFree(pData);
   564    564   }
   565    565   
   566    566   /*

Changes to lsm-test/lsmtest_tdb.c.

   677    677     *ppDb = (TestDb *)pDb;
   678    678     return 0;
   679    679   }
   680    680   /* 
   681    681   ** End wrapper for SQLite.
   682    682   *************************************************************************/
   683    683   
   684         -/*************************************************************************
   685         -** Begin bt wrapper.
   686         -*/
   687         -#include "bt.h"
   688         -
   689         -typedef struct BtDb BtDb;
   690         -struct BtDb {
   691         -  TestDb base;
   692         -  bt_db *pBt;
   693         -
   694         -  /* Space for bt_fetch() results */
   695         -  u8 *aBuffer;
   696         -  int nBuffer;
   697         -};
   698         -
   699         -int bt_close(TestDb *pTestDb){
   700         -  BtDb *p = (BtDb*)pTestDb;
   701         -  free(p->aBuffer);
   702         -  return sqlite4BtClose(p->pBt);
   703         -}
   704         -
   705         -static int btMinTransaction(BtDb *p, int iMin, int *piLevel){
   706         -  int iLevel;
   707         -  int rc = SQLITE4_OK;
   708         -
   709         -  iLevel = sqlite4BtTransactionLevel(p->pBt);
   710         -  if( iLevel<iMin ){ 
   711         -    rc = sqlite4BtBegin(p->pBt, iMin); 
   712         -    *piLevel = iLevel;
   713         -  }else{
   714         -    *piLevel = -1;
   715         -  }
   716         -
   717         -  return rc;
   718         -}
   719         -static int btRestoreTransaction(BtDb *p, int iLevel, int rcin){
   720         -  int rc = rcin;
   721         -  if( iLevel>=0 ){
   722         -    if( rc==SQLITE4_OK ){
   723         -      rc = sqlite4BtCommit(p->pBt, iLevel);
   724         -    }else{
   725         -      sqlite4BtRollback(p->pBt, iLevel);
   726         -    }
   727         -    assert( iLevel==sqlite4BtTransactionLevel(p->pBt) );
   728         -  }
   729         -  return rc;
   730         -}
   731         -
   732         -
   733         -
   734         -int bt_write(TestDb *pTestDb, void *pK, int nK, void *pV, int nV){
   735         -  BtDb *p = (BtDb*)pTestDb;
   736         -  int iLevel;
   737         -  int rc;
   738         -
   739         -  rc = btMinTransaction(p, 2, &iLevel);
   740         -  if( rc==SQLITE4_OK ){
   741         -    rc = sqlite4BtReplace(p->pBt, pK, nK, pV, nV);
   742         -    rc = btRestoreTransaction(p, iLevel, rc);
   743         -  }
   744         -  return rc;
   745         -}
   746         -
   747         -int bt_delete(TestDb *pTestDb, void *pK, int nK){
   748         -  return bt_write(pTestDb, pK, nK, 0, -1);
   749         -}
   750         -
   751         -int bt_delete_range(
   752         -  TestDb *pTestDb, 
   753         -  void *pKey1, int nKey1,
   754         -  void *pKey2, int nKey2
   755         -){
   756         -  BtDb *p = (BtDb*)pTestDb;
   757         -  bt_cursor *pCsr = 0;
   758         -  int rc = SQLITE4_OK;
   759         -  int iLevel;
   760         -
   761         -  rc = btMinTransaction(p, 2, &iLevel);
   762         -  if( rc==SQLITE4_OK ){
   763         -    rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr);
   764         -  }
   765         -  while( rc==SQLITE4_OK ){
   766         -    const void *pK;
   767         -    int n;
   768         -    int nCmp;
   769         -    int res;
   770         -
   771         -    rc = sqlite4BtCsrSeek(pCsr, pKey1, nKey1, BT_SEEK_GE);
   772         -    if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK;
   773         -    if( rc!=SQLITE4_OK ) break;
   774         -
   775         -    rc = sqlite4BtCsrKey(pCsr, &pK, &n);
   776         -    if( rc!=SQLITE4_OK ) break;
   777         -
   778         -    nCmp = MIN(n, nKey1);
   779         -    res = memcmp(pKey1, pK, nCmp);
   780         -    assert( res<0 || (res==0 && nKey1<=n) );
   781         -    if( res==0 && nKey1==n ){
   782         -      rc = sqlite4BtCsrNext(pCsr);
   783         -      if( rc!=SQLITE4_OK ) break;
   784         -      rc = sqlite4BtCsrKey(pCsr, &pK, &n);
   785         -      if( rc!=SQLITE4_OK ) break;
   786         -    }
   787         -
   788         -    nCmp = MIN(n, nKey2);
   789         -    res = memcmp(pKey2, pK, nCmp);
   790         -    if( res<0 || (res==0 && nKey2<=n) ) break;
   791         -    
   792         -    rc = sqlite4BtDelete(pCsr);
   793         -  }
   794         -  if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK;
   795         -
   796         -  sqlite4BtCsrClose(pCsr);
   797         -
   798         -  rc = btRestoreTransaction(p, iLevel, rc);
   799         -  return rc;
   800         -}
   801         -
   802         -int bt_fetch(TestDb *pTestDb, void *pK, int nK, void **ppVal, int *pnVal){
   803         -  BtDb *p = (BtDb*)pTestDb;
   804         -  bt_cursor *pCsr = 0;
   805         -  int iLevel;
   806         -  int rc = SQLITE4_OK;
   807         -
   808         -  iLevel = sqlite4BtTransactionLevel(p->pBt);
   809         -  if( iLevel==0 ){ 
   810         -    rc = sqlite4BtBegin(p->pBt, 1); 
   811         -    if( rc!=SQLITE4_OK ) return rc;
   812         -  }
   813         -
   814         -  rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr);
   815         -  if( rc==SQLITE4_OK ){
   816         -    rc = sqlite4BtCsrSeek(pCsr, pK, nK, BT_SEEK_EQ);
   817         -    if( rc==SQLITE4_OK ){
   818         -      const void *pV = 0;
   819         -      int nV = 0;
   820         -      rc = sqlite4BtCsrData(pCsr, 0, -1, &pV, &nV);
   821         -      if( rc==SQLITE4_OK ){
   822         -        if( nV>p->nBuffer ){
   823         -          free(p->aBuffer);
   824         -          p->aBuffer = (u8*)malloc(nV*2);
   825         -          p->nBuffer = nV*2;
   826         -        }
   827         -        memcpy(p->aBuffer, pV, nV);
   828         -        *pnVal = nV;
   829         -        *ppVal = (void*)(p->aBuffer);
   830         -      }
   831         -
   832         -    }else if( rc==SQLITE4_INEXACT || rc==SQLITE4_NOTFOUND ){
   833         -      *ppVal = 0;
   834         -      *pnVal = -1;
   835         -      rc = SQLITE4_OK;
   836         -    }
   837         -    sqlite4BtCsrClose(pCsr);
   838         -  }
   839         -
   840         -  if( iLevel==0 ) sqlite4BtCommit(p->pBt, 0); 
   841         -  return rc;
   842         -}
   843         -
   844         -static int bt_scan(
   845         -  TestDb *pTestDb,
   846         -  void *pCtx,
   847         -  int bReverse,
   848         -  void *pFirst, int nFirst,
   849         -  void *pLast, int nLast,
   850         -  void (*xCallback)(void *, void *, int , void *, int)
   851         -){
   852         -  BtDb *p = (BtDb*)pTestDb;
   853         -  bt_cursor *pCsr = 0;
   854         -  int rc;
   855         -  int iLevel;
   856         -
   857         -  rc = btMinTransaction(p, 1, &iLevel);
   858         -
   859         -  if( rc==SQLITE4_OK ){
   860         -    rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr);
   861         -  }
   862         -  if( rc==SQLITE4_OK ){
   863         -    if( bReverse ){
   864         -      if( pLast ){
   865         -        rc = sqlite4BtCsrSeek(pCsr, pLast, nLast, BT_SEEK_LE);
   866         -      }else{
   867         -        rc = sqlite4BtCsrLast(pCsr);
   868         -      }
   869         -    }else{
   870         -      rc = sqlite4BtCsrSeek(pCsr, pFirst, nFirst, BT_SEEK_GE);
   871         -    }
   872         -    if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK;
   873         -
   874         -    while( rc==SQLITE4_OK ){
   875         -      const void *pK = 0; int nK = 0;
   876         -      const void *pV = 0; int nV = 0;
   877         -
   878         -      rc = sqlite4BtCsrKey(pCsr, &pK, &nK);
   879         -      if( rc==SQLITE4_OK ){
   880         -        rc = sqlite4BtCsrData(pCsr, 0, -1, &pV, &nV);
   881         -      }
   882         -
   883         -      if( rc!=SQLITE4_OK ) break;
   884         -      if( bReverse ){
   885         -        if( pFirst ){
   886         -          int res;
   887         -          int nCmp = MIN(nK, nFirst);
   888         -          res = memcmp(pFirst, pK, nCmp);
   889         -          if( res>0 || (res==0 && nK<nFirst) ) break;
   890         -        }
   891         -      }else{
   892         -        if( pLast ){
   893         -          int res;
   894         -          int nCmp = MIN(nK, nLast);
   895         -          res = memcmp(pLast, pK, nCmp);
   896         -          if( res<0 || (res==0 && nK>nLast) ) break;
   897         -        }
   898         -      }
   899         -
   900         -      xCallback(pCtx, (void*)pK, nK, (void*)pV, nV);
   901         -      if( bReverse ){
   902         -        rc = sqlite4BtCsrPrev(pCsr);
   903         -      }else{
   904         -        rc = sqlite4BtCsrNext(pCsr);
   905         -      }
   906         -    }
   907         -    if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK;
   908         -
   909         -    sqlite4BtCsrClose(pCsr);
   910         -  }
   911         -
   912         -  rc = btRestoreTransaction(p, iLevel, rc);
   913         -  return rc;
   914         -}
   915         -
   916         -static int bt_begin(TestDb *pTestDb, int iLvl){
   917         -  BtDb *p = (BtDb*)pTestDb;
   918         -  int rc = sqlite4BtBegin(p->pBt, iLvl);
   919         -  return rc;
   920         -}
   921         -
   922         -static int bt_commit(TestDb *pTestDb, int iLvl){
   923         -  BtDb *p = (BtDb*)pTestDb;
   924         -  int rc = sqlite4BtCommit(p->pBt, iLvl);
   925         -  return rc;
   926         -}
   927         -
   928         -static int bt_rollback(TestDb *pTestDb, int iLvl){
   929         -  BtDb *p = (BtDb*)pTestDb;
   930         -  int rc = sqlite4BtRollback(p->pBt, iLvl);
   931         -  return rc;
   932         -}
   933         -
   934         -int bt_open(const char *zFilename, int bClear, TestDb **ppDb){
   935         -  static const DatabaseMethods SqlMethods = {
   936         -    bt_close,
   937         -    bt_write,
   938         -    bt_delete,
   939         -    bt_delete_range,
   940         -    bt_fetch,
   941         -    bt_scan,
   942         -    bt_begin,
   943         -    bt_commit,
   944         -    bt_rollback
   945         -  };
   946         -  BtDb *p = 0;
   947         -  bt_db *pBt = 0;
   948         -  int rc;
   949         -  sqlite4_env *pEnv = sqlite4_env_default();
   950         -
   951         -  if( bClear && zFilename && zFilename[0] ){
   952         -    char *zLog = sqlite3_mprintf("%s-wal", zFilename);
   953         -    unlink(zFilename);
   954         -    unlink(zLog);
   955         -    sqlite3_free(zLog);
   956         -  }
   957         -  
   958         -  rc = sqlite4BtNew(pEnv, sizeof(BtDb), &pBt);
   959         -  if( rc==SQLITE4_OK ){
   960         -    p = (BtDb*)sqlite4BtExtra(pBt);
   961         -    p->base.pMethods = &SqlMethods;
   962         -    p->pBt = pBt;
   963         -    rc = sqlite4BtOpen(pBt, zFilename);
   964         -  }
   965         -
   966         -  if( rc!=SQLITE4_OK && p ){
   967         -    bt_close(&p->base);
   968         -  }
   969         -
   970         -  *ppDb = &p->base;
   971         -  return rc;
   972         -}
   973         -/* 
   974         -** End wrapper for bt.
   975         -*************************************************************************/
   976         -
   977    684   /*************************************************************************
   978    685   ** Begin exported functions.
   979    686   */
   980    687   static struct Lib {
   981    688     const char *zName;
   982    689     const char *zDefaultDb;
   983    690     int (*xOpen)(const char *zFilename, int bClear, TestDb **ppDb);
   984    691   } aLib[] = {
   985         -  { "bt",           "testdb.bt",        bt_open },
          692  +  { "bt",           "testdb.bt",        test_bt_open },
   986    693     { "sqlite3",      "testdb.sqlite",    sql_open },
   987    694     { "lsm_small",    "testdb.lsm_small", test_lsm_small_open },
   988    695     { "lsm_lomem",    "testdb.lsm_lomem", test_lsm_lomem_open },
   989    696   #ifdef HAVE_ZLIB
   990    697     { "lsm_zip",      "testdb.lsm_zip",   test_lsm_zip_open },
   991    698   #endif
   992    699     { "lsm",          "testdb.lsm",       test_lsm_open },

Changes to lsm-test/lsmtest_tdb.h.

   148    148   ** illegal to call them with any other type of database handle specified
   149    149   ** as an argument.
   150    150   */
   151    151   void tdb_lsm_enable_log(TestDb *pDb, int bEnable);
   152    152   void tdb_lsm_application_crash(TestDb *pDb);
   153    153   void tdb_lsm_prepare_system_crash(TestDb *pDb);
   154    154   void tdb_lsm_system_crash(TestDb *pDb);
   155         -
   156         -void tdb_lsm_safety(TestDb *pDb, int eMode);
   157         -void tdb_lsm_prepare_system_crash(TestDb *pDb);
   158    155   void tdb_lsm_prepare_sync_crash(TestDb *pDb, int iSync);
   159    156   
          157  +
          158  +void tdb_lsm_safety(TestDb *pDb, int eMode);
   160    159   void tdb_lsm_config_work_hook(TestDb *pDb, void (*)(lsm_db *, void *), void *);
   161    160   void tdb_lsm_write_hook(TestDb *, void(*)(void*,int,lsm_i64,int,int), void*);
   162         -
   163    161   int tdb_lsm_config_str(TestDb *pDb, const char *zStr);
          162  +
          163  +/*************************************************************************
          164  +** Start of bt specific things. From lsmtest_tdb4.c.
          165  +*/
          166  +
          167  +/*
          168  +** Simulate a system crash during the iSync'th call to xSync(). Passing
          169  +** iSync==1 means crash the next time xSync is called.
          170  +*/
          171  +void tdb_bt_prepare_sync_crash(TestDb *pDb, int iSync);
   164    172   
   165    173   #ifdef __cplusplus
   166    174   }  /* End of the 'extern "C"' block */
   167    175   #endif
   168    176   
   169    177   #endif

Added lsm-test/lsmtest_tdb4.c.

            1  +
            2  +/*
            3  +** This file contains the TestDb bt wrapper.
            4  +*/
            5  +
            6  +#include "lsmtest_tdb.h"
            7  +#include "lsmtest.h"
            8  +#include <unistd.h>
            9  +#include "bt.h"
           10  +
           11  +typedef struct BtDb BtDb;
           12  +typedef struct BtFile BtFile;
           13  +
           14  +/*
           15  +** Each database or log file opened by a database handle is wrapped by
           16  +** an object of the following type.
           17  +*/
           18  +struct BtFile {
           19  +  BtDb *pBt;                      /* Database handle that opened this file */
           20  +  bt_file *pFile;                 /* File handle belonging to underlying VFS */
           21  +  int nSectorSize;                /* Size of sectors in bytes */
           22  +  int nSector;                    /* Allocated size of nSector array */
           23  +  u8 **apSector;                  /* Original sector data */
           24  +};
           25  +
           26  +/*
           27  +** nCrashSync:
           28  +**   If this value is non-zero, then a "crash-test" is running. If
           29  +**   nCrashSync==1, then the crash is simulated during the very next 
           30  +**   call to the xSync() VFS method (on either the db or log file).
           31  +**   If nCrashSync==2, the following call to xSync(), and so on.
           32  +**
           33  +** bCrash:
           34  +**   After a crash is simulated, this variable is set. Any subsequent
           35  +**   attempts to write to a file or modify the file system in any way 
           36  +**   fail once this is set. All the caller can do is close the connection.
           37  +*/
           38  +struct BtDb {
           39  +  TestDb base;                    /* Base class */
           40  +  bt_db *pBt;                     /* bt database handle */
           41  +  sqlite4_env *pEnv;              /* SQLite environment (for malloc/free) */
           42  +  bt_env *pVfs;                   /* Underlying VFS */
           43  +
           44  +  /* Space for bt_fetch() results */
           45  +  u8 *aBuffer;                    /* Space to store results */
           46  +  int nBuffer;                    /* Allocated size of aBuffer[] in bytes */
           47  +
           48  +  /* Stuff used for crash test simulation */
           49  +  BtFile *apFile[2];              /* Database and log files used by pBt */
           50  +  bt_env env;                     /* Private VFS for this object */
           51  +  int nCrashSync;                 /* Number of syncs until crash (see above) */
           52  +  int bCrash;                     /* True once a crash has been simulated */
           53  +};
           54  +
           55  +static int btVfsFullpath(
           56  +  sqlite4_env *pEnv, 
           57  +  bt_env *pVfs, 
           58  +  const char *z, 
           59  +  char **pzOut
           60  +){
           61  +  BtDb *pBt = (BtDb*)pVfs->pVfsCtx;
           62  +  if( pBt->bCrash ) return SQLITE4_IOERR;
           63  +  return pBt->pVfs->xFullpath(pEnv, pBt->pVfs, z, pzOut);
           64  +}
           65  +
           66  +static int btVfsOpen(
           67  +  sqlite4_env *pEnv, 
           68  +  bt_env *pVfs, 
           69  +  const char *zFile, 
           70  +  int flags, bt_file **ppFile
           71  +){
           72  +  BtFile *p;
           73  +  BtDb *pBt = (BtDb*)pVfs->pVfsCtx;
           74  +  int rc;
           75  +
           76  +  if( pBt->bCrash ) return SQLITE4_IOERR;
           77  +
           78  +  p = (BtFile*)testMalloc(sizeof(BtFile));
           79  +  if( !p ) return SQLITE4_NOMEM;
           80  +  assert( pBt->apFile[0]==0 || pBt->apFile[1]==0 );
           81  +  pBt->apFile[pBt->apFile[0]!=0] = p;
           82  +
           83  +  p->pBt = pBt; rc = pBt->pVfs->xOpen(pEnv, pVfs, zFile, flags, &p->pFile);
           84  +  if( rc!=SQLITE4_OK ){
           85  +    testFree(p);
           86  +    p = 0;
           87  +  }
           88  +
           89  +  *ppFile = (bt_file*)p;
           90  +  return rc;
           91  +}
           92  +
           93  +static int btVfsSize(bt_file *pFile, sqlite4_int64 *piRes){
           94  +  BtFile *p = (BtFile*)pFile;
           95  +  if( p->pBt->bCrash ) return SQLITE4_IOERR;
           96  +  return p->pBt->pVfs->xSize(p->pFile, piRes);
           97  +}
           98  +
           99  +static int btVfsRead(bt_file *pFile, sqlite4_int64 iOff, void *pBuf, int nBuf){
          100  +  BtFile *p = (BtFile*)pFile;
          101  +  if( p->pBt->bCrash ) return SQLITE4_IOERR;
          102  +  return p->pBt->pVfs->xRead(p->pFile, iOff, pBuf, nBuf);
          103  +}
          104  +
          105  +static int btFlushSectors(BtFile *p){
          106  +  sqlite4_int64 iSz;
          107  +  int rc;
          108  +  int i;
          109  +  u8 *aTmp = 0;
          110  +
          111  +  rc = p->pBt->pVfs->xSize(p->pFile, &iSz);
          112  +  for(i=0; rc==SQLITE4_OK && i<p->nSector; i++){
          113  +    if( p->pBt->bCrash && p->apSector[i] ){
          114  +
          115  +      /* The system is simulating a crash. There are three choices for
          116  +      ** this sector:
          117  +      **
          118  +      **   1) Leave it as it is (simulating a successful write),
          119  +      **   2) Restore the original data (simulating a lost write),
          120  +      **   3) Populate the disk sector with garbage data.
          121  +      */
          122  +      sqlite4_int64 iSOff = p->nSectorSize*i;
          123  +      int nWrite = MIN(p->nSectorSize, iSz - iSOff);
          124  +
          125  +      if( nWrite ){
          126  +        u8 *aWrite = 0;
          127  +        int iOpt = (testPrngValue(i) % 3) + 1;
          128  +        if( iOpt==1 ){
          129  +          aWrite = p->apSector[i];
          130  +        }else if( iOpt==3 ){
          131  +          if( aTmp==0 ) aTmp = testMalloc(p->nSectorSize);
          132  +          aWrite = aTmp;
          133  +          testPrngArray(i*13, (u32*)aWrite, nWrite/sizeof(u32));
          134  +        }
          135  +
          136  +#if 0
          137  +fprintf(stderr, "handle sector %d with %s\n", i, 
          138  +    iOpt==1 ? "rollback" : iOpt==2 ? "write" : "omit"
          139  +);
          140  +fflush(stderr);
          141  +#endif
          142  +
          143  +        if( aWrite ){
          144  +          rc = p->pBt->pVfs->xWrite(p->pFile, iSOff, aWrite, nWrite);
          145  +        }
          146  +      }
          147  +    }
          148  +    testFree(p->apSector[i]);
          149  +    p->apSector[i] = 0;
          150  +  }
          151  +
          152  +  testFree(aTmp);
          153  +  return rc;
          154  +}
          155  +
          156  +static int btSaveSectors(BtFile *p, sqlite4_int64 iOff, int nBuf){
          157  +  int rc;
          158  +  sqlite4_int64 iSz;              /* Size of file on disk */
          159  +  int iFirst;                     /* First sector affected */
          160  +  int iSector;                    /* Current sector */
          161  +  int iLast;                      /* Last sector affected */
          162  +
          163  +  if( p->nSectorSize==0 ){
          164  +    p->nSectorSize = p->pBt->pVfs->xSectorSize(p->pFile);
          165  +    if( p->nSectorSize<512 ) p->nSectorSize = 512;
          166  +  }
          167  +  iLast = (iOff+nBuf) / p->nSectorSize;
          168  +  iFirst = iOff / p->nSectorSize;
          169  +
          170  +  rc = p->pBt->pVfs->xSize(p->pFile, &iSz);
          171  +  for(iSector=iFirst; rc==SQLITE4_OK && iSector<=iLast; iSector++){
          172  +    int nRead;
          173  +    sqlite4_int64 iSOff = iSector * p->nSectorSize;
          174  +    u8 *aBuf = testMalloc(p->nSectorSize);
          175  +    nRead = MIN(p->nSectorSize, (iSz - iSOff));
          176  +    if( nRead>0 ){
          177  +      rc = p->pBt->pVfs->xRead(p->pFile, iSOff, aBuf, nRead);
          178  +    }
          179  +
          180  +    while( rc==SQLITE4_OK && iSector>=p->nSector ){
          181  +      int nNew = p->nSector + 32;
          182  +      u8 **apNew = (u8**)testMalloc(nNew * sizeof(u8*));
          183  +      memcpy(apNew, p->apSector, p->nSector*sizeof(u8*));
          184  +      testFree(p->apSector);
          185  +      p->apSector = apNew;
          186  +      p->nSector = nNew;
          187  +    }
          188  +
          189  +    p->apSector[iSector] = aBuf;
          190  +  }
          191  +
          192  +  return rc;
          193  +}
          194  +
          195  +static int btVfsWrite(bt_file *pFile, sqlite4_int64 iOff, void *pBuf, int nBuf){
          196  +  BtFile *p = (BtFile*)pFile;
          197  +  if( p->pBt->bCrash ) return SQLITE4_IOERR;
          198  +  if( p->pBt->nCrashSync ){
          199  +    btSaveSectors(p, iOff, nBuf);
          200  +  }
          201  +  return p->pBt->pVfs->xWrite(p->pFile, iOff, pBuf, nBuf);
          202  +}
          203  +
          204  +static int btVfsTruncate(bt_file *pFile, sqlite4_int64 iOff){
          205  +  BtFile *p = (BtFile*)pFile;
          206  +  if( p->pBt->bCrash ) return SQLITE4_IOERR;
          207  +  return p->pBt->pVfs->xTruncate(p->pFile, iOff);
          208  +}
          209  +
          210  +static int btVfsSync(bt_file *pFile){
          211  +  int rc = SQLITE4_OK;
          212  +  BtFile *p = (BtFile*)pFile;
          213  +  BtDb *pBt = p->pBt;
          214  +
          215  +  if( p->pBt->bCrash ) return SQLITE4_IOERR;
          216  +  if( pBt->nCrashSync ){
          217  +    pBt->nCrashSync--;
          218  +    pBt->bCrash = (pBt->nCrashSync==0);
          219  +    if( pBt->bCrash ){
          220  +      btFlushSectors(pBt->apFile[0]);
          221  +      btFlushSectors(pBt->apFile[1]);
          222  +      rc = SQLITE4_IOERR;
          223  +    }else{
          224  +      btFlushSectors(p);
          225  +    }
          226  +  }
          227  +
          228  +  if( rc==SQLITE4_OK ){
          229  +    rc = p->pBt->pVfs->xSync(p->pFile);
          230  +  }
          231  +  return rc;
          232  +}
          233  +
          234  +static int btVfsSectorSize(bt_file *pFile){
          235  +  BtFile *p = (BtFile*)pFile;
          236  +  return p->pBt->pVfs->xSectorSize(p->pFile);
          237  +}
          238  +
          239  +static int btVfsClose(bt_file *pFile){
          240  +  BtFile *p = (BtFile*)pFile;
          241  +  int rc;
          242  +  assert( p->pBt->apFile[0]==p || p->pBt->apFile[1]==p );
          243  +  btFlushSectors(p);
          244  +  testFree(p->apSector);
          245  +  rc = p->pBt->pVfs->xClose(p->pFile);
          246  +  testFree(p);
          247  +  return rc;
          248  +}
          249  +
          250  +static int btVfsUnlink(sqlite4_env *pEnv, bt_env *pVfs, const char *zFile){
          251  +  BtDb *pBt = (BtDb*)pVfs->pVfsCtx;
          252  +  if( pBt->bCrash ) return SQLITE4_IOERR;
          253  +  return pBt->pVfs->xUnlink(pEnv, pBt->pVfs, zFile);
          254  +}
          255  +
          256  +static int btVfsLock(bt_file *pFile, int iLock, int eType){
          257  +  BtFile *p = (BtFile*)pFile;
          258  +  if( p->pBt->bCrash ) return SQLITE4_IOERR;
          259  +  return p->pBt->pVfs->xLock(p->pFile, iLock, eType);
          260  +}
          261  +
          262  +static int btVfsTestLock(bt_file *pFile, int iLock, int nLock, int eType){
          263  +  BtFile *p = (BtFile*)pFile;
          264  +  if( p->pBt->bCrash ) return SQLITE4_IOERR;
          265  +  return p->pBt->pVfs->xTestLock(p->pFile, iLock, nLock, eType);
          266  +}
          267  +
          268  +static int btVfsShmMap(bt_file *pFile, int iChunk, int sz, void **ppOut){
          269  +  BtFile *p = (BtFile*)pFile;
          270  +  if( p->pBt->bCrash ) return SQLITE4_IOERR;
          271  +  return p->pBt->pVfs->xShmMap(p->pFile, iChunk, sz, ppOut);
          272  +}
          273  +
          274  +static void btVfsShmBarrier(bt_file *pFile){
          275  +  BtFile *p = (BtFile*)pFile;
          276  +  return p->pBt->pVfs->xShmBarrier(p->pFile);
          277  +}
          278  +
          279  +static int btVfsShmUnmap(bt_file *pFile, int bDelete){
          280  +  BtFile *p = (BtFile*)pFile;
          281  +  if( p->pBt->bCrash ) return SQLITE4_IOERR;
          282  +  return p->pBt->pVfs->xShmUnmap(p->pFile, bDelete);
          283  +}
          284  +
          285  +static int bt_close(TestDb *pTestDb){
          286  +  BtDb *p = (BtDb*)pTestDb;
          287  +  free(p->aBuffer);
          288  +  return sqlite4BtClose(p->pBt);
          289  +}
          290  +
          291  +static int btMinTransaction(BtDb *p, int iMin, int *piLevel){
          292  +  int iLevel;
          293  +  int rc = SQLITE4_OK;
          294  +
          295  +  iLevel = sqlite4BtTransactionLevel(p->pBt);
          296  +  if( iLevel<iMin ){ 
          297  +    rc = sqlite4BtBegin(p->pBt, iMin); 
          298  +    *piLevel = iLevel;
          299  +  }else{
          300  +    *piLevel = -1;
          301  +  }
          302  +
          303  +  return rc;
          304  +}
          305  +static int btRestoreTransaction(BtDb *p, int iLevel, int rcin){
          306  +  int rc = rcin;
          307  +  if( iLevel>=0 ){
          308  +    if( rc==SQLITE4_OK ){
          309  +      rc = sqlite4BtCommit(p->pBt, iLevel);
          310  +    }else{
          311  +      sqlite4BtRollback(p->pBt, iLevel);
          312  +    }
          313  +    assert( iLevel==sqlite4BtTransactionLevel(p->pBt) );
          314  +  }
          315  +  return rc;
          316  +}
          317  +
          318  +static int bt_write(TestDb *pTestDb, void *pK, int nK, void *pV, int nV){
          319  +  BtDb *p = (BtDb*)pTestDb;
          320  +  int iLevel;
          321  +  int rc;
          322  +
          323  +  rc = btMinTransaction(p, 2, &iLevel);
          324  +  if( rc==SQLITE4_OK ){
          325  +    rc = sqlite4BtReplace(p->pBt, pK, nK, pV, nV);
          326  +    rc = btRestoreTransaction(p, iLevel, rc);
          327  +  }
          328  +  return rc;
          329  +}
          330  +
          331  +static int bt_delete(TestDb *pTestDb, void *pK, int nK){
          332  +  return bt_write(pTestDb, pK, nK, 0, -1);
          333  +}
          334  +
          335  +static int bt_delete_range(
          336  +  TestDb *pTestDb, 
          337  +  void *pKey1, int nKey1,
          338  +  void *pKey2, int nKey2
          339  +){
          340  +  BtDb *p = (BtDb*)pTestDb;
          341  +  bt_cursor *pCsr = 0;
          342  +  int rc = SQLITE4_OK;
          343  +  int iLevel;
          344  +
          345  +  rc = btMinTransaction(p, 2, &iLevel);
          346  +  if( rc==SQLITE4_OK ){
          347  +    rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr);
          348  +  }
          349  +  while( rc==SQLITE4_OK ){
          350  +    const void *pK;
          351  +    int n;
          352  +    int nCmp;
          353  +    int res;
          354  +
          355  +    rc = sqlite4BtCsrSeek(pCsr, pKey1, nKey1, BT_SEEK_GE);
          356  +    if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK;
          357  +    if( rc!=SQLITE4_OK ) break;
          358  +
          359  +    rc = sqlite4BtCsrKey(pCsr, &pK, &n);
          360  +    if( rc!=SQLITE4_OK ) break;
          361  +
          362  +    nCmp = MIN(n, nKey1);
          363  +    res = memcmp(pKey1, pK, nCmp);
          364  +    assert( res<0 || (res==0 && nKey1<=n) );
          365  +    if( res==0 && nKey1==n ){
          366  +      rc = sqlite4BtCsrNext(pCsr);
          367  +      if( rc!=SQLITE4_OK ) break;
          368  +      rc = sqlite4BtCsrKey(pCsr, &pK, &n);
          369  +      if( rc!=SQLITE4_OK ) break;
          370  +    }
          371  +
          372  +    nCmp = MIN(n, nKey2);
          373  +    res = memcmp(pKey2, pK, nCmp);
          374  +    if( res<0 || (res==0 && nKey2<=n) ) break;
          375  +    
          376  +    rc = sqlite4BtDelete(pCsr);
          377  +  }
          378  +  if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK;
          379  +
          380  +  sqlite4BtCsrClose(pCsr);
          381  +
          382  +  rc = btRestoreTransaction(p, iLevel, rc);
          383  +  return rc;
          384  +}
          385  +
          386  +static int bt_fetch(
          387  +  TestDb *pTestDb, 
          388  +  void *pK, int nK, 
          389  +  void **ppVal, int *pnVal
          390  +){
          391  +  BtDb *p = (BtDb*)pTestDb;
          392  +  bt_cursor *pCsr = 0;
          393  +  int iLevel;
          394  +  int rc = SQLITE4_OK;
          395  +
          396  +  iLevel = sqlite4BtTransactionLevel(p->pBt);
          397  +  if( iLevel==0 ){ 
          398  +    rc = sqlite4BtBegin(p->pBt, 1); 
          399  +    if( rc!=SQLITE4_OK ) return rc;
          400  +  }
          401  +
          402  +  rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr);
          403  +  if( rc==SQLITE4_OK ){
          404  +    rc = sqlite4BtCsrSeek(pCsr, pK, nK, BT_SEEK_EQ);
          405  +    if( rc==SQLITE4_OK ){
          406  +      const void *pV = 0;
          407  +      int nV = 0;
          408  +      rc = sqlite4BtCsrData(pCsr, 0, -1, &pV, &nV);
          409  +      if( rc==SQLITE4_OK ){
          410  +        if( nV>p->nBuffer ){
          411  +          free(p->aBuffer);
          412  +          p->aBuffer = (u8*)malloc(nV*2);
          413  +          p->nBuffer = nV*2;
          414  +        }
          415  +        memcpy(p->aBuffer, pV, nV);
          416  +        *pnVal = nV;
          417  +        *ppVal = (void*)(p->aBuffer);
          418  +      }
          419  +
          420  +    }else if( rc==SQLITE4_INEXACT || rc==SQLITE4_NOTFOUND ){
          421  +      *ppVal = 0;
          422  +      *pnVal = -1;
          423  +      rc = SQLITE4_OK;
          424  +    }
          425  +    sqlite4BtCsrClose(pCsr);
          426  +  }
          427  +
          428  +  if( iLevel==0 ) sqlite4BtCommit(p->pBt, 0); 
          429  +  return rc;
          430  +}
          431  +
          432  +static int bt_scan(
          433  +  TestDb *pTestDb,
          434  +  void *pCtx,
          435  +  int bReverse,
          436  +  void *pFirst, int nFirst,
          437  +  void *pLast, int nLast,
          438  +  void (*xCallback)(void *, void *, int , void *, int)
          439  +){
          440  +  BtDb *p = (BtDb*)pTestDb;
          441  +  bt_cursor *pCsr = 0;
          442  +  int rc;
          443  +  int iLevel;
          444  +
          445  +  rc = btMinTransaction(p, 1, &iLevel);
          446  +
          447  +  if( rc==SQLITE4_OK ){
          448  +    rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr);
          449  +  }
          450  +  if( rc==SQLITE4_OK ){
          451  +    if( bReverse ){
          452  +      if( pLast ){
          453  +        rc = sqlite4BtCsrSeek(pCsr, pLast, nLast, BT_SEEK_LE);
          454  +      }else{
          455  +        rc = sqlite4BtCsrLast(pCsr);
          456  +      }
          457  +    }else{
          458  +      rc = sqlite4BtCsrSeek(pCsr, pFirst, nFirst, BT_SEEK_GE);
          459  +    }
          460  +    if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK;
          461  +
          462  +    while( rc==SQLITE4_OK ){
          463  +      const void *pK = 0; int nK = 0;
          464  +      const void *pV = 0; int nV = 0;
          465  +
          466  +      rc = sqlite4BtCsrKey(pCsr, &pK, &nK);
          467  +      if( rc==SQLITE4_OK ){
          468  +        rc = sqlite4BtCsrData(pCsr, 0, -1, &pV, &nV);
          469  +      }
          470  +
          471  +      if( rc!=SQLITE4_OK ) break;
          472  +      if( bReverse ){
          473  +        if( pFirst ){
          474  +          int res;
          475  +          int nCmp = MIN(nK, nFirst);
          476  +          res = memcmp(pFirst, pK, nCmp);
          477  +          if( res>0 || (res==0 && nK<nFirst) ) break;
          478  +        }
          479  +      }else{
          480  +        if( pLast ){
          481  +          int res;
          482  +          int nCmp = MIN(nK, nLast);
          483  +          res = memcmp(pLast, pK, nCmp);
          484  +          if( res<0 || (res==0 && nK>nLast) ) break;
          485  +        }
          486  +      }
          487  +
          488  +      xCallback(pCtx, (void*)pK, nK, (void*)pV, nV);
          489  +      if( bReverse ){
          490  +        rc = sqlite4BtCsrPrev(pCsr);
          491  +      }else{
          492  +        rc = sqlite4BtCsrNext(pCsr);
          493  +      }
          494  +    }
          495  +    if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK;
          496  +
          497  +    sqlite4BtCsrClose(pCsr);
          498  +  }
          499  +
          500  +  rc = btRestoreTransaction(p, iLevel, rc);
          501  +  return rc;
          502  +}
          503  +
          504  +static int bt_begin(TestDb *pTestDb, int iLvl){
          505  +  BtDb *p = (BtDb*)pTestDb;
          506  +  int rc = sqlite4BtBegin(p->pBt, iLvl);
          507  +  return rc;
          508  +}
          509  +
          510  +static int bt_commit(TestDb *pTestDb, int iLvl){
          511  +  BtDb *p = (BtDb*)pTestDb;
          512  +  int rc = sqlite4BtCommit(p->pBt, iLvl);
          513  +  return rc;
          514  +}
          515  +
          516  +static int bt_rollback(TestDb *pTestDb, int iLvl){
          517  +  BtDb *p = (BtDb*)pTestDb;
          518  +  int rc = sqlite4BtRollback(p->pBt, iLvl);
          519  +  return rc;
          520  +}
          521  +
          522  +int test_bt_open(const char *zFilename, int bClear, TestDb **ppDb){
          523  +
          524  +  static const DatabaseMethods SqlMethods = {
          525  +    bt_close,
          526  +    bt_write,
          527  +    bt_delete,
          528  +    bt_delete_range,
          529  +    bt_fetch,
          530  +    bt_scan,
          531  +    bt_begin,
          532  +    bt_commit,
          533  +    bt_rollback
          534  +  };
          535  +  BtDb *p = 0;
          536  +  bt_db *pBt = 0;
          537  +  int rc;
          538  +  sqlite4_env *pEnv = sqlite4_env_default();
          539  +
          540  +  if( bClear && zFilename && zFilename[0] ){
          541  +    char *zLog = sqlite3_mprintf("%s-wal", zFilename);
          542  +    unlink(zFilename);
          543  +    unlink(zLog);
          544  +    sqlite3_free(zLog);
          545  +  }
          546  +  
          547  +  rc = sqlite4BtNew(pEnv, sizeof(BtDb), &pBt);
          548  +  if( rc==SQLITE4_OK ){
          549  +    p = (BtDb*)sqlite4BtExtra(pBt);
          550  +    p->base.pMethods = &SqlMethods;
          551  +    p->pBt = pBt;
          552  +    p->pEnv = pEnv;
          553  +
          554  +    p->env.pVfsCtx = (void*)p;
          555  +    p->env.xFullpath = btVfsFullpath;
          556  +    p->env.xOpen = btVfsOpen;
          557  +    p->env.xSize = btVfsSize;
          558  +    p->env.xRead = btVfsRead;
          559  +    p->env.xWrite = btVfsWrite;
          560  +    p->env.xTruncate = btVfsTruncate;
          561  +    p->env.xSync = btVfsSync;
          562  +    p->env.xSectorSize = btVfsSectorSize;
          563  +    p->env.xClose = btVfsClose;
          564  +    p->env.xUnlink = btVfsUnlink;
          565  +    p->env.xLock = btVfsLock;
          566  +    p->env.xTestLock = btVfsTestLock;
          567  +    p->env.xShmMap = btVfsShmMap;
          568  +    p->env.xShmBarrier = btVfsShmBarrier;
          569  +    p->env.xShmUnmap = btVfsShmUnmap;
          570  +
          571  +    sqlite4BtControl(pBt, BT_CONTROL_GETVFS, (void*)&p->pVfs);
          572  +    sqlite4BtControl(pBt, BT_CONTROL_SETVFS, (void*)&p->env);
          573  +
          574  +    rc = sqlite4BtOpen(pBt, zFilename);
          575  +  }
          576  +
          577  +  if( rc!=SQLITE4_OK && p ){
          578  +    bt_close(&p->base);
          579  +  }
          580  +
          581  +  *ppDb = &p->base;
          582  +  return rc;
          583  +}
          584  +
          585  +void tdb_bt_prepare_sync_crash(TestDb *pTestDb, int iSync){
          586  +  BtDb *p = (BtDb*)pTestDb;
          587  +  assert( pTestDb->pMethods->xClose==bt_close );
          588  +  assert( p->bCrash==0 );
          589  +  p->nCrashSync = iSync;
          590  +}
          591  +

Changes to main.mk.

   302    302                $(TOP)/lsm-test/lsmtest5.c $(TOP)/lsm-test/lsmtest6.c           \
   303    303                $(TOP)/lsm-test/lsmtest7.c $(TOP)/lsm-test/lsmtest8.c           \
   304    304                $(TOP)/lsm-test/lsmtest9.c                                      \
   305    305                $(TOP)/lsm-test/lsmtest_datasource.c \
   306    306                $(TOP)/lsm-test/lsmtest_func.c $(TOP)/lsm-test/lsmtest_io.c     \
   307    307                $(TOP)/lsm-test/lsmtest_main.c $(TOP)/lsm-test/lsmtest_mem.c    \
   308    308                $(TOP)/lsm-test/lsmtest_tdb.c $(TOP)/lsm-test/lsmtest_tdb3.c    \
          309  +             $(TOP)/lsm-test/lsmtest_tdb4.c \
   309    310                $(TOP)/lsm-test/lsmtest_util.c 
   310    311   
   311    312   LSMTESTHDR = $(TOP)/lsm-test/lsmtest.h $(TOP)/lsm-test/lsmtest_tdb.h
   312    313   
   313    314   # This is the default Makefile target.  The objects listed here
   314    315   # are what get build when you type just "make" with no arguments.
   315    316   #
................................................................................
   517    518   test:	testfixture$(EXE) sqlite4$(EXE)
   518    519   	./testfixture$(EXE) $(TOP)/test/src4.test
   519    520   
   520    521   # Rules to build the 'lsmtest' application.
   521    522   #
   522    523   lsmtest$(EXE): libsqlite4.a $(LSMTESTSRC) $(LSMTESTHDR)
   523    524   	$(TCPPX) -c $(TOP)/lsm-test/lsmtest_tdb2.cc
   524         -	$(TCCX) $(LSMTESTSRC) lsmtest_tdb2.o libsqlite4.a -o lsmtest$(EXE) $(THREADLIB) -lsqlite3
          525  +	$(TCCX) $(LSMTESTSRC) lsmtest_tdb2.o libsqlite4.a -o lsmtest$(EXE) $(THREADLIB) -lsqlite3 -llmdb -lstdc++
   525    526   
   526    527   
   527    528   varint$(EXE):	$(TOP)/src/varint.c
   528    529   	$(TCCX) -DVARINT_TOOL -o varint$(EXE) $(TOP)/src/varint.c
   529    530   
   530    531   # The next two rules are used to support the "threadtest" target. Building
   531    532   # threadtest runs a few thread-safety tests that are implemented in C. This

Changes to src/bt.h.

   128    128   ** BT_CONTROL_INFO:
   129    129   **   If the second argument to sqlite4BtControl() is BT_CONTROL_INFO, then
   130    130   **   the third is expected to be a pointer to an instance of type bt_info.
   131    131   **   The "output" buffer must already be initialized. Before 
   132    132   **   sqlite4BtControl() returns it appends debugging information to the
   133    133   **   buffer. The specific information appended depends on the eType and
   134    134   **   pgno member variables.
          135  +**
          136  +** BT_CONTROL_SETVFS:
          137  +**   The third argument is assumed to be a pointer to an instance of type
          138  +**   bt_env. The database handle takes a copy of this pointer (not a copy 
          139  +**   of the object) and uses it for all subsequent IO. It is the 
          140  +**   responsibility of the caller to ensure that the pointer is valid for
          141  +**   the lifetime of the database connection.
          142  +**
          143  +** BT_CONTROL_GETVFS:
          144  +**   The third argument is assumed to be of type (bt_env**). Before 
          145  +**   returning, the value pointed to is populated with a pointer to 
          146  +**   to the current bt_env object.
          147  +**
          148  +** BT_CONTROL_SAFETY:
          149  +**   The third argument is interpreted as a pointer to type (int). If
          150  +**   the value stored in the (int) location is 0, 1 or 2, then the current
          151  +**   b-tree safety level is set to 0, 1 or 2, respectively. Otherwise, the
          152  +**   integer value is set to the current safety level.
   135    153   */
          154  +#define BT_CONTROL_INFO   7706389
          155  +#define BT_CONTROL_SETVFS 7706390
          156  +#define BT_CONTROL_GETVFS 7706391
          157  +#define BT_CONTROL_SAFETY 7706392
          158  +
   136    159   int sqlite4BtControl(bt_db*, int op, void *pArg);
   137    160   
   138         -#define BT_CONTROL_INFO 7706389
   139    161   typedef struct bt_info bt_info;
   140    162   struct bt_info {
   141    163     int eType;
   142    164     unsigned int pgno;
   143    165     sqlite4_buffer output;
   144    166   };
   145    167   
          168  +/*
          169  +** File-system interface.
          170  +*/
          171  +typedef struct bt_env bt_env;
          172  +typedef struct bt_file bt_file;
   146    173   
          174  +/*
          175  +** xFullpath:
          176  +*/
          177  +struct bt_env {
          178  +  void *pVfsCtx;
          179  +  int (*xFullpath)(sqlite4_env*,bt_env*, const char *, char **);
          180  +  int (*xOpen)(sqlite4_env*,bt_env*, const char *, int flags, bt_file**);
          181  +  int (*xSize)(bt_file*, sqlite4_int64*);
          182  +  int (*xRead)(bt_file*, sqlite4_int64, void *, int);
          183  +  int (*xWrite)(bt_file*, sqlite4_int64, void *, int);
          184  +  int (*xTruncate)(bt_file*, sqlite4_int64);
          185  +  int (*xSync)(bt_file*);
          186  +  int (*xSectorSize)(bt_file*);
          187  +  int (*xClose)(bt_file*);
          188  +  int (*xUnlink)(sqlite4_env*,bt_env*, const char *);
          189  +  int (*xLock)(bt_file*, int, int);
          190  +  int (*xTestLock)(bt_file*, int, int, int);
          191  +  int (*xShmMap)(bt_file*, int, int, void **);
          192  +  void (*xShmBarrier)(bt_file*);
          193  +  int (*xShmUnmap)(bt_file*, int);
          194  +};
   147    195   

Changes to src/btInt.h.

   123    123   ** Return a pointer to a buffer containing the name of the pager log file.
   124    124   */
   125    125   #define BT_PAGERFILE_DATABASE 0
   126    126   #define BT_PAGERFILE_LOG      1
   127    127   #define BT_PAGERFILE_SHM      2
   128    128   const char *sqlite4BtPagerFilename(BtPager*, int ePagerfile);
   129    129   
          130  +bt_env *sqlite4BtPagerGetEnv(BtPager*);
          131  +void sqlite4BtPagerSetEnv(BtPager*, bt_env*);
          132  +
          133  +void sqlite4BtPagerSetSafety(BtPager*, int*);
          134  +
   130    135   /*
   131    136   ** End of bt_pager.c interface.
   132    137   *************************************************************************/
   133    138   
   134    139   /*************************************************************************
   135    140   ** File-system interface.
   136    141   */
   137         -typedef struct bt_env bt_env;
   138         -typedef struct bt_file bt_file;
   139         -
   140         -/*
   141         -** xFullpath:
   142         -*/
   143         -struct bt_env {
   144         -  void *pVfsCtx;
   145         -  int (*xFullpath)(sqlite4_env*,bt_env*, const char *, char **);
   146         -  int (*xOpen)(sqlite4_env*,bt_env*, const char *, int flags, bt_file**);
   147         -  int (*xSize)(bt_file*, i64*);
   148         -  int (*xRead)(bt_file*, i64, void *, int);
   149         -  int (*xWrite)(bt_file*, i64, void *, int);
   150         -  int (*xTruncate)(bt_file*, i64);
   151         -  int (*xSync)(bt_file*);
   152         -  int (*xSectorSize)(bt_file*);
   153         -  int (*xClose)(bt_file*);
   154         -  int (*xUnlink)(sqlite4_env*,bt_env*, const char *);
   155         -  int (*xLock)(bt_file*, int, int);
   156         -  int (*xTestLock)(bt_file*, int, int, int);
   157         -  int (*xShmMap)(bt_file*, int, int, void **);
   158         -  void (*xShmBarrier)(bt_file*);
   159         -  int (*xShmUnmap)(bt_file*, int);
   160         -};
   161    142   
   162    143   /* Flags for the 3rd argument to xOpen */
   163    144   #define BT_OPEN_READONLY 0x0001
   164    145   
   165    146   /* Candidate values for the 3rd argument to bt_env.xLock() */
   166    147   #define BT_LOCK_UNLOCK 0
   167    148   #define BT_LOCK_SHARED 1

Changes to src/bt_log.c.

   240    240     u32 *aOut                       /* OUT: Final checksum value output */
   241    241   ){
   242    242     assert( (nByte&0x00000007)==4 && nByte>=8 );
   243    243     btLogChecksum(nativeCksum, a, 8, aIn, aOut);
   244    244     btLogChecksum(nativeCksum, &a[4], nByte-4, aOut, aOut);
   245    245   }
   246    246   
   247         -#define BT_PAGE_DEBUG 0
          247  +#define BT_PAGE_DEBUG 1
   248    248   #define BT_VAL_DEBUG  0
   249    249   
   250    250   static void btDebugTopology(BtLock *pLock, char *zStr, int iSide, u32 *aLog){
   251    251   #if BT_PAGE_DEBUG
   252    252     fprintf(stderr, "%d:%s: (side=%d) %d..%d  %d..%d  %d..%d\n", 
   253    253         pLock->iDebugId, zStr, iSide,
   254    254         (int)aLog[0], (int)aLog[1], (int)aLog[2], 
   255    255         (int)aLog[3], (int)aLog[4], (int)aLog[5]
   256    256     );
   257    257     fflush(stderr);
   258    258   #endif
   259    259   }
   260    260   
          261  +#ifndef NDEBUG
   261    262   void sqlite4BtDebugReadlock(BtLock *pLock, u32 iFirst, u32 iLast){
   262    263   #if BT_PAGE_DEBUG
   263    264     static int nCall = 0;
   264    265     fprintf(stderr, "%d:%d: readlock=(%d..%d)\n",
   265    266         pLock->iDebugId, nCall++, (int)iFirst, (int)iLast
   266    267     );
   267    268     fflush(stderr);
   268    269   #endif
   269    270   }
          271  +#endif
   270    272   
   271    273   #ifndef NDEBUG
   272    274   static void btDebugCheckSnapshot(BtShmHdr *pHdr){
   273    275     u32 *aLog = pHdr->aLog;
   274    276     assert( pHdr->iNextFrame!=1 ||
   275    277         (aLog[0]==0 && aLog[1]==0 && aLog[2]==0 && aLog[3]==0)
   276    278     );
................................................................................
   293    295     fprintf(stderr, "%d:%d: checkpoint safepoint=%d\n",
   294    296         pLock->iDebugId, nCall++, (int)iSafe
   295    297     );
   296    298     fflush(stderr);
   297    299   #endif
   298    300   }
   299    301   #else
   300         -#define btDebugLogSafepoint(x)
          302  +#define btDebugLogSafepoint(x,y)
   301    303   #endif
   302    304   
   303    305   static void btDebugCkptPage(BtLock *pLock, u32 pgno, u8 *aData, int pgsz){
   304    306   #if BT_PAGE_DEBUG
   305    307     static int nCall = 0;
   306    308     u32 aCksum[2];
   307    309     btLogChecksum(1, aData, pgsz, 0, aCksum);
................................................................................
   346    348   static void binToStr(u8 *pIn, int nIn, u8 *pOut, int nOut){
   347    349     int i;
   348    350     int nCopy = MIN(nIn, (nOut-1));
   349    351     for(i=0; i<nCopy; i++){
   350    352       if( isprint(pIn[i]) ){
   351    353         pOut[i] = pIn[i];
   352    354       }else{
   353         -      pOut[i] = ".";
          355  +      pOut[i] = '.';
   354    356       }
   355    357     }
   356    358     pOut[i] = '\0';
   357    359   }
   358    360   void sqlite4BtDebugKV(
   359    361       BtLock *pLock, const char *zStr, u8 *pK, int nK, u8 *pV, int nV
   360    362   ){
................................................................................
   456    458   ** Return the offset of frame iFrame within the log file.
   457    459   */
   458    460   static i64 btLogFrameOffset(BtLog *pLog, int pgsz, u32 iFrame){
   459    461     return 
   460    462         (i64)pLog->snapshot.nSector*2 
   461    463       + (i64)(iFrame-1) * (i64)(pgsz + sizeof(BtFrameHdr));
   462    464   }
          465  +
          466  +static int btLogSyncFile(BtLog *pLog, bt_file *pFd){
          467  +  bt_env *pVfs = pLog->pLock->pVfs;
          468  +  return pVfs->xSync(pFd);
          469  +}
   463    470   
   464    471   static int btLogWriteData(BtLog *pLog, i64 iOff, u8 *aData, int nData){
   465    472     bt_env *pVfs = pLog->pLock->pVfs;
   466    473     return pVfs->xWrite(pLog->pFd, iOff, aData, nData);
   467    474   }
   468    475   
   469    476   static int btLogReadData(BtLog *pLog, i64 iOff, u8 *aData, int nData){
................................................................................
   790    797   static int btLogRecover(BtLog *pLog){
   791    798     bt_env *pVfs = pLog->pLock->pVfs;
   792    799     i64 nByte = 0;                  /* Size of log file on disk */
   793    800     int rc;                         /* Return code */
   794    801     BtWalHdr *pHdr = 0;
   795    802     int iSlot = 0;
   796    803     FrameRecoverCtx ctx = {0, 0};
          804  +  BtWalHdr hdr1;
          805  +  BtWalHdr hdr2;
   797    806   
   798    807     /* Read a log file header from the start of the file. */
   799    808     rc = pVfs->xSize(pLog->pFd, &nByte);
   800    809     if( rc==SQLITE4_OK && nByte>0 ){
   801         -    BtWalHdr hdr1;
   802    810       rc = btLogReadHeader(pLog, 0, &hdr1);
   803    811       if( rc==SQLITE4_OK ){
   804         -      BtWalHdr hdr2;
   805    812         rc = btLogReadHeader(pLog, hdr1.nSector, &hdr2);
   806    813         if( rc==SQLITE4_NOTFOUND ){
   807    814           pHdr = &hdr1;
   808    815         }else if( rc==SQLITE4_OK ){
   809    816           int aGreater[3] = {1, 2, 0};
   810    817           pHdr = ((hdr2.iCnt==aGreater[hdr1.iCnt]) ? &hdr2 : &hdr1);
   811    818         }
................................................................................
   840    847         /* One or more transactions were recovered from the log file. */
   841    848         BtShm *pShm = btLogShm(pLog);
   842    849         pShm->ckpt.iWalHdr = (iSlot<<2) + pHdr->iCnt;
   843    850         pShm->ckpt.iFirstRead = pHdr->iFirstFrame;
   844    851         pShm->ckpt.iFirstRecover = pHdr->iFirstFrame;
   845    852         rc = btLogRollbackRecovery(pLog, &ctx);
   846    853         pLog->snapshot.iNextFrame = ctx.iNextFrame;
          854  +      pLog->snapshot.pgsz = pHdr->nPgsz;
   847    855         assert( pShm->ckpt.iFirstRead>0 );
   848    856       }
   849    857     }
   850    858   
   851    859     if( rc==SQLITE4_OK && ctx.iLast==0 ){
   852    860       /* No transactions were recovered from the log file. */
   853    861       BtDbhdr dbhdr;            /* Database header */
................................................................................
   857    865       ** by the snapshot.  */
   858    866       rc = btLogReadDbhdr(pLog, &dbhdr);
   859    867       pLog->snapshot.nPg = dbhdr.nPg;
   860    868       pLog->snapshot.pgsz = dbhdr.pgsz;
   861    869       pLog->snapshot.iCookie = dbhdr.cookie;
   862    870     }
   863    871   
          872  +  if( rc==SQLITE4_OK ){
          873  +    btDebugTopology(
          874  +        pLog->pLock, "recovered", pLog->snapshot.iHashSide, pLog->snapshot.aLog
          875  +    );
          876  +  }
   864    877     return rc;
   865    878   }
   866    879   
   867    880   /*
   868    881   ** Open the log file for pager pPager. If successful, return the BtLog* 
   869    882   ** handle via output variable *ppLog. If parameter bRecover is true, then
   870    883   ** also run database recovery before returning. In this case, the caller
................................................................................
   996   1009   ** SQLITE4_NOTFOUND.
   997   1010   */
   998   1011   int btLogRead(BtLog *pLog, u32 pgno, u8 *aData, u32 iSafe){
   999   1012     const int pgsz = sqlite4BtPagerPagesize((BtPager*)(pLog->pLock));
  1000   1013     int rc = SQLITE4_NOTFOUND;
  1001   1014     u32 iFrame = 0;
  1002   1015     int i;
  1003         -  int bSeen = (iSafe==0);
  1004   1016   
  1005   1017     u32 *aLog = pLog->snapshot.aLog;
  1006   1018     int iSafeIdx = sqlite4BtLogFrameToIdx(aLog, iSafe);
  1007   1019   
  1008   1020     /* Loop through regions (c), (b) and (a) of the log file. In that order. */
  1009   1021     for(i=2; i>=0 && rc==SQLITE4_NOTFOUND; i--){
  1010   1022       u32 iLo = pLog->snapshot.aLog[i*2+0];
................................................................................
  1075   1087     rc = btLogFindHash(pLog, iSide, iHash, &aHash, &aPgno, &iZero);
  1076   1088     if( rc==SQLITE4_OK ){
  1077   1089       memset(aHash, 0, sizeof(ht_slot)*HASHTABLE_NSLOT);
  1078   1090     }
  1079   1091     return rc;
  1080   1092   }
  1081   1093   
  1082         -/*
  1083         -** Write a frame to the log file.
  1084         -*/
  1085         -int sqlite4BtLogWrite(BtLog *pLog, u32 pgno, u8 *aData, u32 nPg){
         1094  +static int btLogWriteFrame(BtLog *pLog, u32 pgno, u8 *aData, u32 nPg){
  1086   1095     const int pgsz = sqlite4BtPagerPagesize((BtPager*)(pLog->pLock));
  1087         -  int rc = SQLITE4_OK;
         1096  +  u32 *aLog = pLog->snapshot.aLog;
         1097  +  int rc;                         /* Return code */
  1088   1098     u32 iFrame;                     /* Write this frame (numbered from 1) */
  1089         -  BtFrameHdr frame;               /* Header for new frame */
  1090         -  u32 *a;                         /* Pointer to cksum of previous frame */
         1099  +  u32 iNextFrame;                 /* Frame to write following this one */
  1091   1100     i64 iOff;                       /* Offset of log file to write to */
  1092         -  u32 iNextFrame;
  1093         -  u32 *aLog = pLog->snapshot.aLog;
  1094         -
  1095         -  /* If this is a commit frame and the size of the database has changed,
  1096         -  ** ensure that the log file contains at least one copy of page 1 written
  1097         -  ** since the last checkpoint. This is required as a future checkpoint
  1098         -  ** will need to update the nPg field in the database header located on
  1099         -  ** page 1. */
  1100         -  if( nPg && nPg!=pLog->snapshot.nPg ){
  1101         -    BtPager *pPager = (BtPager *)(pLog->pLock);
  1102         -    BtPage *pOne = 0;
  1103         -    rc = sqlite4BtPageGet(pPager, 1, &pOne);
  1104         -    if( rc==SQLITE4_OK ){
  1105         -      rc = sqlite4BtLogWrite(pLog, 1, sqlite4BtPageData(pOne), 0);
  1106         -      sqlite4BtPageRelease(pOne);
  1107         -    }
  1108         -    if( rc!=SQLITE4_OK ) return rc;
  1109         -  }
  1110         -
  1111         -  /* Handle a special case - if the log file is completely empty then
  1112         -  ** this writer must write the first header into the WAL file. */
  1113         -  if( btLogIsEmpty(pLog) ){
  1114         -    BtWalHdr hdr;
  1115         -    memset(&hdr, 0, sizeof(BtWalHdr));
  1116         -
  1117         -    hdr.iMagic = BT_WAL_MAGIC;
  1118         -    hdr.iVersion = BT_WAL_VERSION;
  1119         -    hdr.nSector = pLog->snapshot.nSector;
  1120         -    hdr.nPgsz = pgsz;
  1121         -    hdr.iSalt1 = 22;
  1122         -    hdr.iSalt2 = 23;
  1123         -    hdr.iFirstFrame = 1;
  1124         -
  1125         -    rc = btLogWriteHeader(pLog, 0, &hdr);
  1126         -    if( rc!=SQLITE4_OK ) return rc;
  1127         -
  1128         -    pLog->snapshot.aFrameCksum[0] = hdr.iSalt1;
  1129         -    pLog->snapshot.aFrameCksum[1] = hdr.iSalt2;
  1130         -    pLog->snapshot.iNextFrame = 1;
  1131         -  }
  1132         -  btDebugCheckSnapshot(&pLog->snapshot);
         1101  +  BtFrameHdr frame;               /* Header for new frame */
  1133   1102   
  1134   1103     /* Figure out the offset to write the current frame to. */
  1135   1104     iFrame = pLog->snapshot.iNextFrame;
  1136   1105     iOff = btLogFrameOffset(pLog, pgsz, iFrame);
  1137   1106   
  1138   1107     /* The current frame will be written to location pLog->snapshot.iNextFrame.
  1139   1108     ** This code determines where the following frame will be stored. There
................................................................................
  1161   1130       }
  1162   1131     }
  1163   1132   
  1164   1133     if( rc==SQLITE4_OK ){
  1165   1134       if( iNextFrame & 0x80000000 ){
  1166   1135         rc = SQLITE4_FULL;
  1167   1136       }else{
         1137  +      u32 *a;                     /* Pointer to cksum of previous frame */
  1168   1138   
  1169   1139         /* Populate the frame header object. */
  1170   1140         memset(&frame, 0, sizeof(frame));
  1171   1141         frame.pgno = pgno;
  1172   1142         frame.iNext = iNextFrame;
  1173   1143         frame.nPg = nPg;
  1174   1144         a = pLog->snapshot.aFrameCksum;
................................................................................
  1221   1191       }
  1222   1192   
  1223   1193       aLog[5] = iFrame;
  1224   1194       memcpy(pLog->snapshot.aFrameCksum, frame.aCksum, sizeof(frame.aCksum));
  1225   1195     }
  1226   1196     btDebugCheckSnapshot(&pLog->snapshot);
  1227   1197   
  1228         -  /* If this is a COMMIT, also update the shared shm-header. */
         1198  +  return rc;
         1199  +}
         1200  +
         1201  +/*
         1202  +** Write a frame to the log file.
         1203  +*/
         1204  +int sqlite4BtLogWrite(BtLog *pLog, u32 pgno, u8 *aData, u32 nPg){
         1205  +  const int pgsz = sqlite4BtPagerPagesize((BtPager*)(pLog->pLock));
         1206  +  int rc = SQLITE4_OK;
         1207  +
         1208  +  int nPad = 1;
         1209  +
         1210  +  /* If this is a commit frame and the size of the database has changed,
         1211  +  ** ensure that the log file contains at least one copy of page 1 written
         1212  +  ** since the last checkpoint. This is required as a future checkpoint
         1213  +  ** will need to update the nPg field in the database header located on
         1214  +  ** page 1. */
         1215  +  if( nPg && nPg!=pLog->snapshot.nPg ){
         1216  +    BtPager *pPager = (BtPager *)(pLog->pLock);
         1217  +    BtPage *pOne = 0;
         1218  +    rc = sqlite4BtPageGet(pPager, 1, &pOne);
         1219  +    if( rc==SQLITE4_OK ){
         1220  +      rc = sqlite4BtLogWrite(pLog, 1, sqlite4BtPageData(pOne), 0);
         1221  +      sqlite4BtPageRelease(pOne);
         1222  +    }
         1223  +    if( rc!=SQLITE4_OK ) return rc;
         1224  +  }
         1225  +
         1226  +  /* Handle a special case - if the log file is completely empty then
         1227  +  ** this writer must write the first header into the WAL file. */
         1228  +  if( btLogIsEmpty(pLog) ){
         1229  +    BtWalHdr hdr;
         1230  +    memset(&hdr, 0, sizeof(BtWalHdr));
         1231  +
         1232  +    hdr.iMagic = BT_WAL_MAGIC;
         1233  +    hdr.iVersion = BT_WAL_VERSION;
         1234  +    hdr.nSector = pLog->snapshot.nSector;
         1235  +    hdr.nPgsz = pgsz;
         1236  +    hdr.iSalt1 = 22;
         1237  +    hdr.iSalt2 = 23;
         1238  +    hdr.iFirstFrame = 1;
         1239  +
         1240  +    rc = btLogWriteHeader(pLog, 0, &hdr);
         1241  +    if( rc!=SQLITE4_OK ) return rc;
         1242  +
         1243  +    pLog->snapshot.aFrameCksum[0] = hdr.iSalt1;
         1244  +    pLog->snapshot.aFrameCksum[1] = hdr.iSalt2;
         1245  +    pLog->snapshot.iNextFrame = 1;
         1246  +  }
         1247  +  btDebugCheckSnapshot(&pLog->snapshot);
         1248  +
         1249  +  rc = btLogWriteFrame(pLog, pgno, aData, nPg);
         1250  +
         1251  +  /* If this is a COMMIT, sync the log and update the shared shm-header. */
  1229   1252     if( nPg ){
  1230         -    rc = btLogUpdateSharedHdr(pLog);
         1253  +    int i;
         1254  +    for(i=0; i<nPad && rc==SQLITE4_OK; i++){
         1255  +      rc = btLogWriteFrame(pLog, pgno, aData, nPg);
         1256  +    }
         1257  +    if( rc==SQLITE4_OK ) rc = btLogSyncFile(pLog, pLog->pFd);
         1258  +    if( rc==SQLITE4_OK ) rc = btLogUpdateSharedHdr(pLog);
  1231   1259     }
  1232   1260   
  1233   1261     return rc;
  1234   1262   }
  1235   1263   
  1236   1264   /*
  1237   1265   ** Return true if the checksum in BtShmHdr.aCksum[] matches the rest

Changes to src/bt_main.c.

  2293   2293   /*
  2294   2294   ** Insert a new key/value pair or replace an existing one.
  2295   2295   */
  2296   2296   int sqlite4BtReplace(bt_db *db, const void *pK, int nK, const void *pV, int nV){
  2297   2297     int rc = SQLITE4_OK;
  2298   2298     bt_cursor csr;
  2299   2299   
  2300         -  sqlite4BtDebugKV((BtLock*)db->pPager, "replace", pK, nK, pV, nV);
         2300  +  sqlite4BtDebugKV((BtLock*)db->pPager, "replace", (u8*)pK, nK, (u8*)pV, nV);
  2301   2301   
  2302   2302     btCheckPageRefs(db);
  2303   2303     btCsrSetup(db, &csr);
  2304   2304     rc = btCsrSeek(&csr, pK, nK, BT_SEEK_GE, 1);
  2305   2305     if( rc==SQLITE4_OK ){
  2306   2306       /* The cursor currently points to an entry with key pK/nK. This call
  2307   2307       ** should therefore replace that entry. So delete it and then re-seek
................................................................................
  2354   2354   
  2355   2355   int sqlite4BtGetCookie(bt_db *db, unsigned int *piVal){
  2356   2356     return sqlite4BtPagerGetCookie(db->pPager, piVal);
  2357   2357   }
  2358   2358   
  2359   2359   int sqlite4BtControl(bt_db *db, int op, void *pArg){
  2360   2360     int rc = SQLITE4_OK;
         2361  +
  2361   2362     switch( op ){
  2362   2363       case BT_CONTROL_INFO: {
  2363   2364         bt_info *pInfo = (bt_info*)pArg;
  2364   2365         int iTrans = sqlite4BtTransactionLevel(db);
  2365   2366         if( iTrans==0 ) rc = sqlite4BtBegin(db, 1);
  2366   2367         if( rc==SQLITE4_OK ){
  2367   2368           BtPage *pPg = 0;
................................................................................
  2373   2374             nData = sqlite4BtPagerPagesize(db->pPager);
  2374   2375             btPageToAscii(pInfo->pgno, aData, nData, &pInfo->output);
  2375   2376             sqlite4_buffer_append(&pInfo->output, "", 1);
  2376   2377             sqlite4BtPageRelease(pPg);
  2377   2378           }
  2378   2379           if( iTrans==0 ) rc = sqlite4BtCommit(db, 0);
  2379   2380         }
         2381  +      break;
         2382  +    }
         2383  +
         2384  +    case BT_CONTROL_GETVFS: {
         2385  +      *((bt_env**)pArg) = sqlite4BtPagerGetEnv(db->pPager);
         2386  +      break;
         2387  +    }
         2388  +
         2389  +    case BT_CONTROL_SETVFS: {
         2390  +      sqlite4BtPagerSetEnv(db->pPager, (bt_env*)pArg);
         2391  +      break;
         2392  +    }
         2393  +
         2394  +    case BT_CONTROL_SAFETY: {
         2395  +      int *pInt = (int*)pArg;
         2396  +      sqlite4BtPagerSetSafety(db->pPager, pInt);
         2397  +      break;
  2380   2398       }
  2381   2399     }
  2382   2400   
  2383   2401     return rc;
  2384   2402   }
  2385   2403   

Changes to src/bt_pager.c.

    76     76   
    77     77   /*
    78     78   ** Pager object.
    79     79   **
    80     80   ** nAutoCkpt:
    81     81   **   If a transaction is committed and there are this many frames in the
    82     82   **   log file, automatically run a checkpoint operation.
           83  +**
           84  +** iSafetyLevel:
           85  +**   Current safety level. 0==off, 1==normal, 2=full.
    83     86   */
    84     87   struct BtPager {
    85     88     BtLock btl;                     /* Variables shared with bt_lock module */
    86     89     BtLog *pLog;                    /* Logging module */
    87     90     int iTransactionLevel;          /* Current transaction level (see bt.h) */
           91  +  int iSafetyLevel;               /* Current safety level */
    88     92     char *zFile;                    /* Database file name */
    89     93     int nFile;                      /* Length of string zFile in bytes */
    90     94     BtPageHash hash;                /* Hash table */
    91     95     BtPage *pDirty;                 /* List of all dirty pages */
    92     96     int nTotalRef;                  /* Total number of outstanding page refs */
    93     97     int nAutoCkpt;                  /* Auto-checkpoint when log is this large */
    94     98     int bDoAutoCkpt;                /* Do auto-checkpoint after next unlock */
................................................................................
   667    671     if( p->iTransactionLevel>=iLevel ){
   668    672       if( p->iTransactionLevel>=2 && iLevel<2 ){
   669    673         /* Commit the main write transaction. */
   670    674         rc = btCommitTransaction(p);
   671    675       }
   672    676       p->iTransactionLevel = iLevel;
   673    677       if( iLevel==0 ){
   674         -      rc = btCloseReadTransaction(p);
          678  +      int rc2 = btCloseReadTransaction(p);
          679  +      if( rc==SQLITE4_OK ) rc = rc2;
   675    680       }
   676    681     }
   677    682     return rc;
   678    683   }
   679    684   
   680    685   int sqlite4BtPagerRollback(BtPager *p, int iLevel){
   681    686     int rc = SQLITE4_OK;
................................................................................
   760    765         }else{
   761    766           memset(pRet->aData, 0, p->pgsz);
   762    767         }
   763    768   
   764    769         if( rc==SQLITE4_OK ){
   765    770           rc = btHashAdd(p, pRet);
   766    771         }
          772  +
   767    773         if( rc!=SQLITE4_OK ){
   768    774           btFreePage(p, pRet);
   769    775           pRet = 0;
          776  +      }else{
          777  +        sqlite4BtDebugReadPage(&p->btl, pgno, pRet->aData, p->pgsz);
   770    778         }
   771    779       }
   772         -    sqlite4BtDebugReadPage(&p->btl, pgno, pRet->aData, p->pgsz);
   773    780     }
   774    781   
   775    782     assert( (pRet!=0)==(rc==SQLITE4_OK) );
   776    783     if( rc==SQLITE4_OK ){
   777    784       p->nTotalRef++;
   778    785       pRet->nRef++;
   779    786     }
................................................................................
   904    911         assert( ePagerfile==BT_PAGERFILE_SHM );
   905    912         zTail = "-shm";
   906    913         break;
   907    914     }
   908    915     memcpy(&p->zFile[p->nFile], zTail, strlen(zTail)+1);
   909    916     return p->zFile;
   910    917   }
          918  +
          919  +bt_env *sqlite4BtPagerGetEnv(BtPager *p){
          920  +  return p->btl.pVfs;
          921  +}
          922  +void sqlite4BtPagerSetEnv(BtPager *p, bt_env *pVfs){
          923  +  p->btl.pVfs = pVfs;
          924  +}
          925  +
          926  +void sqlite4BtPagerSetSafety(BtPager *pPager, int *piVal){
          927  +  int iVal = *piVal;
          928  +  if( iVal>=0 && iVal<=2 ){
          929  +    pPager->iSafetyLevel = iVal;
          930  +  }
          931  +  *piVal = pPager->iSafetyLevel;
          932  +}
   911    933   
   912    934   #ifndef NDEBUG
   913    935   int sqlite4BtPagerRefcount(BtPager *p){
   914    936     return p->nTotalRef;
   915    937   }
   916    938   #endif
   917    939   

Changes to src/bt_unix.c.

   176    176     }
   177    177   
   178    178     return rc;
   179    179   }
   180    180   
   181    181   static int btPosixOsSync(bt_file *pFile){
   182    182     int rc = SQLITE4_OK;
   183         -#ifndef LSM_NO_SYNC
          183  +#ifndef SQLITE_NO_SYNC
   184    184     PosixFile *p = (PosixFile *)pFile;
   185    185     int prc = 0;
   186    186   
   187    187   #if 0
   188    188     if( p->pMap ){
   189    189       prc = msync(p->pMap, p->nMap, MS_SYNC);
   190    190     }