Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes to ext/lsm1/lsm-test/lsmtest1.c.

    77     77     int nIter;                      /* Total number of iterations to run */
    78     78   };
    79     79   
    80     80   /*
    81     81   ** Generate a unique name for the test case pTest with database system
    82     82   ** zSystem.
    83     83   */
    84         -static char *getName(const char *zSystem, Datatest1 *pTest){
           84  +static char *getName(const char *zSystem, int bRecover, Datatest1 *pTest){
    85     85     char *zRet;
    86     86     char *zData;
    87     87     zData = testDatasourceName(&pTest->defn);
    88         -  zRet = testMallocPrintf("data.%s.%s.%d.%d", 
    89         -      zSystem, zData, pTest->nRow, pTest->nVerify
           88  +  zRet = testMallocPrintf("data.%s.%s.rec=%d.%d.%d", 
           89  +      zSystem, zData, bRecover, pTest->nRow, pTest->nVerify
    90     90     );
    91     91     testFree(zData);
    92     92     return zRet;
    93     93   }
    94     94   
    95     95   int testControlDb(TestDb **ppDb){
    96     96   #ifdef HAVE_KYOTOCABINET
................................................................................
   246    246   static void printScanCb(
   247    247       void *pCtx, void *pKey, int nKey, void *pVal, int nVal
   248    248   ){
   249    249     printf("%s\n", (char *)pKey);
   250    250     fflush(stdout);
   251    251   }
   252    252   #endif
          253  +
          254  +void testReopenRecover(TestDb **ppDb, int *pRc){
          255  +  if( *pRc==0 ){
          256  +    const char *zLib = tdb_library_name(*ppDb);
          257  +    const char *zDflt = tdb_default_db(zLib);
          258  +    testCopyLsmdb(zDflt, "bak.db");
          259  +    testClose(ppDb);
          260  +    testCopyLsmdb("bak.db", zDflt);
          261  +    *pRc = tdb_open(zLib, 0, 0, ppDb);
          262  +  }
          263  +}
          264  +
   253    265   
   254    266   static void doDataTest1(
   255    267     const char *zSystem,            /* Database system to test */
          268  +  int bRecover,
   256    269     Datatest1 *p,                   /* Structure containing test parameters */
   257    270     int *pRc                        /* OUT: Error code */
   258    271   ){
   259    272     int i;
   260    273     int iDot;
   261    274     int rc = LSM_OK;
   262    275     Datasource *pData;
................................................................................
   273    286       /* Insert some data */
   274    287       testWriteDatasourceRange(pDb, pData, i, p->nVerify, &rc);
   275    288       i += p->nVerify;
   276    289   
   277    290       /* Check that the db content is correct. */
   278    291       testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
   279    292   
   280         -    /* Close and reopen the database. */
   281         -    testReopen(&pDb, &rc);
          293  +    if( bRecover ){
          294  +      testReopenRecover(&pDb, &rc);
          295  +    }else{
          296  +      testReopen(&pDb, &rc);
          297  +    }
   282    298   
   283    299       /* Check that the db content is still correct. */
   284    300       testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
   285    301   
   286    302       /* Update the progress dots... */
   287    303       testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot);
   288    304     }
................................................................................
   295    311       testDeleteDatasourceRange(pDb, pData, i, p->nVerify, &rc);
   296    312       i += p->nVerify;
   297    313   
   298    314       /* Check that the db content is correct. */
   299    315       testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc);
   300    316   
   301    317       /* Close and reopen the database. */
   302         -    testReopen(&pDb, &rc);
          318  +    if( bRecover ){
          319  +      testReopenRecover(&pDb, &rc);
          320  +    }else{
          321  +      testReopen(&pDb, &rc);
          322  +    }
   303    323   
   304    324       /* Check that the db content is still correct. */
   305    325       testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc);
   306    326   
   307    327       /* Update the progress dots... */
   308    328       testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot);
   309    329     }
................................................................................
   335    355       { {DATA_SEQUENTIAL, 5,10,      1000,2000},     1000,  250, 1000, 1},
   336    356       { {DATA_SEQUENTIAL, 5,100,     10000,20000},    100,   25,  100, 1},
   337    357       { {DATA_RANDOM,     10,10,     100,100},     100000, 1000,  100, 0},
   338    358       { {DATA_SEQUENTIAL, 10,10,     100,100},     100000, 1000,  100, 0},
   339    359     };
   340    360   
   341    361     int i;
          362  +  int bRecover;
   342    363   
   343         -  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
   344         -    char *zName = getName(zSystem, &aTest[i]);
   345         -    if( testCaseBegin(pRc, zPattern, "%s", zName) ){
   346         -      doDataTest1(zSystem, &aTest[i], pRc);
          364  +  for(bRecover=0; bRecover<2; bRecover++){
          365  +    if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break;
          366  +    for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
          367  +      char *zName = getName(zSystem, bRecover, &aTest[i]);
          368  +      if( testCaseBegin(pRc, zPattern, "%s", zName) ){
          369  +        doDataTest1(zSystem, bRecover, &aTest[i], pRc);
          370  +      }
          371  +      testFree(zName);
   347    372       }
   348         -    testFree(zName);
   349    373     }
   350    374   }
   351    375   
   352    376   void testCompareDb(
   353    377     Datasource *pData,
   354    378     int nData,
   355    379     int iSeed,
................................................................................
   392    416       testDatasourceEntry(pData, i, &pKey, &nKey, 0, 0);
   393    417       testFetchCompare(pControl, pDb, pKey, nKey, pRc);
   394    418     }
   395    419   }
   396    420   
   397    421   static void doDataTest2(
   398    422     const char *zSystem,            /* Database system to test */
          423  +  int bRecover,
   399    424     Datatest2 *p,                   /* Structure containing test parameters */
   400    425     int *pRc                        /* OUT: Error code */
   401    426   ){
   402    427     TestDb *pDb;
   403    428     TestDb *pControl;
   404    429     Datasource *pData;
   405    430     int i;
................................................................................
   433    458       testDatasourceEntry(pData, i+2000000, &pKey2, &nKey2, 0, 0);
   434    459   
   435    460       testDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2, &rc);
   436    461       testDeleteRange(pControl, pKey1, nKey1, pKey2, nKey2, &rc);
   437    462       testFree(pKey1);
   438    463   
   439    464       testCompareDb(pData, nRange, i, pControl, pDb, &rc);
   440         -    testReopen(&pDb, &rc);
          465  +    if( bRecover ){
          466  +      testReopenRecover(&pDb, &rc);
          467  +    }else{
          468  +      testReopen(&pDb, &rc);
          469  +    }
   441    470       testCompareDb(pData, nRange, i, pControl, pDb, &rc);
   442    471   
   443    472       /* Update the progress dots... */
   444    473       testCaseProgress(i, p->nIter, testCaseNDot(), &iDot);
   445    474     }
   446    475   
   447    476     testClose(&pDb);
   448    477     testClose(&pControl);
   449    478     testDatasourceFree(pData);
   450    479     testCaseFinish(rc);
   451    480     *pRc = rc;
   452    481   }
   453    482   
   454         -static char *getName2(const char *zSystem, Datatest2 *pTest){
          483  +static char *getName2(const char *zSystem, int bRecover, Datatest2 *pTest){
   455    484     char *zRet;
   456    485     char *zData;
   457    486     zData = testDatasourceName(&pTest->defn);
   458         -  zRet = testMallocPrintf("data2.%s.%s.%d.%d.%d", 
   459         -      zSystem, zData, pTest->nRange, pTest->nWrite, pTest->nIter
          487  +  zRet = testMallocPrintf("data2.%s.%s.rec=%d.%d.%d.%d", 
          488  +      zSystem, zData, bRecover, pTest->nRange, pTest->nWrite, pTest->nIter
   460    489     );
   461    490     testFree(zData);
   462    491     return zRet;
   463    492   }
   464    493   
   465    494   void test_data_2(
   466    495     const char *zSystem,            /* Database system name */
................................................................................
   472    501       { {DATA_RANDOM,     20,25,     100,200},   10000,  10,     50   },
   473    502       { {DATA_RANDOM,     20,25,     100,200},   10000,  200,    50   },
   474    503       { {DATA_RANDOM,     20,25,     100,200},   100,    10,     1000 },
   475    504       { {DATA_RANDOM,     20,25,     100,200},   100,    200,    50   },
   476    505     };
   477    506   
   478    507     int i;
          508  +  int bRecover;
   479    509   
   480         -  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
   481         -    char *zName = getName2(zSystem, &aTest[i]);
   482         -    if( testCaseBegin(pRc, zPattern, "%s", zName) ){
   483         -      doDataTest2(zSystem, &aTest[i], pRc);
          510  +  for(bRecover=0; bRecover<2; bRecover++){
          511  +    if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break;
          512  +    for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
          513  +      char *zName = getName2(zSystem, bRecover, &aTest[i]);
          514  +      if( testCaseBegin(pRc, zPattern, "%s", zName) ){
          515  +        doDataTest2(zSystem, bRecover, &aTest[i], pRc);
          516  +      }
          517  +      testFree(zName);
   484    518       }
   485         -    testFree(zName);
   486    519     }
   487    520   }
   488    521   
   489    522   /*************************************************************************
   490    523   ** Test case data3.*
   491    524   */
   492    525   

Changes to ext/lsm1/lsm-test/lsmtest_main.c.

    22     22     return; 
    23     23   }
    24     24   
    25     25   #define testSetError(rc) testSetErrorFunc(rc, pRc, __FILE__, __LINE__)
    26     26   static void testSetErrorFunc(int rc, int *pRc, const char *zFile, int iLine){
    27     27     if( rc ){
    28     28       *pRc = rc;
    29         -    printf("FAILED (%s:%d) rc=%d ", zFile, iLine, rc);
           29  +    fprintf(stderr, "FAILED (%s:%d) rc=%d ", zFile, iLine, rc);
    30     30       test_failed();
    31     31     }
    32     32   }
    33     33   
    34     34   static int lsm_memcmp(u8 *a, u8 *b, int c){
    35     35     int i;
    36     36     for(i=0; i<c; i++){

Changes to ext/lsm1/lsm-test/lsmtest_tdb.c.

   740    740   #endif
   741    741   };
   742    742   
   743    743   const char *tdb_system_name(int i){
   744    744     if( i<0 || i>=ArraySize(aLib) ) return 0;
   745    745     return aLib[i].zName;
   746    746   }
          747  +
          748  +const char *tdb_default_db(const char *zSys){
          749  +  int i;
          750  +  for(i=0; i<ArraySize(aLib); i++){
          751  +    if( strcmp(aLib[i].zName, zSys)==0 ) return aLib[i].zDefaultDb;
          752  +  }
          753  +  return 0;
          754  +}
   747    755   
   748    756   int tdb_open(const char *zLib, const char *zDb, int bClear, TestDb **ppDb){
   749    757     int i;
   750    758     int rc = 1;
   751    759     const char *zSpec = 0;
   752    760   
   753    761     int nLib = 0;

Changes to ext/lsm1/lsm-test/lsmtest_tdb.h.

   121    121     int bReverse,                   /* True to scan in reverse order */
   122    122     void *pKey1, int nKey1,         /* Start of search */
   123    123     void *pKey2, int nKey2,         /* End of search */
   124    124     void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal)
   125    125   );
   126    126   
   127    127   const char *tdb_system_name(int i);
          128  +const char *tdb_default_db(const char *zSys);
   128    129   
   129    130   int tdb_lsm_open(const char *zCfg, const char *zDb, int bClear, TestDb **ppDb);
   130    131   
   131    132   /*
   132    133   ** If the TestDb handle passed as an argument is a wrapper around an LSM
   133    134   ** database, return the LSM handle. Otherwise, if the argument is some other
   134    135   ** database system, return NULL.

Changes to ext/lsm1/lsmInt.h.

   860    860   void lsmLogMessage(lsm_db *, int, const char *, ...);
   861    861   int lsmInfoFreelist(lsm_db *pDb, char **pzOut);
   862    862   
   863    863   /*
   864    864   ** Functions from file "lsm_log.c".
   865    865   */
   866    866   int lsmLogBegin(lsm_db *pDb);
   867         -int lsmLogWrite(lsm_db *, void *, int, void *, int);
          867  +int lsmLogWrite(lsm_db *, int, void *, int, void *, int);
   868    868   int lsmLogCommit(lsm_db *);
   869    869   void lsmLogEnd(lsm_db *pDb, int bCommit);
   870    870   void lsmLogTell(lsm_db *, LogMark *);
   871    871   void lsmLogSeek(lsm_db *, LogMark *);
   872    872   void lsmLogClose(lsm_db *);
   873    873   
   874    874   int lsmLogRecover(lsm_db *);
   875    875   int lsmInfoLogStructure(lsm_db *pDb, char **pzVal);
   876    876   
          877  +/* Valid values for the second argument to lsmLogWrite(). */
          878  +#define LSM_WRITE        0x06
          879  +#define LSM_DELETE       0x08
          880  +#define LSM_DRANGE       0x0A
   877    881   
   878    882   /**************************************************************************
   879    883   ** Functions from file "lsm_shared.c".
   880    884   */
   881    885   
   882    886   int lsmDbDatabaseConnect(lsm_db*, const char *);
   883    887   void lsmDbDatabaseRelease(lsm_db *);

Changes to ext/lsm1/lsm_log.c.

   195    195   #define LSM_LOG_PAD1         0x01
   196    196   #define LSM_LOG_PAD2         0x02
   197    197   #define LSM_LOG_COMMIT       0x03
   198    198   #define LSM_LOG_JUMP         0x04
   199    199   
   200    200   #define LSM_LOG_WRITE        0x06
   201    201   #define LSM_LOG_WRITE_CKSUM  0x07
          202  +
   202    203   #define LSM_LOG_DELETE       0x08
   203    204   #define LSM_LOG_DELETE_CKSUM 0x09
          205  +
          206  +#define LSM_LOG_DRANGE       0x0A
          207  +#define LSM_LOG_DRANGE_CKSUM 0x0B
   204    208   
   205    209   /* Require a checksum every 32KB. */
   206    210   #define LSM_CKSUM_MAXDATA (32*1024)
   207    211   
   208    212   /* Do not wrap a log file smaller than this in bytes. */
   209    213   #define LSM_MIN_LOGWRAP      (128*1024)
   210    214   
................................................................................
   649    653   
   650    654   /*
   651    655   ** Append an LSM_LOG_WRITE (if nVal>=0) or LSM_LOG_DELETE (if nVal<0) 
   652    656   ** record to the database log.
   653    657   */
   654    658   int lsmLogWrite(
   655    659     lsm_db *pDb,                    /* Database handle */
          660  +  int eType,
   656    661     void *pKey, int nKey,           /* Database key to write to log */
   657    662     void *pVal, int nVal            /* Database value (or nVal<0) to write */
   658    663   ){
   659    664     int rc = LSM_OK;
   660    665     LogWriter *pLog;                /* Log object to write to */
   661    666     int nReq;                       /* Bytes of space required in log */
   662    667     int bCksum = 0;                 /* True to embed a checksum in this record */
   663    668   
          669  +  assert( eType==LSM_WRITE || eType==LSM_DELETE || eType==LSM_DRANGE );
          670  +  assert( LSM_LOG_WRITE==LSM_WRITE );
          671  +  assert( LSM_LOG_DELETE==LSM_DELETE );
          672  +  assert( LSM_LOG_DRANGE==LSM_DRANGE );
          673  +  assert( (eType==LSM_LOG_DELETE)==(nVal<0) );
          674  +
   664    675     if( pDb->bUseLog==0 ) return LSM_OK;
   665    676     pLog = pDb->pLogWriter;
   666    677   
   667    678     /* Determine how many bytes of space are required, assuming that a checksum
   668    679     ** will be embedded in this record (even though it may not be).  */
   669    680     nReq = 1 + lsmVarintLen32(nKey) + 8 + nKey;
   670         -  if( nVal>=0 ) nReq += lsmVarintLen32(nVal) + nVal;
          681  +  if( eType!=LSM_LOG_DELETE ) nReq += lsmVarintLen32(nVal) + nVal;
   671    682   
   672    683     /* Jump over the jump region if required. Set bCksum to true to tell the
   673    684     ** code below to include a checksum in the record if either (a) writing
   674    685     ** this record would mean that more than LSM_CKSUM_MAXDATA bytes of data
   675    686     ** have been written to the log since the last checksum, or (b) the jump
   676    687     ** is taken.  */
   677    688     rc = jumpIfRequired(pDb, pLog, nReq, &bCksum);
................................................................................
   683    694     if( rc==LSM_OK ){
   684    695       u8 *a = (u8 *)&pLog->buf.z[pLog->buf.n];
   685    696       
   686    697       /* Write the record header - the type byte followed by either 1 (for
   687    698       ** DELETE) or 2 (for WRITE) varints.  */
   688    699       assert( LSM_LOG_WRITE_CKSUM == (LSM_LOG_WRITE | 0x0001) );
   689    700       assert( LSM_LOG_DELETE_CKSUM == (LSM_LOG_DELETE | 0x0001) );
   690         -    *(a++) = (nVal>=0 ? LSM_LOG_WRITE : LSM_LOG_DELETE) | (u8)bCksum;
          701  +    assert( LSM_LOG_DRANGE_CKSUM == (LSM_LOG_DRANGE | 0x0001) );
          702  +    *(a++) = (u8)eType | (u8)bCksum;
   691    703       a += lsmVarintPut32(a, nKey);
   692         -    if( nVal>=0 ) a += lsmVarintPut32(a, nVal);
          704  +    if( eType!=LSM_LOG_DELETE ) a += lsmVarintPut32(a, nVal);
   693    705   
   694    706       if( bCksum ){
   695    707         pLog->buf.n = (a - (u8 *)pLog->buf.z);
   696    708         rc = logCksumAndFlush(pDb);
   697    709         a = (u8 *)&pLog->buf.z[pLog->buf.n];
   698    710       }
   699    711   
   700    712       memcpy(a, pKey, nKey);
   701    713       a += nKey;
   702         -    if( nVal>=0 ){
          714  +    if( eType!=LSM_LOG_DELETE ){
   703    715         memcpy(a, pVal, nVal);
   704    716         a += nVal;
   705    717       }
   706    718       pLog->buf.n = a - (u8 *)pLog->buf.z;
   707    719       assert( pLog->buf.n<=pLog->buf.nAlloc );
   708    720     }
   709    721   
................................................................................
  1001   1013             case LSM_LOG_PAD2: {
  1002   1014               int nPad;
  1003   1015               logReaderVarint(&reader, &buf1, &nPad, &rc);
  1004   1016               logReaderBlob(&reader, &buf1, nPad, 0, &rc);
  1005   1017               break;
  1006   1018             }
  1007   1019   
         1020  +          case LSM_LOG_DRANGE:
         1021  +          case LSM_LOG_DRANGE_CKSUM:
  1008   1022             case LSM_LOG_WRITE:
  1009   1023             case LSM_LOG_WRITE_CKSUM: {
  1010   1024               int nKey;
  1011   1025               int nVal;
  1012   1026               u8 *aVal;
  1013   1027               logReaderVarint(&reader, &buf1, &nKey, &rc);
  1014   1028               logReaderVarint(&reader, &buf2, &nVal, &rc);
  1015   1029   
  1016         -            if( eType==LSM_LOG_WRITE_CKSUM ){
         1030  +            if( eType==LSM_LOG_WRITE_CKSUM || eType==LSM_LOG_DRANGE_CKSUM ){
  1017   1031                 logReaderCksum(&reader, &buf1, &bEof, &rc);
  1018   1032               }else{
  1019   1033                 bEof = logRequireCksum(&reader, nKey+nVal);
  1020   1034               }
  1021   1035               if( bEof ) break;
  1022   1036   
  1023   1037               logReaderBlob(&reader, &buf1, nKey, 0, &rc);
  1024   1038               logReaderBlob(&reader, &buf2, nVal, &aVal, &rc);
  1025   1039               if( iPass==1 && rc==LSM_OK ){ 
  1026         -              rc = lsmTreeInsert(pDb, (u8 *)buf1.z, nKey, aVal, nVal);
         1040  +              if( eType==LSM_LOG_WRITE || eType==LSM_LOG_WRITE_CKSUM ){
         1041  +                rc = lsmTreeInsert(pDb, (u8 *)buf1.z, nKey, aVal, nVal);
         1042  +              }else{
         1043  +                rc = lsmTreeDelete(pDb, (u8 *)buf1.z, nKey, aVal, nVal);
         1044  +              }
  1027   1045               }
  1028   1046               break;
  1029   1047             }
  1030   1048   
  1031   1049             case LSM_LOG_DELETE:
  1032   1050             case LSM_LOG_DELETE_CKSUM: {
  1033   1051               int nKey; u8 *aKey;

Changes to ext/lsm1/lsm_main.c.

   665    665   
   666    666     if( pDb->nTransOpen==0 ){
   667    667       bCommit = 1;
   668    668       rc = lsm_begin(pDb, 1);
   669    669     }
   670    670   
   671    671     if( rc==LSM_OK ){
   672         -    if( bDeleteRange==0 ){
   673         -      rc = lsmLogWrite(pDb, (void *)pKey, nKey, (void *)pVal, nVal);
   674         -    }else{
   675         -      /* TODO */
   676         -    }
          672  +    int eType = (bDeleteRange ? LSM_DRANGE : (nVal>=0?LSM_WRITE:LSM_DELETE));
          673  +    rc = lsmLogWrite(pDb, eType, (void *)pKey, nKey, (void *)pVal, nVal);
   677    674     }
   678    675   
   679    676     lsmSortedSaveTreeCursors(pDb);
   680    677   
   681    678     if( rc==LSM_OK ){
   682    679       int pgsz = lsmFsPageSize(pDb->pFS);
   683    680       int nQuant = LSM_AUTOWORK_QUANT * pgsz;