SQLite4
Check-in [8670182769]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Have lsmtest use zlib for compression if HAVE_ZLIB is defined. This causes at least one test to fail.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | compression-hooks
Files: files | file ages | folders
SHA1: 867018276943a0a7c15ae4a3b268c703f1060695
User & Date: dan 2012-10-24 19:54:41
Context
2012-10-25
11:08
Fix bug reading page data from a compressed database that occurs when the last page of a segment ends on the last byte of a block. check-in: 549868a020 user: dan tags: compression-hooks
2012-10-24
19:54
Have lsmtest use zlib for compression if HAVE_ZLIB is defined. This causes at least one test to fail. check-in: 8670182769 user: dan tags: compression-hooks
18:33
Fix memory leaks in compressed database mode. check-in: 083e3a6c0f user: dan tags: compression-hooks
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to lsm-test/lsmtest_tdb.c.

   609    609     const char *zName;
   610    610     const char *zDefaultDb;
   611    611     int (*xOpen)(const char *zFilename, int bClear, TestDb **ppDb);
   612    612   } aLib[] = {
   613    613     { "sqlite3",      "testdb.sqlite",    sql_open },
   614    614     { "lsm_small",    "testdb.lsm_small", test_lsm_small_open },
   615    615     { "lsm_lomem",    "testdb.lsm_lomem", test_lsm_lomem_open },
          616  +#ifdef HAVE_ZLIB
   616    617     { "lsm_zip",      "testdb.lsm_zip",   test_lsm_zip_open },
          618  +#endif
   617    619     { "lsm",          "testdb.lsm",       test_lsm_open },
   618    620   #ifdef LSM_MUTEX_PTHREADS
   619    621     { "lsm_mt2",      "testdb.lsm_mt2",   test_lsm_mt2 },
   620    622     { "lsm_mt3",      "testdb.lsm_mt3",   test_lsm_mt3 },
   621    623   #endif
   622    624   #ifdef HAVE_LEVELDB
   623    625     { "leveldb",      "testdb.leveldb",   test_leveldb_open },

Changes to lsm-test/lsmtest_tdb3.c.

   393    393   *************************************************************************/
   394    394   
   395    395   /*************************************************************************
   396    396   **************************************************************************
   397    397   ** Begin test compression hooks.
   398    398   */
   399    399   
          400  +#ifdef HAVE_ZLIB
          401  +#include <zlib.h>
          402  +
   400    403   static int testZipBound(void *pCtx, int nSrc){
   401         -  assert( 0 );
   402         -  return 0;
          404  +  return compressBound(nSrc);
   403    405   }
   404    406   
   405    407   static int testZipCompress(
   406         -  void *pCtx,                    /* Context pointer */
   407         -  char *aOut, int *pnOut,        /* OUT: Buffer containing compressed data */
   408         -  const char *aIn, int nIn       /* Buffer containing input data */
          408  +  void *pCtx,                     /* Context pointer */
          409  +  char *aOut, int *pnOut,         /* OUT: Buffer containing compressed data */
          410  +  const char *aIn, int nIn        /* Buffer containing input data */
   409    411   ){
   410         -  assert( 0 );
   411         -  return 0;
          412  +  uLongf n = *pnOut;              /* In/out buffer size for compress() */
          413  +  int rc;                         /* compress() return code */
          414  + 
          415  +  rc = compress((Bytef*)aOut, &n, (Bytef*)aIn, nIn);
          416  +  *pnOut = n;
          417  +  return (rc==Z_OK ? 0 : LSM_ERROR);
   412    418   }
   413    419   
   414    420   static int testZipUncompress(
   415         -  void *pCtx,                    /* Context pointer */
   416         -  char *aOut, int *pnOut,        /* OUT: Buffer containing uncompressed data */
   417         -  const char *aIn, int nIn       /* Buffer containing input data */
          421  +  void *pCtx,                     /* Context pointer */
          422  +  char *aOut, int *pnOut,         /* OUT: Buffer containing uncompressed data */
          423  +  const char *aIn, int nIn        /* Buffer containing input data */
   418    424   ){
   419         -  assert( 0 );
   420         -  return 0;
          425  +  uLongf n = *pnOut;              /* In/out buffer size for uncompress() */
          426  +  int rc;                         /* uncompress() return code */
          427  +
          428  +  rc = uncompress((Bytef*)aOut, &n, (Bytef*)aIn, nIn);
          429  +  *pnOut = n;
          430  +  return (rc==Z_OK ? 0 : LSM_ERROR);
   421    431   }
   422    432   
   423    433   static int testConfigureCompression(lsm_db *pDb){
   424    434     static lsm_compress zip = {
   425    435       1, sizeof(lsm_compress),
   426    436       0,                            /* Context pointer (unused) */
   427    437       testZipBound,                 /* xBound method */
   428    438       testZipCompress,              /* xCompress method */
   429    439       testZipUncompress             /* xUncompress method */
   430    440     };
   431    441     return lsm_config(pDb, LSM_CONFIG_SET_COMPRESSION, &zip);
   432    442   }
          443  +#endif /* ifdef HAVE_ZLIB */
   433    444   
   434    445   /*
   435    446   ** End test compression hooks.
   436    447   **************************************************************************
   437    448   *************************************************************************/
   438    449   
   439    450   static int test_lsm_close(TestDb *pTestDb){
................................................................................
   686    697       { "use_log",          0, LSM_CONFIG_USE_LOG },
   687    698       { "nmerge",           0, LSM_CONFIG_NMERGE },
   688    699       { "max_freelist",     0, LSM_CONFIG_MAX_FREELIST },
   689    700       { "multi_proc",       0, LSM_CONFIG_MULTIPLE_PROCESSES },
   690    701       { "worker_nmerge",    1, LSM_CONFIG_NMERGE },
   691    702       { "test_no_recovery", 0, TEST_NO_RECOVERY },
   692    703       { "threads",          0, TEST_THREADS },
          704  +#ifdef HAVE_ZLIB
   693    705       { "compression",      0, TEST_COMPRESSION },
          706  +#endif
   694    707       { 0, 0 }
   695    708     };
   696    709     const char *z = zStr;
   697    710     int nThread = 1;
   698    711   
   699    712     assert( db );
   700    713     while( z[0] ){
................................................................................
   738    751             switch( eParam ){
   739    752               case TEST_NO_RECOVERY:
   740    753                 pLsm->bNoRecovery = iVal;
   741    754                 break;
   742    755               case TEST_THREADS:
   743    756                 nThread = iVal;
   744    757                 break;
          758  +#ifdef HAVE_ZLIB
   745    759               case TEST_COMPRESSION:
   746    760                 testConfigureCompression(db);
   747    761                 break;
          762  +#endif
   748    763             }
   749    764           }
   750    765         }
   751    766       }else if( z!=zStart ){
   752    767         goto syntax_error;
   753    768       }
   754    769     }

Changes to src/lsm_file.c.

   168    168     lsm_file *fdDb;                 /* Database file */
   169    169     lsm_file *fdLog;                /* Log file */
   170    170   
   171    171     /* If this is a compressed database, a pointer to the compression methods.
   172    172     ** For an uncompressed database, a NULL pointer.  */
   173    173     lsm_compress *pCompress;
   174    174     u8 *aBuffer;                    /* Buffer to compress into */
          175  +  int nBuffer;                    /* Allocated size of aBuffer[] in bytes */
   175    176   
   176    177     /* mmap() mode things */
   177    178     int bUseMmap;                   /* True to use mmap() to access db file */
   178    179     void *pMap;                     /* Current mapping of database file */
   179    180     i64 nMap;                       /* Bytes mapped at pMap */
   180    181     Page *pFree;
   181    182   
................................................................................
   976    977       return LSM_OK;
   977    978     }
   978    979   
   979    980     rc = fsBlockNext(pFS, fsPageToBlock(pFS, iOff), &iBlk);
   980    981     *piRes = fsFirstPageOnBlock(pFS, iBlk) + iAdd - (iEob - iOff + 1);
   981    982     return rc;
   982    983   }
          984  +
          985  +static int fsAllocateBuffer(FileSystem *pFS){
          986  +  assert( pFS->pCompress );
          987  +  if( pFS->aBuffer==0 ){
          988  +    pFS->nBuffer = pFS->pCompress->xBound(pFS->pCompress->pCtx, pFS->nPagesize);
          989  +    pFS->aBuffer = lsmMalloc(pFS->pEnv, LSM_MAX(pFS->nBuffer, pFS->nPagesize));
          990  +    if( pFS->aBuffer==0 ) return LSM_NOMEM_BKPT;
          991  +  }
          992  +  return LSM_OK;
          993  +}
   983    994   
   984    995   /*
   985    996   ** This function is only called in compressed database mode. It reads and
   986    997   ** uncompresses the compressed data for page pPg from the database and
   987    998   ** populates the pPg->aData[] buffer and pPg->nCompress field.
   988    999   **
   989   1000   ** LSM_OK is returned if successful, or an LSM error code otherwise.
   990   1001   */
   991   1002   static int fsReadPagedata(
   992   1003     FileSystem *pFS,                /* File-system handle */
   993   1004     Page *pPg                       /* Page to read and uncompress data for */
   994   1005   ){
   995         -  i64 iOff;
         1006  +  lsm_compress *p = pFS->pCompress;
         1007  +  i64 iOff = pPg->iPg;
   996   1008     u8 aSz[6];
   997   1009     int rc;
   998   1010   
   999         -  assert( pFS->pCompress && pPg->nCompress==0 );
  1000         -  iOff = pPg->iPg;
         1011  +  assert( p && pPg->nCompress==0 );
         1012  +
         1013  +  if( fsAllocateBuffer(pFS) ) return LSM_NOMEM;
  1001   1014   
  1002   1015     rc = fsReadData(pFS, iOff, aSz, sizeof(aSz));
  1003   1016   
  1004   1017     if( rc==LSM_OK ){
  1005   1018       pPg->nCompress = (int)lsmGetU24(aSz);
  1006   1019       rc = fsAddOffset(pFS, iOff, 3, &iOff);
  1007         -    if( rc==LSM_OK ) rc = fsReadData(pFS, iOff, pPg->aData, pPg->nCompress);
         1020  +    if( rc==LSM_OK ){
         1021  +      if( pPg->nCompress>pFS->nBuffer ){
         1022  +        rc = LSM_CORRUPT_BKPT;
         1023  +      }else{
         1024  +        rc = fsReadData(pFS, iOff, pFS->aBuffer, pPg->nCompress);
         1025  +      }
         1026  +      if( rc==LSM_OK ){
         1027  +        int n = pFS->nBuffer;
         1028  +        rc = p->xUncompress(p->pCtx, 
         1029  +            (char *)pPg->aData, &n, 
         1030  +            (const char *)pFS->aBuffer, pPg->nCompress
         1031  +        );
         1032  +        if( rc==LSM_OK && n!=pPg->nData ){
         1033  +          rc = LSM_CORRUPT_BKPT;
         1034  +        }
         1035  +      }
         1036  +    }
  1008   1037     }
  1009   1038     return rc;
  1010   1039   }
  1011   1040   
  1012   1041   /*
  1013   1042   ** Return a handle for a database page.
  1014   1043   */
................................................................................
  1062   1091           p->iPg = iPg;
  1063   1092           p->nRef = 0;
  1064   1093           p->pFS = pFS;
  1065   1094           assert( p->flags==0 || p->flags==PAGE_FREE );
  1066   1095           if( pFS->pCompress==0 && (fsIsLast(pFS, iPg) || fsIsFirst(pFS, iPg)) ){
  1067   1096             p->flags |= PAGE_SHORT;
  1068   1097           }
         1098  +        p->nData =  pFS->nPagesize - (p->flags & PAGE_SHORT);
  1069   1099   
  1070   1100   #ifdef LSM_DEBUG
  1071   1101           memset(p->aData, 0x56, pFS->nPagesize);
  1072   1102   #endif
  1073   1103           assert( p->pLruNext==0 && p->pLruPrev==0 );
  1074   1104           if( noContent==0 ){
  1075   1105             if( pFS->pCompress ){
................................................................................
  1083   1113           }
  1084   1114   
  1085   1115           /* If the xRead() call was successful (or not attempted), link the
  1086   1116            ** page into the page-cache hash-table. Otherwise, if it failed,
  1087   1117            ** free the buffer. */
  1088   1118           if( rc==LSM_OK ){
  1089   1119             p->pHashNext = pFS->apHash[iHash];
  1090         -          p->nData =  pFS->nPagesize - (p->flags & PAGE_SHORT);
  1091   1120             pFS->apHash[iHash] = p;
  1092   1121           }else{
  1093   1122             fsPageBufferFree(p);
  1094   1123             p = 0;
  1095   1124           }
  1096   1125         }
  1097   1126       }else if( p->nRef==0 ){
................................................................................
  1459   1488   }
  1460   1489   
  1461   1490   /*
  1462   1491   ** Mark the sorted run passed as the second argument as finished. 
  1463   1492   */
  1464   1493   int lsmFsSortedFinish(FileSystem *pFS, Segment *p){
  1465   1494     int rc = LSM_OK;
  1466         -  if( p ){
         1495  +  if( p && p->iLastPg ){
  1467   1496       const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
         1497  +    int iBlk;
  1468   1498   
  1469   1499       /* Check if the last page of this run happens to be the last of a block.
  1470   1500       ** If it is, then an extra block has already been allocated for this run.
  1471   1501       ** Shift this extra block back to the free-block list. 
  1472   1502       **
  1473   1503       ** Otherwise, add the first free page in the last block used by the run
  1474   1504       ** to the lAppend list.
  1475   1505       */
  1476         -    if( (p->iLastPg % nPagePerBlock)==0 ){
  1477         -      Page *pLast;
  1478         -      rc = fsPageGet(pFS, p->iLastPg, 0, &pLast);
  1479         -      if( rc==LSM_OK ){
  1480         -        int iPg = (int)lsmGetU32(&pLast->aData[pFS->nPagesize-4]);
  1481         -        int iBlk = fsPageToBlock(pFS, iPg);
  1482         -        lsmBlockRefree(pFS->pDb, iBlk);
  1483         -        lsmFsPageRelease(pLast);
  1484         -      }
  1485         -    }else{
         1506  +    iBlk = fsPageToBlock(pFS, p->iLastPg);
         1507  +    if( fsLastPageOnBlock(pFS, fsPageToBlock(pFS, p->iLastPg) )!=p->iLastPg ){
  1486   1508         int i;
  1487   1509         u32 *aiAppend = pFS->pDb->pWorker->aiAppend;
  1488   1510         for(i=0; i<LSM_APPLIST_SZ; i++){
  1489   1511           if( aiAppend[i]==0 ){
  1490   1512             aiAppend[i] = p->iLastPg+1;
  1491   1513             break;
  1492   1514           }
         1515  +      }
         1516  +    }else if( pFS->pCompress==0 ){
         1517  +      Page *pLast;
         1518  +      rc = fsPageGet(pFS, p->iLastPg, 0, &pLast);
         1519  +      if( rc==LSM_OK ){
         1520  +        int iPg = (int)lsmGetU32(&pLast->aData[pFS->nPagesize-4]);
         1521  +        lsmBlockRefree(pFS->pDb, fsPageToBlock(pFS, iPg));
         1522  +        lsmFsPageRelease(pLast);
  1493   1523         }
  1494   1524       }
  1495   1525     }
  1496   1526     return rc;
  1497   1527   }
  1498   1528   
  1499   1529   /*
................................................................................
  1647   1677       iRet = iApp;
  1648   1678   
  1649   1679       /* Write as much data as is possible at iApp (usually all of it). */
  1650   1680       if( rc==LSM_OK ){
  1651   1681         int nSpace = fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iApp)) - iApp + 1;
  1652   1682         nWrite = LSM_MIN(nData, nSpace);
  1653   1683         nRem = nData - nWrite;
  1654         -      rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, aData, nWrite);
  1655         -
         1684  +      assert( nWrite>=0 );
         1685  +      if( nWrite!=0 ){
         1686  +        rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, aData, nWrite);
         1687  +      }
  1656   1688         iApp += nWrite;
  1657   1689       }
  1658   1690   
  1659   1691       /* If required, allocate a new block and write the rest of the data
  1660   1692       ** into it. Set the next and previous block pointers to link the new
  1661   1693       ** block to the old.  */
  1662   1694       if( rc==LSM_OK && nRem ){
................................................................................
  1700   1732   ** buffer at pFS->aBuffer. The size of the compressed data is stored in
  1701   1733   ** pPg->nCompress.
  1702   1734   **
  1703   1735   ** If buffer pFS->aBuffer[] has not been allocated then this function
  1704   1736   ** allocates it. If this fails, LSM_NOMEM is returned. Otherwise, LSM_OK.
  1705   1737   */
  1706   1738   static int fsCompressIntoBuffer(FileSystem *pFS, Page *pPg){
  1707         -  /* TODO: Fill in a real version of this function */
         1739  +  lsm_compress *p = pFS->pCompress;
  1708   1740   
  1709         -  if( pFS->aBuffer==0 ){
  1710         -    pFS->aBuffer = lsmMalloc(pFS->pEnv, pFS->nPagesize);
  1711         -    if( pFS->aBuffer==0 ) return LSM_NOMEM_BKPT;
  1712         -  }
  1713         -
         1741  +  if( fsAllocateBuffer(pFS) ) return LSM_NOMEM;
  1714   1742     assert( pPg->nData==pFS->nPagesize );
  1715         -  memcpy(pFS->aBuffer, pPg->aData, pFS->nPagesize);
  1716         -  pPg->nCompress = pFS->nPagesize;
  1717         -  return LSM_OK;
         1743  +
         1744  +  pPg->nCompress = pFS->nBuffer;
         1745  +  return p->xCompress(p->pCtx, 
         1746  +      (char *)pFS->aBuffer, &pPg->nCompress, 
         1747  +      (const char *)pPg->aData, pPg->nData
         1748  +  );
  1718   1749   }
  1719   1750   
  1720   1751   /*
  1721   1752   ** If the page passed as an argument is dirty, update the database file
  1722   1753   ** (or mapping of the database file) with its current contents and mark
  1723   1754   ** the page as clean.
  1724   1755   **