Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Have lsmtest use zlib for compression if HAVE_ZLIB is defined. This causes at least one test to fail. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | compression-hooks |
Files: | files | file ages | folders |
SHA1: |
867018276943a0a7c15ae4a3b268c703 |
User & Date: | dan 2012-10-24 19:54:41.724 |
Context
2012-10-25
| ||
11:08 | Fix bug reading page data from a compressed database that occurs when the last page of a segment ends on the last byte of a block. check-in: 549868a020 user: dan tags: compression-hooks | |
2012-10-24
| ||
19:54 | Have lsmtest use zlib for compression if HAVE_ZLIB is defined. This causes at least one test to fail. check-in: 8670182769 user: dan tags: compression-hooks | |
18:33 | Fix memory leaks in compressed database mode. check-in: 083e3a6c0f user: dan tags: compression-hooks | |
Changes
Changes to lsm-test/lsmtest_tdb.c.
︙ | ︙ | |||
609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 | const char *zName; const char *zDefaultDb; int (*xOpen)(const char *zFilename, int bClear, TestDb **ppDb); } aLib[] = { { "sqlite3", "testdb.sqlite", sql_open }, { "lsm_small", "testdb.lsm_small", test_lsm_small_open }, { "lsm_lomem", "testdb.lsm_lomem", test_lsm_lomem_open }, { "lsm_zip", "testdb.lsm_zip", test_lsm_zip_open }, { "lsm", "testdb.lsm", test_lsm_open }, #ifdef LSM_MUTEX_PTHREADS { "lsm_mt2", "testdb.lsm_mt2", test_lsm_mt2 }, { "lsm_mt3", "testdb.lsm_mt3", test_lsm_mt3 }, #endif #ifdef HAVE_LEVELDB { "leveldb", "testdb.leveldb", test_leveldb_open }, | > > | 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 | const char *zName; const char *zDefaultDb; int (*xOpen)(const char *zFilename, int bClear, TestDb **ppDb); } aLib[] = { { "sqlite3", "testdb.sqlite", sql_open }, { "lsm_small", "testdb.lsm_small", test_lsm_small_open }, { "lsm_lomem", "testdb.lsm_lomem", test_lsm_lomem_open }, #ifdef HAVE_ZLIB { "lsm_zip", "testdb.lsm_zip", test_lsm_zip_open }, #endif { "lsm", "testdb.lsm", test_lsm_open }, #ifdef LSM_MUTEX_PTHREADS { "lsm_mt2", "testdb.lsm_mt2", test_lsm_mt2 }, { "lsm_mt3", "testdb.lsm_mt3", test_lsm_mt3 }, #endif #ifdef HAVE_LEVELDB { "leveldb", "testdb.leveldb", test_leveldb_open }, |
︙ | ︙ |
Changes to lsm-test/lsmtest_tdb3.c.
︙ | ︙ | |||
393 394 395 396 397 398 399 400 | *************************************************************************/ /************************************************************************* ************************************************************************** ** Begin test compression hooks. */ static int testZipBound(void *pCtx, int nSrc){ | > > > < | | | | > > | > > | | | | > > | > > | > | 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 | *************************************************************************/ /************************************************************************* ************************************************************************** ** Begin test compression hooks. */ #ifdef HAVE_ZLIB #include <zlib.h> static int testZipBound(void *pCtx, int nSrc){ return compressBound(nSrc); } static int testZipCompress( void *pCtx, /* Context pointer */ char *aOut, int *pnOut, /* OUT: Buffer containing compressed data */ const char *aIn, int nIn /* Buffer containing input data */ ){ uLongf n = *pnOut; /* In/out buffer size for compress() */ int rc; /* compress() return code */ rc = compress((Bytef*)aOut, &n, (Bytef*)aIn, nIn); *pnOut = n; return (rc==Z_OK ? 0 : LSM_ERROR); } static int testZipUncompress( void *pCtx, /* Context pointer */ char *aOut, int *pnOut, /* OUT: Buffer containing uncompressed data */ const char *aIn, int nIn /* Buffer containing input data */ ){ uLongf n = *pnOut; /* In/out buffer size for uncompress() */ int rc; /* uncompress() return code */ rc = uncompress((Bytef*)aOut, &n, (Bytef*)aIn, nIn); *pnOut = n; return (rc==Z_OK ? 0 : LSM_ERROR); } static int testConfigureCompression(lsm_db *pDb){ static lsm_compress zip = { 1, sizeof(lsm_compress), 0, /* Context pointer (unused) */ testZipBound, /* xBound method */ testZipCompress, /* xCompress method */ testZipUncompress /* xUncompress method */ }; return lsm_config(pDb, LSM_CONFIG_SET_COMPRESSION, &zip); } #endif /* ifdef HAVE_ZLIB */ /* ** End test compression hooks. ************************************************************************** *************************************************************************/ static int test_lsm_close(TestDb *pTestDb){ |
︙ | ︙ | |||
686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 | { "use_log", 0, LSM_CONFIG_USE_LOG }, { "nmerge", 0, LSM_CONFIG_NMERGE }, { "max_freelist", 0, LSM_CONFIG_MAX_FREELIST }, { "multi_proc", 0, LSM_CONFIG_MULTIPLE_PROCESSES }, { "worker_nmerge", 1, LSM_CONFIG_NMERGE }, { "test_no_recovery", 0, TEST_NO_RECOVERY }, { "threads", 0, TEST_THREADS }, { "compression", 0, TEST_COMPRESSION }, { 0, 0 } }; const char *z = zStr; int nThread = 1; assert( db ); while( z[0] ){ | > > | 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 | { "use_log", 0, LSM_CONFIG_USE_LOG }, { "nmerge", 0, LSM_CONFIG_NMERGE }, { "max_freelist", 0, LSM_CONFIG_MAX_FREELIST }, { "multi_proc", 0, LSM_CONFIG_MULTIPLE_PROCESSES }, { "worker_nmerge", 1, LSM_CONFIG_NMERGE }, { "test_no_recovery", 0, TEST_NO_RECOVERY }, { "threads", 0, TEST_THREADS }, #ifdef HAVE_ZLIB { "compression", 0, TEST_COMPRESSION }, #endif { 0, 0 } }; const char *z = zStr; int nThread = 1; assert( db ); while( z[0] ){ |
︙ | ︙ | |||
738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 | switch( eParam ){ case TEST_NO_RECOVERY: pLsm->bNoRecovery = iVal; break; case TEST_THREADS: nThread = iVal; break; case TEST_COMPRESSION: testConfigureCompression(db); break; } } } }else if( z!=zStart ){ goto syntax_error; } } | > > | 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 | switch( eParam ){ case TEST_NO_RECOVERY: pLsm->bNoRecovery = iVal; break; case TEST_THREADS: nThread = iVal; break; #ifdef HAVE_ZLIB case TEST_COMPRESSION: testConfigureCompression(db); break; #endif } } } }else if( z!=zStart ){ goto syntax_error; } } |
︙ | ︙ |
Changes to src/lsm_file.c.
︙ | ︙ | |||
168 169 170 171 172 173 174 175 176 177 178 179 180 181 | lsm_file *fdDb; /* Database file */ lsm_file *fdLog; /* Log file */ /* If this is a compressed database, a pointer to the compression methods. ** For an uncompressed database, a NULL pointer. */ lsm_compress *pCompress; u8 *aBuffer; /* Buffer to compress into */ /* mmap() mode things */ int bUseMmap; /* True to use mmap() to access db file */ void *pMap; /* Current mapping of database file */ i64 nMap; /* Bytes mapped at pMap */ Page *pFree; | > | 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 | lsm_file *fdDb; /* Database file */ lsm_file *fdLog; /* Log file */ /* If this is a compressed database, a pointer to the compression methods. ** For an uncompressed database, a NULL pointer. */ lsm_compress *pCompress; u8 *aBuffer; /* Buffer to compress into */ int nBuffer; /* Allocated size of aBuffer[] in bytes */ /* mmap() mode things */ int bUseMmap; /* True to use mmap() to access db file */ void *pMap; /* Current mapping of database file */ i64 nMap; /* Bytes mapped at pMap */ Page *pFree; |
︙ | ︙ | |||
976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 | return LSM_OK; } rc = fsBlockNext(pFS, fsPageToBlock(pFS, iOff), &iBlk); *piRes = fsFirstPageOnBlock(pFS, iBlk) + iAdd - (iEob - iOff + 1); return rc; } /* ** This function is only called in compressed database mode. It reads and ** uncompresses the compressed data for page pPg from the database and ** populates the pPg->aData[] buffer and pPg->nCompress field. ** ** LSM_OK is returned if successful, or an LSM error code otherwise. */ static int fsReadPagedata( FileSystem *pFS, /* File-system handle */ Page *pPg /* Page to read and uncompress data for */ ){ | > > > > > > > > > > > | | | > | > > > > > > > > > > > > > > > > | 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 | return LSM_OK; } rc = fsBlockNext(pFS, fsPageToBlock(pFS, iOff), &iBlk); *piRes = fsFirstPageOnBlock(pFS, iBlk) + iAdd - (iEob - iOff + 1); return rc; } static int fsAllocateBuffer(FileSystem *pFS){ assert( pFS->pCompress ); if( pFS->aBuffer==0 ){ pFS->nBuffer = pFS->pCompress->xBound(pFS->pCompress->pCtx, pFS->nPagesize); pFS->aBuffer = lsmMalloc(pFS->pEnv, LSM_MAX(pFS->nBuffer, pFS->nPagesize)); if( pFS->aBuffer==0 ) return LSM_NOMEM_BKPT; } return LSM_OK; } /* ** This function is only called in compressed database mode. It reads and ** uncompresses the compressed data for page pPg from the database and ** populates the pPg->aData[] buffer and pPg->nCompress field. ** ** LSM_OK is returned if successful, or an LSM error code otherwise. */ static int fsReadPagedata( FileSystem *pFS, /* File-system handle */ Page *pPg /* Page to read and uncompress data for */ ){ lsm_compress *p = pFS->pCompress; i64 iOff = pPg->iPg; u8 aSz[6]; int rc; assert( p && pPg->nCompress==0 ); if( fsAllocateBuffer(pFS) ) return LSM_NOMEM; rc = fsReadData(pFS, iOff, aSz, sizeof(aSz)); if( rc==LSM_OK ){ pPg->nCompress = (int)lsmGetU24(aSz); rc = fsAddOffset(pFS, iOff, 3, &iOff); if( rc==LSM_OK ){ if( pPg->nCompress>pFS->nBuffer ){ rc = LSM_CORRUPT_BKPT; }else{ rc = fsReadData(pFS, iOff, pFS->aBuffer, pPg->nCompress); } if( rc==LSM_OK ){ int n = pFS->nBuffer; rc = p->xUncompress(p->pCtx, (char *)pPg->aData, &n, (const char *)pFS->aBuffer, pPg->nCompress ); if( rc==LSM_OK && n!=pPg->nData ){ rc = LSM_CORRUPT_BKPT; } } } } return rc; } /* ** Return a handle for a database page. */ |
︙ | ︙ | |||
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 | p->iPg = iPg; p->nRef = 0; p->pFS = pFS; assert( p->flags==0 || p->flags==PAGE_FREE ); if( pFS->pCompress==0 && (fsIsLast(pFS, iPg) || fsIsFirst(pFS, iPg)) ){ p->flags |= PAGE_SHORT; } #ifdef LSM_DEBUG memset(p->aData, 0x56, pFS->nPagesize); #endif assert( p->pLruNext==0 && p->pLruPrev==0 ); if( noContent==0 ){ if( pFS->pCompress ){ | > | 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 | p->iPg = iPg; p->nRef = 0; p->pFS = pFS; assert( p->flags==0 || p->flags==PAGE_FREE ); if( pFS->pCompress==0 && (fsIsLast(pFS, iPg) || fsIsFirst(pFS, iPg)) ){ p->flags |= PAGE_SHORT; } p->nData = pFS->nPagesize - (p->flags & PAGE_SHORT); #ifdef LSM_DEBUG memset(p->aData, 0x56, pFS->nPagesize); #endif assert( p->pLruNext==0 && p->pLruPrev==0 ); if( noContent==0 ){ if( pFS->pCompress ){ |
︙ | ︙ | |||
1083 1084 1085 1086 1087 1088 1089 | } /* If the xRead() call was successful (or not attempted), link the ** page into the page-cache hash-table. Otherwise, if it failed, ** free the buffer. */ if( rc==LSM_OK ){ p->pHashNext = pFS->apHash[iHash]; | < | 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 | } /* If the xRead() call was successful (or not attempted), link the ** page into the page-cache hash-table. Otherwise, if it failed, ** free the buffer. */ if( rc==LSM_OK ){ p->pHashNext = pFS->apHash[iHash]; pFS->apHash[iHash] = p; }else{ fsPageBufferFree(p); p = 0; } } }else if( p->nRef==0 ){ |
︙ | ︙ | |||
1459 1460 1461 1462 1463 1464 1465 | } /* ** Mark the sorted run passed as the second argument as finished. */ int lsmFsSortedFinish(FileSystem *pFS, Segment *p){ int rc = LSM_OK; | | > < < < < < | | < < < > > > > > > > > | 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 | } /* ** Mark the sorted run passed as the second argument as finished. */ int lsmFsSortedFinish(FileSystem *pFS, Segment *p){ int rc = LSM_OK; if( p && p->iLastPg ){ const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); int iBlk; /* Check if the last page of this run happens to be the last of a block. ** If it is, then an extra block has already been allocated for this run. ** Shift this extra block back to the free-block list. ** ** Otherwise, add the first free page in the last block used by the run ** to the lAppend list. */ iBlk = fsPageToBlock(pFS, p->iLastPg); if( fsLastPageOnBlock(pFS, fsPageToBlock(pFS, p->iLastPg) )!=p->iLastPg ){ int i; u32 *aiAppend = pFS->pDb->pWorker->aiAppend; for(i=0; i<LSM_APPLIST_SZ; i++){ if( aiAppend[i]==0 ){ aiAppend[i] = p->iLastPg+1; break; } } }else if( pFS->pCompress==0 ){ Page *pLast; rc = fsPageGet(pFS, p->iLastPg, 0, &pLast); if( rc==LSM_OK ){ int iPg = (int)lsmGetU32(&pLast->aData[pFS->nPagesize-4]); lsmBlockRefree(pFS->pDb, fsPageToBlock(pFS, iPg)); lsmFsPageRelease(pLast); } } } return rc; } /* |
︙ | ︙ | |||
1647 1648 1649 1650 1651 1652 1653 | iRet = iApp; /* Write as much data as is possible at iApp (usually all of it). */ if( rc==LSM_OK ){ int nSpace = fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iApp)) - iApp + 1; nWrite = LSM_MIN(nData, nSpace); nRem = nData - nWrite; | > > | | | 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 | iRet = iApp; /* Write as much data as is possible at iApp (usually all of it). */ if( rc==LSM_OK ){ int nSpace = fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iApp)) - iApp + 1; nWrite = LSM_MIN(nData, nSpace); nRem = nData - nWrite; assert( nWrite>=0 ); if( nWrite!=0 ){ rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, aData, nWrite); } iApp += nWrite; } /* If required, allocate a new block and write the rest of the data ** into it. Set the next and previous block pointers to link the new ** block to the old. */ if( rc==LSM_OK && nRem ){ |
︙ | ︙ | |||
1700 1701 1702 1703 1704 1705 1706 | ** buffer at pFS->aBuffer. The size of the compressed data is stored in ** pPg->nCompress. ** ** If buffer pFS->aBuffer[] has not been allocated then this function ** allocates it. If this fails, LSM_NOMEM is returned. Otherwise, LSM_OK. */ static int fsCompressIntoBuffer(FileSystem *pFS, Page *pPg){ | < | < < < | | | | | > > > | 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 | ** buffer at pFS->aBuffer. The size of the compressed data is stored in ** pPg->nCompress. ** ** If buffer pFS->aBuffer[] has not been allocated then this function ** allocates it. If this fails, LSM_NOMEM is returned. Otherwise, LSM_OK. */ static int fsCompressIntoBuffer(FileSystem *pFS, Page *pPg){ lsm_compress *p = pFS->pCompress; if( fsAllocateBuffer(pFS) ) return LSM_NOMEM; assert( pPg->nData==pFS->nPagesize ); pPg->nCompress = pFS->nBuffer; return p->xCompress(p->pCtx, (char *)pFS->aBuffer, &pPg->nCompress, (const char *)pPg->aData, pPg->nData ); } /* ** If the page passed as an argument is dirty, update the database file ** (or mapping of the database file) with its current contents and mark ** the page as clean. ** |
︙ | ︙ |