Index: src/lsm.h ================================================================== --- src/lsm.h +++ src/lsm.h @@ -22,10 +22,11 @@ /* ** Opaque handle types. */ typedef struct lsm_compress lsm_compress; /* Compression library functions */ +typedef struct lsm_compress_factory lsm_compress_factory; typedef struct lsm_cursor lsm_cursor; /* Database cursor handle */ typedef struct lsm_db lsm_db; /* Database connection handle */ typedef struct lsm_env lsm_env; /* Runtime environment */ typedef struct lsm_file lsm_file; /* OS file handle */ typedef struct lsm_mutex lsm_mutex; /* Mutex handle */ @@ -104,10 +105,12 @@ #define LSM_FULL 13 #define LSM_CANTOPEN 14 #define LSM_PROTOCOL 15 #define LSM_MISUSE 21 +#define LSM_MISMATCH 50 + /* ** CAPI: Creating and Destroying Database Connection Handles ** ** Open and close a database connection handle. */ @@ -118,11 +121,11 @@ ** CAPI: Connecting to a Database */ int lsm_open(lsm_db *pDb, const char *zFilename); /* -** CAPI: Obtaining pointers to databases environments +** CAPI: Obtaining pointers to database environments ** ** Return a pointer to the environment used by the database connection ** passed as the first argument. Assuming the argument is valid, this ** function always returns a valid environment pointer - it cannot fail. */ @@ -253,24 +256,29 @@ ** after lsm_open() has been called results in an LSM_MISUSE error. ** ** LSM_CONFIG_GET_COMPRESSION: ** Query the compression methods used to compress and decompress database ** content. +** +** LSM_CONFIG_SET_COMPRESSION_FACTORY: +** Configure a factory method to be invoked in case of an LSM_MISMATCH +** error. */ -#define LSM_CONFIG_AUTOFLUSH 1 -#define LSM_CONFIG_PAGE_SIZE 2 -#define LSM_CONFIG_SAFETY 3 -#define LSM_CONFIG_BLOCK_SIZE 4 -#define LSM_CONFIG_AUTOWORK 5 -#define LSM_CONFIG_MMAP 7 -#define LSM_CONFIG_USE_LOG 8 -#define LSM_CONFIG_AUTOMERGE 9 -#define LSM_CONFIG_MAX_FREELIST 10 -#define LSM_CONFIG_MULTIPLE_PROCESSES 11 -#define LSM_CONFIG_AUTOCHECKPOINT 12 -#define LSM_CONFIG_SET_COMPRESSION 13 -#define LSM_CONFIG_GET_COMPRESSION 14 +#define LSM_CONFIG_AUTOFLUSH 1 +#define LSM_CONFIG_PAGE_SIZE 2 +#define LSM_CONFIG_SAFETY 3 +#define LSM_CONFIG_BLOCK_SIZE 4 +#define LSM_CONFIG_AUTOWORK 5 +#define LSM_CONFIG_MMAP 7 +#define LSM_CONFIG_USE_LOG 8 +#define LSM_CONFIG_AUTOMERGE 9 +#define LSM_CONFIG_MAX_FREELIST 10 +#define LSM_CONFIG_MULTIPLE_PROCESSES 11 +#define LSM_CONFIG_AUTOCHECKPOINT 12 +#define LSM_CONFIG_SET_COMPRESSION 13 +#define LSM_CONFIG_GET_COMPRESSION 14 +#define LSM_CONFIG_SET_COMPRESSION_FACTORY 15 #define LSM_SAFETY_OFF 0 #define LSM_SAFETY_NORMAL 1 #define LSM_SAFETY_FULL 2 @@ -281,11 +289,21 @@ void *pCtx; unsigned int iId; int (*xBound)(void *, int nSrc); int (*xCompress)(void *, char *, int *, const char *, int); int (*xUncompress)(void *, char *, int *, const char *, int); + void (*xFree)(void *pCtx); +}; + +struct lsm_compress_factory { + void *pCtx; + int (*xFactory)(void *, lsm_db *, unsigned int); + void (*xFree)(void *pCtx); }; + +#define LSM_COMPRESSION_EMPTY 0 +#define LSM_COMPRESSION_NONE 1 /* ** CAPI: Allocating and Freeing Memory ** Index: src/lsmInt.h ================================================================== --- src/lsmInt.h +++ src/lsmInt.h @@ -314,10 +314,11 @@ int nMaxFreelist; /* Configured by LSM_CONFIG_MAX_FREELIST */ int bMmap; /* Configured by LSM_CONFIG_MMAP */ i64 nAutockpt; /* Configured by LSM_CONFIG_AUTOCHECKPOINT */ int bMultiProc; /* Configured by L_C_MULTIPLE_PROCESSES */ lsm_compress compress; /* Compression callbacks */ + lsm_compress_factory factory; /* Compression callback factory */ /* Sub-system handles */ FileSystem *pFS; /* On-disk portion of database */ Database *pDatabase; /* Database shared data */ @@ -524,10 +525,11 @@ ** to read or write a database file on disk. See the description of struct ** Database below for futher details. */ struct Snapshot { Database *pDatabase; /* Database this snapshot belongs to */ + u32 iCmpId; /* Id of compression scheme */ Level *pLevel; /* Pointer to level 0 of snapshot (or NULL) */ i64 iId; /* Snapshot id */ i64 iLogOff; /* Log file offset */ Redirect redirect; /* Block redirection array */ @@ -641,10 +643,12 @@ /************************************************************************** ** Start of functions from "lsm_file.c". */ int lsmFsOpen(lsm_db *, const char *); void lsmFsClose(FileSystem *); + +int lsmFsConfigure(lsm_db *db); int lsmFsBlockSize(FileSystem *); void lsmFsSetBlockSize(FileSystem *, int); int lsmFsPageSize(FileSystem *); Index: src/lsm_ckpt.c ================================================================== --- src/lsm_ckpt.c +++ src/lsm_ckpt.c @@ -30,15 +30,16 @@ ** ** 1. The checkpoint id MSW. ** 2. The checkpoint id LSW. ** 3. The number of integer values in the entire checkpoint, including ** the two checksum values. -** 4. The total number of blocks in the database. -** 5. The block size. -** 6. The number of levels. -** 7. The nominal database page size. -** 8. The number of pages (in total) written to the database file. +** 4. The compression scheme id. +** 5. The total number of blocks in the database. +** 6. The block size. +** 7. The number of levels. +** 8. The nominal database page size. +** 9. The number of pages (in total) written to the database file. ** ** Log pointer: ** ** 1. The log offset MSW. ** 2. The log offset LSW. @@ -172,28 +173,29 @@ static const int one = 1; #define LSM_LITTLE_ENDIAN (*(u8 *)(&one)) /* Sizes, in integers, of various parts of the checkpoint. */ -#define CKPT_HDR_SIZE 8 +#define CKPT_HDR_SIZE 9 #define CKPT_LOGPTR_SIZE 4 #define CKPT_APPENDLIST_SIZE (LSM_APPLIST_SZ * 2) /* A #define to describe each integer in the checkpoint header. */ #define CKPT_HDR_ID_MSW 0 #define CKPT_HDR_ID_LSW 1 #define CKPT_HDR_NCKPT 2 -#define CKPT_HDR_NBLOCK 3 -#define CKPT_HDR_BLKSZ 4 -#define CKPT_HDR_NLEVEL 5 -#define CKPT_HDR_PGSZ 6 -#define CKPT_HDR_NWRITE 7 - -#define CKPT_HDR_LO_MSW 8 -#define CKPT_HDR_LO_LSW 9 -#define CKPT_HDR_LO_CKSUM1 10 -#define CKPT_HDR_LO_CKSUM2 11 +#define CKPT_HDR_CMPID 3 +#define CKPT_HDR_NBLOCK 4 +#define CKPT_HDR_BLKSZ 5 +#define CKPT_HDR_NLEVEL 6 +#define CKPT_HDR_PGSZ 7 +#define CKPT_HDR_NWRITE 8 + +#define CKPT_HDR_LO_MSW 9 +#define CKPT_HDR_LO_LSW 10 +#define CKPT_HDR_LO_CKSUM1 11 +#define CKPT_HDR_LO_CKSUM2 12 typedef struct CkptBuffer CkptBuffer; /* ** Dynamic buffer used to accumulate data for a checkpoint. @@ -450,10 +452,11 @@ /* Write the checkpoint header */ assert( iId>=0 ); ckptSetValue(&ckpt, CKPT_HDR_ID_MSW, (u32)(iId>>32), &rc); ckptSetValue(&ckpt, CKPT_HDR_ID_LSW, (u32)(iId&0xFFFFFFFF), &rc); ckptSetValue(&ckpt, CKPT_HDR_NCKPT, iOut+2, &rc); + ckptSetValue(&ckpt, CKPT_HDR_CMPID, pSnap->iCmpId, &rc); ckptSetValue(&ckpt, CKPT_HDR_NBLOCK, pSnap->nBlock, &rc); ckptSetValue(&ckpt, CKPT_HDR_BLKSZ, lsmFsBlockSize(pFS), &rc); ckptSetValue(&ckpt, CKPT_HDR_NLEVEL, nLevel, &rc); ckptSetValue(&ckpt, CKPT_HDR_PGSZ, lsmFsPageSize(pFS), &rc); ckptSetValue(&ckpt, CKPT_HDR_NWRITE, pSnap->nWrite, &rc); @@ -759,23 +762,24 @@ ** Initialize the shared-memory header with an empty snapshot. This function ** is called when no valid snapshot can be found in the database header. */ static void ckptLoadEmpty(lsm_db *pDb){ u32 aCkpt[] = { - 0, /* CKPT_HDR_ID_MSW */ - 10, /* CKPT_HDR_ID_LSW */ - 0, /* CKPT_HDR_NCKPT */ - 0, /* CKPT_HDR_NBLOCK */ - 0, /* CKPT_HDR_BLKSZ */ - 0, /* CKPT_HDR_NLEVEL */ - 0, /* CKPT_HDR_PGSZ */ - 0, /* CKPT_HDR_OVFL */ - 0, /* CKPT_HDR_NWRITE */ - 0, 0, 1234, 5678, /* The log pointer and initial checksum */ - 0,0,0,0, 0,0,0,0, /* The append list */ - 0, /* The free block list */ - 0, 0 /* Space for checksum values */ + 0, /* CKPT_HDR_ID_MSW */ + 10, /* CKPT_HDR_ID_LSW */ + 0, /* CKPT_HDR_NCKPT */ + LSM_COMPRESSION_EMPTY, /* CKPT_HDR_CMPID */ + 0, /* CKPT_HDR_NBLOCK */ + 0, /* CKPT_HDR_BLKSZ */ + 0, /* CKPT_HDR_NLEVEL */ + 0, /* CKPT_HDR_PGSZ */ + 0, /* CKPT_HDR_OVFL */ + 0, /* CKPT_HDR_NWRITE */ + 0, 0, 1234, 5678, /* The log pointer and initial checksum */ + 0,0,0,0, 0,0,0,0, /* The append list */ + 0, /* The free block list */ + 0, 0 /* Space for checksum values */ }; u32 nCkpt = array_size(aCkpt); ShmHeader *pShm = pDb->pShmhdr; aCkpt[CKPT_HDR_NCKPT] = nCkpt; @@ -919,10 +923,16 @@ } } rc = lsmCheckpointDeserialize(pDb, 1, pShm->aSnap1, &pDb->pWorker); if( pDb->pWorker ) pDb->pWorker->pDatabase = pDb->pDatabase; + + if( rc==LSM_OK && pDb->pWorker->iCmpId!=pDb->compress.iId ){ + lsmFreeSnapshot(pDb->pEnv, pDb->pWorker); + rc = LSM_MISMATCH; + pDb->pWorker = 0; + } #if 0 assert( rc!=LSM_OK || lsmFsIntegrityCheck(pDb) ); #endif return rc; @@ -948,10 +958,15 @@ pNew->iId = lsmCheckpointId(aCkpt, 0); pNew->nBlock = aCkpt[CKPT_HDR_NBLOCK]; pNew->nWrite = aCkpt[CKPT_HDR_NWRITE]; rc = ckptLoadLevels(pDb, aCkpt, &iIn, nLevel, &pNew->pLevel); pNew->iLogOff = lsmCheckpointLogOffset(aCkpt); + + pNew->iCmpId = aCkpt[CKPT_HDR_CMPID]; + if( pNew->iCmpId==LSM_COMPRESSION_EMPTY ){ + pNew->iCmpId = pDb->compress.iId; + } /* Make a copy of the append-list */ for(i=0; iaiAppend[i] = ckptRead64(a); Index: src/lsm_file.c ================================================================== --- src/lsm_file.c +++ src/lsm_file.c @@ -508,15 +508,10 @@ pFS->nPagesize = LSM_DFLT_PAGE_SIZE; pFS->nBlocksize = LSM_DFLT_BLOCK_SIZE; pFS->nMetasize = 4 * 1024; pFS->pDb = pDb; pFS->pEnv = pDb->pEnv; - if( pDb->compress.xCompress ){ - pFS->pCompress = &pDb->compress; - }else{ - pFS->bUseMmap = pDb->bMmap; - } /* Make a copy of the database and log file names. */ memcpy(pFS->zDb, zDb, nDb+1); memcpy(pFS->zLog, zDb, nDb); memcpy(&pFS->zLog[nDb], "-log", 5); @@ -549,10 +544,60 @@ } pDb->pFS = pFS; return rc; } + +/* +** Configure the file-system object according to the current values of +** the LSM_CONFIG_MMAP and LSM_CONFIG_SET_COMPRESSION options. +*/ +int lsmFsConfigure(lsm_db *db){ + FileSystem *pFS = db->pFS; + lsm_env *pEnv = pFS->pEnv; + Page *pPg; + + assert( pFS->nOut==0 ); + assert( pFS->pWaiting==0 ); + + /* Reset any compression/decompression buffers already allocated */ + lsmFree(pEnv, pFS->aIBuffer); + lsmFree(pEnv, pFS->aOBuffer); + pFS->nBuffer = 0; + + /* Unmap the file, if it is currently mapped */ + if( pFS->pMap ){ + lsmEnvRemap(pEnv, pFS->fdDb, -1, &pFS->pMap, &pFS->nMap); + pFS->bUseMmap = 0; + } + + /* Free all allocate page structures */ + pPg = pFS->pLruFirst; + while( pPg ){ + Page *pNext = pPg->pLruNext; + if( pPg->flags & PAGE_FREE ) lsmFree(pEnv, pPg->aData); + lsmFree(pEnv, pPg); + pPg = pNext; + } + + /* Zero pointers that point to deleted page objects */ + pFS->nCacheAlloc = 0; + pFS->pLruFirst = 0; + pFS->pLruLast = 0; + pFS->pFree = 0; + + /* Configure the FileSystem object */ + if( db->compress.xCompress ){ + pFS->pCompress = &db->compress; + pFS->bUseMmap = 0; + }else{ + pFS->pCompress = 0; + pFS->bUseMmap = db->bMmap; + } + + return LSM_OK; +} /* ** Close and destroy a FileSystem object. */ void lsmFsClose(FileSystem *pFS){ Index: src/lsm_main.c ================================================================== --- src/lsm_main.c +++ src/lsm_main.c @@ -94,10 +94,11 @@ pDb->bUseLog = LSM_DFLT_USE_LOG; pDb->iReader = -1; pDb->bMultiProc = LSM_DFLT_MULTIPLE_PROCESSES; pDb->bMmap = LSM_DFLT_MMAP; pDb->xLog = xLog; + pDb->compress.iId = LSM_COMPRESSION_NONE; return LSM_OK; } lsm_env *lsm_get_env(lsm_db *pDb){ assert( pDb->pEnv ); @@ -192,10 +193,16 @@ lsmFreeSnapshot(pDb->pEnv, pDb->pClient); pDb->pClient = 0; lsmDbDatabaseRelease(pDb); lsmLogClose(pDb); lsmFsClose(pDb->pFS); + + /* Invoke any destructors registered for the compression or + ** compression factory callbacks. */ + if( pDb->factory.xFree ) pDb->factory.xFree(pDb->factory.pCtx); + if( pDb->compress.xFree ) pDb->compress.xFree(pDb->compress.pCtx); + lsmFree(pDb->pEnv, pDb->rollback.aArray); lsmFree(pDb->pEnv, pDb->aTrans); lsmFree(pDb->pEnv, pDb->apShm); lsmFree(pDb->pEnv, pDb); } @@ -335,16 +342,32 @@ break; } case LSM_CONFIG_SET_COMPRESSION: { lsm_compress *p = va_arg(ap, lsm_compress *); - if( pDb->pDatabase ){ - /* If lsm_open() has been called, this call is against the rules. */ + if( pDb->iReader>=0 ){ + /* May not change compression schemes with an open transaction */ rc = LSM_MISUSE_BKPT; }else{ - memcpy(&pDb->compress, p, sizeof(lsm_compress)); + if( p->xBound==0 ){ + memset(&pDb->compress, 0, sizeof(lsm_compress)); + pDb->compress.iId = LSM_COMPRESSION_NONE; + }else{ + memcpy(&pDb->compress, p, sizeof(lsm_compress)); + } + rc = lsmFsConfigure(pDb); + } + break; + } + + case LSM_CONFIG_SET_COMPRESSION_FACTORY: { + lsm_compress_factory *p = va_arg(ap, lsm_compress_factory *); + if( pDb->factory.xFree ){ + /* Invoke any destructor belonging to the current factory. */ + pDb->factory.xFree(pDb->factory.pCtx); } + memcpy(&pDb->factory, p, sizeof(lsm_compress_factory)); break; } case LSM_CONFIG_GET_COMPRESSION: { lsm_compress *p = va_arg(ap, lsm_compress *); Index: src/lsm_shared.c ================================================================== --- src/lsm_shared.c +++ src/lsm_shared.c @@ -432,10 +432,13 @@ rc = lsmFsOpen(pDb, zName); } if( rc==LSM_OK ){ rc = doDbConnect(pDb); } + if( rc==LSM_OK ){ + rc = lsmFsConfigure(pDb); + } return rc; } static void dbDeferClose(lsm_db *pDb){ @@ -973,14 +976,22 @@ if( pDb->pClient==0 ){ rc = lsmCheckpointDeserialize(pDb, 0, pDb->aSnapshot,&pDb->pClient); } assert( (rc==LSM_OK)==(pDb->pClient!=0) ); assert( pDb->iReader>=0 ); + + /* Check that the client has the right compression hooks loaded. + ** If not, set rc to LSM_MISMATCH. */ + assert( rc!=LSM_OK || pDb->pClient->iCmpId!=LSM_COMPRESSION_EMPTY ); + if( rc==LSM_OK && pDb->pClient->iCmpId!=pDb->compress.iId ){ + rc = LSM_MISMATCH; + } }else{ rc = lsmReleaseReadlock(pDb); } } + if( rc==LSM_BUSY ){ rc = LSM_OK; } } #if 0 Index: src/lsm_unix.c ================================================================== --- src/lsm_unix.c +++ src/lsm_unix.c @@ -192,22 +192,24 @@ munmap(p->pMap, p->nMap); *ppOut = p->pMap = 0; *pnOut = p->nMap = 0; } - memset(&buf, 0, sizeof(buf)); - prc = fstat(p->fd, &buf); - if( prc!=0 ) return LSM_IOERR_BKPT; - iSz = buf.st_size; - if( iSzfd, iSz); - if( prc!=0 ) return LSM_IOERR_BKPT; - } - - p->pMap = mmap(0, iSz, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0); - p->nMap = iSz; + if( iMin>=0 ){ + memset(&buf, 0, sizeof(buf)); + prc = fstat(p->fd, &buf); + if( prc!=0 ) return LSM_IOERR_BKPT; + iSz = buf.st_size; + if( iSzfd, iSz); + if( prc!=0 ) return LSM_IOERR_BKPT; + } + + p->pMap = mmap(0, iSz, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0); + p->nMap = iSz; + } *ppOut = p->pMap; *pnOut = p->nMap; return LSM_OK; } ADDED test/lsm4.test Index: test/lsm4.test ================================================================== --- /dev/null +++ test/lsm4.test @@ -0,0 +1,57 @@ +# 2013 February 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix lsm4 +db close + +proc db_fetch {db key} { + db csr_open csr + csr seek $key eq + set ret [csr value] + csr close + set ret +} + +do_test 1.1 { + lsm_open db test.db + db config {set_compression noop} + db write 1 abc + db write 2 def + db close +} {} + +do_test 1.2 { + lsm_open db test.db + db config {set_compression noop} + list [db_fetch db 1] [db_fetch db 2] +} {abc def} + +do_test 1.3 { + db close + lsm_open db test.db + db config {set_compression rle} + list [catch {db_fetch db 1} msg] $msg +} {1 {error in lsm_csr_open() - 50}} + +do_test 1.4 { + db close + lsm_open db test.db +breakpoint + list [catch {db_fetch db 1} msg] $msg +} {1 {error in lsm_csr_open() - 50}} + + + +finish_test + Index: test/test_lsm.c ================================================================== --- test/test_lsm.c +++ test/test_lsm.c @@ -7,11 +7,10 @@ ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* -** */ #include #include "lsm.h" #include "sqlite4.h" @@ -18,10 +17,137 @@ #include #include extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite4 **ppDb); extern const char *sqlite4TestErrorName(int); + +/************************************************************************* +*/ +#define ENCRYPTION_XOR_MASK 0xa3b2bbb6 +static int testCompressEncBound(void *pCtx, int nSrc){ + return nSrc; +} +static int testCompressEncCompress( + void *pCtx, + char *pOut, int *pnOut, + const char *pIn, int nIn +){ + int i; + unsigned int *aIn = (unsigned int *)pOut; + unsigned int *aOut = (unsigned int *)pIn; + + assert( (nIn%4)==0 ); + for(i=0; i<(nIn/4); i++){ + aOut[i] = (aIn[i] ^ ENCRYPTION_XOR_MASK); + } + *pnOut = nIn; + + return LSM_OK; +} +static int testCompressEncUncompress( + void *pCtx, + char *pOut, int *pnOut, + const char *pIn, int nIn +){ + return testCompressEncUncompress(pCtx, pOut, pnOut, pIn, nIn); +} +static void testCompressEncFree(void *pCtx){ + /* no-op */ +} +/* +** End of compression routines "encrypt". +*************************************************************************/ + +/************************************************************************* +*/ +static int testCompressRleBound(void *pCtx, int nSrc){ + return nSrc*2; +} +static int testCompressRleCompress( + void *pCtx, + char *pOut, int *pnOut, + const char *pIn, int nIn +){ + int iOut = 0; + int i; + char c; + int n; + + c = pIn[0]; + n = 1; + for(i=1; i
  • Database Runtime Environment
  • LSM Error Codes
  • Creating and Destroying Database Connection Handles
  • Connecting to a Database -
  • Obtaining pointers to databases environments +
  • Obtaining pointers to database environments
  • Configuring a database connection.
  • Compression and/or Encryption Hooks
  • Allocating and Freeing Memory
  • Querying a Connection For Operational Data
  • Opening and Closing Write Transactions @@ -65,10 +65,11 @@ lsm_rollback lsm_work

    All LSM API Types

    lsm_compress +lsm_compress lsm_env

    All LSM API Constants

    LSM_BUSY LSM_CANTOPEN @@ -76,26 +77,27 @@ LSM_CONFIG_AUTOFLUSH LSM_CONFIG_AUTOMERGE LSM_CONFIG_AUTOWORK LSM_CONFIG_BLOCK_SIZE LSM_CONFIG_GET_COMPRESSION -LSM_CONFIG_LOG_SIZE LSM_CONFIG_MAX_FREELIST LSM_CONFIG_MMAP LSM_CONFIG_MULTIPLE_PROCESSES LSM_CONFIG_PAGE_SIZE LSM_CONFIG_SAFETY LSM_CONFIG_SET_COMPRESSION +LSM_CONFIG_SET_COMPRESSION_FACTORY LSM_CONFIG_USE_LOG LSM_CORRUPT LSM_ERROR LSM_FULL LSM_INFO_ARRAY_PAGES LSM_INFO_ARRAY_STRUCTURE LSM_INFO_CHECKPOINT_SIZE LSM_INFO_DB_STRUCTURE LSM_INFO_FREELIST +LSM_INFO_FREELIST_SIZE LSM_INFO_LOG_STRUCTURE LSM_INFO_NREAD LSM_INFO_NWRITE LSM_INFO_PAGE_ASCII_DUMP LSM_INFO_PAGE_HEX_DUMP @@ -180,49 +182,63 @@

    Open and close a database connection handle.

    Connecting to a Database

    int lsm_open(lsm_db *pDb, const char *zFilename); -

    Obtaining pointers to databases environments

    +

    Obtaining pointers to database environments

    lsm_env *lsm_get_env(lsm_db *pDb); lsm_env *lsm_default_env(void);

    Return a pointer to the environment used by the database connection passed as the first argument. Assuming the argument is valid, this function always returns a valid environment pointer - it cannot fail. The lsm_default_env() function returns a pointer to the default LSM environment for the current platform. -

    Configuring a database connection.

    +

    Configuring a database connection.

    int lsm_config(lsm_db *, int, ...); -#define LSM_CONFIG_AUTOFLUSH 1 -#define LSM_CONFIG_PAGE_SIZE 2 -#define LSM_CONFIG_SAFETY 3 -#define LSM_CONFIG_BLOCK_SIZE 4 -#define LSM_CONFIG_AUTOWORK 5 -#define LSM_CONFIG_LOG_SIZE 6 -#define LSM_CONFIG_MMAP 7 -#define LSM_CONFIG_USE_LOG 8 -#define LSM_CONFIG_AUTOMERGE 9 -#define LSM_CONFIG_MAX_FREELIST 10 -#define LSM_CONFIG_MULTIPLE_PROCESSES 11 -#define LSM_CONFIG_AUTOCHECKPOINT 12 -#define LSM_CONFIG_SET_COMPRESSION 13 -#define LSM_CONFIG_GET_COMPRESSION 14 +#define LSM_CONFIG_AUTOFLUSH 1 +#define LSM_CONFIG_PAGE_SIZE 2 +#define LSM_CONFIG_SAFETY 3 +#define LSM_CONFIG_BLOCK_SIZE 4 +#define LSM_CONFIG_AUTOWORK 5 +#define LSM_CONFIG_MMAP 7 +#define LSM_CONFIG_USE_LOG 8 +#define LSM_CONFIG_AUTOMERGE 9 +#define LSM_CONFIG_MAX_FREELIST 10 +#define LSM_CONFIG_MULTIPLE_PROCESSES 11 +#define LSM_CONFIG_AUTOCHECKPOINT 12 +#define LSM_CONFIG_SET_COMPRESSION 13 +#define LSM_CONFIG_GET_COMPRESSION 14 +#define LSM_CONFIG_SET_COMPRESSION_FACTORY 15 #define LSM_SAFETY_OFF 0 #define LSM_SAFETY_NORMAL 1 #define LSM_SAFETY_FULL 2

    The lsm_config() function is used to configure a database connection. The following values may be passed as the second argument to lsm_config(). -

    LSM_CONFIG_AUTOFLUSH
    A read/write integer parameter. This value determines the maximum amount -of space (in bytes) used to accumulate writes in main-memory before -they are flushed to a level 0 segment. +

    LSM_CONFIG_AUTOFLUSH
    A read/write integer parameter. +

    This value determines the amount of data allowed to accumulate in a +live in-memory tree before it is marked as old. After committing a +transaction, a connection checks if the size of the live in-memory tree, +including data structure overhead, is greater than the value of this +option in KB. If it is, and there is not already an old in-memory tree, +the live in-memory tree is marked as old. +

    The maximum allowable value is 1048576 (1GB). There is no minimum +value. If this parameter is set to zero, then an attempt is made to +mark the live in-memory tree as old after each transaction is committed. +

    The default value is 1024 (1MB).

    LSM_CONFIG_PAGE_SIZE
    A read/write integer parameter. This parameter may only be set before lsm_open() has been called. -

    LSM_CONFIG_BLOCK_SIZE
    A read/write integer parameter. This parameter may only be set before -lsm_open() has been called. -

    LSM_CONFIG_LOG_SIZE
    A read/write integer parameter. +

    LSM_CONFIG_BLOCK_SIZE
    A read/write integer parameter. +

    This parameter may only be set before lsm_open() has been called. It +must be set to a power of two between 64 and 65536, inclusive (block +sizes between 64KB and 64MB). +

    If the connection creates a new database, the block size of the new +database is set to the value of this option in KB. After lsm_open() +has been called, querying this parameter returns the actual block +size of the opened database. +

    The default value is 1024 (1MB blocks).

    LSM_CONFIG_SAFETY
    A read/write integer parameter. Valid values are 0, 1 (the default) and 2. This parameter determines how robust the database is in the face of a system crash (e.g. a power failure or operating system crash). As follows:

    0 (off): No robustness. A system crash may corrupt the database. @@ -232,10 +248,21 @@

    2 (full): Full robustness. A system crash may not corrupt the database file. Following recovery the database file contains all successfully committed transactions.

    LSM_CONFIG_AUTOWORK
    A read/write integer parameter.

    LSM_CONFIG_AUTOCHECKPOINT
    A read/write integer parameter. +

    If this option is set to non-zero value N, then a checkpoint is +automatically attempted after each N KB of data have been written to +the database file. +

    The amount of uncheckpointed data already written to the database file +is a global parameter. After performing database work (writing to the +database file), the process checks if the total amount of uncheckpointed +data exceeds the value of this paramter. If so, a checkpoint is performed. +This means that this option may cause the connection to perform a +checkpoint even if the current connection has itself written very little +data into the database file. +

    The default value is 2048 (checkpoint every 2MB).

    LSM_CONFIG_MMAP
    A read/write integer parameter. True to use mmap() to access the database file. False otherwise.

    LSM_CONFIG_USE_LOG
    A read/write boolean parameter. True (the default) to use the log file normally. False otherwise.

    LSM_CONFIG_AUTOMERGE
    A read/write integer parameter. The minimum number of segments to @@ -257,17 +284,25 @@ structures contents.

    This option may only be used before lsm_open() is called. Invoking it after lsm_open() has been called results in an LSM_MISUSE error.

    LSM_CONFIG_GET_COMPRESSION
    Query the compression methods used to compress and decompress database content. -

    Compression and/or Encryption Hooks

    +

    LSM_CONFIG_SET_COMPRESSION_FACTORY
    Configure a factory method to be invoked in case of an LSM_MISMATCH +error. +

    Compression and/or Encryption Hooks

    struct lsm_compress { void *pCtx; unsigned int iId; int (*xBound)(void *, int nSrc); int (*xCompress)(void *, char *, int *, const char *, int); int (*xUncompress)(void *, char *, int *, const char *, int); + void (*xFree)(void *pCtx); +}; +struct lsm_compress_factory { + void *pCtx; + int (*xFactory)(void *, lsm_db *, u32); + void (*xFree)(void *pCtx); };

    Allocating and Freeing Memory

    void *lsm_malloc(lsm_env*, size_t); void *lsm_realloc(lsm_env*, void *, size_t); @@ -274,11 +309,11 @@ void lsm_free(lsm_env*, void *);

    Invoke the memory allocation functions that belong to environment pEnv. Or the system defaults if no memory allocation functions have been registered. -

    Querying a Connection For Operational Data

    +

    Querying a Connection For Operational Data

    int lsm_info(lsm_db *, int, ...); #define LSM_INFO_NWRITE 1 #define LSM_INFO_NREAD 2 #define LSM_INFO_DB_STRUCTURE 3 #define LSM_INFO_LOG_STRUCTURE 4 @@ -287,10 +322,11 @@ #define LSM_INFO_PAGE_HEX_DUMP 7 #define LSM_INFO_FREELIST 8 #define LSM_INFO_ARRAY_PAGES 9 #define LSM_INFO_CHECKPOINT_SIZE 10 #define LSM_INFO_TREE_SIZE 11 +#define LSM_INFO_FREELIST_SIZE 12

    Query a database connection for operational statistics or data. The following values may be passed as the second argument to lsm_info().

    LSM_INFO_NWRITE
    The third parameter should be of type (int *). The location pointed to by the third parameter is set to the number of 4KB pages written to @@ -352,11 +388,11 @@ string should be eventually freed by the caller using lsm_free().

    The Tcl structure returned is a list containing one element for each free block in the database. The element itself consists of two integers - the block number and the id of the snapshot that freed it.

    LSM_INFO_CHECKPOINT_SIZE
    The third argument should be of type (int *). The location pointed to -by this argument is populated with the number of bytes written to the +by this argument is populated with the number of KB written to the database file since the most recent checkpoint.

    LSM_INFO_TREE_SIZE
    If this value is passed as the second argument to an lsm_info() call, it should be followed by two arguments of type (int *) (for a total of four arguments).

    At any time, there are either one or two tree structures held in shared @@ -365,11 +401,11 @@ information on them). One tree structure - the current tree - is used to accumulate new data written to the database. The other tree structure - the old tree - is a read-only tree holding older data and may be flushed to disk at any time.

    Assuming no error occurs, the location pointed to by the first of the two -(int *) arguments is set to the size of the old in-memory tree in bytes. +(int *) arguments is set to the size of the old in-memory tree in KB. The second is set to the size of the current, or live in-memory tree.

    Opening and Closing Write Transactions

    int lsm_begin(lsm_db *pDb, int iLevel); int lsm_commit(lsm_db *pDb, int iLevel); int lsm_rollback(lsm_db *pDb, int iLevel); @@ -407,21 +443,21 @@ Delete all database entries with keys that are greater than (pKey1/nKey1) and smaller than (pKey2/nKey2). Note that keys (pKey1/nKey1) and (pKey2/nKey2) themselves, if they exist in the database, are not deleted.

    Return LSM_OK if successful, or an LSM error code otherwise.

    Explicit Database Work and Checkpointing

    -int lsm_work(lsm_db *pDb, int nMerge, int nPage, int *pnWrite); +int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite); int lsm_flush(lsm_db *pDb); -int lsm_checkpoint(lsm_db *pDb, int *pnByte); +int lsm_checkpoint(lsm_db *pDb, int *pnKB);

    This function is called by a thread to work on the database structure. Attempt to checkpoint the current database snapshot. Return an LSM error code if an error occurs or LSM_OK otherwise.

    If the current snapshot has already been checkpointed, calling this -function is a no-op. In this case if pnByte is not NULL, *pnByte is +function is a no-op. In this case if pnKB is not NULL, *pnKB is set to 0. Or, if the current snapshot is successfully checkpointed -by this function and pbCkpt is not NULL, *pnByte is set to the number +by this function and pbKB is not NULL, *pnKB is set to the number of bytes written to the database file since the previous checkpoint (the same measure as returned by the LSM_INFO_CHECKPOINT_SIZE query).

    Opening and Closing Database Cursors

    int lsm_csr_open(lsm_db *pDb, lsm_cursor **ppCsr); int lsm_csr_close(lsm_cursor *pCsr); Index: www/lsmusr.wiki ================================================================== --- www/lsmusr.wiki +++ www/lsmusr.wiki @@ -1148,11 +1148,11 @@ clients should set the LSM_CONFIG_AUTOWORK parameter to zero. int rc; lsm_db *db; - int nCkpt = 4*1024*1024; + int nCkpt = 4*1024; /* 4096KB == 4MB */ /* Open a database connection to database "test.db". ** ** Configure the connection to automatically checkpoint the database after ** writing each 4MB of data to it (instead of the default 2MB). As well @@ -1166,11 +1166,11 @@ while( 1 ){ int nWrite; /* Attempt up to 512KB of work. Set nWrite to the number of bytes ** actually written to disk. */ - rc = lsm_work(db, 2, 512*1024, &nWrite); + rc = lsm_work(db, 2, 512, &nWrite); if( rc!=LSM_OK && rc!=LSM_BUSY ){ /* Anything other than LSM_OK or LSM_BUSY is a problem. LSM_BUSY ** indicates that some other client has taken the WORKER lock. Any ** other error indicates something has gone quite wrong. */ lsm_close(db);