Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Fixes to lsm_work() to bring it into line with documentation: (a) third argument is a number of KB, not a number of db pages, and (b) passing -1 as the third argument means "do as much work as possible". |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA1: |
2f8966cd9c167b06195a14821a5f8c78 |
| User & Date: | dan 2013-02-05 18:06:45 |
Context
|
2013-02-05
| ||
| 18:32 | Fix lsm_checkpoint() and some lsm_info() calls to match documented behaviour. check-in: 89e314d98b user: dan tags: trunk | |
| 18:06 | Fixes to lsm_work() to bring it into line with documentation: (a) third argument is a number of KB, not a number of db pages, and (b) passing -1 as the third argument means "do as much work as possible". check-in: 2f8966cd9c user: dan tags: trunk | |
| 16:03 | Change the BLOCK_SIZE, AUTOFLUSH and AUTOCHECKPOINT lsm_config() options to work in KB instead of bytes. This matches the user guide. check-in: 340c9dcd22 user: dan tags: trunk | |
Changes
Changes to lsm-test/lsmtest_func.c.
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 .. 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 .. 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 |
int do_work(int nArg, char **azArg){
struct Option {
const char *zName;
} aOpt [] = {
{ "-nmerge" },
{ "-npage" },
{ 0 }
};
lsm_db *pDb;
int rc;
int i;
const char *zDb;
int nMerge = 1;
int nWork = (1<<30);
if( nArg==0 ) goto usage;
zDb = azArg[nArg-1];
for(i=0; i<(nArg-1); i++){
int iSel;
rc = testArgSelect(aOpt, "option", azArg[i], &iSel);
if( rc ) return rc;
................................................................................
i++;
if( i==(nArg-1) ) goto usage;
nMerge = atoi(azArg[i]);
break;
case 1:
i++;
if( i==(nArg-1) ) goto usage;
nWork = atoi(azArg[i]);
break;
}
}
rc = lsm_new(0, &pDb);
if( rc!=LSM_OK ){
testPrintError("lsm_open(): rc=%d\n", rc);
................................................................................
testPrintError("lsm_open(): rc=%d\n", rc);
}else{
int n = -1;
lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &n);
n = n*2;
lsm_config(pDb, LSM_CONFIG_AUTOCHECKPOINT, &n);
rc = lsm_work(pDb, nMerge, nWork, 0);
if( rc!=LSM_OK ){
testPrintError("lsm_work(): rc=%d\n", rc);
}
}
}
if( rc==LSM_OK ){
rc = lsm_checkpoint(pDb, 0);
|
| | | | |
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 .. 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 .. 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 |
int do_work(int nArg, char **azArg){
struct Option {
const char *zName;
} aOpt [] = {
{ "-nmerge" },
{ "-nkb" },
{ 0 }
};
lsm_db *pDb;
int rc;
int i;
const char *zDb;
int nMerge = 1;
int nKB = (1<<30);
if( nArg==0 ) goto usage;
zDb = azArg[nArg-1];
for(i=0; i<(nArg-1); i++){
int iSel;
rc = testArgSelect(aOpt, "option", azArg[i], &iSel);
if( rc ) return rc;
................................................................................
i++;
if( i==(nArg-1) ) goto usage;
nMerge = atoi(azArg[i]);
break;
case 1:
i++;
if( i==(nArg-1) ) goto usage;
nKB = atoi(azArg[i]);
break;
}
}
rc = lsm_new(0, &pDb);
if( rc!=LSM_OK ){
testPrintError("lsm_open(): rc=%d\n", rc);
................................................................................
testPrintError("lsm_open(): rc=%d\n", rc);
}else{
int n = -1;
lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &n);
n = n*2;
lsm_config(pDb, LSM_CONFIG_AUTOCHECKPOINT, &n);
rc = lsm_work(pDb, nMerge, nKB, 0);
if( rc!=LSM_OK ){
testPrintError("lsm_work(): rc=%d\n", rc);
}
}
}
if( rc==LSM_OK ){
rc = lsm_checkpoint(pDb, 0);
|
Changes to src/lsm.h.
487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 |
);
/*
** CAPI: Explicit Database Work and Checkpointing
**
** This function is called by a thread to work on the database structure.
*/
int lsm_work(lsm_db *pDb, int nMerge, int nPage, int *pnWrite);
int lsm_flush(lsm_db *pDb);
/*
** Attempt to checkpoint the current database snapshot. Return an LSM
** error code if an error occurs or LSM_OK otherwise.
**
|
| |
487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 |
);
/*
** CAPI: Explicit Database Work and Checkpointing
**
** This function is called by a thread to work on the database structure.
*/
int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite);
int lsm_flush(lsm_db *pDb);
/*
** Attempt to checkpoint the current database snapshot. Return an LSM
** error code if an error occurs or LSM_OK otherwise.
**
|
Changes to src/lsm_sorted.c.
5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 .... 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 .... 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 |
static int doLsmWork(lsm_db *pDb, int nMerge, int nPage, int *pnWrite){
int rc = LSM_OK; /* Return code */
int nWrite = 0; /* Number of pages written */
assert( nMerge>=1 );
if( nPage>0 ){
int bCkpt = 0;
do {
int nThis = 0;
bCkpt = 0;
rc = doLsmSingleWork(pDb, 0, nMerge, nPage-nWrite, &nThis, &bCkpt);
nWrite += nThis;
if( rc==LSM_OK && bCkpt ){
rc = lsm_checkpoint(pDb, 0);
}
}while( rc==LSM_OK && (nWrite<nPage && bCkpt) );
}
if( pnWrite ){
if( rc==LSM_OK ){
*pnWrite = nWrite;
}else{
*pnWrite = 0;
................................................................................
}
return rc;
}
/*
** Perform work to merge database segments together.
*/
int lsm_work(lsm_db *pDb, int nMerge, int nPage, int *pnWrite){
/* This function may not be called if pDb has an open read or write
** transaction. Return LSM_MISUSE if an application attempts this. */
if( pDb->nTransOpen || pDb->pCsr ) return LSM_MISUSE_BKPT;
if( nMerge<=0 ) nMerge = pDb->nMerge;
return doLsmWork(pDb, nMerge, nPage, pnWrite);
}
int lsm_flush(lsm_db *db){
int rc;
if( db->nTransOpen>0 || db->pCsr ){
rc = LSM_MISUSE_BKPT;
................................................................................
int nRemaining; /* Units of work to do before returning */
nRemaining = nUnit * nDepth;
#ifdef LSM_LOG_WORK
lsmLogMessage(pDb, rc, "lsmSortedAutoWork(): %d*%d = %d pages",
nUnit, nDepth, nRemaining);
#endif
rc = doLsmWork(pDb, pDb->nMerge, nRemaining, 0);
if( rc==LSM_BUSY ) rc = LSM_OK;
if( bRestore && pDb->pCsr ){
lsmFreeSnapshot(pDb->pEnv, pDb->pClient);
pDb->pClient = 0;
rc = lsmCheckpointLoad(pDb, 0);
|
| > > | | | > > > > < > > > > > > > > > | > > > > > > > |
5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 .... 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 .... 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 |
static int doLsmWork(lsm_db *pDb, int nMerge, int nPage, int *pnWrite){
int rc = LSM_OK; /* Return code */
int nWrite = 0; /* Number of pages written */
assert( nMerge>=1 );
if( nPage!=0 ){
int bCkpt = 0;
do {
int nThis = 0;
int nReq = (nPage>=0) ? (nPage-nWrite) : ((int)0x7FFFFFFF);
bCkpt = 0;
rc = doLsmSingleWork(pDb, 0, nMerge, nReq, &nThis, &bCkpt);
nWrite += nThis;
if( rc==LSM_OK && bCkpt ){
rc = lsm_checkpoint(pDb, 0);
}
}while( rc==LSM_OK && bCkpt && (nWrite<nPage || nPage<0) );
}
if( pnWrite ){
if( rc==LSM_OK ){
*pnWrite = nWrite;
}else{
*pnWrite = 0;
................................................................................
}
return rc;
}
/*
** Perform work to merge database segments together.
*/
int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite){
int rc; /* Return code */
int nPgsz; /* Nominal page size in bytes */
int nPage; /* Equivalent of nKB in pages */
int nWrite = 0; /* Number of pages written */
/* This function may not be called if pDb has an open read or write
** transaction. Return LSM_MISUSE if an application attempts this. */
if( pDb->nTransOpen || pDb->pCsr ) return LSM_MISUSE_BKPT;
if( nMerge<=0 ) nMerge = pDb->nMerge;
/* Convert from KB to pages */
nPgsz = lsmFsPageSize(pDb->pFS);
if( nKB>=0 ){
nPage = ((i64)nKB * 1024 + nPgsz - 1) / nPgsz;
}else{
nPage = -1;
}
rc = doLsmWork(pDb, nMerge, nPage, &nWrite);
if( pnWrite ){
/* Convert back from pages to KB */
*pnWrite = (int)(((i64)nWrite * 1024 + nPgsz - 1) / nPgsz);
}
return rc;
}
int lsm_flush(lsm_db *db){
int rc;
if( db->nTransOpen>0 || db->pCsr ){
rc = LSM_MISUSE_BKPT;
................................................................................
int nRemaining; /* Units of work to do before returning */
nRemaining = nUnit * nDepth;
#ifdef LSM_LOG_WORK
lsmLogMessage(pDb, rc, "lsmSortedAutoWork(): %d*%d = %d pages",
nUnit, nDepth, nRemaining);
#endif
assert( nRemaining>=0 );
rc = doLsmWork(pDb, pDb->nMerge, nRemaining, 0);
if( rc==LSM_BUSY ) rc = LSM_OK;
if( bRestore && pDb->pCsr ){
lsmFreeSnapshot(pDb->pEnv, pDb->pClient);
pDb->pClient = 0;
rc = lsmCheckpointLoad(pDb, 0);
|
Changes to tool/lsmview.tcl.
397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 |
set data [string trim [exec_lsmtest_show -c $myCfg $myDb array-pages $iFirst]]
$myText delete 0.0 end
# Delete the existing tree entries.
$myTree delete [$myTree children {}]
set nBlksz [exec_lsmtest_show -c $myCfg $myDb blocksize]
set nPgsz [exec_lsmtest_show -c $myCfg $myDb pagesize]
if {[regexp {c=1} $myCfg] || [regexp {co=1} $myCfg]
|| [regexp {com=1} $myCfg] || [regexp {comp=1} $myCfg]
|| [regexp {compr=1} $myCfg] || [regexp {compres=1} $myCfg]
|| [regexp {compress=1} $myCfg] || [regexp {compressi=1} $myCfg]
|| [regexp {compressio=1} $myCfg] || [regexp {compression=1} $myCfg]
|
| |
397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 |
set data [string trim [exec_lsmtest_show -c $myCfg $myDb array-pages $iFirst]]
$myText delete 0.0 end
# Delete the existing tree entries.
$myTree delete [$myTree children {}]
set nBlksz [expr [exec_lsmtest_show -c $myCfg $myDb blocksize] * 1024]
set nPgsz [exec_lsmtest_show -c $myCfg $myDb pagesize]
if {[regexp {c=1} $myCfg] || [regexp {co=1} $myCfg]
|| [regexp {com=1} $myCfg] || [regexp {comp=1} $myCfg]
|| [regexp {compr=1} $myCfg] || [regexp {compres=1} $myCfg]
|| [regexp {compress=1} $myCfg] || [regexp {compressi=1} $myCfg]
|| [regexp {compressio=1} $myCfg] || [regexp {compression=1} $myCfg]
|
Changes to www/lsmusr.wiki.
1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 |
<p>Database optimization transforms the contents of database file so that
the following are true:
<ul>
<li> <p>All database content is stored in a single
<a href=#architectural_overview>segment</a>. This makes the
database effectively equivalent to an optimally packed b-tree stucture
for search operations - minimizing the number of disk sectors that need
to be visted when searching the database.
<li> <p>The database file contains no (or as little as possible) free space.
In other words, it is no larger than required to contain the single
segment.
</ul>
<p>In order to optimize the database, lsm_work() should be called with
the nMerge argument set to 1 and the third parameter set to a negative value
(interpreted as - keep working until there is no more work to do). For
example:
<verbatim>
rc = lsm_work(db, 1, -1, 0);
</verbatim>
<p><span style=color:red>todo: the -1 as the 3rd argument above is currently
not supported</span>
<p>When optimizing the database as above, either the LSM_CONFIG_AUTOCHECKPOINT
parameter should be set to a non-zero value or lsm_checkpoint() should be
called periodically. Otherwise, no checkpoints will be performed, preventing
the library from reusing any space occupied by old segments even after their
content has been merged into the new segment. The result - a database file that
is optimized, except that it is up to twice as large as it otherwise would be.
|
| | < < < < |
1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 |
<p>Database optimization transforms the contents of database file so that
the following are true:
<ul>
<li> <p>All database content is stored in a single
<a href=#architectural_overview>segment</a>. This makes the
data structure equivalent to an optimally packed b-tree stucture
for search operations - minimizing the number of disk sectors that need
to be visted when searching the database.
<li> <p>The database file contains no (or as little as possible) free space.
In other words, it is no larger than required to contain the single
segment.
</ul>
<p>In order to optimize the database, lsm_work() should be called with
the nMerge argument set to 1 and the third parameter set to a negative value
(interpreted as "keep working until there is no more work to do"). For
example:
<verbatim>
rc = lsm_work(db, 1, -1, 0);
</verbatim>
<p>When optimizing the database as above, either the LSM_CONFIG_AUTOCHECKPOINT
parameter should be set to a non-zero value or lsm_checkpoint() should be
called periodically. Otherwise, no checkpoints will be performed, preventing
the library from reusing any space occupied by old segments even after their
content has been merged into the new segment. The result - a database file that
is optimized, except that it is up to twice as large as it otherwise would be.
|