SQLite4
Check-in [2f8966cd9c]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fixes to lsm_work() to bring it into line with documentation: (a) third argument is a number of KB, not a number of db pages, and (b) passing -1 as the third argument means "do as much work as possible".
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 2f8966cd9c167b06195a14821a5f8c78f4724a94
User & Date: dan 2013-02-05 18:06:45
Context
2013-02-05
18:32
Fix lsm_checkpoint() and some lsm_info() calls to match documented behaviour. check-in: 89e314d98b user: dan tags: trunk
18:06
Fixes to lsm_work() to bring it into line with documentation: (a) third argument is a number of KB, not a number of db pages, and (b) passing -1 as the third argument means "do as much work as possible". check-in: 2f8966cd9c user: dan tags: trunk
16:03
Change the BLOCK_SIZE, AUTOFLUSH and AUTOCHECKPOINT lsm_config() options to work in KB instead of bytes. This matches the user guide. check-in: 340c9dcd22 user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to lsm-test/lsmtest_func.c.

3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
..
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
..
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61


int do_work(int nArg, char **azArg){
  struct Option {
    const char *zName;
  } aOpt [] = {
    { "-nmerge" },
    { "-npage" },
    { 0 }
  };

  lsm_db *pDb;
  int rc;
  int i;
  const char *zDb;
  int nMerge = 1;
  int nWork = (1<<30);

  if( nArg==0 ) goto usage;
  zDb = azArg[nArg-1];
  for(i=0; i<(nArg-1); i++){
    int iSel;
    rc = testArgSelect(aOpt, "option", azArg[i], &iSel);
    if( rc ) return rc;
................................................................................
        i++;
        if( i==(nArg-1) ) goto usage;
        nMerge = atoi(azArg[i]);
        break;
      case 1:
        i++;
        if( i==(nArg-1) ) goto usage;
        nWork = atoi(azArg[i]);
        break;
    }
  }

  rc = lsm_new(0, &pDb);
  if( rc!=LSM_OK ){
    testPrintError("lsm_open(): rc=%d\n", rc);
................................................................................
      testPrintError("lsm_open(): rc=%d\n", rc);
    }else{
      int n = -1;
      lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &n);
      n = n*2;
      lsm_config(pDb, LSM_CONFIG_AUTOCHECKPOINT, &n);

      rc = lsm_work(pDb, nMerge, nWork, 0);
      if( rc!=LSM_OK ){
        testPrintError("lsm_work(): rc=%d\n", rc);
      }
    }
  }
  if( rc==LSM_OK ){
    rc = lsm_checkpoint(pDb, 0);







|








|







 







|







 







|







3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
..
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
..
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61


int do_work(int nArg, char **azArg){
  struct Option {
    const char *zName;
  } aOpt [] = {
    { "-nmerge" },
    { "-nkb" },
    { 0 }
  };

  lsm_db *pDb;
  int rc;
  int i;
  const char *zDb;
  int nMerge = 1;
  int nKB = (1<<30);

  if( nArg==0 ) goto usage;
  zDb = azArg[nArg-1];
  for(i=0; i<(nArg-1); i++){
    int iSel;
    rc = testArgSelect(aOpt, "option", azArg[i], &iSel);
    if( rc ) return rc;
................................................................................
        i++;
        if( i==(nArg-1) ) goto usage;
        nMerge = atoi(azArg[i]);
        break;
      case 1:
        i++;
        if( i==(nArg-1) ) goto usage;
        nKB = atoi(azArg[i]);
        break;
    }
  }

  rc = lsm_new(0, &pDb);
  if( rc!=LSM_OK ){
    testPrintError("lsm_open(): rc=%d\n", rc);
................................................................................
      testPrintError("lsm_open(): rc=%d\n", rc);
    }else{
      int n = -1;
      lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &n);
      n = n*2;
      lsm_config(pDb, LSM_CONFIG_AUTOCHECKPOINT, &n);

      rc = lsm_work(pDb, nMerge, nKB, 0);
      if( rc!=LSM_OK ){
        testPrintError("lsm_work(): rc=%d\n", rc);
      }
    }
  }
  if( rc==LSM_OK ){
    rc = lsm_checkpoint(pDb, 0);

Changes to src/lsm.h.

487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
);

/*
** CAPI: Explicit Database Work and Checkpointing
**
** This function is called by a thread to work on the database structure.
*/
int lsm_work(lsm_db *pDb, int nMerge, int nPage, int *pnWrite);

int lsm_flush(lsm_db *pDb);

/*
** Attempt to checkpoint the current database snapshot. Return an LSM
** error code if an error occurs or LSM_OK otherwise.
**







|







487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
);

/*
** CAPI: Explicit Database Work and Checkpointing
**
** This function is called by a thread to work on the database structure.
*/
int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite);

int lsm_flush(lsm_db *pDb);

/*
** Attempt to checkpoint the current database snapshot. Return an LSM
** error code if an error occurs or LSM_OK otherwise.
**

Changes to src/lsm_sorted.c.

5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206


5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
....
5222
5223
5224
5225
5226
5227
5228
5229




5230
5231
5232
5233
5234
5235









5236






5237
5238
5239
5240
5241
5242
5243
....
5297
5298
5299
5300
5301
5302
5303

5304
5305
5306
5307
5308
5309
5310

static int doLsmWork(lsm_db *pDb, int nMerge, int nPage, int *pnWrite){
  int rc = LSM_OK;                /* Return code */
  int nWrite = 0;                 /* Number of pages written */

  assert( nMerge>=1 );

  if( nPage>0 ){
    int bCkpt = 0;
    do {
      int nThis = 0;


      bCkpt = 0;
      rc = doLsmSingleWork(pDb, 0, nMerge, nPage-nWrite, &nThis, &bCkpt);
      nWrite += nThis;
      if( rc==LSM_OK && bCkpt ){
        rc = lsm_checkpoint(pDb, 0);
      }
    }while( rc==LSM_OK && (nWrite<nPage && bCkpt) );
  }

  if( pnWrite ){
    if( rc==LSM_OK ){
      *pnWrite = nWrite;
    }else{
      *pnWrite = 0;
................................................................................
  }
  return rc;
}

/*
** Perform work to merge database segments together.
*/
int lsm_work(lsm_db *pDb, int nMerge, int nPage, int *pnWrite){





  /* This function may not be called if pDb has an open read or write
  ** transaction. Return LSM_MISUSE if an application attempts this.  */
  if( pDb->nTransOpen || pDb->pCsr ) return LSM_MISUSE_BKPT;

  if( nMerge<=0 ) nMerge = pDb->nMerge;









  return doLsmWork(pDb, nMerge, nPage, pnWrite);






}

int lsm_flush(lsm_db *db){
  int rc;

  if( db->nTransOpen>0 || db->pCsr ){
    rc = LSM_MISUSE_BKPT;
................................................................................
    int nRemaining;               /* Units of work to do before returning */

    nRemaining = nUnit * nDepth;
#ifdef LSM_LOG_WORK
    lsmLogMessage(pDb, rc, "lsmSortedAutoWork(): %d*%d = %d pages", 
        nUnit, nDepth, nRemaining);
#endif

    rc = doLsmWork(pDb, pDb->nMerge, nRemaining, 0);
    if( rc==LSM_BUSY ) rc = LSM_OK;

    if( bRestore && pDb->pCsr ){
      lsmFreeSnapshot(pDb->pEnv, pDb->pClient);
      pDb->pClient = 0;
      rc = lsmCheckpointLoad(pDb, 0);







|



>
>

|




|







 







|
>
>
>
>




<

>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>







 







>







5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
....
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239

5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
....
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331

static int doLsmWork(lsm_db *pDb, int nMerge, int nPage, int *pnWrite){
  int rc = LSM_OK;                /* Return code */
  int nWrite = 0;                 /* Number of pages written */

  assert( nMerge>=1 );

  if( nPage!=0 ){
    int bCkpt = 0;
    do {
      int nThis = 0;
      int nReq = (nPage>=0) ? (nPage-nWrite) : ((int)0x7FFFFFFF);

      bCkpt = 0;
      rc = doLsmSingleWork(pDb, 0, nMerge, nReq, &nThis, &bCkpt);
      nWrite += nThis;
      if( rc==LSM_OK && bCkpt ){
        rc = lsm_checkpoint(pDb, 0);
      }
    }while( rc==LSM_OK && bCkpt && (nWrite<nPage || nPage<0) );
  }

  if( pnWrite ){
    if( rc==LSM_OK ){
      *pnWrite = nWrite;
    }else{
      *pnWrite = 0;
................................................................................
  }
  return rc;
}

/*
** Perform work to merge database segments together.
*/
int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite){
  int rc;                         /* Return code */
  int nPgsz;                      /* Nominal page size in bytes */
  int nPage;                      /* Equivalent of nKB in pages */
  int nWrite = 0;                 /* Number of pages written */

  /* This function may not be called if pDb has an open read or write
  ** transaction. Return LSM_MISUSE if an application attempts this.  */
  if( pDb->nTransOpen || pDb->pCsr ) return LSM_MISUSE_BKPT;

  if( nMerge<=0 ) nMerge = pDb->nMerge;

  /* Convert from KB to pages */
  nPgsz = lsmFsPageSize(pDb->pFS);
  if( nKB>=0 ){
    nPage = ((i64)nKB * 1024 + nPgsz - 1) / nPgsz;
  }else{
    nPage = -1;
  }

  rc = doLsmWork(pDb, nMerge, nPage, &nWrite);
  
  if( pnWrite ){
    /* Convert back from pages to KB */
    *pnWrite = (int)(((i64)nWrite * 1024 + nPgsz - 1) / nPgsz);
  }
  return rc;
}

int lsm_flush(lsm_db *db){
  int rc;

  if( db->nTransOpen>0 || db->pCsr ){
    rc = LSM_MISUSE_BKPT;
................................................................................
    int nRemaining;               /* Units of work to do before returning */

    nRemaining = nUnit * nDepth;
#ifdef LSM_LOG_WORK
    lsmLogMessage(pDb, rc, "lsmSortedAutoWork(): %d*%d = %d pages", 
        nUnit, nDepth, nRemaining);
#endif
    assert( nRemaining>=0 );
    rc = doLsmWork(pDb, pDb->nMerge, nRemaining, 0);
    if( rc==LSM_BUSY ) rc = LSM_OK;

    if( bRestore && pDb->pCsr ){
      lsmFreeSnapshot(pDb->pEnv, pDb->pClient);
      pDb->pClient = 0;
      rc = lsmCheckpointLoad(pDb, 0);

Changes to tool/lsmview.tcl.

397
398
399
400
401
402
403
404
405
406
407
408
409
410
411

  set data [string trim [exec_lsmtest_show -c $myCfg $myDb array-pages $iFirst]]
  $myText delete 0.0 end

  # Delete the existing tree entries.
  $myTree delete [$myTree children {}]

  set nBlksz [exec_lsmtest_show -c $myCfg $myDb blocksize]
  set nPgsz  [exec_lsmtest_show -c $myCfg $myDb pagesize]

  if {[regexp {c=1} $myCfg]          || [regexp {co=1} $myCfg]
   || [regexp {com=1} $myCfg]        || [regexp {comp=1} $myCfg]
   || [regexp {compr=1} $myCfg]      || [regexp {compres=1} $myCfg]
   || [regexp {compress=1} $myCfg]   || [regexp {compressi=1} $myCfg]
   || [regexp {compressio=1} $myCfg] || [regexp {compression=1} $myCfg]







|







397
398
399
400
401
402
403
404
405
406
407
408
409
410
411

  set data [string trim [exec_lsmtest_show -c $myCfg $myDb array-pages $iFirst]]
  $myText delete 0.0 end

  # Delete the existing tree entries.
  $myTree delete [$myTree children {}]

  set nBlksz [expr [exec_lsmtest_show -c $myCfg $myDb blocksize] * 1024]
  set nPgsz  [exec_lsmtest_show -c $myCfg $myDb pagesize]

  if {[regexp {c=1} $myCfg]          || [regexp {co=1} $myCfg]
   || [regexp {com=1} $myCfg]        || [regexp {comp=1} $myCfg]
   || [regexp {compr=1} $myCfg]      || [regexp {compres=1} $myCfg]
   || [regexp {compress=1} $myCfg]   || [regexp {compressi=1} $myCfg]
   || [regexp {compressio=1} $myCfg] || [regexp {compression=1} $myCfg]

Changes to www/lsmusr.wiki.

1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322

<p>Database optimization transforms the contents of database file so that
the following are true:

<ul>
  <li> <p>All database content is stored in a single 
       <a href=#architectural_overview>segment</a>. This makes the
       database effectively equivalent to an optimally packed b-tree stucture
       for search operations - minimizing the number of disk sectors that need
       to be visted when searching the database.

  <li> <p>The database file contains no (or as little as possible) free space.
       In other words, it is no larger than required to contain the single
       segment.
</ul>

<p>In order to optimize the database, lsm_work() should be called with 
the nMerge argument set to 1 and the third parameter set to a negative value
(interpreted as - keep working until there is no more work to do). For 
example:

<verbatim>
  rc = lsm_work(db, 1, -1, 0);
</verbatim>

<p><span style=color:red>todo: the -1 as the 3rd argument above is currently
not supported</span>

<p>When optimizing the database as above, either the LSM_CONFIG_AUTOCHECKPOINT
parameter should be set to a non-zero value or lsm_checkpoint() should be
called periodically. Otherwise, no checkpoints will be performed, preventing
the library from reusing any space occupied by old segments even after their
content has been merged into the new segment. The result - a database file that
is optimized, except that it is up to twice as large as it otherwise would be.











|










|






<
<
<






<



1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309



1310
1311
1312
1313
1314
1315

1316
1317
1318

<p>Database optimization transforms the contents of database file so that
the following are true:

<ul>
  <li> <p>All database content is stored in a single 
       <a href=#architectural_overview>segment</a>. This makes the
       data structure equivalent to an optimally packed b-tree stucture
       for search operations - minimizing the number of disk sectors that need
       to be visted when searching the database.

  <li> <p>The database file contains no (or as little as possible) free space.
       In other words, it is no larger than required to contain the single
       segment.
</ul>

<p>In order to optimize the database, lsm_work() should be called with 
the nMerge argument set to 1 and the third parameter set to a negative value
(interpreted as "keep working until there is no more work to do"). For 
example:

<verbatim>
  rc = lsm_work(db, 1, -1, 0);
</verbatim>




<p>When optimizing the database as above, either the LSM_CONFIG_AUTOCHECKPOINT
parameter should be set to a non-zero value or lsm_checkpoint() should be
called periodically. Otherwise, no checkpoints will be performed, preventing
the library from reusing any space occupied by old segments even after their
content has been merged into the new segment. The result - a database file that
is optimized, except that it is up to twice as large as it otherwise would be.