SQLite4
Check-in [77b1401c13]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Limit the number of levels that may exist within the database file.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | multi-process
Files: files | file ages | folders
SHA1: 77b1401c13a68d505f0c4156a156fd9d0f8b54cf
User & Date: dan 2012-08-28 19:06:05
Context
2012-08-29
19:45
Fix some of the problems with very large free-block lists. check-in: dc91e55841 user: dan tags: multi-process
2012-08-28
19:06
Limit the number of levels that may exist within the database file. check-in: 77b1401c13 user: dan tags: multi-process
16:50
Fix an OOB read caused by an implicit UNIQUE index on the same columns as the primary key of a table. check-in: 57b62e32f6 user: dan tags: multi-process
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/lsmInt.h.

474
475
476
477
478
479
480

481
482
483
484
485
486
487
i64 lsmCheckpointLogOffset(u32 *);
int lsmCheckpointPgsz(u32 *);
int lsmCheckpointBlksz(u32 *);
void lsmCheckpointLogoffset(u32 *aCkpt, DbLog *pLog);
void lsmCheckpointZeroLogoffset(lsm_db *);

int lsmCheckpointSaveWorker(lsm_db *pDb, int);



/* 
** Functions from file "lsm_tree.c".
*/
int lsmTreeNew(lsm_env *, int (*)(void *, int, void *, int), Tree **ppTree);
void lsmTreeRelease(lsm_env *, Tree *);







>







474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
i64 lsmCheckpointLogOffset(u32 *);
int lsmCheckpointPgsz(u32 *);
int lsmCheckpointBlksz(u32 *);
void lsmCheckpointLogoffset(u32 *aCkpt, DbLog *pLog);
void lsmCheckpointZeroLogoffset(lsm_db *);

int lsmCheckpointSaveWorker(lsm_db *pDb, int);
int lsmDatabaseFull(lsm_db *pDb);


/* 
** Functions from file "lsm_tree.c".
*/
int lsmTreeNew(lsm_env *, int (*)(void *, int, void *, int), Tree **ppTree);
void lsmTreeRelease(lsm_env *, Tree *);

Changes to src/lsm_ckpt.c.

84
85
86
87
88
89
90







91
92
93
94
95
96
97
...
942
943
944
945
946
947
948

























949
950
951
952
953
954
955
**
**     1. First page of array,
**     2. Last page of array,
**     3. Root page of array (or 0),
**     4. Size of array in pages,
*/








/*
** OVERSIZED CHECKPOINT BLOBS:
**
** There are two slots allocated for checkpoints at the start of each
** database file. Each are 4096 bytes in size, so may accommodate
** checkpoints that consist of up to 1024 32-bit integers. Normally,
** this is enough.
................................................................................
    lsmFreeSnapshot(pDb->pEnv, pNew);
    pNew = 0;
  }

  *ppSnap = pNew;
  return rc;
}


























/*
** The connection passed as the only argument is currently the worker
** connection. Some work has been performed on the database by the connection,
** but no new snapshot has been written into shared memory.
**
** This function updates the shared-memory worker and client snapshots with







>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
...
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
**
**     1. First page of array,
**     2. Last page of array,
**     3. Root page of array (or 0),
**     4. Size of array in pages,
*/

/*
** A limit on the number of rhs segments that may be present in the database
** file. Defining this limit ensures that all level records fit within
** the 4096 byte limit for checkpoint blobs.
*/
#define LSM_CKPT_MAX_LEVELS 40

/*
** OVERSIZED CHECKPOINT BLOBS:
**
** There are two slots allocated for checkpoints at the start of each
** database file. Each are 4096 bytes in size, so may accommodate
** checkpoints that consist of up to 1024 32-bit integers. Normally,
** this is enough.
................................................................................
    lsmFreeSnapshot(pDb->pEnv, pNew);
    pNew = 0;
  }

  *ppSnap = pNew;
  return rc;
}

/*
** Connection pDb must be the worker connection in order to call this
** function. It returns true if the database already contains the maximum
** number of levels or false otherwise.
**
** This is used when flushing the in-memory tree to disk. If the database
** is already full, then the caller should invoke lsm_work() or similar
** until it is not full before creating a new level by flushing the in-memory
** tree to disk. Limiting the number of levels in the database ensures that
** the records describing them always fit within the checkpoint blob.
*/
int lsmDatabaseFull(lsm_db *pDb){
  Level *p;
  int nRhs = 0;

  assert( lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL) );
  assert( pDb->pWorker );

  for(p=pDb->pWorker->pLevel; p; p=p->pNext){
    nRhs += (p->nRight ? p->nRight : 1);
  }

  return (nRhs >= LSM_CKPT_MAX_LEVELS);
}

/*
** The connection passed as the only argument is currently the worker
** connection. Some work has been performed on the database by the connection,
** but no new snapshot has been written into shared memory.
**
** This function updates the shared-memory worker and client snapshots with

Changes to src/lsm_main.c.

280
281
282
283
284
285
286



287
288
289
290
291
292
293
  if( rc==LSM_OK ){
    rc = lsmSaveCursors(pDb);
  }

  if( rc==LSM_OK && pDb->bAutowork ){
    rc = lsmSortedAutoWork(pDb, LSM_AUTOWORK_QUANT);
  }




  /* Write the contents of the in-memory tree into the database file and 
  ** update the worker snapshot accordingly. Then flush the contents of 
  ** the db file to disk too. No calls to fsync() are made here - just 
  ** write().  */
  if( rc==LSM_OK ) bOvfl = lsmCheckpointOverflow(pDb, &nLsmLevel);
  if( rc==LSM_OK ) rc = lsmSortedFlushTree(pDb, nLsmLevel, bOvfl);







>
>
>







280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
  if( rc==LSM_OK ){
    rc = lsmSaveCursors(pDb);
  }

  if( rc==LSM_OK && pDb->bAutowork ){
    rc = lsmSortedAutoWork(pDb, LSM_AUTOWORK_QUANT);
  }
  while( rc==LSM_OK && lsmDatabaseFull(pDb) ){
    rc = lsmSortedAutoWork(pDb, LSM_AUTOWORK_QUANT);
  }

  /* Write the contents of the in-memory tree into the database file and 
  ** update the worker snapshot accordingly. Then flush the contents of 
  ** the db file to disk too. No calls to fsync() are made here - just 
  ** write().  */
  if( rc==LSM_OK ) bOvfl = lsmCheckpointOverflow(pDb, &nLsmLevel);
  if( rc==LSM_OK ) rc = lsmSortedFlushTree(pDb, nLsmLevel, bOvfl);

Added test/ckpt1.test.































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
# 2012 August 29
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# The tests in this file focus on testing that very large checkpoints
# (those that occur when the database contains an unusually large number 
# of levels or free blocks) are handled correctly.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix ckpt1

# Check that lsm_config(AUTOWORK) seems to be connected to something.
#
do_test 1.1 { sqlite4_lsm_config db main autowork  0  } 0
do_test 1.2 { sqlite4_lsm_config db main autowork  1  } 1
do_test 1.3 { sqlite4_lsm_config db main autowork -1  } 1
do_test 1.4 { sqlite4_lsm_config db main autowork  0  } 0
do_test 1.5 { sqlite4_lsm_config db main autowork -1  } 0


set nLevel 200
do_execsql_test 2.0 { CREATE TABLE t1(a INTEGER PRIMARY KEY, b INTEGER UNIQUE) }
do_test 2.1 {
  for {set i 1} {$i <= $nLevel} {incr i} {
    db close
    sqlite4 db test.db
    sqlite4_lsm_config db main autowork 0
    db eval { INSERT INTO t1 VALUES($i, $i || $i) }
  }
  db eval { 
    SELECT count(*) FROM t1;
    PRAGMA integrity_check;
  }
} [list $nLevel ok]




finish_test

Changes to test/permutations.test.

129
130
131
132
133
134
135

136


137
138
139
140
141
142
143
#   quick
#   full
#
lappend ::testsuitelist xxx

test_suite "src4" -prefix "" -description {
} -files {

  simple.test log1.test log2.test log3.test csr1.test



  aggerror.test
  attach.test
  autoindex1.test
  badutf.test
  between.test
  bigrow.test







>
|
>
>







129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
#   quick
#   full
#
lappend ::testsuitelist xxx

test_suite "src4" -prefix "" -description {
} -files {
  simple.test 
  log1.test log2.test log3.test 
  csr1.test
  ckpt1.test

  aggerror.test
  attach.test
  autoindex1.test
  badutf.test
  between.test
  bigrow.test

Changes to test/test_lsm.c.

33
34
35
36
37
38
39

40
41
42
43
44
45
46
    int iVal;
  } aParam[] = {
    { "log-size",       LSM_CONFIG_LOG_SIZE }, 
    { "safety",         LSM_CONFIG_SAFETY }, 
    { "write-buffer",   LSM_CONFIG_WRITE_BUFFER }, 
    { "mmap",           LSM_CONFIG_MMAP }, 
    { "page-size",      LSM_CONFIG_PAGE_SIZE }, 

    { 0, 0 }
  };

  const char *zDb;                /* objv[1] as a string */
  const char *zName;              /* objv[2] as a string */
  int iParam;                     /* Second argument for lsm_config() */
  int iConfig = -1;               /* Third argument for lsm_config() */







>







33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
    int iVal;
  } aParam[] = {
    { "log-size",       LSM_CONFIG_LOG_SIZE }, 
    { "safety",         LSM_CONFIG_SAFETY }, 
    { "write-buffer",   LSM_CONFIG_WRITE_BUFFER }, 
    { "mmap",           LSM_CONFIG_MMAP }, 
    { "page-size",      LSM_CONFIG_PAGE_SIZE }, 
    { "autowork",       LSM_CONFIG_AUTOWORK }, 
    { 0, 0 }
  };

  const char *zDb;                /* objv[1] as a string */
  const char *zName;              /* objv[2] as a string */
  int iParam;                     /* Second argument for lsm_config() */
  int iConfig = -1;               /* Third argument for lsm_config() */