Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Limit the number of levels that may exist within the database file. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | multi-process |
Files: | files | file ages | folders |
SHA1: |
77b1401c13a68d505f0c4156a156fd9d |
User & Date: | dan 2012-08-28 19:06:05.666 |
Context
2012-08-29
| ||
19:45 | Fix some of the problems with very large free-block lists. check-in: dc91e55841 user: dan tags: multi-process | |
2012-08-28
| ||
19:06 | Limit the number of levels that may exist within the database file. check-in: 77b1401c13 user: dan tags: multi-process | |
16:50 | Fix an OOB read caused by an implicit UNIQUE index on the same columns as the primary key of a table. check-in: 57b62e32f6 user: dan tags: multi-process | |
Changes
Changes to src/lsmInt.h.
︙ | ︙ | |||
474 475 476 477 478 479 480 481 482 483 484 485 486 487 | i64 lsmCheckpointLogOffset(u32 *); int lsmCheckpointPgsz(u32 *); int lsmCheckpointBlksz(u32 *); void lsmCheckpointLogoffset(u32 *aCkpt, DbLog *pLog); void lsmCheckpointZeroLogoffset(lsm_db *); int lsmCheckpointSaveWorker(lsm_db *pDb, int); /* ** Functions from file "lsm_tree.c". */ int lsmTreeNew(lsm_env *, int (*)(void *, int, void *, int), Tree **ppTree); void lsmTreeRelease(lsm_env *, Tree *); | > | 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 | i64 lsmCheckpointLogOffset(u32 *); int lsmCheckpointPgsz(u32 *); int lsmCheckpointBlksz(u32 *); void lsmCheckpointLogoffset(u32 *aCkpt, DbLog *pLog); void lsmCheckpointZeroLogoffset(lsm_db *); int lsmCheckpointSaveWorker(lsm_db *pDb, int); int lsmDatabaseFull(lsm_db *pDb); /* ** Functions from file "lsm_tree.c". */ int lsmTreeNew(lsm_env *, int (*)(void *, int, void *, int), Tree **ppTree); void lsmTreeRelease(lsm_env *, Tree *); |
︙ | ︙ |
Changes to src/lsm_ckpt.c.
︙ | ︙ | |||
84 85 86 87 88 89 90 91 92 93 94 95 96 97 | ** ** 1. First page of array, ** 2. Last page of array, ** 3. Root page of array (or 0), ** 4. Size of array in pages, */ /* ** OVERSIZED CHECKPOINT BLOBS: ** ** There are two slots allocated for checkpoints at the start of each ** database file. Each are 4096 bytes in size, so may accommodate ** checkpoints that consist of up to 1024 32-bit integers. Normally, ** this is enough. | > > > > > > > | 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | ** ** 1. First page of array, ** 2. Last page of array, ** 3. Root page of array (or 0), ** 4. Size of array in pages, */ /* ** A limit on the number of rhs segments that may be present in the database ** file. Defining this limit ensures that all level records fit within ** the 4096 byte limit for checkpoint blobs. */ #define LSM_CKPT_MAX_LEVELS 40 /* ** OVERSIZED CHECKPOINT BLOBS: ** ** There are two slots allocated for checkpoints at the start of each ** database file. Each are 4096 bytes in size, so may accommodate ** checkpoints that consist of up to 1024 32-bit integers. Normally, ** this is enough. |
︙ | ︙ | |||
942 943 944 945 946 947 948 949 950 951 952 953 954 955 | lsmFreeSnapshot(pDb->pEnv, pNew); pNew = 0; } *ppSnap = pNew; return rc; } /* ** The connection passed as the only argument is currently the worker ** connection. Some work has been performed on the database by the connection, ** but no new snapshot has been written into shared memory. ** ** This function updates the shared-memory worker and client snapshots with | > > > > > > > > > > > > > > > > > > > > > > > > > | 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 | lsmFreeSnapshot(pDb->pEnv, pNew); pNew = 0; } *ppSnap = pNew; return rc; } /* ** Connection pDb must be the worker connection in order to call this ** function. It returns true if the database already contains the maximum ** number of levels or false otherwise. ** ** This is used when flushing the in-memory tree to disk. If the database ** is already full, then the caller should invoke lsm_work() or similar ** until it is not full before creating a new level by flushing the in-memory ** tree to disk. Limiting the number of levels in the database ensures that ** the records describing them always fit within the checkpoint blob. */ int lsmDatabaseFull(lsm_db *pDb){ Level *p; int nRhs = 0; assert( lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL) ); assert( pDb->pWorker ); for(p=pDb->pWorker->pLevel; p; p=p->pNext){ nRhs += (p->nRight ? p->nRight : 1); } return (nRhs >= LSM_CKPT_MAX_LEVELS); } /* ** The connection passed as the only argument is currently the worker ** connection. Some work has been performed on the database by the connection, ** but no new snapshot has been written into shared memory. ** ** This function updates the shared-memory worker and client snapshots with |
︙ | ︙ |
Changes to src/lsm_main.c.
︙ | ︙ | |||
280 281 282 283 284 285 286 287 288 289 290 291 292 293 | if( rc==LSM_OK ){ rc = lsmSaveCursors(pDb); } if( rc==LSM_OK && pDb->bAutowork ){ rc = lsmSortedAutoWork(pDb, LSM_AUTOWORK_QUANT); } /* Write the contents of the in-memory tree into the database file and ** update the worker snapshot accordingly. Then flush the contents of ** the db file to disk too. No calls to fsync() are made here - just ** write(). */ if( rc==LSM_OK ) bOvfl = lsmCheckpointOverflow(pDb, &nLsmLevel); if( rc==LSM_OK ) rc = lsmSortedFlushTree(pDb, nLsmLevel, bOvfl); | > > > | 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 | if( rc==LSM_OK ){ rc = lsmSaveCursors(pDb); } if( rc==LSM_OK && pDb->bAutowork ){ rc = lsmSortedAutoWork(pDb, LSM_AUTOWORK_QUANT); } while( rc==LSM_OK && lsmDatabaseFull(pDb) ){ rc = lsmSortedAutoWork(pDb, LSM_AUTOWORK_QUANT); } /* Write the contents of the in-memory tree into the database file and ** update the worker snapshot accordingly. Then flush the contents of ** the db file to disk too. No calls to fsync() are made here - just ** write(). */ if( rc==LSM_OK ) bOvfl = lsmCheckpointOverflow(pDb, &nLsmLevel); if( rc==LSM_OK ) rc = lsmSortedFlushTree(pDb, nLsmLevel, bOvfl); |
︙ | ︙ |
Added test/ckpt1.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | # 2012 August 29 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # The tests in this file focus on testing that very large checkpoints # (those that occur when the database contains an unusually large number # of levels or free blocks) are handled correctly. # set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix ckpt1 # Check that lsm_config(AUTOWORK) seems to be connected to something. # do_test 1.1 { sqlite4_lsm_config db main autowork 0 } 0 do_test 1.2 { sqlite4_lsm_config db main autowork 1 } 1 do_test 1.3 { sqlite4_lsm_config db main autowork -1 } 1 do_test 1.4 { sqlite4_lsm_config db main autowork 0 } 0 do_test 1.5 { sqlite4_lsm_config db main autowork -1 } 0 set nLevel 200 do_execsql_test 2.0 { CREATE TABLE t1(a INTEGER PRIMARY KEY, b INTEGER UNIQUE) } do_test 2.1 { for {set i 1} {$i <= $nLevel} {incr i} { db close sqlite4 db test.db sqlite4_lsm_config db main autowork 0 db eval { INSERT INTO t1 VALUES($i, $i || $i) } } db eval { SELECT count(*) FROM t1; PRAGMA integrity_check; } } [list $nLevel ok] finish_test |
Changes to test/permutations.test.
︙ | ︙ | |||
129 130 131 132 133 134 135 | # quick # full # lappend ::testsuitelist xxx test_suite "src4" -prefix "" -description { } -files { | > | > > | 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 | # quick # full # lappend ::testsuitelist xxx test_suite "src4" -prefix "" -description { } -files { simple.test log1.test log2.test log3.test csr1.test ckpt1.test aggerror.test attach.test autoindex1.test badutf.test between.test bigrow.test |
︙ | ︙ |
Changes to test/test_lsm.c.
︙ | ︙ | |||
33 34 35 36 37 38 39 40 41 42 43 44 45 46 | int iVal; } aParam[] = { { "log-size", LSM_CONFIG_LOG_SIZE }, { "safety", LSM_CONFIG_SAFETY }, { "write-buffer", LSM_CONFIG_WRITE_BUFFER }, { "mmap", LSM_CONFIG_MMAP }, { "page-size", LSM_CONFIG_PAGE_SIZE }, { 0, 0 } }; const char *zDb; /* objv[1] as a string */ const char *zName; /* objv[2] as a string */ int iParam; /* Second argument for lsm_config() */ int iConfig = -1; /* Third argument for lsm_config() */ | > | 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | int iVal; } aParam[] = { { "log-size", LSM_CONFIG_LOG_SIZE }, { "safety", LSM_CONFIG_SAFETY }, { "write-buffer", LSM_CONFIG_WRITE_BUFFER }, { "mmap", LSM_CONFIG_MMAP }, { "page-size", LSM_CONFIG_PAGE_SIZE }, { "autowork", LSM_CONFIG_AUTOWORK }, { 0, 0 } }; const char *zDb; /* objv[1] as a string */ const char *zName; /* objv[2] as a string */ int iParam; /* Second argument for lsm_config() */ int iConfig = -1; /* Third argument for lsm_config() */ |
︙ | ︙ |