ADDED ext/rbu/rbuexlock.test Index: ext/rbu/rbuexlock.test ================================================================== --- /dev/null +++ ext/rbu/rbuexlock.test @@ -0,0 +1,207 @@ +# 2021 November 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rbuexlock + +db close +sqlite3_shutdown +sqlite3_config_uri 1 + +# Create a simple RBU database. That expects to write to a table: +# +# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +# +proc create_rbu {filename} { + forcedelete $filename + sqlite3 rbu1 $filename + rbu1 eval { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(10, random(), random(), 0); + INSERT INTO data_t1 VALUES(20, random(), random(), 0); + INSERT INTO data_t1 VALUES(30, random(), random(), 0); + INSERT INTO data_t1 VALUES(40, random(), random(), 0); + INSERT INTO data_t1 VALUES(50, random(), random(), 0); + INSERT INTO data_t1 VALUES(60, random(), random(), 0); + INSERT INTO data_t1 VALUES(70, random(), random(), 0); + INSERT INTO data_t1 VALUES(80, random(), random(), 0); + } + rbu1 close + return $filename +} + +reset_db + +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b INT, c INT); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + INSERT INTO t1 VALUES(1, 2, 3); +} +create_rbu rbu1.db + +do_test 1.1.0 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=1 rbu1.db + rbu step +} SQLITE_OK +do_catchsql_test 1.1.1 { SELECT * FROM t1 } {0 {1 2 3}} + +do_test 1.2.0 { + for {set ii 0} {$ii < 10} {incr ii} { + rbu step + } + rbu step +} SQLITE_OK +do_catchsql_test 1.2.1 { SELECT * FROM t1 } {0 {1 2 3}} +do_test 1.2.2 { + db eval {PRAGMA journal_mode} +} {delete} + +do_test 1.3.0 { + while {[file exists test.db-wal]==0} { + rbu step + } +} {} +do_catchsql_test 1.3.1 { SELECT * FROM t1 } {1 {database is locked}} +do_test 1.3.2 { + db eval {PRAGMA journal_mode} +} {delete} + + +do_test 1.4.0 { + rbu step +} SQLITE_OK +do_catchsql_test 1.4.1 { SELECT * FROM t1 } {1 {database is locked}} +do_test 1.4.2 { + db eval {PRAGMA journal_mode} +} {delete} + + +rbu close + +do_test 1.5.0 { + file exists test.db-wal +} {1} +do_test 1.5.1 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=1 rbu1.db + file exists test.db-wal +} 1 +do_catchsql_test 1.5.2 { SELECT * FROM t1 } {1 {database is locked}} +do_test 1.5.2 { + db eval {PRAGMA journal_mode} +} {delete} + + +do_test 1.6.0 { + rbu step +} SQLITE_OK +do_catchsql_test 1.6.1 { SELECT * FROM t1 } {1 {database is locked}} +do_test 1.6.2 { + db eval {PRAGMA journal_mode} +} {delete} + +do_test 1.7.0 { + while {[rbu step]=="SQLITE_OK"} {} + rbu close +} SQLITE_DONE +do_catchsql_test 1.7.2 { SELECT count(*) FROM t1 } {0 9} +do_test 1.7.2 { + db eval {PRAGMA journal_mode} +} {delete} + +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a PRIMARY KEY, b INT, c INT); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + INSERT INTO t1 VALUES(1, 2, 3); +} +create_rbu rbu1.db + +do_test 2.1.0 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=0 rbu1.db + rbu step +} SQLITE_OK +do_catchsql_test 2.1.1 { SELECT * FROM t1 } {0 {1 2 3}} + +do_test 2.2.0 { + for {set ii 0} {$ii < 10} {incr ii} { + rbu step + } + rbu step +} SQLITE_OK +do_catchsql_test 2.2.1 { SELECT * FROM t1 } {0 {1 2 3}} + +do_test 2.3.0 { + while {[file exists test.db-wal]==0} { + rbu step + } +} {} +do_test 2.3.1 { + llength [db eval {SELECT * FROM t1}] +} {27} +do_test 2.3.2 { + db eval {PRAGMA journal_mode} +} {wal} + +do_test 2.4.0 { + rbu step +} SQLITE_OK +do_test 2.4.1 { + llength [db eval {SELECT * FROM t1}] +} {27} +do_test 2.4.2 { + db eval {PRAGMA journal_mode} +} {wal} + +rbu close + +do_test 2.5.0 { + db eval {PRAGMA journal_mode} +} {wal} +do_execsql_test 2.5.1 { + DELETE FROM t1; +} {} + +create_rbu rbu1.db +do_test 3.1.0 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=0 rbu1.db + rbu step +} SQLITE_ERROR + +do_test 3.1.1 { + set rc [catch {rbu close} msg] + lappend rc $msg +} {1 {SQLITE_ERROR - cannot update wal mode database}} +db eval {PRAGMA journal_mode=DELETE} + +create_rbu rbu1.db +do_test 3.2.0 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=0 rbu1.db + rbu step +} SQLITE_OK + +do_test 3.3.1 { + set rc [catch {rbu close} msg] + lappend rc $msg +} {0 SQLITE_OK} + +db close +create_rbu rbu1.db +do_test 3.4.0 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=0 rbu1.db + rbu step +} SQLITE_OK +rbu close + + +finish_test Index: ext/rbu/sqlite3rbu.c ================================================================== --- ext/rbu/sqlite3rbu.c +++ ext/rbu/sqlite3rbu.c @@ -107,10 +107,17 @@ ** Swap two objects of type TYPE. */ #if !defined(SQLITE_AMALGAMATION) # define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} #endif + +/* +** Name of the URI option that causes RBU to take an exclusive lock as +** part of the incremental checkpoint operation. +*/ +#define RBU_EXCLUSIVE_CHECKPOINT "rbu_exclusive_checkpoint" + /* ** The rbu_state table is used to save the state of a partially applied ** update so that it can be resumed later. The table consists of integer ** keys mapped to values as follows: @@ -2754,17 +2761,23 @@ /* ** Open the database handle and attach the RBU database as "rbu". If an ** error occurs, leave an error code and message in the RBU handle. +** +** If argument dbMain is not NULL, then it is a database handle already +** open on the target database. Use this handle instead of opening a new +** one. */ -static void rbuOpenDatabase(sqlite3rbu *p, int *pbRetry){ +static void rbuOpenDatabase(sqlite3rbu *p, sqlite3 *dbMain, int *pbRetry){ assert( p->rc || (p->dbMain==0 && p->dbRbu==0) ); assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 ); + assert( dbMain==0 || rbuIsVacuum(p)==0 ); /* Open the RBU database */ p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); + p->dbMain = dbMain; if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); if( p->zState==0 ){ const char *zFile = sqlite3_db_filename(p->dbRbu, "main"); @@ -3126,19 +3139,35 @@ p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff); } /* -** Take an EXCLUSIVE lock on the database file. +** Take an EXCLUSIVE lock on the database file. Return SQLITE_OK if +** successful, or an SQLite error code otherwise. +*/ +static int rbuLockDatabase(sqlite3 *db){ + int rc = SQLITE_OK; + sqlite3_file *fd = 0; + sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd); + + if( fd->pMethods ){ + rc = fd->pMethods->xLock(fd, SQLITE_LOCK_SHARED); + if( rc==SQLITE_OK ){ + rc = fd->pMethods->xLock(fd, SQLITE_LOCK_EXCLUSIVE); + } + } + return rc; +} + +/* +** Return true if the database handle passed as the only argument +** was opened with the rbu_exclusive_checkpoint=1 URI parameter +** specified. Or false otherwise. */ -static void rbuLockDatabase(sqlite3rbu *p){ - sqlite3_file *pReal = p->pTargetFd->pReal; - assert( p->rc==SQLITE_OK ); - p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_SHARED); - if( p->rc==SQLITE_OK ){ - p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_EXCLUSIVE); - } +static int rbuExclusiveCheckpoint(sqlite3 *db){ + const char *zUri = sqlite3_db_filename(db, 0); + return sqlite3_uri_boolean(zUri, RBU_EXCLUSIVE_CHECKPOINT, 0); } #if defined(_WIN32_WCE) static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){ int nChar; @@ -3192,22 +3221,28 @@ ** in WAL mode). So no other connection may be writing the db. ** ** In order to ensure that there are no database readers, an EXCLUSIVE ** lock is obtained here before the *-oal is moved to *-wal. */ - rbuLockDatabase(p); + sqlite3 *dbMain = 0; + rbuFileSuffix3(zBase, zWal); + rbuFileSuffix3(zBase, zOal); + + /* Re-open the databases. */ + rbuObjIterFinalize(&p->objiter); + sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); + p->dbMain = 0; + p->dbRbu = 0; + + dbMain = rbuOpenDbhandle(p, p->zTarget, 1); + if( dbMain ){ + assert( p->rc==SQLITE_OK ); + p->rc = rbuLockDatabase(dbMain); + } + if( p->rc==SQLITE_OK ){ - rbuFileSuffix3(zBase, zWal); - rbuFileSuffix3(zBase, zOal); - - /* Re-open the databases. */ - rbuObjIterFinalize(&p->objiter); - sqlite3_close(p->dbRbu); - sqlite3_close(p->dbMain); - p->dbMain = 0; - p->dbRbu = 0; - #if defined(_WIN32_WCE) { LPWSTR zWideOal; LPWSTR zWideWal; @@ -3230,15 +3265,23 @@ } } #else p->rc = rename(zOal, zWal) ? SQLITE_IOERR : SQLITE_OK; #endif + } - if( p->rc==SQLITE_OK ){ - rbuOpenDatabase(p, 0); - rbuSetupCheckpoint(p, 0); - } + if( p->rc!=SQLITE_OK + || rbuIsVacuum(p) + || rbuExclusiveCheckpoint(dbMain)==0 + ){ + sqlite3_close(dbMain); + dbMain = 0; + } + + if( p->rc==SQLITE_OK ){ + rbuOpenDatabase(p, dbMain, 0); + rbuSetupCheckpoint(p, 0); } } sqlite3_free(zWal); sqlite3_free(zOal); @@ -3985,13 +4028,13 @@ ** to be a wal-mode db. But, this may have happened due to an earlier ** RBU vacuum operation leaving an old wal file in the directory. ** If this is the case, it will have been checkpointed and deleted ** when the handle was closed and a second attempt to open the ** database may succeed. */ - rbuOpenDatabase(p, &bRetry); + rbuOpenDatabase(p, 0, &bRetry); if( bRetry ){ - rbuOpenDatabase(p, 0); + rbuOpenDatabase(p, 0, 0); } } if( p->rc==SQLITE_OK ){ pState = rbuLoadState(p); @@ -4082,10 +4125,18 @@ } } }else if( p->eStage==RBU_STAGE_MOVE ){ /* no-op */ }else if( p->eStage==RBU_STAGE_CKPT ){ + if( !rbuIsVacuum(p) && rbuExclusiveCheckpoint(p->dbMain) ){ + /* If the rbu_exclusive_checkpoint=1 URI parameter was specified + ** and an incremental checkpoint is being resumed, attempt an + ** exclusive lock on the db file. If this fails, so be it. */ + p->eStage = RBU_STAGE_DONE; + rbuLockDatabase(p->dbMain); + p->eStage = RBU_STAGE_CKPT; + } rbuSetupCheckpoint(p, pState); }else if( p->eStage==RBU_STAGE_DONE ){ p->rc = SQLITE_DONE; }else{ p->rc = SQLITE_CORRUPT; @@ -4119,11 +4170,10 @@ const char *zTarget, const char *zRbu, const char *zState ){ if( zTarget==0 || zRbu==0 ){ return rbuMisuseError(); } - /* TODO: Check that zTarget and zRbu are non-NULL */ return openRbuHandle(zTarget, zRbu, zState); } /* ** Open a handle to begin or resume an RBU VACUUM operation.