Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -44,10 +44,13 @@ ** plus implementations of sqlite3_os_init() and sqlite3_os_end(). */ #include "sqliteInt.h" #if SQLITE_OS_UNIX /* This file is used on unix only */ +/* Turn this feature on in all builds for now */ +#define SQLITE_MUTEXFREE_SHMLOCK 1 + /* ** There are various methods for file locking used for concurrency ** control: ** ** 1. POSIX locking (the default), @@ -4237,11 +4240,41 @@ #ifdef SQLITE_DEBUG u8 exclMask; /* Mask of exclusive locks held */ u8 sharedMask; /* Mask of shared locks held */ u8 nextShmId; /* Next available unixShm.id value */ #endif + +#ifdef SQLITE_MUTEXFREE_SHMLOCK + /* In unix-excl mode, if SQLITE_MUTEXFREE_SHMLOCK is defined, all locks + ** are stored in the following 64-bit value. There are in total 8 + ** shm-locking slots, each of which are assigned 8-bits from the 64-bit + ** value. The least-significant 8 bits correspond to shm-locking slot + ** 0, and so on. + ** + ** If the 8-bits corresponding to a shm-locking locking slot are set to + ** 0xFF, then a write-lock is held on the slot. Or, if they are set to + ** a non-zero value smaller than 0xFF, then they represent the total + ** number of read-locks held on the slot. There is no way to distinguish + ** between a write-lock and 255 read-locks. */ + u64 lockmask; +#endif }; + +/* +** Atomic CAS primitive used in multi-process mode. Equivalent to: +** +** int unixCompareAndSwap(u32 *ptr, u32 oldval, u32 newval){ +** if( *ptr==oldval ){ +** *ptr = newval; +** return 1; +** } +** return 0; +** } +*/ +#define unixCompareAndSwap(ptr,oldval,newval) \ + __sync_bool_compare_and_swap(ptr,oldval,newval) + /* ** Structure used internally by this VFS to record the state of an ** open shared memory connection. ** @@ -4798,10 +4831,62 @@ assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 ); assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 ); mask = (1<<(ofst+n)) - (1<1 || mask==(1<pInode->bProcessLock ){ + + while( 1 ){ + u64 lockmask = pShmNode->lockmask; + u64 newmask = lockmask; + int i; + for(i=ofst; i> (ix8)) & 0xFF; + if( flags & SQLITE_SHM_UNLOCK ){ + if( flags & SQLITE_SHM_EXCLUSIVE ){ + if( p->exclMask & (1 << i) ){ + newmask = newmask & ~((u64)0xFF<sharedMask & (1 << i) ){ + newmask = newmask & ~((u64)0xFF<exclMask & (1 << i))==0 ){ + newmask = newmask | ((u64)0xFF<sharedMask & (1 << i))==0 ){ + newmask = newmask & ~((u64)0xFF<lockmask, lockmask, newmask) ) break; + } + + if( flags & SQLITE_SHM_UNLOCK ){ + p->sharedMask &= ~mask; + p->exclMask &= ~mask; + }else if( flags & SQLITE_SHM_EXCLUSIVE ){ + p->exclMask |= mask; + }else{ + p->sharedMask |= mask; + } + + return SQLITE_OK; + } +#endif + sqlite3_mutex_enter(pShmNode->pShmMutex); if( flags & SQLITE_SHM_UNLOCK ){ u16 allMask = 0; /* Mask of locks held by siblings */ /* See if any siblings hold this same lock */ Index: src/test_vfs.c ================================================================== --- src/test_vfs.c +++ src/test_vfs.c @@ -1560,12 +1560,66 @@ bad_args: Tcl_WrongNumArgs(interp, 1, objv, "VFSNAME ?-noshm BOOL? ?-fullshm BOOL? ?-default BOOL? ?-mxpathname INT? ?-szosfile INT? ?-iversion INT?"); return TCL_ERROR; } + +extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); +extern const char *sqlite3ErrName(int); + +/* +** tclcmd: vfs_shmlock DB DBNAME (shared|exclusive) (lock|unlock) OFFSET N +*/ +static int SQLITE_TCLAPI test_vfs_shmlock( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *azArg1[] = {"shared", "exclusive", 0}; + const char *azArg2[] = {"lock", "unlock", 0}; + sqlite3 *db = 0; + int rc = SQLITE_OK; + const char *zDbname = 0; + int iArg1 = 0; + int iArg2 = 0; + int iOffset = 0; + int n = 0; + sqlite3_file *pFd; + + if( objc!=7 ){ + Tcl_WrongNumArgs(interp, 1, objv, + "DB DBNAME (shared|exclusive) (lock|unlock) OFFSET N" + ); + return TCL_ERROR; + } + + zDbname = Tcl_GetString(objv[2]); + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) + || Tcl_GetIndexFromObj(interp, objv[3], azArg1, "ARG", 0, &iArg1) + || Tcl_GetIndexFromObj(interp, objv[4], azArg2, "ARG", 0, &iArg2) + || Tcl_GetIntFromObj(interp, objv[5], &iOffset) + || Tcl_GetIntFromObj(interp, objv[6], &n) + ){ + return TCL_ERROR; + } + + sqlite3_file_control(db, zDbname, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd); + if( pFd==0 ){ + return TCL_ERROR; + } + rc = pFd->pMethods->xShmLock(pFd, iOffset, n, + (iArg1==0 ? SQLITE_SHM_SHARED : SQLITE_SHM_EXCLUSIVE) + | (iArg2==0 ? SQLITE_SHM_LOCK : SQLITE_SHM_UNLOCK) + ); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + return TCL_OK; +} + int Sqlitetestvfs_Init(Tcl_Interp *interp){ Tcl_CreateObjCommand(interp, "testvfs", testvfs_cmd, 0, 0); + Tcl_CreateObjCommand(interp, "vfs_shmlock", test_vfs_shmlock, 0, 0); return TCL_OK; } #endif Index: test/lock_common.tcl ================================================================== --- test/lock_common.tcl +++ test/lock_common.tcl @@ -13,22 +13,24 @@ # processes) to test locking. # proc do_multiclient_test {varname script} { - foreach code [list { + foreach {tn code} [list 1 { if {[info exists ::G(valgrind)]} { db close ; continue } set ::code2_chan [launch_testfixture] set ::code3_chan [launch_testfixture] proc code2 {tcl} { testfixture $::code2_chan $tcl } proc code3 {tcl} { testfixture $::code3_chan $tcl } - set tn 1 - } { + } 2 { proc code2 {tcl} { uplevel #0 $tcl } proc code3 {tcl} { uplevel #0 $tcl } - set tn 2 }] { + # Do not run multi-process tests with the unix-excl VFS. + # + if {$tn==1 && [permutation]=="unix-excl"} continue + faultsim_delete_and_reopen proc code1 {tcl} { uplevel #0 $tcl } # Open connections [db2] and [db3]. Depending on which iteration this Index: test/permutations.test ================================================================== --- test/permutations.test +++ test/permutations.test @@ -604,10 +604,20 @@ set ::G(perm:sqlite3_args) [list -vfs fs] } -files { conflict.test insert.test insert2.test insert3.test rollback.test select1.test select2.test select3.test } + +# Run some tests using the "unix-excl" VFS. +# +test_suite "unix-excl" -description { + Run some tests using the "unix-excl" VFS +} -initialize { + set ::G(perm:sqlite3_args) [list -vfs unix-excl] +} -files { + shmlock.test +} # Run some tests using UTF-16 databases. # test_suite "utf16" -description { Run tests using UTF-16 databases ADDED test/shmlock.test Index: test/shmlock.test ================================================================== --- /dev/null +++ test/shmlock.test @@ -0,0 +1,111 @@ +# 2018 December 6 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix shmlock + +ifcapable !wal {finish_test ; return } + +sqlite3 db2 test.db +sqlite3 db3 test.db + +do_execsql_test 1.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); +} {wal} +do_test 1.1 { execsql { SELECT * FROM t1 } db2 } {1 2} +do_test 1.2 { execsql { SELECT * FROM t1 } db3 } {1 2} + +foreach {tn dbhandle cmd res} { + 1 db {shared lock 7 1} OK + 2 db2 {exclusive lock 7 1} BUSY + 3 db {shared unlock 7 1} OK + 4 db2 {exclusive lock 7 1} OK + 5 db {shared lock 7 1} BUSY + 6 db {exclusive lock 7 1} BUSY + 7 db2 {exclusive unlock 7 1} OK + + 8 db {exclusive lock 0 8} OK + 9 db {exclusive unlock 0 8} OK + 10 db2 {exclusive lock 0 8} OK + 11 db2 {exclusive unlock 0 8} OK + + 12 db {shared lock 0 1} OK + 13 db2 {shared lock 0 1} OK + 14 db3 {shared lock 0 1} OK + 15 db3 {shared unlock 0 1} OK + 16 db3 {exclusive lock 0 1} BUSY + 17 db2 {shared unlock 0 1} OK + 18 db3 {exclusive lock 0 1} BUSY + 19 db {shared unlock 0 1} OK + 20 db3 {exclusive lock 0 1} OK + 21 db3 {exclusive unlock 0 1} OK + + 22 db {shared lock 3 1} OK + 23 db2 {exclusive lock 2 2} BUSY + 24 db {shared lock 2 1} OK + 25 db2 {exclusive lock 0 5} BUSY + 26 db2 {exclusive lock 0 4} BUSY + 27 db2 {exclusive lock 0 3} BUSY + 28 db {shared unlock 3 1} OK + 29 db2 {exclusive lock 2 2} BUSY + 28 db {shared unlock 2 1} OK + 29 db2 {exclusive lock 2 2} OK + 29 db2 {exclusive unlock 2 2} OK +} { + do_test 1.3.$tn [list vfs_shmlock $dbhandle main {*}$cmd] "SQLITE_$res" +} + +db close +db2 close +db3 close + +if {[permutation]=="unix-excl"} { + do_test 2.0 { + for {set i 0} {$i < 256} {incr i} { + sqlite3 db$i test.db + execsql { SELECT * FROM t1 } db$i + } + for {set i 0} {$i < 255} {incr i} { + set rc [vfs_shmlock db$i main shared lock 4 1] + if {$rc != "SQLITE_OK"} { error $rc } + } + + vfs_shmlock db255 main shared lock 4 1 + } {SQLITE_BUSY} + + do_test 2.1 { vfs_shmlock db255 main exclusive lock 4 1 } SQLITE_BUSY + do_test 2.2 { vfs_shmlock db0 main shared unlock 4 1 } SQLITE_OK + do_test 2.3 { vfs_shmlock db255 main shared lock 4 1 } SQLITE_OK + do_test 2.4 { vfs_shmlock db255 main shared unlock 4 1 } SQLITE_OK + do_test 2.5 { vfs_shmlock db255 main exclusive lock 4 1 } SQLITE_BUSY + + do_test 2.6 { + for {set i 1} {$i < 255} {incr i} { + set rc [vfs_shmlock db255 main exclusive lock 4 1] + if {$rc != "SQLITE_BUSY"} { error $rc } + set rc [vfs_shmlock db$i main shared unlock 4 1] + if {$rc != "SQLITE_OK"} { error $rc } + } + + vfs_shmlock db255 main exclusive lock 4 1 + } {SQLITE_OK} + + vfs_shmlock db255 main exclusive unlock 4 1 +} + +finish_test + + Index: test/wal.test ================================================================== --- test/wal.test +++ test/wal.test @@ -1295,55 +1295,57 @@ # the data is present and the database is not corrupt. # # At one point, SQLite was failing to grow the mapping of the wal-index # file in step 3 and the checkpoint was corrupting the database file. # -do_test wal-20.1 { - catch {db close} - forcedelete test.db test.db-wal test.db-journal - sqlite3 db test.db - execsql { - PRAGMA journal_mode = WAL; - CREATE TABLE t1(x); - INSERT INTO t1 VALUES(randomblob(900)); - SELECT count(*) FROM t1; - } -} {wal 1} -do_test wal-20.2 { - set ::buddy [launch_testfixture] - testfixture $::buddy { - sqlite3 db test.db - db transaction { db eval { - PRAGMA wal_autocheckpoint = 0; - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 128 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 256 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 512 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 1024 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2048 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4096 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8192 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16384 */ - } } - } -} {0} -do_test wal-20.3 { - close $::buddy - execsql { PRAGMA wal_checkpoint } - execsql { SELECT count(*) FROM t1 } -} {16384} -do_test wal-20.4 { - db close - sqlite3 db test.db - execsql { SELECT count(*) FROM t1 } -} {16384} -integrity_check wal-20.5 +if {[permutation]!="unix-excl"} { + do_test wal-20.1 { + catch {db close} + forcedelete test.db test.db-wal test.db-journal + sqlite3 db test.db + execsql { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(randomblob(900)); + SELECT count(*) FROM t1; + } + } {wal 1} + do_test wal-20.2 { + set ::buddy [launch_testfixture] + testfixture $::buddy { + sqlite3 db test.db + db transaction { db eval { + PRAGMA wal_autocheckpoint = 0; + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 128 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 256 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 512 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 1024 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2048 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4096 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8192 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16384 */ + } } + } + } {0} + do_test wal-20.3 { + close $::buddy + execsql { PRAGMA wal_checkpoint } + execsql { SELECT count(*) FROM t1 } + } {16384} + do_test wal-20.4 { + db close + sqlite3 db test.db + execsql { SELECT count(*) FROM t1 } + } {16384} + integrity_check wal-20.5 +} catch { db2 close } catch { db close } do_test wal-21.1 {