Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Experiment with using an atomic CPU primitive instead of a mutex for intra-process locking with the unix-excl VFS. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | mutexfree-shmlock |
Files: | files | file ages | folders |
SHA3-256: |
8f4cb9dd3396bcaaf85dcdb4e3ae3c96 |
User & Date: | dan 2018-12-06 18:58:58.578 |
Context
2018-12-10
| ||
09:45 | Avoid a mutex in-and-out in unixShmBarrier() on this branch. Use __sync_synchronize() instead. (check-in: a8c5fd86ce user: dan tags: mutexfree-shmlock) | |
2018-12-06
| ||
20:26 | Add the mutexfree-shmlock experiment to the begin-concurrent-pnu branch. (check-in: 3aead209b4 user: dan tags: begin-concurrent-pnu-mutexfree-shmlock) | |
18:58 | Experiment with using an atomic CPU primitive instead of a mutex for intra-process locking with the unix-excl VFS. (check-in: 8f4cb9dd33 user: dan tags: mutexfree-shmlock) | |
03:59 | Invoking the sqlite3_trace() or sqlite3_trace_v2() interfaces cancels any sqlite3_profile() that is running. (check-in: ec63d3506b user: drh tags: trunk) | |
Changes
Changes to src/os_unix.c.
︙ | ︙ | |||
42 43 44 45 46 47 48 49 50 51 52 53 54 55 | ** * Locking primitives for the proxy uber-locking-method. (MacOSX only) ** * Definitions of sqlite3_vfs objects for all locking methods ** plus implementations of sqlite3_os_init() and sqlite3_os_end(). */ #include "sqliteInt.h" #if SQLITE_OS_UNIX /* This file is used on unix only */ /* ** There are various methods for file locking used for concurrency ** control: ** ** 1. POSIX locking (the default), ** 2. No locking, ** 3. Dot-file locking, | > > > | 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | ** * Locking primitives for the proxy uber-locking-method. (MacOSX only) ** * Definitions of sqlite3_vfs objects for all locking methods ** plus implementations of sqlite3_os_init() and sqlite3_os_end(). */ #include "sqliteInt.h" #if SQLITE_OS_UNIX /* This file is used on unix only */ /* Turn this feature on in all builds for now */ #define SQLITE_MUTEXFREE_SHMLOCK 1 /* ** There are various methods for file locking used for concurrency ** control: ** ** 1. POSIX locking (the default), ** 2. No locking, ** 3. Dot-file locking, |
︙ | ︙ | |||
4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 | int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ #ifdef SQLITE_DEBUG u8 exclMask; /* Mask of exclusive locks held */ u8 sharedMask; /* Mask of shared locks held */ u8 nextShmId; /* Next available unixShm.id value */ #endif }; /* ** Structure used internally by this VFS to record the state of an ** open shared memory connection. ** ** The following fields are initialized when this object is created and ** are read-only thereafter: | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 | int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ #ifdef SQLITE_DEBUG u8 exclMask; /* Mask of exclusive locks held */ u8 sharedMask; /* Mask of shared locks held */ u8 nextShmId; /* Next available unixShm.id value */ #endif #ifdef SQLITE_MUTEXFREE_SHMLOCK /* In unix-excl mode, if SQLITE_MUTEXFREE_SHMLOCK is defined, all locks ** are stored in the following 64-bit value. There are in total 8 ** shm-locking slots, each of which are assigned 8-bits from the 64-bit ** value. The least-significant 8 bits correspond to shm-locking slot ** 0, and so on. ** ** If the 8-bits corresponding to a shm-locking locking slot are set to ** 0xFF, then a write-lock is held on the slot. Or, if they are set to ** a non-zero value smaller than 0xFF, then they represent the total ** number of read-locks held on the slot. There is no way to distinguish ** between a write-lock and 255 read-locks. */ u64 lockmask; #endif }; /* ** Atomic CAS primitive used in multi-process mode. Equivalent to: ** ** int unixCompareAndSwap(u32 *ptr, u32 oldval, u32 newval){ ** if( *ptr==oldval ){ ** *ptr = newval; ** return 1; ** } ** return 0; ** } */ #define unixCompareAndSwap(ptr,oldval,newval) \ __sync_bool_compare_and_swap(ptr,oldval,newval) /* ** Structure used internally by this VFS to record the state of an ** open shared memory connection. ** ** The following fields are initialized when this object is created and ** are read-only thereafter: |
︙ | ︙ | |||
4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 | || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 ); assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 ); mask = (1<<(ofst+n)) - (1<<ofst); assert( n>1 || mask==(1<<ofst) ); sqlite3_mutex_enter(pShmNode->pShmMutex); if( flags & SQLITE_SHM_UNLOCK ){ u16 allMask = 0; /* Mask of locks held by siblings */ /* See if any siblings hold this same lock */ for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ if( pX==p ) continue; | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 | || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 ); assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 ); mask = (1<<(ofst+n)) - (1<<ofst); assert( n>1 || mask==(1<<ofst) ); #ifdef SQLITE_MUTEXFREE_SHMLOCK if( pDbFd->pInode->bProcessLock ){ while( 1 ){ u64 lockmask = pShmNode->lockmask; u64 newmask = lockmask; int i; for(i=ofst; i<n+ofst; i++){ int ix8 = i*8; u8 v = (lockmask >> (ix8)) & 0xFF; if( flags & SQLITE_SHM_UNLOCK ){ if( flags & SQLITE_SHM_EXCLUSIVE ){ if( p->exclMask & (1 << i) ){ newmask = newmask & ~((u64)0xFF<<ix8); } }else{ if( p->sharedMask & (1 << i) ){ newmask = newmask & ~((u64)0xFF<<ix8) | ((u64)(v-1)<<ix8); } } }else{ if( flags & SQLITE_SHM_EXCLUSIVE ){ if( v ) return SQLITE_BUSY; if( (p->exclMask & (1 << i))==0 ){ newmask = newmask | ((u64)0xFF<<ix8); } }else{ if( v==0xFF ) return SQLITE_BUSY; if( (p->sharedMask & (1 << i))==0 ){ newmask = newmask & ~((u64)0xFF<<ix8) | ((u64)(v+1)<<ix8); } } } } if( unixCompareAndSwap(&pShmNode->lockmask, lockmask, newmask) ) break; } if( flags & SQLITE_SHM_UNLOCK ){ p->sharedMask &= ~mask; p->exclMask &= ~mask; }else if( flags & SQLITE_SHM_EXCLUSIVE ){ p->exclMask |= mask; }else{ p->sharedMask |= mask; } return SQLITE_OK; } #endif sqlite3_mutex_enter(pShmNode->pShmMutex); if( flags & SQLITE_SHM_UNLOCK ){ u16 allMask = 0; /* Mask of locks held by siblings */ /* See if any siblings hold this same lock */ for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ if( pX==p ) continue; |
︙ | ︙ |
Changes to src/test_vfs.c.
︙ | ︙ | |||
1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 | return TCL_OK; bad_args: Tcl_WrongNumArgs(interp, 1, objv, "VFSNAME ?-noshm BOOL? ?-fullshm BOOL? ?-default BOOL? ?-mxpathname INT? ?-szosfile INT? ?-iversion INT?"); return TCL_ERROR; } int Sqlitetestvfs_Init(Tcl_Interp *interp){ Tcl_CreateObjCommand(interp, "testvfs", testvfs_cmd, 0, 0); return TCL_OK; } #endif | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 | return TCL_OK; bad_args: Tcl_WrongNumArgs(interp, 1, objv, "VFSNAME ?-noshm BOOL? ?-fullshm BOOL? ?-default BOOL? ?-mxpathname INT? ?-szosfile INT? ?-iversion INT?"); return TCL_ERROR; } extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); extern const char *sqlite3ErrName(int); /* ** tclcmd: vfs_shmlock DB DBNAME (shared|exclusive) (lock|unlock) OFFSET N */ static int SQLITE_TCLAPI test_vfs_shmlock( void * clientData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[] ){ const char *azArg1[] = {"shared", "exclusive", 0}; const char *azArg2[] = {"lock", "unlock", 0}; sqlite3 *db = 0; int rc = SQLITE_OK; const char *zDbname = 0; int iArg1 = 0; int iArg2 = 0; int iOffset = 0; int n = 0; sqlite3_file *pFd; if( objc!=7 ){ Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME (shared|exclusive) (lock|unlock) OFFSET N" ); return TCL_ERROR; } zDbname = Tcl_GetString(objv[2]); if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) || Tcl_GetIndexFromObj(interp, objv[3], azArg1, "ARG", 0, &iArg1) || Tcl_GetIndexFromObj(interp, objv[4], azArg2, "ARG", 0, &iArg2) || Tcl_GetIntFromObj(interp, objv[5], &iOffset) || Tcl_GetIntFromObj(interp, objv[6], &n) ){ return TCL_ERROR; } sqlite3_file_control(db, zDbname, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd); if( pFd==0 ){ return TCL_ERROR; } rc = pFd->pMethods->xShmLock(pFd, iOffset, n, (iArg1==0 ? SQLITE_SHM_SHARED : SQLITE_SHM_EXCLUSIVE) | (iArg2==0 ? SQLITE_SHM_LOCK : SQLITE_SHM_UNLOCK) ); Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); return TCL_OK; } int Sqlitetestvfs_Init(Tcl_Interp *interp){ Tcl_CreateObjCommand(interp, "testvfs", testvfs_cmd, 0, 0); Tcl_CreateObjCommand(interp, "vfs_shmlock", test_vfs_shmlock, 0, 0); return TCL_OK; } #endif |
Changes to test/lock_common.tcl.
︙ | ︙ | |||
11 12 13 14 15 16 17 | # This file contains code used by several different test scripts. The # code in this file allows testfixture to control another process (or # processes) to test locking. # proc do_multiclient_test {varname script} { | | < | < > > > > | 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | # This file contains code used by several different test scripts. The # code in this file allows testfixture to control another process (or # processes) to test locking. # proc do_multiclient_test {varname script} { foreach {tn code} [list 1 { if {[info exists ::G(valgrind)]} { db close ; continue } set ::code2_chan [launch_testfixture] set ::code3_chan [launch_testfixture] proc code2 {tcl} { testfixture $::code2_chan $tcl } proc code3 {tcl} { testfixture $::code3_chan $tcl } } 2 { proc code2 {tcl} { uplevel #0 $tcl } proc code3 {tcl} { uplevel #0 $tcl } }] { # Do not run multi-process tests with the unix-excl VFS. # if {$tn==1 && [permutation]=="unix-excl"} continue faultsim_delete_and_reopen proc code1 {tcl} { uplevel #0 $tcl } # Open connections [db2] and [db3]. Depending on which iteration this # is, the connections may be created in this interpreter, or in # interpreters running in other OS processes. As such, the [db2] and [db3] |
︙ | ︙ |
Changes to test/permutations.test.
︙ | ︙ | |||
602 603 604 605 606 607 608 609 610 611 612 613 614 615 | Run some tests using the "test_onefile.c" demo } -initialize { set ::G(perm:sqlite3_args) [list -vfs fs] } -files { conflict.test insert.test insert2.test insert3.test rollback.test select1.test select2.test select3.test } # Run some tests using UTF-16 databases. # test_suite "utf16" -description { Run tests using UTF-16 databases } -presql { pragma encoding = 'UTF-16' | > > > > > > > > > > | 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 | Run some tests using the "test_onefile.c" demo } -initialize { set ::G(perm:sqlite3_args) [list -vfs fs] } -files { conflict.test insert.test insert2.test insert3.test rollback.test select1.test select2.test select3.test } # Run some tests using the "unix-excl" VFS. # test_suite "unix-excl" -description { Run some tests using the "unix-excl" VFS } -initialize { set ::G(perm:sqlite3_args) [list -vfs unix-excl] } -files { shmlock.test } # Run some tests using UTF-16 databases. # test_suite "utf16" -description { Run tests using UTF-16 databases } -presql { pragma encoding = 'UTF-16' |
︙ | ︙ |
Added test/shmlock.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | # 2018 December 6 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix shmlock ifcapable !wal {finish_test ; return } sqlite3 db2 test.db sqlite3 db3 test.db do_execsql_test 1.0 { PRAGMA journal_mode = wal; CREATE TABLE t1(a, b); INSERT INTO t1 VALUES(1, 2); } {wal} do_test 1.1 { execsql { SELECT * FROM t1 } db2 } {1 2} do_test 1.2 { execsql { SELECT * FROM t1 } db3 } {1 2} foreach {tn dbhandle cmd res} { 1 db {shared lock 7 1} OK 2 db2 {exclusive lock 7 1} BUSY 3 db {shared unlock 7 1} OK 4 db2 {exclusive lock 7 1} OK 5 db {shared lock 7 1} BUSY 6 db {exclusive lock 7 1} BUSY 7 db2 {exclusive unlock 7 1} OK 8 db {exclusive lock 0 8} OK 9 db {exclusive unlock 0 8} OK 10 db2 {exclusive lock 0 8} OK 11 db2 {exclusive unlock 0 8} OK 12 db {shared lock 0 1} OK 13 db2 {shared lock 0 1} OK 14 db3 {shared lock 0 1} OK 15 db3 {shared unlock 0 1} OK 16 db3 {exclusive lock 0 1} BUSY 17 db2 {shared unlock 0 1} OK 18 db3 {exclusive lock 0 1} BUSY 19 db {shared unlock 0 1} OK 20 db3 {exclusive lock 0 1} OK 21 db3 {exclusive unlock 0 1} OK 22 db {shared lock 3 1} OK 23 db2 {exclusive lock 2 2} BUSY 24 db {shared lock 2 1} OK 25 db2 {exclusive lock 0 5} BUSY 26 db2 {exclusive lock 0 4} BUSY 27 db2 {exclusive lock 0 3} BUSY 28 db {shared unlock 3 1} OK 29 db2 {exclusive lock 2 2} BUSY 28 db {shared unlock 2 1} OK 29 db2 {exclusive lock 2 2} OK 29 db2 {exclusive unlock 2 2} OK } { do_test 1.3.$tn [list vfs_shmlock $dbhandle main {*}$cmd] "SQLITE_$res" } db close db2 close db3 close if {[permutation]=="unix-excl"} { do_test 2.0 { for {set i 0} {$i < 256} {incr i} { sqlite3 db$i test.db execsql { SELECT * FROM t1 } db$i } for {set i 0} {$i < 255} {incr i} { set rc [vfs_shmlock db$i main shared lock 4 1] if {$rc != "SQLITE_OK"} { error $rc } } vfs_shmlock db255 main shared lock 4 1 } {SQLITE_BUSY} do_test 2.1 { vfs_shmlock db255 main exclusive lock 4 1 } SQLITE_BUSY do_test 2.2 { vfs_shmlock db0 main shared unlock 4 1 } SQLITE_OK do_test 2.3 { vfs_shmlock db255 main shared lock 4 1 } SQLITE_OK do_test 2.4 { vfs_shmlock db255 main shared unlock 4 1 } SQLITE_OK do_test 2.5 { vfs_shmlock db255 main exclusive lock 4 1 } SQLITE_BUSY do_test 2.6 { for {set i 1} {$i < 255} {incr i} { set rc [vfs_shmlock db255 main exclusive lock 4 1] if {$rc != "SQLITE_BUSY"} { error $rc } set rc [vfs_shmlock db$i main shared unlock 4 1] if {$rc != "SQLITE_OK"} { error $rc } } vfs_shmlock db255 main exclusive lock 4 1 } {SQLITE_OK} vfs_shmlock db255 main exclusive unlock 4 1 } finish_test |
Changes to test/wal.test.
︙ | ︙ | |||
1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 | # # 3. Using connection 1, checkpoint the database. Make sure all # the data is present and the database is not corrupt. # # At one point, SQLite was failing to grow the mapping of the wal-index # file in step 3 and the checkpoint was corrupting the database file. # do_test wal-20.1 { catch {db close} forcedelete test.db test.db-wal test.db-journal sqlite3 db test.db execsql { PRAGMA journal_mode = WAL; CREATE TABLE t1(x); | > | 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 | # # 3. Using connection 1, checkpoint the database. Make sure all # the data is present and the database is not corrupt. # # At one point, SQLite was failing to grow the mapping of the wal-index # file in step 3 and the checkpoint was corrupting the database file. # if {[permutation]!="unix-excl"} { do_test wal-20.1 { catch {db close} forcedelete test.db test.db-wal test.db-journal sqlite3 db test.db execsql { PRAGMA journal_mode = WAL; CREATE TABLE t1(x); |
︙ | ︙ | |||
1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 | } {16384} do_test wal-20.4 { db close sqlite3 db test.db execsql { SELECT count(*) FROM t1 } } {16384} integrity_check wal-20.5 catch { db2 close } catch { db close } do_test wal-21.1 { faultsim_delete_and_reopen execsql { | > | 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 | } {16384} do_test wal-20.4 { db close sqlite3 db test.db execsql { SELECT count(*) FROM t1 } } {16384} integrity_check wal-20.5 } catch { db2 close } catch { db close } do_test wal-21.1 { faultsim_delete_and_reopen execsql { |
︙ | ︙ |