Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch opfs-lock-without-xlock Excluding Merge-Ins
This is equivalent to a diff from ae43e97087 to a7fe91afca
2022-11-10
| ||
13:22 | Automatically relinquish implicitly-acquired OPFS file locks during VFS idle time in an attempt to help alleviate cross-tab locking contention like that described in forum post 58a377083cd24a. (check-in: 8daf24ff73 user: stephan tags: trunk) | |
13:14 | Rework automatically acquired OPFS locks to be released during idle time. This eliminates the performance hit reported in [46304ba057707c]. (Closed-Leaf check-in: a7fe91afca user: stephan tags: opfs-lock-without-xlock) | |
11:35 | OPFS: if an op which needs a lock is called when no lock has been obtained, automatically lock it at the start of the op and unlock it at the end of that op. This is an attempt to alleviate the cross-tab contention described in forum post 58a377083cd24a but it increases speedtest1 run time by approximately 4x. Perhaps auto-lock can be combined with the older idle-time-based auto-unlock to unlock such locks (but not those from xLock()) to improve this? (check-in: 46304ba057 user: stephan tags: opfs-lock-without-xlock) | |
2022-11-09
| ||
11:17 | Better handle an error in the fts5 integrity-check code. dbsqlfuzz e87c62f9b67ea21aebdc36ab71cab7cc3eda8dc3. (check-in: ae43e97087 user: dan tags: trunk) | |
11:02 | Additional defense against corrupt database files in dbdata.c. (check-in: 2e70d1e5c9 user: drh tags: trunk) | |
Changes to ext/wasm/api/sqlite3-api-opfs.js.
︙ | ︙ | |||
463 464 465 466 467 468 469 | default: toss("Invalid type ID:",tid); } }; /** Returns an array of the deserialized state stored by the most recent serialize() operation (from from this thread or the | | > > | | 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 | default: toss("Invalid type ID:",tid); } }; /** Returns an array of the deserialized state stored by the most recent serialize() operation (from from this thread or the counterpart thread), or null if the serialization buffer is empty. If passed a truthy argument, the serialization buffer is cleared after deserialization. */ state.s11n.deserialize = function(clear=false){ ++metrics.s11n.deserialize.count; const t = performance.now(); const argc = viewU8[0]; const rc = argc ? [] : null; if(argc){ const typeIds = []; let offset = 1, i, n, v; |
︙ | ︙ | |||
490 491 492 493 494 495 496 497 498 499 500 501 502 503 | offset += 4; v = textDecoder.decode(viewU8.slice(offset, offset+n)); offset += n; } rc.push(v); } } //log("deserialize:",argc, rc); metrics.s11n.deserialize.time += performance.now() - t; return rc; }; /** Serializes all arguments to the shared buffer for consumption | > | 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 | offset += 4; v = textDecoder.decode(viewU8.slice(offset, offset+n)); offset += n; } rc.push(v); } } if(clear) viewU8[0] = 0; //log("deserialize:",argc, rc); metrics.s11n.deserialize.time += performance.now() - t; return rc; }; /** Serializes all arguments to the shared buffer for consumption |
︙ | ︙ |
Changes to ext/wasm/api/sqlite3-opfs-async-proxy.js.
︙ | ︙ | |||
40 41 42 43 44 45 46 47 48 49 50 51 52 53 | } /** Will hold state copied to this object from the syncronous side of this API. */ const state = Object.create(null); /** verbose: 0 = no logging output 1 = only errors 2 = warnings and errors 3 = debug, warnings, and errors | > | 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 | } /** Will hold state copied to this object from the syncronous side of this API. */ const state = Object.create(null); /** verbose: 0 = no logging output 1 = only errors 2 = warnings and errors 3 = debug, warnings, and errors |
︙ | ︙ | |||
92 93 94 95 96 97 98 | metrics, "\nTotal of",n,"op(s) for",t,"ms", "approx",w,"ms spent waiting on OPFS APIs."); console.log("Serialization metrics:",metrics.s11n); }; /** | | | | | | > > > > > > > > > > > > > > | 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 | metrics, "\nTotal of",n,"op(s) for",t,"ms", "approx",w,"ms spent waiting on OPFS APIs."); console.log("Serialization metrics:",metrics.s11n); }; /** __openFiles is a map of sqlite3_file pointers (integers) to metadata related to a given OPFS file handles. The pointers are, in this side of the interface, opaque file handle IDs provided by the synchronous part of this constellation. Each value is an object with a structure demonstrated in the xOpen() impl. */ const __openFiles = Object.create(null); /** __autoLocks is a Set of sqlite3_file pointers (integers) which were "auto-locked". i.e. those for which we obtained a sync access handle without an explicit xLock() call. Such locks will be released during db connection idle time, whereas a sync access handle obtained via xLock(), or subsequently xLock()'d after auto-acquisition, will not be released until xUnlock() is called. Maintenance reminder: if we relinquish auto-locks at the end of the operation which acquires them, we pay a massive performance penalty: speedtest1 benchmarks take up to 4x as long. By delaying the lock release until idle time, the hit is negligible. */ const __autoLocks = new Set(); /** Expects an OPFS file path. It gets resolved, such that ".." components are properly expanded, and returned. If the 2nd arg is true, the result is returned as an array of path elements, else an absolute path string is returned. */ |
︙ | ︙ | |||
187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 | } warn("Error getting sync handle. Waiting",ms, "ms and trying again.",fh.filenameAbs,e); Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); } } log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms'); } return fh.syncHandle; }; /** If the given file-holding object has a sync handle attached to it, that handle is remove and asynchronously closed. Though it may sound sensible to continue work as soon as the close() returns (noting that it's asynchronous), doing so can cause operations performed soon afterwards, e.g. a call to getSyncHandle() to fail because they may happen out of order from the close(). OPFS does not guaranty that the actual order of operations is retained in such cases. i.e. always "await" on the result of this function. */ const closeSyncHandle = async (fh)=>{ if(fh.syncHandle){ log("Closing sync handle for",fh.filenameAbs); const h = fh.syncHandle; delete fh.syncHandle; return h.close(); } }; /** Stores the given value at state.sabOPView[state.opIds.rc] and then Atomics.notify()'s it. */ const storeAndNotify = (opName, value)=>{ log(opName+"() => notify(",value,")"); | > > > > > > > > > > > > > > > > > > > > > > > > | 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 | } warn("Error getting sync handle. Waiting",ms, "ms and trying again.",fh.filenameAbs,e); Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); } } log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms'); if(!fh.xLock){ __autoLocks.add(fh.fid); log("Auto-locked",fh.fid,fh.filenameAbs); } } return fh.syncHandle; }; /** If the given file-holding object has a sync handle attached to it, that handle is remove and asynchronously closed. Though it may sound sensible to continue work as soon as the close() returns (noting that it's asynchronous), doing so can cause operations performed soon afterwards, e.g. a call to getSyncHandle() to fail because they may happen out of order from the close(). OPFS does not guaranty that the actual order of operations is retained in such cases. i.e. always "await" on the result of this function. */ const closeSyncHandle = async (fh)=>{ if(fh.syncHandle){ log("Closing sync handle for",fh.filenameAbs); const h = fh.syncHandle; delete fh.syncHandle; delete fh.xLock; __autoLocks.delete(fh.fid); return h.close(); } }; /** A proxy for closeSyncHandle() which is guaranteed to not throw. This function is part of a lock/unlock step in functions which require a sync access handle but may be called without xLock() having been called first. Such calls need to release that handle to avoid locking the file for all of time. This is an _attempt_ at reducing cross-tab contention but it may prove to be more of a problem than a solution and may need to be removed. */ const closeSyncHandleNoThrow = async (fh)=>{ try{await closeSyncHandle(fh)} catch(e){ warn("closeSyncHandleNoThrow() ignoring:",e,fh); } }; /** Stores the given value at state.sabOPView[state.opIds.rc] and then Atomics.notify()'s it. */ const storeAndNotify = (opName, value)=>{ log(opName+"() => notify(",value,")"); |
︙ | ︙ | |||
338 339 340 341 342 343 344 345 346 | } storeAndNotify('xAccess', rc); mTimeEnd(); }, xClose: async function(fid/*sqlite3_file pointer*/){ const opName = 'xClose'; mTimeStart(opName); const fh = __openFiles[fid]; let rc = 0; | > | | 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 | } storeAndNotify('xAccess', rc); mTimeEnd(); }, xClose: async function(fid/*sqlite3_file pointer*/){ const opName = 'xClose'; mTimeStart(opName); __autoLocks.delete(fid); const fh = __openFiles[fid]; let rc = 0; wTimeStart(opName); if(fh){ delete __openFiles[fid]; await closeSyncHandle(fh); if(fh.deleteOnClose){ try{ await fh.dirHandle.removeEntry(fh.filenamePart) } catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) } } |
︙ | ︙ | |||
418 419 420 421 422 423 424 425 426 | mTimeEnd(); }, xLock: async function(fid/*sqlite3_file pointer*/, lockType/*SQLITE_LOCK_...*/){ mTimeStart('xLock'); const fh = __openFiles[fid]; let rc = 0; if( !fh.syncHandle ){ wTimeStart('xLock'); | > > > | > | > | 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 | mTimeEnd(); }, xLock: async function(fid/*sqlite3_file pointer*/, lockType/*SQLITE_LOCK_...*/){ mTimeStart('xLock'); const fh = __openFiles[fid]; let rc = 0; const oldLockType = fh.xLock; fh.xLock = lockType; if( !fh.syncHandle ){ wTimeStart('xLock'); try { await getSyncHandle(fh); __autoLocks.delete(fid); }catch(e){ state.s11n.storeException(1,e); rc = state.sq3Codes.SQLITE_IOERR_LOCK; fh.xLock = oldLockType; } wTimeEnd(); } storeAndNotify('xLock',rc); mTimeEnd(); }, xOpen: async function(fid/*sqlite3_file pointer*/, filename, |
︙ | ︙ | |||
457 458 459 460 461 462 463 464 465 466 467 468 469 470 | wa-sqlite, at this point, grabs a SyncAccessHandle and assigns it to the syncHandle prop of the file state object, but only for certain cases and it's unclear why it places that limitation on it. */ wTimeEnd(); __openFiles[fid] = Object.assign(Object.create(null),{ filenameAbs: filename, filenamePart: filenamePart, dirHandle: hDir, fileHandle: hFile, sabView: state.sabFileBufView, readOnly: create ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), | > | 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 | wa-sqlite, at this point, grabs a SyncAccessHandle and assigns it to the syncHandle prop of the file state object, but only for certain cases and it's unclear why it places that limitation on it. */ wTimeEnd(); __openFiles[fid] = Object.assign(Object.create(null),{ fid: fid, filenameAbs: filename, filenamePart: filenamePart, dirHandle: hDir, fileHandle: hFile, sabView: state.sabFileBufView, readOnly: create ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), |
︙ | ︙ | |||
606 607 608 609 610 611 612 | case TypeIds.number.id: return TypeIds.number; case TypeIds.bigint.id: return TypeIds.bigint; case TypeIds.boolean.id: return TypeIds.boolean; case TypeIds.string.id: return TypeIds.string; default: toss("Invalid type ID:",tid); } }; | | | 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 | case TypeIds.number.id: return TypeIds.number; case TypeIds.bigint.id: return TypeIds.bigint; case TypeIds.boolean.id: return TypeIds.boolean; case TypeIds.string.id: return TypeIds.string; default: toss("Invalid type ID:",tid); } }; state.s11n.deserialize = function(clear=false){ ++metrics.s11n.deserialize.count; const t = performance.now(); const argc = viewU8[0]; const rc = argc ? [] : null; if(argc){ const typeIds = []; let offset = 1, i, n, v; |
︙ | ︙ | |||
631 632 633 634 635 636 637 638 639 640 641 642 643 644 | offset += 4; v = textDecoder.decode(viewU8.slice(offset, offset+n)); offset += n; } rc.push(v); } } //log("deserialize:",argc, rc); metrics.s11n.deserialize.time += performance.now() - t; return rc; }; state.s11n.serialize = function(...args){ const t = performance.now(); ++metrics.s11n.serialize.count; | > | 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 | offset += 4; v = textDecoder.decode(viewU8.slice(offset, offset+n)); offset += n; } rc.push(v); } } if(clear) viewU8[0] = 0; //log("deserialize:",argc, rc); metrics.s11n.deserialize.time += performance.now() - t; return rc; }; state.s11n.serialize = function(...args){ const t = performance.now(); ++metrics.s11n.serialize.count; |
︙ | ︙ | |||
697 698 699 700 701 702 703 | o.f = vi; } /** waitTime is how long (ms) to wait for each Atomics.wait(). We need to wake up periodically to give the thread a chance to do other things. */ | | > > > > > > > > | | | | > | 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 | o.f = vi; } /** waitTime is how long (ms) to wait for each Atomics.wait(). We need to wake up periodically to give the thread a chance to do other things. */ const waitTime = 500; while(!flagAsyncShutdown){ try { if('timed-out'===Atomics.wait( state.sabOPView, state.opIds.whichOp, 0, waitTime )){ if(__autoLocks.size){ /* Release all auto-locks. */ for(const fid of __autoLocks){ const fh = __openFiles[fid]; await closeSyncHandleNoThrow(fh); log("Auto-unlocked",fid,fh.filenameAbs); } } continue; } const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); Atomics.store(state.sabOPView, state.opIds.whichOp, 0); const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); const args = state.s11n.deserialize( true /* clear s11n to keep the caller from confusing this with an exception string written by the upcoming operation */ ) || []; //warn("waitLoop() whichOp =",opId, hnd, args); if(hnd.f) await hnd.f(...args); else error("Missing callback for opId",opId); }catch(e){ error('in waitLoop():',e); } } |
︙ | ︙ |
Changes to ext/wasm/module-symbols.html.
︙ | ︙ | |||
185 186 187 188 189 190 191 | <p> <code>SQLITE_...</code> compilation options used in this build of <code>sqlite3.wasm</code>... </p> <div id='list-compile-options' class='pseudolist wide2'></div> </div><!-- .initially-hidden --> | | | 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 | <p> <code>SQLITE_...</code> compilation options used in this build of <code>sqlite3.wasm</code>... </p> <div id='list-compile-options' class='pseudolist wide2'></div> </div><!-- .initially-hidden --> <script src="jswasm/sqlite3.js">/* This tag MUST be inside the fossil-doc block so that this part can work without modification in the wasm docs repo. */</script> <script>(async function(){ const eNew = (tag,parent)=>{ const e = document.createElement(tag); if(parent) parent.appendChild(e); return e; |
︙ | ︙ |
Changes to ext/wasm/speedtest1-worker.html.
︙ | ︙ | |||
165 166 167 168 169 170 171 | const nbspPad = function(str,len=21){ if(str.length===len) return str; else if(str.length>len) return str.substr(0,len); const a = []; a.length = len - str.length; return str+a.join(' '); }; | > > | > > < | 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 | const nbspPad = function(str,len=21){ if(str.length===len) return str; else if(str.length>len) return str.substr(0,len); const a = []; a.length = len - str.length; return str+a.join(' '); }; const urlParams = new URL(self.location.href).searchParams; const W = new Worker( "speedtest1-worker.js?sqlite3.dir=jswasm"+ (urlParams.has('opfs-verbose') ? '&opfs-verbose' : '') ); const mPost = function(msgType,payload){ W.postMessage({type: msgType, data: payload}); }; const eFlags = E('#select-flags'); const eSelectedFlags = E('#toolbar-selected-flags'); const eLinkMainThread = E('#link-main-thread'); const eLinkWasmfs = E('#link-wasmfs'); const eLinkKvvfs = E('#link-kvvfs'); const getSelectedFlags = ()=>{ const f = Array.prototype.map.call(eFlags.selectedOptions, (v)=>v.value); [ 'size', 'vfs', 'journal', 'cachesize' ].forEach(function(k){ if(urlParams.has(k)) f.push('--'+k, urlParams.get(k)); }); |
︙ | ︙ |