SQLite

Changes On Branch opfs-lock-without-xlock
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch opfs-lock-without-xlock Excluding Merge-Ins

This is equivalent to a diff from ae43e97087 to a7fe91afca

2022-11-10
13:22
Automatically relinquish implicitly-acquired OPFS file locks during VFS idle time in an attempt to help alleviate cross-tab locking contention like that described in forum post 58a377083cd24a. (check-in: 8daf24ff73 user: stephan tags: trunk)
13:14
Rework automatically acquired OPFS locks to be released during idle time. This eliminates the performance hit reported in [46304ba057707c]. (Closed-Leaf check-in: a7fe91afca user: stephan tags: opfs-lock-without-xlock)
11:35
OPFS: if an op which needs a lock is called when no lock has been obtained, automatically lock it at the start of the op and unlock it at the end of that op. This is an attempt to alleviate the cross-tab contention described in forum post 58a377083cd24a but it increases speedtest1 run time by approximately 4x. Perhaps auto-lock can be combined with the older idle-time-based auto-unlock to unlock such locks (but not those from xLock()) to improve this? (check-in: 46304ba057 user: stephan tags: opfs-lock-without-xlock)
2022-11-09
11:17
Better handle an error in the fts5 integrity-check code. dbsqlfuzz e87c62f9b67ea21aebdc36ab71cab7cc3eda8dc3. (check-in: ae43e97087 user: dan tags: trunk)
11:02
Additional defense against corrupt database files in dbdata.c. (check-in: 2e70d1e5c9 user: drh tags: trunk)

Changes to ext/wasm/api/sqlite3-api-opfs.js.
463
464
465
466
467
468
469
470


471
472
473
474
475
476
477
478
479
            default: toss("Invalid type ID:",tid);
        }
      };

      /**
         Returns an array of the deserialized state stored by the most
         recent serialize() operation (from from this thread or the
         counterpart thread), or null if the serialization buffer is empty.


      */
      state.s11n.deserialize = function(){
        ++metrics.s11n.deserialize.count;
        const t = performance.now();
        const argc = viewU8[0];
        const rc = argc ? [] : null;
        if(argc){
          const typeIds = [];
          let offset = 1, i, n, v;







|
>
>

|







463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
            default: toss("Invalid type ID:",tid);
        }
      };

      /**
         Returns an array of the deserialized state stored by the most
         recent serialize() operation (from from this thread or the
         counterpart thread), or null if the serialization buffer is
         empty.  If passed a truthy argument, the serialization buffer
         is cleared after deserialization.
      */
      state.s11n.deserialize = function(clear=false){
        ++metrics.s11n.deserialize.count;
        const t = performance.now();
        const argc = viewU8[0];
        const rc = argc ? [] : null;
        if(argc){
          const typeIds = [];
          let offset = 1, i, n, v;
490
491
492
493
494
495
496

497
498
499
500
501
502
503
              offset += 4;
              v = textDecoder.decode(viewU8.slice(offset, offset+n));
              offset += n;
            }
            rc.push(v);
          }
        }

        //log("deserialize:",argc, rc);
        metrics.s11n.deserialize.time += performance.now() - t;
        return rc;
      };

      /**
         Serializes all arguments to the shared buffer for consumption







>







492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
              offset += 4;
              v = textDecoder.decode(viewU8.slice(offset, offset+n));
              offset += n;
            }
            rc.push(v);
          }
        }
        if(clear) viewU8[0] = 0;
        //log("deserialize:",argc, rc);
        metrics.s11n.deserialize.time += performance.now() - t;
        return rc;
      };

      /**
         Serializes all arguments to the shared buffer for consumption
Changes to ext/wasm/api/sqlite3-opfs-async-proxy.js.
40
41
42
43
44
45
46

47
48
49
50
51
52
53
}

/**
   Will hold state copied to this object from the syncronous side of
   this API.
*/
const state = Object.create(null);

/**
   verbose:

   0 = no logging output
   1 = only errors
   2 = warnings and errors
   3 = debug, warnings, and errors







>







40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
}

/**
   Will hold state copied to this object from the syncronous side of
   this API.
*/
const state = Object.create(null);

/**
   verbose:

   0 = no logging output
   1 = only errors
   2 = warnings and errors
   3 = debug, warnings, and errors
92
93
94
95
96
97
98
99
100
101
102
103
104
105














106
107
108
109
110
111
112
              metrics,
              "\nTotal of",n,"op(s) for",t,"ms",
              "approx",w,"ms spent waiting on OPFS APIs.");
  console.log("Serialization metrics:",metrics.s11n);
};

/**
   Map of sqlite3_file pointers (integers) to metadata related to a
   given OPFS file handles. The pointers are, in this side of the
   interface, opaque file handle IDs provided by the synchronous
   part of this constellation. Each value is an object with a structure
   demonstrated in the xOpen() impl.
*/
const __openFiles = Object.create(null);















/**
   Expects an OPFS file path. It gets resolved, such that ".."
   components are properly expanded, and returned. If the 2nd arg is
   true, the result is returned as an array of path elements, else an
   absolute path string is returned.
*/







|
|
|
|
|


>
>
>
>
>
>
>
>
>
>
>
>
>
>







93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
              metrics,
              "\nTotal of",n,"op(s) for",t,"ms",
              "approx",w,"ms spent waiting on OPFS APIs.");
  console.log("Serialization metrics:",metrics.s11n);
};

/**
   __openFiles is a map of sqlite3_file pointers (integers) to
   metadata related to a given OPFS file handles. The pointers are, in
   this side of the interface, opaque file handle IDs provided by the
   synchronous part of this constellation. Each value is an object
   with a structure demonstrated in the xOpen() impl.
*/
const __openFiles = Object.create(null);
/**
   __autoLocks is a Set of sqlite3_file pointers (integers) which were
   "auto-locked".  i.e. those for which we obtained a sync access
   handle without an explicit xLock() call. Such locks will be
   released during db connection idle time, whereas a sync access
   handle obtained via xLock(), or subsequently xLock()'d after
   auto-acquisition, will not be released until xUnlock() is called.

   Maintenance reminder: if we relinquish auto-locks at the end of the
   operation which acquires them, we pay a massive performance
   penalty: speedtest1 benchmarks take up to 4x as long. By delaying
   the lock release until idle time, the hit is negligible.
*/
const __autoLocks = new Set();

/**
   Expects an OPFS file path. It gets resolved, such that ".."
   components are properly expanded, and returned. If the 2nd arg is
   true, the result is returned as an array of path elements, else an
   absolute path string is returned.
*/
187
188
189
190
191
192
193




194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212


213
214
215


















216
217
218
219
220
221
222
        }
        warn("Error getting sync handle. Waiting",ms,
              "ms and trying again.",fh.filenameAbs,e);
        Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms);
      }
    }
    log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms');




  }
  return fh.syncHandle;
};

/**
   If the given file-holding object has a sync handle attached to it,
   that handle is remove and asynchronously closed. Though it may
   sound sensible to continue work as soon as the close() returns
   (noting that it's asynchronous), doing so can cause operations
   performed soon afterwards, e.g. a call to getSyncHandle() to fail
   because they may happen out of order from the close(). OPFS does
   not guaranty that the actual order of operations is retained in
   such cases. i.e.  always "await" on the result of this function.
*/
const closeSyncHandle = async (fh)=>{
  if(fh.syncHandle){
    log("Closing sync handle for",fh.filenameAbs);
    const h = fh.syncHandle;
    delete fh.syncHandle;


    return h.close();
  }
};



















/**
   Stores the given value at state.sabOPView[state.opIds.rc] and then
   Atomics.notify()'s it.
*/
const storeAndNotify = (opName, value)=>{
  log(opName+"() => notify(",value,")");







>
>
>
>



















>
>



>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
        }
        warn("Error getting sync handle. Waiting",ms,
              "ms and trying again.",fh.filenameAbs,e);
        Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms);
      }
    }
    log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms');
    if(!fh.xLock){
      __autoLocks.add(fh.fid);
      log("Auto-locked",fh.fid,fh.filenameAbs);
    }
  }
  return fh.syncHandle;
};

/**
   If the given file-holding object has a sync handle attached to it,
   that handle is remove and asynchronously closed. Though it may
   sound sensible to continue work as soon as the close() returns
   (noting that it's asynchronous), doing so can cause operations
   performed soon afterwards, e.g. a call to getSyncHandle() to fail
   because they may happen out of order from the close(). OPFS does
   not guaranty that the actual order of operations is retained in
   such cases. i.e.  always "await" on the result of this function.
*/
const closeSyncHandle = async (fh)=>{
  if(fh.syncHandle){
    log("Closing sync handle for",fh.filenameAbs);
    const h = fh.syncHandle;
    delete fh.syncHandle;
    delete fh.xLock;
    __autoLocks.delete(fh.fid);
    return h.close();
  }
};

/**
   A proxy for closeSyncHandle() which is guaranteed to not throw.

   This function is part of a lock/unlock step in functions which
   require a sync access handle but may be called without xLock()
   having been called first. Such calls need to release that
   handle to avoid locking the file for all of time. This is an
   _attempt_ at reducing cross-tab contention but it may prove
   to be more of a problem than a solution and may need to be
   removed.
*/
const closeSyncHandleNoThrow = async (fh)=>{
  try{await closeSyncHandle(fh)}
  catch(e){
    warn("closeSyncHandleNoThrow() ignoring:",e,fh);
  }
};

/**
   Stores the given value at state.sabOPView[state.opIds.rc] and then
   Atomics.notify()'s it.
*/
const storeAndNotify = (opName, value)=>{
  log(opName+"() => notify(",value,")");
338
339
340
341
342
343
344

345
346
347
348
349
350
351
352
353
354
    }
    storeAndNotify('xAccess', rc);
    mTimeEnd();
  },
  xClose: async function(fid/*sqlite3_file pointer*/){
    const opName = 'xClose';
    mTimeStart(opName);

    const fh = __openFiles[fid];
    let rc = 0;
    wTimeStart('xClose');
    if(fh){
      delete __openFiles[fid];
      await closeSyncHandle(fh);
      if(fh.deleteOnClose){
        try{ await fh.dirHandle.removeEntry(fh.filenamePart) }
        catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) }
      }







>


|







377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
    }
    storeAndNotify('xAccess', rc);
    mTimeEnd();
  },
  xClose: async function(fid/*sqlite3_file pointer*/){
    const opName = 'xClose';
    mTimeStart(opName);
    __autoLocks.delete(fid);
    const fh = __openFiles[fid];
    let rc = 0;
    wTimeStart(opName);
    if(fh){
      delete __openFiles[fid];
      await closeSyncHandle(fh);
      if(fh.deleteOnClose){
        try{ await fh.dirHandle.removeEntry(fh.filenamePart) }
        catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) }
      }
418
419
420
421
422
423
424


425
426

427

428
429
430

431
432
433
434
435
436
437
    mTimeEnd();
  },
  xLock: async function(fid/*sqlite3_file pointer*/,
                        lockType/*SQLITE_LOCK_...*/){
    mTimeStart('xLock');
    const fh = __openFiles[fid];
    let rc = 0;


    if( !fh.syncHandle ){
      wTimeStart('xLock');

      try { await getSyncHandle(fh) }

      catch(e){
        state.s11n.storeException(1,e);
        rc = state.sq3Codes.SQLITE_IOERR_LOCK;

      }
      wTimeEnd();
    }
    storeAndNotify('xLock',rc);
    mTimeEnd();
  },
  xOpen: async function(fid/*sqlite3_file pointer*/, filename,







>
>


>
|
>
|


>







458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
    mTimeEnd();
  },
  xLock: async function(fid/*sqlite3_file pointer*/,
                        lockType/*SQLITE_LOCK_...*/){
    mTimeStart('xLock');
    const fh = __openFiles[fid];
    let rc = 0;
    const oldLockType = fh.xLock;
    fh.xLock = lockType;
    if( !fh.syncHandle ){
      wTimeStart('xLock');
      try {
        await getSyncHandle(fh);
        __autoLocks.delete(fid);
      }catch(e){
        state.s11n.storeException(1,e);
        rc = state.sq3Codes.SQLITE_IOERR_LOCK;
        fh.xLock = oldLockType;
      }
      wTimeEnd();
    }
    storeAndNotify('xLock',rc);
    mTimeEnd();
  },
  xOpen: async function(fid/*sqlite3_file pointer*/, filename,
457
458
459
460
461
462
463

464
465
466
467
468
469
470
         wa-sqlite, at this point, grabs a SyncAccessHandle and
         assigns it to the syncHandle prop of the file state
         object, but only for certain cases and it's unclear why it
         places that limitation on it.
      */
      wTimeEnd();
      __openFiles[fid] = Object.assign(Object.create(null),{

        filenameAbs: filename,
        filenamePart: filenamePart,
        dirHandle: hDir,
        fileHandle: hFile,
        sabView: state.sabFileBufView,
        readOnly: create
          ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags),







>







502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
         wa-sqlite, at this point, grabs a SyncAccessHandle and
         assigns it to the syncHandle prop of the file state
         object, but only for certain cases and it's unclear why it
         places that limitation on it.
      */
      wTimeEnd();
      __openFiles[fid] = Object.assign(Object.create(null),{
        fid: fid,
        filenameAbs: filename,
        filenamePart: filenamePart,
        dirHandle: hDir,
        fileHandle: hFile,
        sabView: state.sabFileBufView,
        readOnly: create
          ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags),
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
      case TypeIds.number.id: return TypeIds.number;
      case TypeIds.bigint.id: return TypeIds.bigint;
      case TypeIds.boolean.id: return TypeIds.boolean;
      case TypeIds.string.id: return TypeIds.string;
      default: toss("Invalid type ID:",tid);
    }
  };
  state.s11n.deserialize = function(){
    ++metrics.s11n.deserialize.count;
    const t = performance.now();
    const argc = viewU8[0];
    const rc = argc ? [] : null;
    if(argc){
      const typeIds = [];
      let offset = 1, i, n, v;







|







652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
      case TypeIds.number.id: return TypeIds.number;
      case TypeIds.bigint.id: return TypeIds.bigint;
      case TypeIds.boolean.id: return TypeIds.boolean;
      case TypeIds.string.id: return TypeIds.string;
      default: toss("Invalid type ID:",tid);
    }
  };
  state.s11n.deserialize = function(clear=false){
    ++metrics.s11n.deserialize.count;
    const t = performance.now();
    const argc = viewU8[0];
    const rc = argc ? [] : null;
    if(argc){
      const typeIds = [];
      let offset = 1, i, n, v;
631
632
633
634
635
636
637

638
639
640
641
642
643
644
          offset += 4;
          v = textDecoder.decode(viewU8.slice(offset, offset+n));
          offset += n;
        }
        rc.push(v);
      }
    }

    //log("deserialize:",argc, rc);
    metrics.s11n.deserialize.time += performance.now() - t;
    return rc;
  };
  state.s11n.serialize = function(...args){
    const t = performance.now();
    ++metrics.s11n.serialize.count;







>







677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
          offset += 4;
          v = textDecoder.decode(viewU8.slice(offset, offset+n));
          offset += n;
        }
        rc.push(v);
      }
    }
    if(clear) viewU8[0] = 0;
    //log("deserialize:",argc, rc);
    metrics.s11n.deserialize.time += performance.now() - t;
    return rc;
  };
  state.s11n.serialize = function(...args){
    const t = performance.now();
    ++metrics.s11n.serialize.count;
697
698
699
700
701
702
703
704
705
706
707
708
709








710
711
712
713
714
715
716
717
718

719
720
721
722
723
724
725
    o.f = vi;
  }
  /**
     waitTime is how long (ms) to wait for each Atomics.wait().
     We need to wake up periodically to give the thread a chance
     to do other things.
  */
  const waitTime = 1000;
  while(!flagAsyncShutdown){
    try {
      if('timed-out'===Atomics.wait(
        state.sabOPView, state.opIds.whichOp, 0, waitTime
      )){








        continue;
      }
      const opId = Atomics.load(state.sabOPView, state.opIds.whichOp);
      Atomics.store(state.sabOPView, state.opIds.whichOp, 0);
      const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId);
      const args = state.s11n.deserialize() || [];
      state.s11n.serialize(/* clear s11n to keep the caller from
                              confusing this with an exception string
                              written by the upcoming operation */);

      //warn("waitLoop() whichOp =",opId, hnd, args);
      if(hnd.f) await hnd.f(...args);
      else error("Missing callback for opId",opId);
    }catch(e){
      error('in waitLoop():',e);
    }
  }







|





>
>
>
>
>
>
>
>





|
|
|
|
>







744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
    o.f = vi;
  }
  /**
     waitTime is how long (ms) to wait for each Atomics.wait().
     We need to wake up periodically to give the thread a chance
     to do other things.
  */
  const waitTime = 500;
  while(!flagAsyncShutdown){
    try {
      if('timed-out'===Atomics.wait(
        state.sabOPView, state.opIds.whichOp, 0, waitTime
      )){
        if(__autoLocks.size){
          /* Release all auto-locks. */
          for(const fid of __autoLocks){
            const fh = __openFiles[fid];
            await closeSyncHandleNoThrow(fh);
            log("Auto-unlocked",fid,fh.filenameAbs);
          }
        }
        continue;
      }
      const opId = Atomics.load(state.sabOPView, state.opIds.whichOp);
      Atomics.store(state.sabOPView, state.opIds.whichOp, 0);
      const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId);
      const args = state.s11n.deserialize(
        true /* clear s11n to keep the caller from confusing this with
                an exception string written by the upcoming
                operation */
      ) || [];
      //warn("waitLoop() whichOp =",opId, hnd, args);
      if(hnd.f) await hnd.f(...args);
      else error("Missing callback for opId",opId);
    }catch(e){
      error('in waitLoop():',e);
    }
  }
Changes to ext/wasm/module-symbols.html.
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
    <p>
      <code>SQLITE_...</code> compilation options used in this build
      of <code>sqlite3.wasm</code>...
    </p>
    <div id='list-compile-options' class='pseudolist wide2'></div>

  </div><!-- .initially-hidden -->
  <script src="jswasm/sqlite3.js">/* This tag MUST be in side the
  fossil-doc block so that this part can work without modification in
  the wasm docs repo.  */</script>
  <script>(async function(){
    const eNew = (tag,parent)=>{
        const e = document.createElement(tag);
        if(parent) parent.appendChild(e);
        return e;







|







185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
    <p>
      <code>SQLITE_...</code> compilation options used in this build
      of <code>sqlite3.wasm</code>...
    </p>
    <div id='list-compile-options' class='pseudolist wide2'></div>

  </div><!-- .initially-hidden -->
  <script src="jswasm/sqlite3.js">/* This tag MUST be inside the
  fossil-doc block so that this part can work without modification in
  the wasm docs repo.  */</script>
  <script>(async function(){
    const eNew = (tag,parent)=>{
        const e = document.createElement(tag);
        if(parent) parent.appendChild(e);
        return e;
Changes to ext/wasm/speedtest1-worker.html.
165
166
167
168
169
170
171


172


173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
    const nbspPad = function(str,len=21){
        if(str.length===len) return str;
        else if(str.length>len) return str.substr(0,len);
        const a = []; a.length = len - str.length;
        return str+a.join('&nbsp;');
    };



    const W = new Worker("speedtest1-worker.js?sqlite3.dir=jswasm");


    const mPost = function(msgType,payload){
        W.postMessage({type: msgType, data: payload});
    };

    const eFlags = E('#select-flags');
    const eSelectedFlags = E('#toolbar-selected-flags');
    const eLinkMainThread = E('#link-main-thread');
    const eLinkWasmfs = E('#link-wasmfs');
    const eLinkKvvfs = E('#link-kvvfs');
    const urlParams = new URL(self.location.href).searchParams;
    const getSelectedFlags = ()=>{
        const f = Array.prototype.map.call(eFlags.selectedOptions, (v)=>v.value);
        [
            'size', 'vfs', 'journal', 'cachesize'
        ].forEach(function(k){
            if(urlParams.has(k)) f.push('--'+k, urlParams.get(k));
        });







>
>
|
>
>









<







165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185

186
187
188
189
190
191
192
    const nbspPad = function(str,len=21){
        if(str.length===len) return str;
        else if(str.length>len) return str.substr(0,len);
        const a = []; a.length = len - str.length;
        return str+a.join('&nbsp;');
    };

    const urlParams = new URL(self.location.href).searchParams;
    const W = new Worker(
        "speedtest1-worker.js?sqlite3.dir=jswasm"+
            (urlParams.has('opfs-verbose') ? '&opfs-verbose' : '')
    );
    const mPost = function(msgType,payload){
        W.postMessage({type: msgType, data: payload});
    };

    const eFlags = E('#select-flags');
    const eSelectedFlags = E('#toolbar-selected-flags');
    const eLinkMainThread = E('#link-main-thread');
    const eLinkWasmfs = E('#link-wasmfs');
    const eLinkKvvfs = E('#link-kvvfs');

    const getSelectedFlags = ()=>{
        const f = Array.prototype.map.call(eFlags.selectedOptions, (v)=>v.value);
        [
            'size', 'vfs', 'journal', 'cachesize'
        ].forEach(function(k){
            if(urlParams.has(k)) f.push('--'+k, urlParams.get(k));
        });