SQLite

Check-in [40e60f570d]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Get speedtest1.js working with WASMFS/OPFS.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fiddle-opfs
Files: files | file ages | folders
SHA3-256: 40e60f570d4f489d58d12e27c1c067b41d6c5a5e374c5fce0baa8881ef183216
User & Date: stephan 2022-09-06 20:17:15.425
Context
2022-09-06
23:04
Add a note about Emscripten's -sSINGLE_FILE flag, why it would be nice, and why we can't use it. (check-in: 5ea0623630 user: stephan tags: fiddle-opfs)
20:17
Get speedtest1.js working with WASMFS/OPFS. (check-in: 40e60f570d user: stephan tags: fiddle-opfs)
16:47
Initial build of speedtest1.wasm and speedtest1.html with which to run it. (check-in: 4441535e3e user: stephan tags: fiddle-opfs)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/wasm/GNUmakefile.
255
256
257
258
259
260
261

262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320

321
322

323
324

325
326
327

328
329
330
331

332
333
334
335
336
337











338
339
340
341
342
343
344
345
346
347




348
349
350
351
352
353
354
355
356
357
358
359
########################################################################
# Maintenance reminder: the output .js and .wasm files of emcc must be
# in _this_ dir, rather than a subdir, or else parts of the generated
# code get confused and cannot load property (namely, the
# sqlite3.worker.js generated in conjunction with -sWASMFS).
sqlite3.js := sqlite3.js
sqlite3.wasm := sqlite3.wasm

$(dir.api)/sqlite3-wasm.o: emcc.cflags += $(SQLITE_OPT)
$(dir.api)/sqlite3-wasm.o: $(dir.top)/sqlite3.c
$(dir.api)/wasm_util.o: emcc.cflags += $(SQLITE_OPT)
sqlite3.wasm.c := $(dir.api)/sqlite3-wasm.c \
    $(dir.jacc)/jaccwabyt_test.c
# ^^^ FIXME (how?): jaccwabyt_test.c is only needed for the test apps,
# so we don't really want to include it in release builds. However, we
# want to test the release builds with those apps, so we cannot simply
# elide that file in release builds. That component is critical to the
# VFS bindings so needs to be tested along with the core APIs.
ifneq (,$(filter -sWASMFS,$(emcc.jsflags)))
  $(dir.api)/sqlite3-wasm.o: emcc.cflags+=-DSQLITE_WASM_OPFS
endif
define WASM_C_COMPILE
$(1).o := $$(subst .c,.o,$(1))
sqlite3.wasm.obj += $$($(1).o)
$$($(1).o): $$(MAKEFILE) $(1)
	$$(emcc.bin) $$(emcc_opt) $$(emcc.flags) $$(emcc.cflags) -c $(1) -o $$@
CLEAN_FILES += $$($(1).o)
endef
$(foreach c,$(sqlite3.wasm.c),$(eval $(call WASM_C_COMPILE,$(c))))
$(sqlite3.js): 
$(sqlite3.js): $(MAKEFILE) $(sqlite3.wasm.obj) \
    EXPORTED_FUNCTIONS.api \
    $(post-js.js)
	$(emcc.bin) -o $(sqlite3.js) $(emcc_opt) $(emcc.flags) $(emcc.jsflags) $(sqlite3.wasm.obj)
	chmod -x $(sqlite3.wasm)
ifneq (,$(wasm-strip))
	$(wasm-strip) $(sqlite3.wasm)
endif
	@ls -la $@ $(sqlite3.wasm)

CLEAN_FILES += $(sqlite3.js) $(sqlite3.wasm)
all: $(sqlite3.js)
wasm: $(sqlite3.js)
# End main Emscripten-based module build
########################################################################

########################################################################
# Bits for use with batch-runner.js...
dir.sql := sql
speedtest1 := ../../speedtest1
speedtest1.c := ../../test/speedtest1.c
speedtest1.sql := $(dir.sql)/speedtest1.sql
$(speedtest1):
	$(MAKE) -C ../.. speedtest1
$(speedtest1.sql): $(speedtest1)
	$(speedtest1) --script $@
batch-runner.list: $(MAKEFILE) $(speedtest1.sql) $(dir.sql)/000-mandelbrot.sql
	bash split-speedtest1-script.sh $(dir.sql)/speedtest1.sql
	ls -1 $(dir.sql)/*.sql | grep -v speedtest1.sql | sort > $@
clean-batch:
	rm -f batch-runner.list $(dir.sql)/speedtest1*.sql
# ^^^ we don't do this along with 'clean' because we clean/rebuild on
# a regular basis with different -Ox flags and rebuilding the batch
# pieces each time is an unnecessary time sink.
batch: batch-runner.list
all: batch


speedtest1.js := speedtest1.js
$(speedtest1.js): emcc.cflags+=

$(speedtest1.js): $(speedtest1.c) $(MAKEFILE)
	$(emcc.bin) -g $(emcc_opt) \

        -sINVOKE_RUN=0 \
        --no-entry \
        -sALLOW_TABLE_GROWTH \

        -sABORTING_MALLOC \
        -sINITIAL_MEMORY=128450560 \
        -sSTRICT_JS \
        -sENVIRONMENT=web \

        -sMODULARIZE \
        -sEXPORT_NAME=sqlite3Speedtest1InitModule \
        -Wno-limited-postlink-optimizations \
        -sEXPORTED_FUNCTIONS=_main,_malloc,_free \
        -sDYNAMIC_EXECUTION=0 \
        --minify 0 \











        -I. -I$(dir.top) \
        -DSQLITE_THREADSAFE=0 \
        -DSQLITE_TEMP_STORE=3 \
        -DSQLITE_OMIT_UTF16 \
        -DSQLITE_OMIT_DEPRECATED \
        -DSQLITE_OMIT_SHARED_CACHE \
        '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \
        -DSQLITE_SPEEDTEST1_WASM \
        $(emcc_flags_wasmfs) \
        -o $@ $(speedtest1.c) $(sqlite3.c) -lm





speedtest1: $(speedtest1.js)
all: $(speedtest1.js)
CLEAN_FILES += $(speedtest1.js) $(subst .js,.wasm,$(speedtest1.js))

########################################################################
# fiddle_remote is the remote destination for the fiddle app. It
# must be a [user@]HOST:/path for rsync.
# Note that the target "should probably" contain a symlink of
# index.html -> fiddle.html.
fiddle_remote ?=
ifeq (,$(fiddle_remote))







>
|
|

|







|








|


















|


















|
>
|
|
>
|
|
>
|
|
<
>
|
|
|
<
>
|
|
|
|
|
|
>
>
>
>
>
>
>
>
>
>
>








<
|
>
>
>
>



|
|







255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330

331
332
333
334

335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360

361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
########################################################################
# Maintenance reminder: the output .js and .wasm files of emcc must be
# in _this_ dir, rather than a subdir, or else parts of the generated
# code get confused and cannot load property (namely, the
# sqlite3.worker.js generated in conjunction with -sWASMFS).
sqlite3.js := sqlite3.js
sqlite3.wasm := sqlite3.wasm
sqlite3-wasm.o := $(dir.api)/sqlite3-wasm.o
$(sqlite3-wasm.o): emcc.cflags += $(SQLITE_OPT)
$(sqlite3-wasm.o): $(dir.top)/sqlite3.c
$(dir.api)/wasm_util.o: emcc.cflags += $(SQLITE_OPT)
sqlite3-wasm.c := $(dir.api)/sqlite3-wasm.c \
    $(dir.jacc)/jaccwabyt_test.c
# ^^^ FIXME (how?): jaccwabyt_test.c is only needed for the test apps,
# so we don't really want to include it in release builds. However, we
# want to test the release builds with those apps, so we cannot simply
# elide that file in release builds. That component is critical to the
# VFS bindings so needs to be tested along with the core APIs.
ifneq (,$(filter -sWASMFS,$(emcc.jsflags)))
  $(sqlite3-wasm.o): emcc.cflags+=-DSQLITE_WASM_OPFS
endif
define WASM_C_COMPILE
$(1).o := $$(subst .c,.o,$(1))
sqlite3.wasm.obj += $$($(1).o)
$$($(1).o): $$(MAKEFILE) $(1)
	$$(emcc.bin) $$(emcc_opt) $$(emcc.flags) $$(emcc.cflags) -c $(1) -o $$@
CLEAN_FILES += $$($(1).o)
endef
$(foreach c,$(sqlite3-wasm.c),$(eval $(call WASM_C_COMPILE,$(c))))
$(sqlite3.js): 
$(sqlite3.js): $(MAKEFILE) $(sqlite3.wasm.obj) \
    EXPORTED_FUNCTIONS.api \
    $(post-js.js)
	$(emcc.bin) -o $(sqlite3.js) $(emcc_opt) $(emcc.flags) $(emcc.jsflags) $(sqlite3.wasm.obj)
	chmod -x $(sqlite3.wasm)
ifneq (,$(wasm-strip))
	$(wasm-strip) $(sqlite3.wasm)
endif
	@ls -la $@ $(sqlite3.wasm)

CLEAN_FILES += $(sqlite3.js) $(sqlite3.wasm)
all: $(sqlite3.js)
wasm: $(sqlite3.js)
# End main Emscripten-based module build
########################################################################

########################################################################
# batch-runner.js...
dir.sql := sql
speedtest1 := ../../speedtest1
speedtest1.c := ../../test/speedtest1.c
speedtest1.sql := $(dir.sql)/speedtest1.sql
$(speedtest1):
	$(MAKE) -C ../.. speedtest1
$(speedtest1.sql): $(speedtest1)
	$(speedtest1) --script $@
batch-runner.list: $(MAKEFILE) $(speedtest1.sql) $(dir.sql)/000-mandelbrot.sql
	bash split-speedtest1-script.sh $(dir.sql)/speedtest1.sql
	ls -1 $(dir.sql)/*.sql | grep -v speedtest1.sql | sort > $@
clean-batch:
	rm -f batch-runner.list $(dir.sql)/speedtest1*.sql
# ^^^ we don't do this along with 'clean' because we clean/rebuild on
# a regular basis with different -Ox flags and rebuilding the batch
# pieces each time is an unnecessary time sink.
batch: batch-runner.list
all: batch
# end batch-runner.js
########################################################################
# speedtest1.js...
emcc.speedtest1-flags := -g $(emcc_opt)
ifneq (0,$(ENABLE_WASMFS))
  emcc.speedtest1-flags += -pthread -sWASMFS -sPTHREAD_POOL_SIZE=2
  emcc.speedtest1-flags += -DSQLITE_WASM_OPFS
endif
emcc.speedtest1-flags += -sINVOKE_RUN=0
#emcc.speedtest1-flags += --no-entry

emcc.speedtest1-flags += -flto
emcc.speedtest1-flags += -sABORTING_MALLOC
emcc.speedtest1-flags += -sINITIAL_MEMORY=128450560
emcc.speedtest1-flags += -sSTRICT_JS

emcc.speedtest1-flags += $(emcc.environment)
emcc.speedtest1-flags += -sMODULARIZE
emcc.speedtest1-flags += -sEXPORT_NAME=sqlite3Speedtest1InitModule
emcc.speedtest1-flags += -Wno-limited-postlink-optimizations
emcc.speedtest1-flags += -sEXPORTED_FUNCTIONS=_main,_malloc,_free,_sqlite3_wasm_vfs_unlink,_sqlite3_wasm_init_opfs
emcc.speedtest1-flags += -sDYNAMIC_EXECUTION=0
emcc.speedtest1-flags += --minify 0

speedtest1.js := speedtest1.js
speedtest1.wasm := $(subst .js,.wasm,$(speedtest1.js))
$(speedtest1.js): emcc.cflags+=
# speedtest1 notes re. sqlite3-wasm.o vs sqlite3-wasm.c: building against
# the latter (predictably) results in a slightly faster binary, but we're
# close enough to the target speed requirements that the 500ms makes a
# difference.
$(speedtest1.js): $(speedtest1.c) $(sqlite3-wasm.c) $(MAKEFILE)
	$(emcc.bin) \
        $(emcc.speedtest1-flags) \
        -I. -I$(dir.top) \
        -DSQLITE_THREADSAFE=0 \
        -DSQLITE_TEMP_STORE=3 \
        -DSQLITE_OMIT_UTF16 \
        -DSQLITE_OMIT_DEPRECATED \
        -DSQLITE_OMIT_SHARED_CACHE \
        '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \
        -DSQLITE_SPEEDTEST1_WASM \

        -o $@ $(speedtest1.c) $(sqlite3-wasm.c) -lm
ifneq (,$(wasm-strip))
	$(wasm-strip) $(speedtest1.wasm)
endif
	ls -la $@ $(speedtest1.wasm)

speedtest1: $(speedtest1.js)
all: $(speedtest1.js)
CLEAN_FILES += $(speedtest1.js) $(speedtest1.wasm)
# end speedtest1.js
########################################################################
# fiddle_remote is the remote destination for the fiddle app. It
# must be a [user@]HOST:/path for rsync.
# Note that the target "should probably" contain a symlink of
# index.html -> fiddle.html.
fiddle_remote ?=
ifeq (,$(fiddle_remote))
Changes to ext/wasm/api/sqlite3-wasm.c.
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
    /* mkdir() simply hangs when called from fiddle app. Cause is
       not yet determined but the hypothesis is an init-order
       issue. */
    /* Note that this check and is not robust but it will
       hypothetically suffice for the transient wasm-based virtual
       filesystem we're currently running in. */
    const int rc = wasmfs_create_directory(zMountPoint, 0777, pOpfs);
    emscripten_console_logf("OPFS mkdir rc=%d", rc);
    if(rc) return SQLITE_IOERR;
  }
  return pOpfs ? 0 : SQLITE_NOMEM;
}
#else
int sqlite3_wasm_init_opfs(void){
  return SQLITE_NOTFOUND;







|







469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
    /* mkdir() simply hangs when called from fiddle app. Cause is
       not yet determined but the hypothesis is an init-order
       issue. */
    /* Note that this check and is not robust but it will
       hypothetically suffice for the transient wasm-based virtual
       filesystem we're currently running in. */
    const int rc = wasmfs_create_directory(zMountPoint, 0777, pOpfs);
    emscripten_console_logf("OPFS mkdir(%s) rc=%d", zMountPoint, rc);
    if(rc) return SQLITE_IOERR;
  }
  return pOpfs ? 0 : SQLITE_NOMEM;
}
#else
int sqlite3_wasm_init_opfs(void){
  return SQLITE_NOTFOUND;
Changes to ext/wasm/common/whwasmutil.js.
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
     arguably more efficient because it will hypothetically free the
     wrapper function quickly.
  */
  target.xCallWrapped = function(fname, resultType, argTypes, ...args){
    if(Array.isArray(arguments[3])) args = arguments[3];
    return target.xWrap(fname, resultType, argTypes||[]).apply(null, args||[]);
  };
  
  return target;
};

/**
   yawl (Yet Another Wasm Loader) provides very basic wasm loader.
   It requires a config object:








|







1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
     arguably more efficient because it will hypothetically free the
     wrapper function quickly.
  */
  target.xCallWrapped = function(fname, resultType, argTypes, ...args){
    if(Array.isArray(arguments[3])) args = arguments[3];
    return target.xWrap(fname, resultType, argTypes||[]).apply(null, args||[]);
  };

  return target;
};

/**
   yawl (Yet Another Wasm Loader) provides very basic wasm loader.
   It requires a config object:

Changes to ext/wasm/speedtest1.html.
27
28
29
30
31
32
33
34





35






















36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

62
63
64
65





66

67
68
69
70
71
72
73
74
75


76

77
78
79
80

81
82
83
84
85
86
87
88
89
90
91
    <div>Output is sent to the dev console because we cannot update the UI while the
      speedtest is running unless/until we move the speedtest to a worker thread.</div>
    <hr>
    <div id='test-output'></div>
    <script src="common/whwasmutil.js"></script>
    <script src="common/SqliteTestUtil.js"></script>
    <script src="speedtest1.js"></script>
    <script>





      (function(){






















        const eOut = document.querySelector('#test-output');
        const logHtml2 = async function(cssClass,...args){
          const ln = document.createElement('div');
          if(cssClass) ln.classList.add(cssClass);
          ln.append(document.createTextNode(args.join(' ')));
          eOut.append(ln);
          //this.e.output.lastElementChild.scrollIntoViewIfNeeded();
        };
        const doHtmlOutput = false
        /* can't update DOM while speedtest is running unless we run
           speedtest in a worker thread. */;
        const logHtml = (...args)=>{
          console.log(...args);
          if(doHtmlOutput) logHtml2('', ...args);
        };
        const logErr = function(...args){
          console.error(...args);
          if(doHtmlOutput) this.logHtml2('error', ...args);
        };

        const runTests = function(EmscriptenModule){
          console.log("Module inited.");
          const wasm = {
            exports: EmscriptenModule.asm,
            alloc: (n)=>EmscriptenModule._malloc(n),
            dealloc: (m)=>EmscriptenModule._free(m)

          };
          //console.debug('Emscripten Module =',EmscriptenModule);
          //console.debug('wasm =',wasm);
          self.WhWasmUtilInstaller(wasm);





          const scope = wasm.scopedAllocPush();

          try{
            const maxArgc = 12;
            const aArgs = [
              // TODO: accept flags via URL arguments and/or a
              // UI control. A multi-SELECT element should do
              // nicely.
              "speedtest1",
              "--memdb",
              "--stats"


            ];

            wasm.xCall('__main_argc_argv', aArgs.length,
                       wasm.scopedAllocMainArgv(aArgs));
          }finally{
            wasm.scopedAllocPop(scope);

          }
        };

        self.sqlite3TestModule.print = logHtml;
        self.sqlite3TestModule.printErr = logErr;
        sqlite3Speedtest1InitModule(self.sqlite3TestModule).then(function(M){
          setTimeout(()=>runTests(M), 100);
        });
      })();
    </script>
  </body>







|
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

|









|

|



|



|



|
>

<


>
>
>
>
>

>

<
|




|
|
>
>

>
|
|


>



|







27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

91
92
93
94
95
96
97
98
99
100

101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
    <div>Output is sent to the dev console because we cannot update the UI while the
      speedtest is running unless/until we move the speedtest to a worker thread.</div>
    <hr>
    <div id='test-output'></div>
    <script src="common/whwasmutil.js"></script>
    <script src="common/SqliteTestUtil.js"></script>
    <script src="speedtest1.js"></script>
    <script>(function(){
        /**
           If this environment contains OPFS, this function initializes it and
           returns the name of the dir on which OPFS is mounted, else it returns
           an empty string.
        */
        const opfsDir = function f(wasmUtil){
          if(undefined !== f._) return f._;
          const pdir = '/persistent';
          if( !self.FileSystemHandle
              || !self.FileSystemDirectoryHandle
              || !self.FileSystemFileHandle){
            return f._ = "";
          }
          try{
            if(0===wasmUtil.xCallWrapped(
              'sqlite3_wasm_init_opfs', 'i32', ['string'], pdir
            )){
              return f._ = pdir;
            }else{
              return f._ = "";
            }
          }catch(e){
            // sqlite3_wasm_init_opfs() is not available
            return f._ = "";
          }
        };
        opfsDir._ = undefined;

        const eOut = document.querySelector('#test-output');
        const log2 = async function(cssClass,...args){
          const ln = document.createElement('div');
          if(cssClass) ln.classList.add(cssClass);
          ln.append(document.createTextNode(args.join(' ')));
          eOut.append(ln);
          //this.e.output.lastElementChild.scrollIntoViewIfNeeded();
        };
        const doHtmlOutput = false
        /* can't update DOM while speedtest is running unless we run
           speedtest in a worker thread. */;
        const log = (...args)=>{
          console.log(...args);
          if(doHtmlOutput) log2('', ...args);
        };
        const logErr = function(...args){
          console.error(...args);
          if(doHtmlOutput) log2('error', ...args);
        };

        const runTests = function(EmscriptenModule){
          console.log("Module inited.",EmscriptenModule);
          const wasm = {
            exports: EmscriptenModule.asm,
            alloc: (n)=>EmscriptenModule._malloc(n),
            dealloc: (m)=>EmscriptenModule._free(m),
            memory: EmscriptenModule.asm.memory || EmscriptenModule.wasmMemory
          };

          //console.debug('wasm =',wasm);
          self.WhWasmUtilInstaller(wasm);
          const unlink = wasm.xWrap("sqlite3_wasm_vfs_unlink", "int", ["string"]);
          const pDir = opfsDir(wasm);
          if(pDir){
            console.warn("Persistent storage:",pDir);
          }
          const scope = wasm.scopedAllocPush();
          const dbFile = 0 ? "" : pDir+"/speedtest1.db";
          try{

            const argv = [
              // TODO: accept flags via URL arguments and/or a
              // UI control. A multi-SELECT element should do
              // nicely.
              "speedtest1",
              "--singlethread",
              //"--stats",
              "--memdb", // note that memdb trumps the filename arg
              dbFile
            ];
            console.log("argv =",argv);
            wasm.xCall('__main_argc_argv', argv.length,
                       wasm.scopedAllocMainArgv(argv));
          }finally{
            wasm.scopedAllocPop(scope);
            if(pDir) unlink(dbFile);
          }
        };

        self.sqlite3TestModule.print = log;
        self.sqlite3TestModule.printErr = logErr;
        sqlite3Speedtest1InitModule(self.sqlite3TestModule).then(function(M){
          setTimeout(()=>runTests(M), 100);
        });
      })();
    </script>
  </body>