Cannot reproduce here.
I copied your lz4 files into the ext/lsm1 directory of SQLite, applied the patch below to build the shell tool with lsm and lz4 support built-in, then ran the following multiple times:
<verbatim>
CREATE VIRTUAL TABLE cdx USING lsm1 ('/home/dan/work/sqlite/bld4/lz4_weetest.lsm', key, TEXT, cdxline);
insert into cdx (key, cdxline) values ('ai,ak)/20200621000621','ai,ak)/ 20200621000621 http://ak.ai/ text/html 200 7OJCBOXLSJRUTLFALBSMSD5XHAWSMGO3 - - 5907 12823972 ARCHIVEIT-9810-DAILY-JOB1205229-SEED2290404-20200620230127543-00000-h3.warc.gz');
insert into cdx (key, cdxline) values ('ai,ak)/20200621000621','ai,ak)/ 20200621000621 http://ak.ai/ text/html 200 7OJCBOXLSJRUTLFALBSMSD5XHAWSMGO3 - - 5907 12823972 ARCHIVEIT-9810-DAILY-JOB1205229-SEED2290404-20200620230127543-00000-h3.warc.gz');
insert into cdx (key, cdxline) values ('ai,ak)/20200621000622','ai,ak)/ 20200621000621 http://ak.ai/ text/html 200 7OJCBOXLSJRUTLFALBSMSD5XHAWSMGO3 - - 5907 12823972 ARCHIVEIT-9810-DAILY-JOB1205229-SEED2290404-20200620230127543-00000-h3.warc.gz');
insert into cdx (key, cdxline) values ('ai,ak)/20200621000623','ai,ak)/ 20200621000621 http://ak.ai/ text/html 200 7OJCBOXLSJRUTLFALBSMSD5XHAWSMGO3 - - 5907 12823972 ARCHIVEIT-9810-DAILY-JOB1205229-SEED2290404-20200620230127543-00000-h3.warc.gz');
insert into cdx (key, cdxline) values ('ai,ak)/20200621000625','ai,ak)/ 20200621000621 http://ak.ai/ text/html 200 7OJCBOXLSJRUTLFALBSMSD5XHAWSMGO3 - - 5907 12823972 ARCHIVEIT-9810-DAILY-JOB1205229-SEED2290404-20200620230127543-00000-h3.warc.gz');
insert into cdx (key, cdxline) values ('ai,ak)/20200621000639','ai,ak)/ 20200621000621 http://ak.ai/ text/html 200 7OJCBOXLSJRUTLFALBSMSD5XHAWSMGO3 - - 5907 12823972 ARCHIVEIT-9810-DAILY-JOB1205229-SEED2290404-20200620230127543-00000-h3.warc.gz');
</verbatim>
but, no error. Not under valgrind or with asan either. Any ideas?
<verbatim>
Index: ext/lsm1/Makefile
==================================================================
--- ext/lsm1/Makefile
+++ ext/lsm1/Makefile
@@ -11,10 +11,11 @@
LSMOBJ = \
lsm_ckpt.o \
lsm_file.o \
lsm_log.o \
+ lsm-lz4.o \
lsm_main.o \
lsm_mem.o \
lsm_mutex.o \
lsm_shared.o \
lsm_sorted.o \
@@ -21,15 +22,20 @@
lsm_str.o \
lsm_tree.o \
lsm_unix.o \
lsm_win32.o \
lsm_varint.o \
- lsm_vtab.o
+ lsm_vtab.o \
+ lz4.o
LSMHDR = \
$(LSMDIR)/lsm.h \
- $(LSMDIR)/lsmInt.h
+ $(LSMDIR)/lsmInt.h \
+ $(LSMDIR)/lsm-lz4.h \
+ $(LSMDIR)/lz4.h \
+ $(LSMDIR)/lz4_decoder.h \
+ $(LSMDIR)/lz4_encoder.h
LSMTESTSRC = $(LSMDIR)/lsm-test/lsmtest1.c $(LSMDIR)/lsm-test/lsmtest2.c \
$(LSMDIR)/lsm-test/lsmtest3.c $(LSMDIR)/lsm-test/lsmtest4.c \
$(LSMDIR)/lsm-test/lsmtest5.c $(LSMDIR)/lsm-test/lsmtest6.c \
$(LSMDIR)/lsm-test/lsmtest7.c $(LSMDIR)/lsm-test/lsmtest8.c \
@@ -41,16 +47,17 @@
$(LSMDIR)/lsm-test/lsmtest_util.c $(LSMDIR)/lsm-test/lsmtest_win32.c
# all: lsm.so
-LSMOPTS += -fPIC -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR) -DHAVE_ZLIB
+LSMOPTS += -fPIC -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR) -DHAVE_ZLIB -DSQLITE_CORE
+LZ4_OPTS = -DUSE_LSM_LZ4_COMPRESSOR
lsm.so: $(LSMOBJ)
$(TCCX) -shared -fPIC -o lsm.so $(LSMOBJ)
%.o: $(LSMDIR)/%.c $(LSMHDR) sqlite3.h
- $(TCCX) $(LSMOPTS) -c $<
+ $(TCCX) $(LSMOPTS) $(LZ4_OPTS) -c $<
lsmtest$(EXE): $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) sqlite3.o
# $(TCPPX) -c $(TOP)/lsm-test/lsmtest_tdb2.cc
- $(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB) -lz
+ $(TCCX) $(LSMOPTS) $(LZ4_OPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB) -lz
Index: ext/lsm1/lsm_vtab.c
==================================================================
--- ext/lsm1/lsm_vtab.c
+++ ext/lsm1/lsm_vtab.c
@@ -90,10 +90,11 @@
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#include "lsm.h"
#include <assert.h>
#include <string.h>
+#include "lsm-lz4.h"
/* Forward declaration of subclasses of virtual table objects */
typedef struct lsm1_vtab lsm1_vtab;
typedef struct lsm1_cursor lsm1_cursor;
typedef struct lsm1_vblob lsm1_vblob;
@@ -263,10 +264,16 @@
rc = lsm_new(0, &pNew->pDb);
if( rc ){
*pzErr = sqlite3_mprintf("lsm_new failed with error code %d", rc);
rc = SQLITE_ERROR;
goto connect_failed;
+ }
+ rc = lsm_lz4_config(pNew->pDb);
+ if( rc ){
+ *pzErr = sqlite3_mprintf("lsm_lz4_config failed with error code %d", rc);
+ rc = SQLITE_ERROR;
+ goto connect_failed;
}
zFilename = sqlite3_mprintf("%s", argv[3]);
lsm1Dequote(zFilename);
rc = lsm_open(pNew->pDb, zFilename);
sqlite3_free(zFilename);
Index: main.mk
==================================================================
--- main.mk
+++ main.mk
@@ -48,10 +48,13 @@
TCCX += -I$(TOP)/ext/async -I$(TOP)/ext/userauth
TCCX += -I$(TOP)/ext/session
TCCX += -I$(TOP)/ext/fts5
THREADLIB += $(LIBS)
+LSMDIR=$(TOP)/ext/lsm1/
+include $(LSMDIR)/Makefile
+
# Object files for the SQLite library.
#
LIBOBJ+= vdbe.o parse.o \
alter.o analyze.o attach.o auth.o \
backup.o bitvec.o btmutex.o btree.o build.o \
@@ -74,11 +77,11 @@
table.o threads.o tokenize.o treeview.o trigger.o \
update.o upsert.o userauth.o util.o vacuum.o \
vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbesort.o \
vdbetrace.o vdbevtab.o \
wal.o walker.o where.o wherecode.o whereexpr.o \
- utf.o vtab.o window.o
+ utf.o vtab.o window.o $(LSMOBJ)
LIBOBJ += sqlite3session.o
# All of the source code files.
#
@@ -549,11 +552,11 @@
# This is the default Makefile target. The objects listed here
# are what get build when you type just "make" with no arguments.
#
all: sqlite3.h libsqlite3.a sqlite3$(EXE)
-libsqlite3.a: $(LIBOBJ)
+libsqlite3.a: $(LIBOBJ)
$(AR) libsqlite3.a $(LIBOBJ)
$(RANLIB) libsqlite3.a
sqlite3$(EXE): shell.c libsqlite3.a sqlite3.h
$(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) $(SHELL_OPT) \
Index: src/main.c
==================================================================
--- src/main.c
+++ src/main.c
@@ -3350,10 +3350,12 @@
/* Load automatic extensions - extensions that have been registered
** using the sqlite3_automatic_extension() API.
*/
if( rc==SQLITE_OK ){
+ int sqlite3_lsm_init(sqlite3 *, char **, const sqlite3_api_routines *);
+ sqlite3_lsm_init(db, 0, 0);
sqlite3AutoLoadExtensions(db);
rc = sqlite3_errcode(db);
if( rc!=SQLITE_OK ){
goto opendb_out;
}
</verbatim>