/ Check-in [cb721d0b]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge latest trunk into this branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | schemalint
Files: files | file ages | folders
SHA3-256: cb721d0b36268a7b0ef493fa4d7f6bcbaa9ead8b1990e3c3fae015fa1d545226
User & Date: dan 2017-03-31 11:20:20
Context
2017-04-04
04:23
Add the sqlite3_whereinfo_hook() API - an experimental API replacing the DBCONFIG_WHEREINFO hack on this branch. check-in: a54aef35 user: dan tags: schemalint
2017-03-31
11:20
Merge latest trunk into this branch. check-in: cb721d0b user: dan tags: schemalint
08:00
Update shell6.test to account for the fact that tests are now run in a separate directory. check-in: 1e3622de user: dan tags: schemalint
2017-03-30
17:13
Declare the Lemon-generated parser object as itself. (Duh) check-in: c8000e94 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

    26     26   # will run on the target platform.  (BCC and TCC are usually the
    27     27   # same unless your are cross-compiling.)  Separate CC and CFLAGS macros
    28     28   # are provide so that these aspects of the build process can be changed
    29     29   # on the "make" command-line.  Ex:  "make CC=clang CFLAGS=-fsanitize=undefined"
    30     30   #
    31     31   CC = @CC@
    32     32   CFLAGS = @CPPFLAGS@ @CFLAGS@
    33         -TCC = $(CC) $(CFLAGS) -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/fts3
           33  +TCC = ${CC} ${CFLAGS} -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/icu
           34  +TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session
    34     35   
    35     36   # Define this for the autoconf-based build, so that the code knows it can
    36     37   # include the generated config.h
    37     38   # 
    38     39   TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite
    39     40   
    40     41   # Define -DNDEBUG to compile without debugging (i.e., for production usage)
................................................................................
   175    176            func.lo global.lo hash.lo \
   176    177            icu.lo insert.lo json1.lo legacy.lo loadext.lo \
   177    178            main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
   178    179            memjournal.lo \
   179    180            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
   180    181            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
   181    182            pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
   182         -         random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \
          183  +         random.lo resolve.lo rowset.lo rtree.lo \
          184  +         sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
   183    185            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
   184    186            update.lo util.lo vacuum.lo \
   185    187            vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
   186    188            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
   187    189            utf.lo vtab.lo
   188    190   
   189    191   # Object files for the amalgamation.
................................................................................
   338    340   SRC += \
   339    341     $(TOP)/ext/icu/sqliteicu.h \
   340    342     $(TOP)/ext/icu/icu.c
   341    343   SRC += \
   342    344     $(TOP)/ext/rtree/rtree.h \
   343    345     $(TOP)/ext/rtree/rtree.c
   344    346   SRC += \
          347  +  $(TOP)/ext/session/sqlite3session.c \
          348  +  $(TOP)/ext/session/sqlite3session.h
          349  +SRC += \
   345    350     $(TOP)/ext/rbu/sqlite3rbu.h \
   346    351     $(TOP)/ext/rbu/sqlite3rbu.c
   347    352   SRC += \
   348    353     $(TOP)/ext/misc/json1.c
   349         -
   350         -
   351    354   
   352    355   # Generated source code files
   353    356   #
   354    357   SRC += \
   355    358     keywordhash.h \
   356    359     opcodes.c \
   357    360     opcodes.h \
................................................................................
   375    378     $(TOP)/src/test_autoext.c \
   376    379     $(TOP)/src/test_async.c \
   377    380     $(TOP)/src/test_backup.c \
   378    381     $(TOP)/src/test_bestindex.c \
   379    382     $(TOP)/src/test_blob.c \
   380    383     $(TOP)/src/test_btree.c \
   381    384     $(TOP)/src/test_config.c \
          385  +  $(TOP)/src/test_delete.c \
   382    386     $(TOP)/src/test_demovfs.c \
   383    387     $(TOP)/src/test_devsym.c \
   384    388     $(TOP)/src/test_fs.c \
   385    389     $(TOP)/src/test_func.c \
   386    390     $(TOP)/src/test_hexio.c \
   387    391     $(TOP)/src/test_init.c \
   388    392     $(TOP)/src/test_intarray.c \
................................................................................
   401    405     $(TOP)/src/test_syscall.c \
   402    406     $(TOP)/src/test_tclvar.c \
   403    407     $(TOP)/src/test_thread.c \
   404    408     $(TOP)/src/test_vfs.c \
   405    409     $(TOP)/src/test_windirent.c \
   406    410     $(TOP)/src/test_wsd.c       \
   407    411     $(TOP)/ext/fts3/fts3_term.c \
   408         -  $(TOP)/ext/fts3/fts3_test.c \
          412  +  $(TOP)/ext/fts3/fts3_test.c  \
          413  +  $(TOP)/ext/session/test_session.c \
   409    414     $(TOP)/ext/rbu/test_rbu.c 
   410    415   
   411    416   # Statically linked extensions
   412    417   #
   413    418   TESTSRC += \
   414    419     $(TOP)/ext/misc/amatch.c \
          420  +  $(TOP)/ext/misc/carray.c \
   415    421     $(TOP)/ext/misc/closure.c \
          422  +  $(TOP)/ext/misc/csv.c \
   416    423     $(TOP)/ext/misc/eval.c \
   417    424     $(TOP)/ext/misc/fileio.c \
   418    425     $(TOP)/ext/misc/fuzzer.c \
   419    426     $(TOP)/ext/fts5/fts5_tcl.c \
   420    427     $(TOP)/ext/fts5/fts5_test_mi.c \
   421    428     $(TOP)/ext/fts5/fts5_test_tok.c \
   422    429     $(TOP)/ext/misc/ieee754.c \
   423    430     $(TOP)/ext/misc/nextchar.c \
   424    431     $(TOP)/ext/misc/percentile.c \
   425    432     $(TOP)/ext/misc/regexp.c \
          433  +  $(TOP)/ext/misc/remember.c \
   426    434     $(TOP)/ext/misc/series.c \
   427    435     $(TOP)/ext/misc/spellfix.c \
   428    436     $(TOP)/ext/misc/totype.c \
   429    437     $(TOP)/ext/misc/wholenumber.c
   430    438   
   431    439   # Source code to the library files needed by the test fixture
   432    440   #
................................................................................
   470    478     parse.c \
   471    479     $(TOP)/ext/fts3/fts3.c \
   472    480     $(TOP)/ext/fts3/fts3_aux.c \
   473    481     $(TOP)/ext/fts3/fts3_expr.c \
   474    482     $(TOP)/ext/fts3/fts3_term.c \
   475    483     $(TOP)/ext/fts3/fts3_tokenizer.c \
   476    484     $(TOP)/ext/fts3/fts3_write.c \
   477         -  $(TOP)/ext/async/sqlite3async.c
          485  +  $(TOP)/ext/async/sqlite3async.c \
          486  +  $(TOP)/ext/session/sqlite3session.c 
   478    487   
   479    488   # Header files used by all library source files.
   480    489   #
   481    490   HDR = \
   482    491      $(TOP)/src/btree.h \
   483    492      $(TOP)/src/btreeInt.h \
   484    493      $(TOP)/src/hash.h \
................................................................................
   529    538   
   530    539   # executables needed for testing
   531    540   #
   532    541   TESTPROGS = \
   533    542     testfixture$(TEXE) \
   534    543     sqlite3$(TEXE) \
   535    544     sqlite3_analyzer$(TEXE) \
   536         -  sqldiff$(TEXE)
          545  +  sqldiff$(TEXE) \
          546  +  dbhash$(TEXE)
   537    547   
   538    548   # Databases containing fuzzer test cases
   539    549   #
   540    550   FUZZDATA = \
   541    551     $(TOP)/test/fuzzdata1.db \
   542    552     $(TOP)/test/fuzzdata2.db \
   543    553     $(TOP)/test/fuzzdata3.db \
   544         -  $(TOP)/test/fuzzdata4.db
          554  +  $(TOP)/test/fuzzdata4.db \
          555  +  $(TOP)/test/fuzzdata5.db
   545    556   
   546    557   # Standard options to testfixture
   547    558   #
   548    559   TESTOPTS = --verbose=file --output=test-out.txt
   549    560   
   550    561   # Extra compiler options for various shell tools
   551    562   #
   552    563   SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
          564  +# SHELL_OPT += -DSQLITE_ENABLE_FTS5
   553    565   SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
          566  +SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
   554    567   FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
   555         -FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
          568  +FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
          569  +FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
          570  +FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
          571  +DBFUZZ_OPT = 
   556    572   
   557    573   # This is the default Makefile target.  The objects listed here
   558    574   # are what get build when you type just "make" with no arguments.
   559    575   #
   560    576   all:	sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la)
   561    577   
   562    578   Makefile: $(TOP)/Makefile.in
................................................................................
   577    593   		-avoid-version
   578    594   
   579    595   sqlite3$(TEXE):	$(TOP)/src/shell.c sqlite3.c
   580    596   	$(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \
   581    597   		$(TOP)/src/shell.c sqlite3.c \
   582    598   		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
   583    599   
   584         -sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
   585         -	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS)
          600  +sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h
          601  +	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS)
          602  +
          603  +dbhash$(TEXE):	$(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h
          604  +	$(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS)
          605  +
          606  +scrub$(TEXE):	$(TOP)/ext/misc/scrub.c sqlite3.lo
          607  +	$(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \
          608  +		$(TOP)/ext/misc/scrub.c sqlite3.lo $(TLIBS)
   586    609   
   587    610   srcck1$(BEXE):	$(TOP)/tool/srcck1.c
   588    611   	$(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c
   589    612   
   590    613   sourcetest:	srcck1$(BEXE) sqlite3.c
   591    614   	./srcck1 sqlite3.c
   592    615   
   593    616   fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
   594    617   	$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
   595    618   	  $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)
   596    619   
   597         -fuzzcheck$(TEXE):	$(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h
   598         -	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS)
          620  +fuzzcheck$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h
          621  +	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
   599    622   
   600         -mptester$(TEXE):	sqlite3.c $(TOP)/mptest/mptest.c
   601         -	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \
          623  +ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
          624  +	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
          625  +             $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
          626  +
          627  +dbfuzz$(TEXE):	$(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h
          628  +	$(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS)
          629  +
          630  +mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
          631  +	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
   602    632   		$(TLIBS) -rpath "$(libdir)"
   603    633   
   604    634   MPTEST1=./mptester$(TEXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20
   605    635   MPTEST2=./mptester$(TEXE) mptest.db $(TOP)/mptest/multiwrite01.test --repeat 20
   606    636   mptest:	mptester$(TEXE)
   607    637   	rm -f mptest.db
   608    638   	$(MPTEST1) --journalmode DELETE
................................................................................
   630    660   	mv vdbe.new tsrc/vdbe.c
   631    661   	cp fts5.c fts5.h tsrc
   632    662   	touch .target_source
   633    663   
   634    664   sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl
   635    665   	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl
   636    666   	cp tsrc/shell.c tsrc/sqlite3ext.h .
          667  +	cp $(TOP)/ext/session/sqlite3session.h .
   637    668   
   638    669   sqlite3ext.h:	.target_source
   639    670   	cp tsrc/sqlite3ext.h .
   640    671   
   641    672   tclsqlite3.c:	sqlite3.c
   642    673   	echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c
   643    674   	cat sqlite3.c >>tclsqlite3.c
................................................................................
   993   1024   
   994   1025   fts3_write.lo:	$(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
   995   1026   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c
   996   1027   
   997   1028   rtree.lo:	$(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
   998   1029   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
   999   1030   
         1031  +sqlite3session.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
         1032  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c
         1033  +
  1000   1034   json1.lo:	$(TOP)/ext/misc/json1.c
  1001   1035   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c
  1002   1036   
  1003   1037   # FTS5 things
  1004   1038   #
  1005   1039   FTS5_SRC = \
  1006   1040      $(TOP)/ext/fts5/fts5.h \
................................................................................
  1018   1052      $(TOP)/ext/fts5/fts5_unicode2.c \
  1019   1053      $(TOP)/ext/fts5/fts5_varint.c \
  1020   1054      $(TOP)/ext/fts5/fts5_vocab.c  \
  1021   1055   
  1022   1056   fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
  1023   1057   	cp $(TOP)/ext/fts5/fts5parse.y .
  1024   1058   	rm -f fts5parse.h
  1025         -	./lemon $(OPTS) fts5parse.y
         1059  +	./lemon$(BEXE) $(OPTS) fts5parse.y
  1026   1060   
  1027   1061   fts5parse.h: fts5parse.c
  1028   1062   
  1029   1063   fts5.c: $(FTS5_SRC)
  1030   1064   	$(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl
  1031   1065   	cp $(TOP)/ext/fts5/fts5.h .
  1032   1066   
................................................................................
  1077   1111   
  1078   1112   fastfuzztest:	fuzzcheck$(TEXE) $(FUZZDATA)
  1079   1113   	./fuzzcheck$(TEXE) --limit-mem 100M $(FUZZDATA)
  1080   1114   
  1081   1115   valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA)
  1082   1116   	valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M --timeout 600 $(FUZZDATA)
  1083   1117   
         1118  +# The veryquick.test TCL tests.
         1119  +#
         1120  +tcltest:	./testfixture$(TEXE)
         1121  +	./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
         1122  +
  1084   1123   # Minimal testing that runs in less than 3 minutes
  1085   1124   #
  1086   1125   quicktest:	./testfixture$(TEXE)
  1087   1126   	./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS)
  1088   1127   
  1089   1128   # This is the common case.  Run many tests that do not take too long,
  1090   1129   # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
  1091   1130   #
  1092         -test:	$(TESTPROGS) sourcetest fastfuzztest
  1093         -	./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
         1131  +test:	fastfuzztest sourcetest $(TESTPROGS) tcltest
  1094   1132   
  1095   1133   # Run a test using valgrind.  This can take a really long time
  1096   1134   # because valgrind is so much slower than a native machine.
  1097   1135   #
  1098   1136   valgrindtest:	$(TESTPROGS) valgrindfuzz
  1099   1137   	OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind $(TESTOPTS)
  1100   1138   
................................................................................
  1113   1151   	echo "static const char *zMainloop = " >> $@
  1114   1152   	$(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@
  1115   1153   	echo "; return zMainloop; }" >> $@
  1116   1154   
  1117   1155   sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
  1118   1156   	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
  1119   1157   
         1158  +dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
         1159  +	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
         1160  +           $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
         1161  +
  1120   1162   showdb$(TEXE):	$(TOP)/tool/showdb.c sqlite3.lo
  1121   1163   	$(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS)
  1122   1164   
  1123   1165   showstat4$(TEXE):	$(TOP)/tool/showstat4.c sqlite3.lo
  1124   1166   	$(LTLINK) -o $@ $(TOP)/tool/showstat4.c sqlite3.lo $(TLIBS)
  1125   1167   
  1126   1168   showjournal$(TEXE):	$(TOP)/tool/showjournal.c sqlite3.lo
  1127   1169   	$(LTLINK) -o $@ $(TOP)/tool/showjournal.c sqlite3.lo $(TLIBS)
  1128   1170   
  1129   1171   showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
  1130   1172   	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)
  1131   1173   
         1174  +changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
         1175  +	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
         1176  +
  1132   1177   rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo
  1133   1178   	$(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS)
  1134   1179   
  1135   1180   LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
  1136   1181   	$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c
  1137   1182   
  1138         -wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.c
  1139         -	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS)
         1183  +wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.lo
         1184  +	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS)
  1140   1185   
  1141         -speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.lo
  1142         -	$(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS)
         1186  +speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.c
         1187  +	$(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS)
         1188  +
         1189  +KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ
         1190  +
         1191  +kvtest$(TEXE):	$(TOP)/test/kvtest.c sqlite3.c
         1192  +	$(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS)
  1143   1193   
  1144   1194   rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo 
  1145   1195   	$(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS)
  1146   1196   
  1147   1197   loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
  1148   1198   	$(LTLINK) $(TOP)/tool/loadfts.c libsqlite3.la -o $@ $(TLIBS)
  1149   1199   
  1150   1200   # This target will fail if the SQLite amalgamation contains any exported
  1151   1201   # symbols that do not begin with "sqlite3_". It is run as part of the
  1152   1202   # releasetest.tcl script.
  1153   1203   #
         1204  +VALIDIDS=' sqlite3(changeset|changegroup|session)?_'
  1154   1205   checksymbols: sqlite3.lo
  1155         -	nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0
         1206  +	nm -g --defined-only sqlite3.lo | egrep -v $(VALIDIDS); test $$? -ne 0
  1156   1207   	echo '0 errors out of 1 tests'
  1157   1208   
  1158   1209   # Build the amalgamation-autoconf package.  The amalamgation-tarball target builds
  1159   1210   # a tarball named for the version number.  Ex:  sqlite-autoconf-3110000.tar.gz.
  1160   1211   # The snapshot-tarball target builds a tarball named by the SHA1 hash
  1161   1212   #
  1162   1213   amalgamation-tarball: sqlite3.c
................................................................................
  1211   1262   clean:	
  1212   1263   	rm -f *.lo *.la *.o sqlite3$(TEXE) libsqlite3.la
  1213   1264   	rm -f sqlite3.h opcodes.*
  1214   1265   	rm -rf .libs .deps
  1215   1266   	rm -f lemon$(BEXE) lempar.c parse.* sqlite*.tar.gz
  1216   1267   	rm -f mkkeywordhash$(BEXE) keywordhash.h
  1217   1268   	rm -f *.da *.bb *.bbg gmon.out
  1218         -	rm -rf quota2a quota2b quota2c
  1219   1269   	rm -rf tsrc .target_source
  1220   1270   	rm -f tclsqlite3$(TEXE)
  1221   1271   	rm -f testfixture$(TEXE) test.db
  1222   1272   	rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE)
  1223   1273   	rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE)
  1224         -	rm -f wordcount$(TEXE)
         1274  +	rm -f wordcount$(TEXE) changeset$(TEXE)
  1225   1275   	rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def
  1226   1276   	rm -f sqlite3.c
  1227   1277   	rm -f sqlite3rc.h
  1228   1278   	rm -f shell.c sqlite3ext.h
  1229   1279   	rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c
  1230   1280   	rm -f sqlite-*-output.vsix
  1231   1281   	rm -f mptester mptester.exe
  1232   1282   	rm -f rbu rbu.exe
  1233   1283   	rm -f srcck1 srcck1.exe
  1234   1284   	rm -f fuzzershell fuzzershell.exe
  1235   1285   	rm -f fuzzcheck fuzzcheck.exe
  1236   1286   	rm -f sqldiff sqldiff.exe
         1287  +	rm -f dbhash dbhash.exe
  1237   1288   	rm -f fts5.* fts5parse.*
  1238   1289   
  1239   1290   distclean:	clean
  1240   1291   	rm -f config.h config.log config.status libtool Makefile sqlite3.pc
  1241   1292   
  1242   1293   #
  1243   1294   # Windows section

Changes to Makefile.msc.

    19     19   # <</mark>>
    20     20   
    21     21   # Set this non-0 to enable full warnings (-W4, etc) when compiling.
    22     22   #
    23     23   !IFNDEF USE_FULLWARN
    24     24   USE_FULLWARN = 0
    25     25   !ENDIF
           26  +
           27  +# Set this non-0 to enable treating warnings as errors (-WX, etc) when
           28  +# compiling.
           29  +#
           30  +!IFNDEF USE_FATAL_WARN
           31  +USE_FATAL_WARN = 0
           32  +!ENDIF
           33  +
           34  +# Set this non-0 to enable full runtime error checks (-RTC1, etc).  This
           35  +# has no effect if (any) optimizations are enabled.
           36  +#
           37  +!IFNDEF USE_RUNTIME_CHECKS
           38  +USE_RUNTIME_CHECKS = 0
           39  +!ENDIF
           40  +
           41  +# Set this non-0 to create a SQLite amalgamation file that excludes the
           42  +# various built-in extensions.
           43  +#
           44  +!IFNDEF MINIMAL_AMALGAMATION
           45  +MINIMAL_AMALGAMATION = 0
           46  +!ENDIF
    26     47   
    27     48   # Set this non-0 to use "stdcall" calling convention for the core library
    28     49   # and shell executable.
    29     50   #
    30     51   !IFNDEF USE_STDCALL
    31     52   USE_STDCALL = 0
    32     53   !ENDIF
................................................................................
   193    214   # Enable use of available compiler optimizations?  Normally, this should be
   194    215   # non-zero.  Setting this to zero, thus disabling all compiler optimizations,
   195    216   # can be useful for testing.
   196    217   #
   197    218   !IFNDEF OPTIMIZATIONS
   198    219   OPTIMIZATIONS = 2
   199    220   !ENDIF
          221  +
          222  +# Set this to non-0 to enable support for the session extension.
          223  +#
          224  +!IFNDEF SESSION
          225  +SESSION = 0
          226  +!ENDIF
   200    227   
   201    228   # Set the source code file to be used by executables and libraries when
   202    229   # they need the amalgamation.
   203    230   #
   204    231   !IFNDEF SQLITE3C
   205    232   !IF $(SPLIT_AMALGAMATION)!=0
   206    233   SQLITE3C = sqlite3-all.c
................................................................................
   253    280   !IF $(FOR_WIN10)!=0
   254    281   SQLITE3EXEPDB =
   255    282   !ELSE
   256    283   SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
   257    284   !ENDIF
   258    285   !ENDIF
   259    286   
          287  +# <<mark>>
          288  +# These are the names of the customized Tcl header files used by various parts
          289  +# of this makefile when the stdcall calling convention is in use.  It is not
          290  +# used for any other purpose.
          291  +#
          292  +!IFNDEF SQLITETCLH
          293  +SQLITETCLH = sqlite_tcl.h
          294  +!ENDIF
          295  +
          296  +!IFNDEF SQLITETCLDECLSH
          297  +SQLITETCLDECLSH = sqlite_tclDecls.h
          298  +!ENDIF
          299  +
          300  +# These are the additional targets that the targets that integrate with the
          301  +# Tcl library should depend on when compiling, etc.
          302  +#
          303  +!IFNDEF SQLITE_TCL_DEP
          304  +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
          305  +SQLITE_TCL_DEP = $(SQLITETCLDECLSH) $(SQLITETCLH)
          306  +!ELSE
          307  +SQLITE_TCL_DEP =
          308  +!ENDIF
          309  +!ENDIF
          310  +# <</mark>>
          311  +
   260    312   # These are the "standard" SQLite compilation options used when compiling for
   261    313   # the Windows platform.
   262    314   #
   263    315   !IFNDEF OPT_FEATURE_FLAGS
          316  +!IF $(MINIMAL_AMALGAMATION)==0
   264    317   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
   265    318   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
          319  +!ENDIF
   266    320   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
   267    321   !ENDIF
          322  +
          323  +# Should the session extension be enabled?  If so, add compilation options
          324  +# to enable it.
          325  +#
          326  +!IF $(SESSION)!=0
          327  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1
          328  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1
          329  +!ENDIF
   268    330   
   269    331   # These are the "extended" SQLite compilation options used when compiling for
   270    332   # the Windows 10 platform.
   271    333   #
   272    334   !IFNDEF EXT_FEATURE_FLAGS
   273    335   !IF $(FOR_WIN10)!=0
   274    336   EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS4=1
................................................................................
   433    495   # same unless your are cross-compiling.)
   434    496   #
   435    497   !IF $(USE_FULLWARN)!=0
   436    498   TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
   437    499   !ELSE
   438    500   TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
   439    501   !ENDIF
          502  +
          503  +# Check if warnings should be treated as errors when compiling.
          504  +#
          505  +!IF $(USE_FATAL_WARN)!=0
          506  +TCC = $(TCC) -WX
          507  +!ENDIF
   440    508   
   441    509   TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src -fp:precise
   442    510   RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS)
   443    511   
   444    512   # Check if we want to use the "stdcall" calling convention when compiling.
   445    513   # This is not supported by the compilers for non-x86 platforms.  It should
   446    514   # also be noted here that building any target with these "stdcall" options
   447    515   # will most likely fail if the Tcl library is also required.  This is due
   448    516   # to how the Tcl library functions are declared and exported (i.e. without
   449    517   # an explicit calling convention, which results in "cdecl").
   450    518   #
   451    519   !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
   452    520   !IF "$(PLATFORM)"=="x86"
   453         -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
   454         -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
          521  +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
          522  +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
          523  +# <<mark>>
          524  +TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl
          525  +# <</mark>>
   455    526   !ELSE
   456    527   !IFNDEF PLATFORM
   457         -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
   458         -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
          528  +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
          529  +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
          530  +# <<mark>>
          531  +TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl
          532  +# <</mark>>
   459    533   !ELSE
   460    534   CORE_CCONV_OPTS =
   461    535   SHELL_CCONV_OPTS =
          536  +# <<mark>>
          537  +TEST_CCONV_OPTS =
          538  +# <</mark>>
   462    539   !ENDIF
   463    540   !ENDIF
   464    541   !ELSE
   465    542   CORE_CCONV_OPTS =
   466    543   SHELL_CCONV_OPTS =
          544  +# <<mark>>
          545  +TEST_CCONV_OPTS =
          546  +# <</mark>>
   467    547   !ENDIF
   468    548   
   469    549   # These are additional compiler options used for the core library.
   470    550   #
   471    551   !IFNDEF CORE_COMPILE_OPTS
   472    552   !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
   473    553   CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) -DSQLITE_API=__declspec(dllexport)
................................................................................
   603    683   # builds, we need to make sure the compiler can find these.
   604    684   #
   605    685   !IF $(USE_AMALGAMATION)==0
   606    686   TCC = $(TCC) -I$(TOP)\ext\fts3
   607    687   RCC = $(RCC) -I$(TOP)\ext\fts3
   608    688   TCC = $(TCC) -I$(TOP)\ext\rtree
   609    689   RCC = $(RCC) -I$(TOP)\ext\rtree
          690  +TCC = $(TCC) -I$(TOP)\ext\session
          691  +RCC = $(RCC) -I$(TOP)\ext\session
   610    692   !ENDIF
   611    693   
   612    694   # The mksqlite3c.tcl script accepts some options on the command
   613    695   # line.  When compiling with debugging enabled, some of these
   614    696   # options are necessary in order to allow debugging symbols to
   615    697   # work correctly with Visual Studio when using the amalgamation.
   616    698   #
          699  +!IFNDEF MKSQLITE3C_TOOL
          700  +!IF $(MINIMAL_AMALGAMATION)!=0
          701  +MKSQLITE3C_TOOL = $(TOP)\tool\mksqlite3c-noext.tcl
          702  +!ELSE
          703  +MKSQLITE3C_TOOL = $(TOP)\tool\mksqlite3c.tcl
          704  +!ENDIF
          705  +!ENDIF
          706  +
   617    707   !IFNDEF MKSQLITE3C_ARGS
   618    708   !IF $(DEBUG)>1
   619    709   MKSQLITE3C_ARGS = --linemacros
   620    710   !ELSE
   621    711   MKSQLITE3C_ARGS =
          712  +!ENDIF
          713  +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
          714  +MKSQLITE3C_ARGS = $(MKSQLITE3C_ARGS) --useapicall
          715  +!ENDIF
          716  +!ENDIF
          717  +
          718  +# The mksqlite3h.tcl script accepts some options on the command line.
          719  +# When compiling with stdcall support, some of these options are
          720  +# necessary.
          721  +#
          722  +!IFNDEF MKSQLITE3H_ARGS
          723  +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
          724  +MKSQLITE3H_ARGS = --useapicall
          725  +!ELSE
          726  +MKSQLITE3H_ARGS =
   622    727   !ENDIF
   623    728   !ENDIF
   624    729   # <</mark>>
   625    730   
   626    731   # Define -DNDEBUG to compile without debugging (i.e., for production usage)
   627    732   # Omitting the define will cause extra debugging code to be inserted and
   628    733   # includes extra comments when "EXPLAIN stmt" is used.
................................................................................
   637    742   TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1
   638    743   RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1
   639    744   !ENDIF
   640    745   
   641    746   !IF $(DEBUG)>2
   642    747   TCC = $(TCC) -DSQLITE_DEBUG=1
   643    748   RCC = $(RCC) -DSQLITE_DEBUG=1
          749  +!IF $(DYNAMIC_SHELL)==0
          750  +TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
          751  +RCC = $(RCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
          752  +!ENDIF
   644    753   !ENDIF
   645    754   
   646    755   !IF $(DEBUG)>4 || $(OSTRACE)!=0
   647    756   TCC = $(TCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1
   648    757   RCC = $(RCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1
   649    758   !ENDIF
   650    759   
................................................................................
   698    807   !ENDIF
   699    808   
   700    809   !IFNDEF TCLLIBDIR
   701    810   TCLLIBDIR = c:\tcl\lib
   702    811   !ENDIF
   703    812   
   704    813   !IFNDEF LIBTCL
   705         -LIBTCL = tcl85.lib
          814  +LIBTCL = tcl86.lib
   706    815   !ENDIF
   707    816   
   708    817   !IFNDEF LIBTCLSTUB
   709         -LIBTCLSTUB = tclstub85.lib
          818  +LIBTCLSTUB = tclstub86.lib
   710    819   !ENDIF
   711    820   
   712    821   !IFNDEF LIBTCLPATH
   713    822   LIBTCLPATH = c:\tcl\bin
   714    823   !ENDIF
   715    824   
   716    825   # The locations of the ICU header and library files.  These variables
................................................................................
   732    841   
   733    842   # This is the command to use for tclsh - normally just "tclsh", but we may
   734    843   # know the specific version we want to use.  This variable (TCLSH_CMD) may be
   735    844   # overridden via the environment prior to running nmake in order to select a
   736    845   # specific Tcl shell to use.
   737    846   #
   738    847   !IFNDEF TCLSH_CMD
   739         -TCLSH_CMD = tclsh85
          848  +TCLSH_CMD = tclsh
   740    849   !ENDIF
   741    850   # <</mark>>
   742    851   
   743    852   # Compiler options needed for programs that use the readline() library.
   744    853   #
   745    854   !IFNDEF READLINE_FLAGS
   746    855   READLINE_FLAGS = -DHAVE_READLINE=0
................................................................................
   815    924   
   816    925   # If optimizations are enabled or disabled (either implicitly or
   817    926   # explicitly), add the necessary flags.
   818    927   #
   819    928   !IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0
   820    929   TCC = $(TCC) -Od
   821    930   BCC = $(BCC) -Od
          931  +!IF $(USE_RUNTIME_CHECKS)!=0
          932  +TCC = $(TCC) -RTC1
          933  +BCC = $(BCC) -RTC1
          934  +!ENDIF
   822    935   !ELSEIF $(OPTIMIZATIONS)>=3
   823    936   TCC = $(TCC) -Ox
   824    937   BCC = $(BCC) -Ox
   825    938   !ELSEIF $(OPTIMIZATIONS)==2
   826    939   TCC = $(TCC) -O2
   827    940   BCC = $(BCC) -O2
   828    941   !ELSEIF $(OPTIMIZATIONS)==1
................................................................................
   987   1100            func.lo global.lo hash.lo \
   988   1101            icu.lo insert.lo legacy.lo loadext.lo \
   989   1102            main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
   990   1103            memjournal.lo \
   991   1104            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
   992   1105            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
   993   1106            pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
   994         -         random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \
         1107  +         random.lo resolve.lo rowset.lo rtree.lo \
         1108  +         sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
   995   1109            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
   996   1110            update.lo util.lo vacuum.lo \
   997   1111            vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
   998   1112            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
   999   1113            utf.lo vtab.lo
  1000   1114   # <</mark>>
  1001   1115   
................................................................................
  1174   1288     $(TOP)\ext\fts3\fts3_tokenizer1.c \
  1175   1289     $(TOP)\ext\fts3\fts3_tokenize_vtab.c \
  1176   1290     $(TOP)\ext\fts3\fts3_unicode.c \
  1177   1291     $(TOP)\ext\fts3\fts3_unicode2.c \
  1178   1292     $(TOP)\ext\fts3\fts3_write.c \
  1179   1293     $(TOP)\ext\icu\icu.c \
  1180   1294     $(TOP)\ext\rtree\rtree.c \
         1295  +  $(TOP)\ext\session\sqlite3session.c \
  1181   1296     $(TOP)\ext\rbu\sqlite3rbu.c \
  1182   1297     $(TOP)\ext\misc\json1.c
  1183   1298   
  1184   1299   # Extension header files, part 1.
  1185   1300   #
  1186   1301   SRC08 = \
  1187   1302     $(TOP)\ext\fts1\fts1.h \
................................................................................
  1196   1311   SRC09 = \
  1197   1312     $(TOP)\ext\fts3\fts3.h \
  1198   1313     $(TOP)\ext\fts3\fts3Int.h \
  1199   1314     $(TOP)\ext\fts3\fts3_hash.h \
  1200   1315     $(TOP)\ext\fts3\fts3_tokenizer.h \
  1201   1316     $(TOP)\ext\icu\sqliteicu.h \
  1202   1317     $(TOP)\ext\rtree\rtree.h \
  1203         -  $(TOP)\ext\rbu\sqlite3rbu.h
         1318  +  $(TOP)\ext\rbu\sqlite3rbu.h \
         1319  +  $(TOP)\ext\session\sqlite3session.h
  1204   1320   
  1205   1321   # Generated source code files
  1206   1322   #
  1207   1323   SRC10 = \
  1208   1324     opcodes.c \
  1209   1325     parse.c
  1210   1326   
................................................................................
  1211   1327   # Generated header files
  1212   1328   #
  1213   1329   SRC11 = \
  1214   1330     keywordhash.h \
  1215   1331     opcodes.h \
  1216   1332     parse.h \
  1217   1333     $(SQLITE3H)
         1334  +
         1335  +# Generated Tcl header files
         1336  +#
         1337  +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
         1338  +SRC12 = \
         1339  +  $(SQLITETCLH) \
         1340  +  $(SQLITETCLDECLSH)
         1341  +!ELSE
         1342  +SRC12 =
         1343  +!ENDIF
  1218   1344   
  1219   1345   # All source code files.
  1220   1346   #
  1221   1347   SRC = $(SRC00) $(SRC01) $(SRC02) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11)
  1222   1348   
  1223   1349   # Source code to the test files.
  1224   1350   #
................................................................................
  1235   1361     $(TOP)\src\test_autoext.c \
  1236   1362     $(TOP)\src\test_async.c \
  1237   1363     $(TOP)\src\test_backup.c \
  1238   1364     $(TOP)\src\test_bestindex.c \
  1239   1365     $(TOP)\src\test_blob.c \
  1240   1366     $(TOP)\src\test_btree.c \
  1241   1367     $(TOP)\src\test_config.c \
         1368  +  $(TOP)\src\test_delete.c \
  1242   1369     $(TOP)\src\test_demovfs.c \
  1243   1370     $(TOP)\src\test_devsym.c \
  1244   1371     $(TOP)\src\test_fs.c \
  1245   1372     $(TOP)\src\test_func.c \
  1246   1373     $(TOP)\src\test_hexio.c \
  1247   1374     $(TOP)\src\test_init.c \
  1248   1375     $(TOP)\src\test_intarray.c \
................................................................................
  1262   1389     $(TOP)\src\test_tclvar.c \
  1263   1390     $(TOP)\src\test_thread.c \
  1264   1391     $(TOP)\src\test_vfs.c \
  1265   1392     $(TOP)\src\test_windirent.c \
  1266   1393     $(TOP)\src\test_wsd.c \
  1267   1394     $(TOP)\ext\fts3\fts3_term.c \
  1268   1395     $(TOP)\ext\fts3\fts3_test.c \
  1269         -  $(TOP)\ext\rbu\test_rbu.c
         1396  +  $(TOP)\ext\rbu\test_rbu.c \
         1397  +  $(TOP)\ext\session\test_session.c
  1270   1398   
  1271   1399   # Statically linked extensions.
  1272   1400   #
  1273   1401   TESTEXT = \
  1274   1402     $(TOP)\ext\misc\amatch.c \
         1403  +  $(TOP)\ext\misc\carray.c \
  1275   1404     $(TOP)\ext\misc\closure.c \
         1405  +  $(TOP)\ext\misc\csv.c \
  1276   1406     $(TOP)\ext\misc\eval.c \
  1277   1407     $(TOP)\ext\misc\fileio.c \
  1278   1408     $(TOP)\ext\misc\fuzzer.c \
  1279   1409     $(TOP)\ext\fts5\fts5_tcl.c \
  1280   1410     $(TOP)\ext\fts5\fts5_test_mi.c \
  1281   1411     $(TOP)\ext\fts5\fts5_test_tok.c \
  1282   1412     $(TOP)\ext\misc\ieee754.c \
  1283   1413     $(TOP)\ext\misc\nextchar.c \
  1284   1414     $(TOP)\ext\misc\percentile.c \
  1285   1415     $(TOP)\ext\misc\regexp.c \
         1416  +  $(TOP)\ext\misc\remember.c \
  1286   1417     $(TOP)\ext\misc\series.c \
  1287   1418     $(TOP)\ext\misc\spellfix.c \
  1288   1419     $(TOP)\ext\misc\totype.c \
  1289   1420     $(TOP)\ext\misc\wholenumber.c
  1290   1421   
  1291   1422   # Source code to the library files needed by the test fixture
         1423  +# (non-amalgamation)
  1292   1424   #
  1293   1425   TESTSRC2 = \
  1294   1426     $(SRC00) \
  1295   1427     $(SRC01) \
  1296   1428     $(SRC06) \
  1297   1429     $(SRC07) \
  1298   1430     $(SRC10) \
................................................................................
  1314   1446      $(TOP)\src\os_setup.h \
  1315   1447      $(TOP)\src\os_win.h \
  1316   1448      $(TOP)\src\pager.h \
  1317   1449      $(TOP)\src\pcache.h \
  1318   1450      parse.h \
  1319   1451      $(TOP)\src\pragma.h \
  1320   1452      $(SQLITE3H) \
  1321         -   $(TOP)\src\sqlite3ext.h \
         1453  +   sqlite3ext.h \
  1322   1454      $(TOP)\src\sqliteInt.h \
  1323   1455      $(TOP)\src\sqliteLimit.h \
  1324   1456      $(TOP)\src\vdbe.h \
  1325   1457      $(TOP)\src\vdbeInt.h \
  1326   1458      $(TOP)\src\vxworks.h \
  1327   1459      $(TOP)\src\whereInt.h
  1328   1460   
................................................................................
  1343   1475     $(TOP)\ext\fts3\fts3_tokenizer.h
  1344   1476   EXTHDR = $(EXTHDR) \
  1345   1477     $(TOP)\ext\rtree\rtree.h
  1346   1478   EXTHDR = $(EXTHDR) \
  1347   1479     $(TOP)\ext\icu\sqliteicu.h
  1348   1480   EXTHDR = $(EXTHDR) \
  1349   1481     $(TOP)\ext\rtree\sqlite3rtree.h
         1482  +EXTHDR = $(EXTHDR) \
         1483  +  $(TOP)\ext\session\sqlite3session.h
  1350   1484   
  1351   1485   # executables needed for testing
  1352   1486   #
  1353   1487   TESTPROGS = \
  1354   1488     testfixture.exe \
  1355   1489     $(SQLITE3EXE) \
  1356   1490     sqlite3_analyzer.exe \
  1357         -  sqldiff.exe
         1491  +  sqldiff.exe \
         1492  +  dbhash.exe
  1358   1493   
  1359   1494   # Databases containing fuzzer test cases
  1360   1495   #
  1361   1496   FUZZDATA = \
  1362   1497     $(TOP)\test\fuzzdata1.db \
  1363   1498     $(TOP)\test\fuzzdata2.db \
  1364   1499     $(TOP)\test\fuzzdata3.db \
  1365         -  $(TOP)\test\fuzzdata4.db
         1500  +  $(TOP)\test\fuzzdata4.db \
         1501  +  $(TOP)\test\fuzzdata5.db
  1366   1502   # <</mark>>
  1367   1503   
  1368   1504   # Additional compiler options for the shell.  These are only effective
  1369   1505   # when the shell is not being dynamically linked.
  1370   1506   #
  1371   1507   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
  1372   1508   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS
................................................................................
  1373   1509   !ENDIF
  1374   1510   
  1375   1511   # <<mark>>
  1376   1512   # Extra compiler options for various test tools.
  1377   1513   #
  1378   1514   MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5
  1379   1515   FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
  1380         -FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
         1516  +FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000
         1517  +FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c
         1518  +OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
         1519  +DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
         1520  +KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
         1521  +DBSELFTEST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
         1522  +ST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0
  1381   1523   
  1382   1524   # Standard options to testfixture.
  1383   1525   #
  1384   1526   TESTOPTS = --verbose=file --output=test-out.txt
  1385   1527   
  1386   1528   # Extra targets for the "all" target that require Tcl.
  1387   1529   #
................................................................................
  1416   1558   $(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
  1417   1559   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
  1418   1560   
  1419   1561   # <<block2>>
  1420   1562   sqlite3.def:	libsqlite3.lib
  1421   1563   	echo EXPORTS > sqlite3.def
  1422   1564   	dumpbin /all libsqlite3.lib \
  1423         -		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_[^@]*)(?:@\d+)?$$" \1 \
         1565  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset)?_[^@]*)(?:@\d+)?$$" \1 \
  1424   1566   		| sort >> sqlite3.def
  1425   1567   # <</block2>>
  1426   1568   
  1427   1569   $(SQLITE3EXE):	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
  1428   1570   	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \
  1429   1571   		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
  1430   1572   
  1431   1573   # <<mark>>
  1432   1574   sqldiff.exe:	$(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
  1433   1575   	$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1434   1576   
         1577  +dbhash.exe:	$(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H)
         1578  +	$(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         1579  +
         1580  +scrub.exe:	$(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H)
         1581  +	$(LTLINK) $(NO_WARN) $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         1582  +
  1435   1583   srcck1.exe:	$(TOP)\tool\srcck1.c
  1436   1584   	$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c
  1437   1585   
  1438   1586   sourcetest:	srcck1.exe sqlite3.c
  1439   1587   	srcck1.exe sqlite3.c
  1440   1588   
  1441   1589   fuzzershell.exe:	$(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H)
  1442   1590   	$(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1443   1591   
  1444         -fuzzcheck.exe:	$(TOP)\test\fuzzcheck.c $(SQLITE3C) $(SQLITE3H)
  1445         -	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         1592  +dbfuzz.exe:	$(TOP)\test\dbfuzz.c $(SQLITE3C) $(SQLITE3H)
         1593  +	$(LTLINK) $(NO_WARN) $(DBFUZZ_COMPILE_OPTS) $(TOP)\test\dbfuzz.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         1594  +
         1595  +fuzzcheck.exe:	$(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H)
         1596  +	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         1597  +
         1598  +ossshell.exe:	$(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H)
         1599  +	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1446   1600   
  1447   1601   mptester.exe:	$(TOP)\mptest\mptest.c $(SQLITE3C) $(SQLITE3H)
  1448   1602   	$(LTLINK) $(NO_WARN) $(MPTESTER_COMPILE_OPTS) $(TOP)\mptest\mptest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1449   1603   
  1450   1604   MPTEST1 = mptester mptest.db $(TOP)\mptest\crash01.test --repeat 20
  1451   1605   MPTEST2 = mptester mptest.db $(TOP)\mptest\multiwrite01.test --repeat 20
  1452   1606   
................................................................................
  1463   1617   
  1464   1618   # This target creates a directory named "tsrc" and fills it with
  1465   1619   # copies of all of the C source code and header files needed to
  1466   1620   # build on the target system.  Some of the C source code and header
  1467   1621   # files are automatically generated.  This target takes care of
  1468   1622   # all that automatic generation.
  1469   1623   #
  1470         -.target_source:	$(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c
         1624  +.target_source:	$(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c $(SQLITE_TCL_DEP)
  1471   1625   	-rmdir /Q/S tsrc 2>NUL
  1472   1626   	-mkdir tsrc
  1473   1627   	for %i in ($(SRC00)) do copy /Y %i tsrc
  1474   1628   	for %i in ($(SRC01)) do copy /Y %i tsrc
  1475   1629   	for %i in ($(SRC02)) do copy /Y %i tsrc
  1476   1630   	for %i in ($(SRC03)) do copy /Y %i tsrc
  1477   1631   	for %i in ($(SRC04)) do copy /Y %i tsrc
................................................................................
  1478   1632   	for %i in ($(SRC05)) do copy /Y %i tsrc
  1479   1633   	for %i in ($(SRC06)) do copy /Y %i tsrc
  1480   1634   	for %i in ($(SRC07)) do copy /Y %i tsrc
  1481   1635   	for %i in ($(SRC08)) do copy /Y %i tsrc
  1482   1636   	for %i in ($(SRC09)) do copy /Y %i tsrc
  1483   1637   	for %i in ($(SRC10)) do copy /Y %i tsrc
  1484   1638   	for %i in ($(SRC11)) do copy /Y %i tsrc
         1639  +	for %i in ($(SRC12)) do copy /Y %i tsrc
  1485   1640   	copy /Y fts5.c tsrc
  1486   1641   	copy /Y fts5.h tsrc
  1487   1642   	del /Q tsrc\sqlite.h.in tsrc\parse.y 2>NUL
  1488   1643   	$(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl $(OPTS) < tsrc\vdbe.c > vdbe.new
  1489   1644   	move vdbe.new tsrc\vdbe.c
  1490   1645   	echo > .target_source
  1491   1646   
  1492         -sqlite3.c:	.target_source sqlite3ext.h $(TOP)\tool\mksqlite3c.tcl
  1493         -	$(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl $(MKSQLITE3C_ARGS)
         1647  +sqlite3.c:	.target_source sqlite3ext.h $(MKSQLITE3C_TOOL)
         1648  +	$(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS)
  1494   1649   	copy tsrc\shell.c .
         1650  +	copy $(TOP)\ext\session\sqlite3session.h .
  1495   1651   
  1496   1652   sqlite3-all.c:	sqlite3.c $(TOP)\tool\split-sqlite3c.tcl
  1497   1653   	$(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl
  1498   1654   # <</mark>>
  1499   1655   
  1500   1656   # Rule to build the amalgamation
  1501   1657   #
................................................................................
  1758   1914   
  1759   1915   wherecode.lo:	$(TOP)\src\wherecode.c $(HDR)
  1760   1916   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\wherecode.c
  1761   1917   
  1762   1918   whereexpr.lo:	$(TOP)\src\whereexpr.c $(HDR)
  1763   1919   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\whereexpr.c
  1764   1920   
  1765         -tclsqlite.lo:	$(TOP)\src\tclsqlite.c $(HDR)
         1921  +tclsqlite.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
  1766   1922   	$(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
  1767   1923   
  1768         -tclsqlite-shell.lo:	$(TOP)\src\tclsqlite.c $(HDR)
         1924  +tclsqlite-shell.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
  1769   1925   	$(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
  1770   1926   
  1771   1927   tclsqlite3.exe:	tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
  1772   1928   	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
  1773   1929   
  1774   1930   # Rules to build opcodes.c and opcodes.h
  1775   1931   #
................................................................................
  1787   1943   	del /Q parse.y parse.h parse.h.temp 2>NUL
  1788   1944   	copy $(TOP)\src\parse.y .
  1789   1945   	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) parse.y
  1790   1946   	move parse.h parse.h.temp
  1791   1947   	$(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h
  1792   1948   
  1793   1949   $(SQLITE3H):	$(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION
  1794         -	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H)
         1950  +	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) $(MKSQLITE3H_ARGS)
  1795   1951   
  1796   1952   sqlite3ext.h:	.target_source
  1797         -	copy tsrc\sqlite3ext.h .
         1953  +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
         1954  +	type tsrc\sqlite3ext.h | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*\)" "(SQLITE_CALLBACK *)" \
         1955  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*" "(SQLITE_APICALL *" > sqlite3ext.h
         1956  +	copy /Y sqlite3ext.h tsrc\sqlite3ext.h
         1957  +!ELSE
         1958  +	copy /Y tsrc\sqlite3ext.h sqlite3ext.h
         1959  +!ENDIF
  1798   1960   
  1799   1961   mkkeywordhash.exe:	$(TOP)\tool\mkkeywordhash.c
  1800   1962   	$(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) \
  1801   1963   		$(TOP)\tool\mkkeywordhash.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS)
  1802   1964   
  1803   1965   keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
  1804   1966   	.\mkkeywordhash.exe > keywordhash.h
................................................................................
  1866   2028   
  1867   2029   fts3_write.lo:	$(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR)
  1868   2030   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_write.c
  1869   2031   
  1870   2032   rtree.lo:	$(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR)
  1871   2033   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c
  1872   2034   
         2035  +sqlite3session.lo:	$(TOP)\ext\session\sqlite3session.c $(HDR) $(EXTHDR)
         2036  +	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\session\sqlite3session.c
         2037  +
  1873   2038   # FTS5 things
  1874   2039   #
  1875   2040   FTS5_SRC = \
  1876   2041      $(TOP)\ext\fts5\fts5.h \
  1877   2042      $(TOP)\ext\fts5\fts5Int.h \
  1878   2043      $(TOP)\ext\fts5\fts5_aux.c \
  1879   2044      $(TOP)\ext\fts5\fts5_buffer.c \
................................................................................
  1920   2085   # hidden when the library is built via the amalgamation).
  1921   2086   #
  1922   2087   TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
  1923   2088   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
  1924   2089   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
  1925   2090   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
  1926   2091   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
         2092  +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
  1927   2093   
  1928   2094   TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
  1929   2095   TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C)
  1930   2096   !IF $(USE_AMALGAMATION)==0
  1931   2097   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0)
  1932   2098   !ELSE
  1933   2099   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1)
  1934   2100   !ENDIF
  1935   2101   
  1936         -testfixture.exe:	$(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR)
         2102  +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
         2103  +sqlite_tclDecls.h:
         2104  +	echo #ifndef SQLITE_TCLAPI > $(SQLITETCLDECLSH)
         2105  +	echo #  define SQLITE_TCLAPI >> $(SQLITETCLDECLSH)
         2106  +	echo #endif >> $(SQLITETCLDECLSH)
         2107  +	type "$(TCLINCDIR)\tclDecls.h" \
         2108  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "^(EXTERN(?: CONST\d+?)?\s+?[^\(]*?\s+?)Tcl_" "\1 SQLITE_TCLAPI Tcl_" \
         2109  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "^(EXTERN\s+?(?:void|VOID)\s+?)TclFreeObj" "\1 SQLITE_TCLAPI TclFreeObj" \
         2110  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*tcl_" "(SQLITE_TCLAPI *tcl_" \
         2111  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*tclFreeObj" "(SQLITE_TCLAPI *tclFreeObj" \
         2112  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*" "(SQLITE_TCLAPI *" >> $(SQLITETCLDECLSH)
         2113  +
         2114  +sqlite_tcl.h:
         2115  +	type "$(TCLINCDIR)\tcl.h" | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact tclDecls.h sqlite_tclDecls.h \
         2116  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "typedef (.*?)\(Tcl_" "typedef \1 (SQLITE_TCLAPI Tcl_" \
         2117  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "void (*freeProc)" "void (SQLITE_TCLAPI *freeProc)" \
         2118  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*findProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *findProc)" \
         2119  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*createProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *createProc)" >> $(SQLITETCLH)
         2120  +!ENDIF
         2121  +
         2122  +testfixture.exe:	$(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP)
  1937   2123   	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \
  1938   2124   		-DBUILD_sqlite -I$(TCLINCDIR) \
  1939   2125   		$(TESTFIXTURE_SRC) \
  1940   2126   		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
  1941   2127   
  1942   2128   extensiontest:	testfixture.exe testloadext.dll
  1943   2129   	@set PATH=$(LIBTCLPATH);$(PATH)
................................................................................
  1978   2164   	@set PATH=$(LIBTCLPATH);$(PATH)
  1979   2165   	.\testfixture.exe $(TOP)\test\veryquick.test $(TESTOPTS)
  1980   2166   
  1981   2167   smoketest:	$(TESTPROGS)
  1982   2168   	@set PATH=$(LIBTCLPATH);$(PATH)
  1983   2169   	.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)
  1984   2170   
  1985         -sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
         2171  +sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(SQLITE_TCL_DEP)
  1986   2172   	echo #define TCLSH 2 > $@
  1987   2173   	echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@
  1988   2174   	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@
  1989   2175   	echo static const char *tclsh_main_loop(void){ >> $@
  1990   2176   	echo static const char *zMainloop = >> $@
  1991   2177   	$(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\spaceanal.tcl >> $@
  1992   2178   	echo ; return zMainloop; } >> $@
  1993   2179   
  1994   2180   sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
  1995   2181   	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
  1996   2182   		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
  1997   2183   
         2184  +dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H)
         2185  +	$(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \
         2186  +		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS)
         2187  +
  1998   2188   testloadext.lo:	$(TOP)\src\test_loadext.c
  1999   2189   	$(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c
  2000   2190   
  2001   2191   testloadext.dll:	testloadext.lo
  2002   2192   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo
  2003   2193   
  2004   2194   showdb.exe:	$(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H)
  2005         -	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2195  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2006   2196   		$(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2007   2197   
  2008   2198   showstat4.exe:	$(TOP)\tool\showstat4.c $(SQLITE3C) $(SQLITE3H)
  2009         -	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2199  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2010   2200   		$(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2011   2201   
  2012   2202   showjournal.exe:	$(TOP)\tool\showjournal.c $(SQLITE3C) $(SQLITE3H)
  2013         -	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2203  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2014   2204   		$(TOP)\tool\showjournal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2015   2205   
  2016   2206   showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
  2017         -	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2207  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2018   2208   		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2019   2209   
         2210  +changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
         2211  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
         2212  +		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
         2213  +		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         2214  +
  2020   2215   fts3view.exe:	$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H)
  2021         -	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2216  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2022   2217   		$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2023   2218   
  2024   2219   rollback-test.exe:	$(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H)
  2025         -	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2220  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2026   2221   		$(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2027   2222   
  2028   2223   LogEst.exe:	$(TOP)\tool\logest.c $(SQLITE3H)
  2029         -	$(LTLINK) $(NO_WARN) -Fe$@ $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS)
         2224  +	$(LTLINK) $(NO_WARN) $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS)
  2030   2225   
  2031   2226   wordcount.exe:	$(TOP)\test\wordcount.c $(SQLITE3C) $(SQLITE3H)
  2032         -	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2227  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2033   2228   		$(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2034   2229   
  2035   2230   speedtest1.exe:	$(TOP)\test\speedtest1.c $(SQLITE3C) $(SQLITE3H)
  2036         -	$(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2231  +	$(LTLINK) $(NO_WARN) $(ST_COMPILE_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION \
  2037   2232   		$(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2038   2233   
         2234  +kvtest.exe:	$(TOP)\test\kvtest.c $(SQLITE3C) $(SQLITE3H)
         2235  +	$(LTLINK) $(NO_WARN) $(KV_COMPILE_OPTS) \
         2236  +		$(TOP)\test\kvtest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         2237  +
         2238  +dbselftest.exe:	$(TOP)\test\dbselftest.c $(SQLITE3C) $(SQLITE3H)
         2239  +	$(LTLINK) $(NO_WARN) $(DBSELFTEST_COMPILE_OPTS) $(TOP)\test\dbselftest.c $(SQLITE3C)
         2240  +
  2039   2241   rbu.exe:	$(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H)
  2040         -	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ \
         2242  +	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU \
  2041   2243   		$(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2042   2244   
  2043   2245   moreclean:	clean
  2044   2246   	del /Q $(SQLITE3C) $(SQLITE3H) 2>NUL
  2045   2247   # <</mark>>
  2046   2248   
  2047   2249   clean:
................................................................................
  2052   2254   	del /Q sqlite3.c sqlite3.h 2>NUL
  2053   2255   	del /Q opcodes.c opcodes.h 2>NUL
  2054   2256   	del /Q lemon.* lempar.c parse.* 2>NUL
  2055   2257   	del /Q mkkeywordhash.* keywordhash.h 2>NUL
  2056   2258   	del /Q notasharedlib.* 2>NUL
  2057   2259   	-rmdir /Q/S .deps 2>NUL
  2058   2260   	-rmdir /Q/S .libs 2>NUL
  2059         -	-rmdir /Q/S quota2a 2>NUL
  2060         -	-rmdir /Q/S quota2b 2>NUL
  2061         -	-rmdir /Q/S quota2c 2>NUL
  2062   2261   	-rmdir /Q/S tsrc 2>NUL
  2063   2262   	del /Q .target_source 2>NUL
  2064         -	del /Q tclsqlite3.exe 2>NUL
         2263  +	del /Q tclsqlite3.exe $(SQLITETCLH) $(SQLITETCLDECLSH) 2>NUL
  2065   2264   	del /Q testloadext.dll 2>NUL
  2066   2265   	del /Q testfixture.exe test.db 2>NUL
  2067         -	del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL
         2266  +	del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe dbdump.exe 2>NUL
         2267  +	del /Q changeset.exe 2>NUL
  2068   2268   	del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL
  2069   2269   	del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL
  2070   2270   	del /Q sqlite3.c sqlite3-*.c 2>NUL
  2071   2271   	del /Q sqlite3rc.h 2>NUL
  2072         -	del /Q shell.c sqlite3ext.h 2>NUL
         2272  +	del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL
  2073   2273   	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
  2074   2274   	del /Q sqlite-*-output.vsix 2>NUL
  2075         -	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL
         2275  +	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL
  2076   2276   	del /Q fts5.* fts5parse.* 2>NUL
  2077   2277   # <</mark>>

Changes to README.md.

     4      4   engine.  Some test scripts are also include.  However, many other test scripts
     5      5   and most of the documentation are managed separately.
     6      6   
     7      7   If you are reading this on a Git mirror someplace, you are doing it wrong.
     8      8   The [official repository](https://www.sqlite.org/src/) is better.  Go there
     9      9   now.
    10     10   
           11  +## Obtaining The Code
           12  +
           13  +SQLite sources are managed using the
           14  +[Fossil](https://www.fossil-scm.org/), a distributed version control system
           15  +that was specifically designed to support SQLite development.
           16  +If you do not want to use Fossil, you can download tarballs or ZIP
           17  +archives as follows:
           18  +
           19  +  *  Lastest trunk check-in:
           20  +     <https://www.sqlite.org/src/tarball/sqlite.tar.gz> or
           21  +     <https://www.sqlite.org/src/zip/sqlite.zip>.
           22  +
           23  +  *  Latest release:
           24  +     <https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release> or
           25  +     <https://www.sqlite.org/src/zip/sqlite.zip?r=release>.
           26  +
           27  +  *  For other check-ins, substitute an appropriate branch name or
           28  +     tag or hash prefix for "release" in the URLs of the previous
           29  +     bullet.  Or browse the [timeline](https://www.sqlite.org/src/timeline)
           30  +     to locate the check-in desired, click on its information page link,
           31  +     then click on the "Tarball" or "ZIP Archive" links on the information
           32  +     page.
           33  +
           34  +If you do want to use Fossil to check out the source tree, 
           35  +first install Fossil version 2.0 or later.
           36  +(Source tarballs and precompiled binaries available
           37  +[here](https://www.fossil-scm.org/fossil/uv/download.html).)
           38  +Then run commands like this:
           39  +
           40  +        mkdir ~/sqlite
           41  +        cd ~/sqlite
           42  +        fossil clone https://www.sqlite.org/src sqlite.fossil
           43  +        fossil open sqlite.fossil
           44  +    
           45  +After setting up a repository using the steps above, you can always
           46  +update to the lastest version using:
           47  +
           48  +        fossil update trunk   ;# latest trunk check-in
           49  +        fossil update release ;# latest official release
           50  +
           51  +Or type "fossil ui" to get a web-based user interface.
           52  +
    11     53   ## Compiling
    12     54   
    13     55   First create a directory in which to place
    14     56   the build products.  It is recommended, but not required, that the
    15     57   build directory be separate from the source directory.  Cd into the
    16     58   build directory and then from the build directory run the configure
    17     59   script found at the root of the source tree.  Then run "make".
    18     60   
    19     61   For example:
    20     62   
    21         -    tar xzf sqlite.tar.gz    ;#  Unpack the source tree into "sqlite"
    22         -    mkdir bld                ;#  Build will occur in a sibling directory
    23         -    cd bld                   ;#  Change to the build directory
    24         -    ../sqlite/configure      ;#  Run the configure script
    25         -    make                     ;#  Run the makefile.
    26         -    make sqlite3.c           ;#  Build the "amalgamation" source file
    27         -    make test                ;#  Run some tests (requires Tcl)
           63  +        tar xzf sqlite.tar.gz    ;#  Unpack the source tree into "sqlite"
           64  +        mkdir bld                ;#  Build will occur in a sibling directory
           65  +        cd bld                   ;#  Change to the build directory
           66  +        ../sqlite/configure      ;#  Run the configure script
           67  +        make                     ;#  Run the makefile.
           68  +        make sqlite3.c           ;#  Build the "amalgamation" source file
           69  +        make test                ;#  Run some tests (requires Tcl)
    28     70   
    29     71   See the makefile for additional targets.
    30     72   
    31     73   The configure script uses autoconf 2.61 and libtool.  If the configure
    32     74   script does not work out for you, there is a generic makefile named
    33     75   "Makefile.linux-gcc" in the top directory of the source tree that you
    34     76   can copy and edit to suit your needs.  Comments on the generic makefile
................................................................................
    39     81   On Windows, all applicable build products can be compiled with MSVC.
    40     82   First open the command prompt window associated with the desired compiler
    41     83   version (e.g. "Developer Command Prompt for VS2013").  Next, use NMAKE
    42     84   with the provided "Makefile.msc" to build one of the supported targets.
    43     85   
    44     86   For example:
    45     87   
    46         -    mkdir bld
    47         -    cd bld
    48         -    nmake /f Makefile.msc TOP=..\sqlite
    49         -    nmake /f Makefile.msc sqlite3.c TOP=..\sqlite
    50         -    nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite
    51         -    nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite
    52         -    nmake /f Makefile.msc test TOP=..\sqlite
           88  +        mkdir bld
           89  +        cd bld
           90  +        nmake /f Makefile.msc TOP=..\sqlite
           91  +        nmake /f Makefile.msc sqlite3.c TOP=..\sqlite
           92  +        nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite
           93  +        nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite
           94  +        nmake /f Makefile.msc test TOP=..\sqlite
    53     95   
    54     96   There are several build options that can be set via the NMAKE command
    55     97   line.  For example, to build for WinRT, simply add "FOR_WINRT=1" argument
    56     98   to the "sqlite3.dll" command line above.  When debugging into the SQLite
    57     99   code, adding the "DEBUG=1" argument to one of the above command lines is
    58    100   recommended.
    59    101   

Changes to VERSION.

     1         -3.12.0
            1  +3.19.0

Changes to autoconf/Makefile.am.

     1      1   
     2         -AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
            2  +AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
     3      3   
     4      4   lib_LTLIBRARIES = libsqlite3.la
     5      5   libsqlite3_la_SOURCES = sqlite3.c
     6      6   libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8
     7      7   
     8      8   bin_PROGRAMS = sqlite3
     9      9   sqlite3_SOURCES = shell.c sqlite3.h

Changes to autoconf/Makefile.msc.

    19     19   
    20     20   
    21     21   # Set this non-0 to enable full warnings (-W4, etc) when compiling.
    22     22   #
    23     23   !IFNDEF USE_FULLWARN
    24     24   USE_FULLWARN = 0
    25     25   !ENDIF
           26  +
           27  +# Set this non-0 to enable treating warnings as errors (-WX, etc) when
           28  +# compiling.
           29  +#
           30  +!IFNDEF USE_FATAL_WARN
           31  +USE_FATAL_WARN = 0
           32  +!ENDIF
           33  +
           34  +# Set this non-0 to enable full runtime error checks (-RTC1, etc).  This
           35  +# has no effect if (any) optimizations are enabled.
           36  +#
           37  +!IFNDEF USE_RUNTIME_CHECKS
           38  +USE_RUNTIME_CHECKS = 0
           39  +!ENDIF
           40  +
           41  +# Set this non-0 to create a SQLite amalgamation file that excludes the
           42  +# various built-in extensions.
           43  +#
           44  +!IFNDEF MINIMAL_AMALGAMATION
           45  +MINIMAL_AMALGAMATION = 0
           46  +!ENDIF
    26     47   
    27     48   # Set this non-0 to use "stdcall" calling convention for the core library
    28     49   # and shell executable.
    29     50   #
    30     51   !IFNDEF USE_STDCALL
    31     52   USE_STDCALL = 0
    32     53   !ENDIF
................................................................................
   178    199   # Enable use of available compiler optimizations?  Normally, this should be
   179    200   # non-zero.  Setting this to zero, thus disabling all compiler optimizations,
   180    201   # can be useful for testing.
   181    202   #
   182    203   !IFNDEF OPTIMIZATIONS
   183    204   OPTIMIZATIONS = 2
   184    205   !ENDIF
          206  +
          207  +# Set this to non-0 to enable support for the session extension.
          208  +#
          209  +!IFNDEF SESSION
          210  +SESSION = 0
          211  +!ENDIF
   185    212   
   186    213   # Set the source code file to be used by executables and libraries when
   187    214   # they need the amalgamation.
   188    215   #
   189    216   !IFNDEF SQLITE3C
   190    217   !IF $(SPLIT_AMALGAMATION)!=0
   191    218   SQLITE3C = sqlite3-all.c
................................................................................
   237    264   !IFNDEF SQLITE3EXEPDB
   238    265   !IF $(FOR_WIN10)!=0
   239    266   SQLITE3EXEPDB =
   240    267   !ELSE
   241    268   SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
   242    269   !ENDIF
   243    270   !ENDIF
          271  +
   244    272   
   245    273   # These are the "standard" SQLite compilation options used when compiling for
   246    274   # the Windows platform.
   247    275   #
   248    276   !IFNDEF OPT_FEATURE_FLAGS
          277  +!IF $(MINIMAL_AMALGAMATION)==0
   249    278   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
   250    279   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
          280  +!ENDIF
   251    281   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
   252    282   !ENDIF
          283  +
          284  +# Should the session extension be enabled?  If so, add compilation options
          285  +# to enable it.
          286  +#
          287  +!IF $(SESSION)!=0
          288  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1
          289  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1
          290  +!ENDIF
   253    291   
   254    292   # These are the "extended" SQLite compilation options used when compiling for
   255    293   # the Windows 10 platform.
   256    294   #
   257    295   !IFNDEF EXT_FEATURE_FLAGS
   258    296   !IF $(FOR_WIN10)!=0
   259    297   EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS4=1
................................................................................
   418    456   # same unless your are cross-compiling.)
   419    457   #
   420    458   !IF $(USE_FULLWARN)!=0
   421    459   TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
   422    460   !ELSE
   423    461   TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
   424    462   !ENDIF
          463  +
          464  +# Check if warnings should be treated as errors when compiling.
          465  +#
          466  +!IF $(USE_FATAL_WARN)!=0
          467  +TCC = $(TCC) -WX
          468  +!ENDIF
   425    469   
   426    470   TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -fp:precise
   427    471   RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS)
   428    472   
   429    473   # Check if we want to use the "stdcall" calling convention when compiling.
   430    474   # This is not supported by the compilers for non-x86 platforms.  It should
   431    475   # also be noted here that building any target with these "stdcall" options
   432    476   # will most likely fail if the Tcl library is also required.  This is due
   433    477   # to how the Tcl library functions are declared and exported (i.e. without
   434    478   # an explicit calling convention, which results in "cdecl").
   435    479   #
   436    480   !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
   437    481   !IF "$(PLATFORM)"=="x86"
   438         -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
   439         -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
          482  +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
          483  +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
   440    484   !ELSE
   441    485   !IFNDEF PLATFORM
   442         -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
   443         -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
          486  +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
          487  +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
   444    488   !ELSE
   445    489   CORE_CCONV_OPTS =
   446    490   SHELL_CCONV_OPTS =
   447    491   !ENDIF
   448    492   !ENDIF
   449    493   !ELSE
   450    494   CORE_CCONV_OPTS =
................................................................................
   597    641   TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1
   598    642   RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1
   599    643   !ENDIF
   600    644   
   601    645   !IF $(DEBUG)>2
   602    646   TCC = $(TCC) -DSQLITE_DEBUG=1
   603    647   RCC = $(RCC) -DSQLITE_DEBUG=1
          648  +!IF $(DYNAMIC_SHELL)==0
          649  +TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
          650  +RCC = $(RCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
          651  +!ENDIF
   604    652   !ENDIF
   605    653   
   606    654   !IF $(DEBUG)>4 || $(OSTRACE)!=0
   607    655   TCC = $(TCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1
   608    656   RCC = $(RCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1
   609    657   !ENDIF
   610    658   
................................................................................
   722    770   
   723    771   # If optimizations are enabled or disabled (either implicitly or
   724    772   # explicitly), add the necessary flags.
   725    773   #
   726    774   !IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0
   727    775   TCC = $(TCC) -Od
   728    776   BCC = $(BCC) -Od
          777  +!IF $(USE_RUNTIME_CHECKS)!=0
          778  +TCC = $(TCC) -RTC1
          779  +BCC = $(BCC) -RTC1
          780  +!ENDIF
   729    781   !ELSEIF $(OPTIMIZATIONS)>=3
   730    782   TCC = $(TCC) -Ox
   731    783   BCC = $(BCC) -Ox
   732    784   !ELSEIF $(OPTIMIZATIONS)==2
   733    785   TCC = $(TCC) -O2
   734    786   BCC = $(BCC) -O2
   735    787   !ELSEIF $(OPTIMIZATIONS)==1
................................................................................
   898    950   
   899    951   Replace.exe:
   900    952   	$(CSC) /target:exe $(TOP)\Replace.cs
   901    953   
   902    954   sqlite3.def:	Replace.exe $(LIBOBJ)
   903    955   	echo EXPORTS > sqlite3.def
   904    956   	dumpbin /all $(LIBOBJ) \
   905         -		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
          957  +		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
   906    958   		| sort >> sqlite3.def
   907    959   
   908    960   $(SQLITE3EXE):	$(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
   909    961   	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \
   910    962   		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
   911    963   
   912    964   

Changes to autoconf/configure.ac.

    26     26   # Check for library functions that SQLite can optionally use.
    27     27   AC_CHECK_FUNCS([fdatasync usleep fullfsync localtime_r gmtime_r])
    28     28   AC_FUNC_STRERROR_R
    29     29   
    30     30   AC_CONFIG_FILES([Makefile sqlite3.pc])
    31     31   AC_SUBST(BUILD_CFLAGS)
    32     32   
    33         -#-----------------------------------------------------------------------
           33  +#-------------------------------------------------------------------------
           34  +# Two options to enable readline compatible libraries: 
           35  +#
    34     36   #   --enable-editline
    35     37   #   --enable-readline
    36     38   #
    37         -AC_ARG_ENABLE(editline, [AS_HELP_STRING(
    38         -  [--enable-editline], 
    39         -  [use BSD libedit])], 
    40         -  [], [enable_editline=yes])
    41         -AC_ARG_ENABLE(readline, [AS_HELP_STRING(
    42         -  [--enable-readline], 
    43         -  [use readline])], 
    44         -  [], [enable_readline=no])
    45         -if test x"$enable_editline" != xno ; then
    46         -  sLIBS=$LIBS
    47         -  LIBS=""
    48         -  AC_SEARCH_LIBS([readline],[edit],[enable_readline=no],[enable_editline=no])
    49         -  READLINE_LIBS=$LIBS
    50         -  if test x"$LIBS" != "x"; then
    51         -     AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
    52         -  else
    53         -    unset ac_cv_search_readline
    54         -  fi
    55         -  LIBS=$sLIBS
    56         -fi
    57         -if test x"$enable_readline" != xno ; then
    58         -  sLIBS=$LIBS
    59         -  LIBS=""
    60         -  AC_SEARCH_LIBS(tgetent, curses ncurses ncursesw, [], [])
    61         -  AC_SEARCH_LIBS(readline, readline, [], [enable_readline=no])
    62         -  AC_CHECK_FUNCS(readline, [], [])
    63         -  READLINE_LIBS=$LIBS
    64         -  LIBS=$sLIBS
    65         -fi
           39  +# Both are enabled by default. If, after command line processing both are
           40  +# still enabled, the script searches for editline first and automatically
           41  +# disables readline if it is found. So, to use readline explicitly, the
           42  +# user must pass "--disable-editline". To disable command line editing
           43  +# support altogether, "--disable-editline --disable-readline".
           44  +#
           45  +# When searching for either library, check for headers before libraries 
           46  +# as some distros supply packages that contain libraries but not header
           47  +# files, which come as a separate development package.
           48  +#
           49  +AC_ARG_ENABLE(editline, [AS_HELP_STRING([--enable-editline],[use BSD libedit])])
           50  +AC_ARG_ENABLE(readline, [AS_HELP_STRING([--enable-readline],[use readline])])
           51  +
           52  +AS_IF([ test x"$enable_editline" != xno ],[
           53  +  AC_CHECK_HEADERS([editline/readline.h],[
           54  +    sLIBS=$LIBS
           55  +    LIBS=""
           56  +    AC_SEARCH_LIBS([readline],[edit],[
           57  +      AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
           58  +      READLINE_LIBS="$LIBS -ltinfo"
           59  +      enable_readline=no
           60  +    ],[],[-ltinfo])
           61  +    AS_UNSET(ac_cv_search_readline)
           62  +    LIBS=$sLIBS
           63  +  ])
           64  +])
           65  +
           66  +AS_IF([ test x"$enable_readline" != xno ],[
           67  +  AC_CHECK_HEADERS([readline/readline.h],[
           68  +    sLIBS=$LIBS
           69  +    LIBS=""
           70  +    AC_SEARCH_LIBS(tgetent, termcap curses ncurses ncursesw, [], [])
           71  +    AC_SEARCH_LIBS(readline,[readline edit], [
           72  +      AC_DEFINE([HAVE_READLINE],1,Define to use readline or wrapper)
           73  +      READLINE_LIBS=$LIBS
           74  +    ])
           75  +    LIBS=$sLIBS
           76  +  ])
           77  +])
           78  +
    66     79   AC_SUBST(READLINE_LIBS)
    67     80   #-----------------------------------------------------------------------
    68     81   
    69     82   #-----------------------------------------------------------------------
    70     83   #   --enable-threadsafe
    71     84   #
    72     85   AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING(
................................................................................
    99    112   
   100    113   #-----------------------------------------------------------------------
   101    114   #   --enable-fts5
   102    115   #
   103    116   AC_ARG_ENABLE(fts5, [AS_HELP_STRING(
   104    117     [--enable-fts5], [include fts5 support [default=no]])], 
   105    118     [], [enable_fts5=no])
   106         -if test x"$enable_fts5" == "xyes"; then
          119  +if test x"$enable_fts5" = "xyes"; then
   107    120     AC_SEARCH_LIBS(log, m)
   108    121     FTS5_FLAGS=-DSQLITE_ENABLE_FTS5
   109    122   fi
   110    123   AC_SUBST(FTS5_FLAGS)
   111    124   #-----------------------------------------------------------------------
   112    125   
   113    126   #-----------------------------------------------------------------------
   114    127   #   --enable-json1
   115    128   #
   116    129   AC_ARG_ENABLE(json1, [AS_HELP_STRING(
   117    130     [--enable-json1], [include json1 support [default=no]])], 
   118    131     [], [enable_json1=no])
   119         -if test x"$enable_json1" == "xyes"; then
          132  +if test x"$enable_json1" = "xyes"; then
   120    133     JSON1_FLAGS=-DSQLITE_ENABLE_JSON1
   121    134   fi
   122    135   AC_SUBST(JSON1_FLAGS)
   123    136   #-----------------------------------------------------------------------
          137  +
          138  +#-----------------------------------------------------------------------
          139  +#   --enable-session
          140  +#
          141  +AC_ARG_ENABLE(session, [AS_HELP_STRING(
          142  +  [--enable-session], [enable the session extension [default=no]])], 
          143  +  [], [enable_session=no])
          144  +if test x"$enable_session" = "xyes"; then
          145  +  SESSION_FLAGS="-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
          146  +fi
          147  +AC_SUBST(SESSION_FLAGS)
          148  +#-----------------------------------------------------------------------
   124    149   
   125    150   #-----------------------------------------------------------------------
   126    151   #   --enable-static-shell
   127    152   #
   128    153   AC_ARG_ENABLE(static-shell, [AS_HELP_STRING(
   129    154     [--enable-static-shell], 
   130    155     [statically link libsqlite3 into shell tool [default=yes]])], 
   131    156     [], [enable_static_shell=yes])
   132         -if test x"$enable_static_shell" == "xyes"; then
          157  +if test x"$enable_static_shell" = "xyes"; then
   133    158     EXTRA_SHELL_OBJ=sqlite3-sqlite3.$OBJEXT
   134    159   else
   135    160     EXTRA_SHELL_OBJ=libsqlite3.la
   136    161   fi
   137    162   AC_SUBST(EXTRA_SHELL_OBJ)
   138    163   #-----------------------------------------------------------------------
   139    164   

Changes to autoconf/tea/configure.ac.

    74     74   TEA_ADD_SOURCES([tclsqlite3.c])
    75     75   TEA_ADD_HEADERS([])
    76     76   TEA_ADD_INCLUDES([-I\"`\${CYGPATH} \${srcdir}/generic`\"])
    77     77   TEA_ADD_LIBS([])
    78     78   TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS3=1])
    79     79   TEA_ADD_CFLAGS([-DSQLITE_3_SUFFIX_ONLY=1])
    80     80   TEA_ADD_CFLAGS([-DSQLITE_ENABLE_RTREE=1])
    81         -TEA_ADD_CFLAGS([-DSQLITE_OMIT_DEPRECATED=1])
    82     81   TEA_ADD_STUB_SOURCES([])
    83     82   TEA_ADD_TCL_SOURCES([])
    84     83   
    85     84   #--------------------------------------------------------------------
    86     85   # The --with-system-sqlite causes the TCL bindings to SQLite to use
    87     86   # the system shared library for SQLite rather than statically linking
    88     87   # against its own private copy.  This is dangerous and leads to

Changes to configure.

     1      1   #! /bin/sh
     2      2   # Guess values for system-dependent variables and create Makefiles.
     3         -# Generated by GNU Autoconf 2.69 for sqlite 3.12.0.
            3  +# Generated by GNU Autoconf 2.69 for sqlite 3.19.0.
     4      4   #
     5      5   #
     6      6   # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
     7      7   #
     8      8   #
     9      9   # This configure script is free software; the Free Software Foundation
    10     10   # gives unlimited permission to copy, distribute and modify it.
................................................................................
   722    722   subdirs=
   723    723   MFLAGS=
   724    724   MAKEFLAGS=
   725    725   
   726    726   # Identity of this package.
   727    727   PACKAGE_NAME='sqlite'
   728    728   PACKAGE_TARNAME='sqlite'
   729         -PACKAGE_VERSION='3.12.0'
   730         -PACKAGE_STRING='sqlite 3.12.0'
          729  +PACKAGE_VERSION='3.19.0'
          730  +PACKAGE_STRING='sqlite 3.19.0'
   731    731   PACKAGE_BUGREPORT=''
   732    732   PACKAGE_URL=''
   733    733   
   734    734   # Factoring default headers for most tests.
   735    735   ac_includes_default="\
   736    736   #include <stdio.h>
   737    737   #ifdef HAVE_SYS_TYPES_H
................................................................................
   899    899   enable_editline
   900    900   enable_readline
   901    901   with_readline_lib
   902    902   with_readline_inc
   903    903   enable_debug
   904    904   enable_amalgamation
   905    905   enable_load_extension
          906  +enable_memsys5
          907  +enable_memsys3
   906    908   enable_fts3
   907    909   enable_fts4
   908    910   enable_fts5
   909    911   enable_json1
   910    912   enable_rtree
          913  +enable_session
   911    914   enable_gcov
   912    915   '
   913    916         ac_precious_vars='build_alias
   914    917   host_alias
   915    918   target_alias
   916    919   CC
   917    920   CFLAGS
................................................................................
  1456   1459   #
  1457   1460   # Report the --help message.
  1458   1461   #
  1459   1462   if test "$ac_init_help" = "long"; then
  1460   1463     # Omit some internal or obsolete options to make the list less imposing.
  1461   1464     # This message is too long to be a string in the A/UX 3.1 sh.
  1462   1465     cat <<_ACEOF
  1463         -\`configure' configures sqlite 3.12.0 to adapt to many kinds of systems.
         1466  +\`configure' configures sqlite 3.19.0 to adapt to many kinds of systems.
  1464   1467   
  1465   1468   Usage: $0 [OPTION]... [VAR=VALUE]...
  1466   1469   
  1467   1470   To assign environment variables (e.g., CC, CFLAGS...), specify them as
  1468   1471   VAR=VALUE.  See below for descriptions of some of the useful variables.
  1469   1472   
  1470   1473   Defaults for the options are specified in brackets.
................................................................................
  1521   1524     --build=BUILD     configure for building on BUILD [guessed]
  1522   1525     --host=HOST       cross-compile to build programs to run on HOST [BUILD]
  1523   1526   _ACEOF
  1524   1527   fi
  1525   1528   
  1526   1529   if test -n "$ac_init_help"; then
  1527   1530     case $ac_init_help in
  1528         -     short | recursive ) echo "Configuration of sqlite 3.12.0:";;
         1531  +     short | recursive ) echo "Configuration of sqlite 3.19.0:";;
  1529   1532      esac
  1530   1533     cat <<\_ACEOF
  1531   1534   
  1532   1535   Optional Features:
  1533   1536     --disable-option-checking  ignore unrecognized --enable/--with options
  1534   1537     --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  1535   1538     --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
  1547   1550     --enable-editline       enable BSD editline support
  1548   1551     --disable-readline      disable readline support
  1549   1552     --enable-debug          enable debugging & verbose explain
  1550   1553     --disable-amalgamation  Disable the amalgamation and instead build all files
  1551   1554                             separately
  1552   1555     --disable-load-extension
  1553   1556                             Disable loading of external extensions
         1557  +  --enable-memsys5        Enable MEMSYS5
         1558  +  --enable-memsys3        Enable MEMSYS3
  1554   1559     --enable-fts3           Enable the FTS3 extension
  1555   1560     --enable-fts4           Enable the FTS4 extension
  1556   1561     --enable-fts5           Enable the FTS5 extension
  1557   1562     --enable-json1          Enable the JSON1 extension
  1558   1563     --enable-rtree          Enable the RTREE extension
         1564  +  --enable-session        Enable the SESSION extension
  1559   1565     --enable-gcov           Enable coverage testing using gcov
  1560   1566   
  1561   1567   Optional Packages:
  1562   1568     --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
  1563   1569     --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
  1564   1570     --with-pic              try to use only PIC/non-PIC objects [default=use
  1565   1571                             both]
................................................................................
  1642   1648       cd "$ac_pwd" || { ac_status=$?; break; }
  1643   1649     done
  1644   1650   fi
  1645   1651   
  1646   1652   test -n "$ac_init_help" && exit $ac_status
  1647   1653   if $ac_init_version; then
  1648   1654     cat <<\_ACEOF
  1649         -sqlite configure 3.12.0
         1655  +sqlite configure 3.19.0
  1650   1656   generated by GNU Autoconf 2.69
  1651   1657   
  1652   1658   Copyright (C) 2012 Free Software Foundation, Inc.
  1653   1659   This configure script is free software; the Free Software Foundation
  1654   1660   gives unlimited permission to copy, distribute and modify it.
  1655   1661   _ACEOF
  1656   1662     exit
................................................................................
  2061   2067     eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
  2062   2068   
  2063   2069   } # ac_fn_c_check_header_mongrel
  2064   2070   cat >config.log <<_ACEOF
  2065   2071   This file contains any messages produced by compilers while
  2066   2072   running configure, to aid debugging if configure makes a mistake.
  2067   2073   
  2068         -It was created by sqlite $as_me 3.12.0, which was
         2074  +It was created by sqlite $as_me 3.19.0, which was
  2069   2075   generated by GNU Autoconf 2.69.  Invocation command line was
  2070   2076   
  2071   2077     $ $0 $@
  2072   2078   
  2073   2079   _ACEOF
  2074   2080   exec 5>>config.log
  2075   2081   {
................................................................................
  3919   3925   { $as_echo "$as_me:${as_lineno-$LINENO}: checking the name lister ($NM) interface" >&5
  3920   3926   $as_echo_n "checking the name lister ($NM) interface... " >&6; }
  3921   3927   if ${lt_cv_nm_interface+:} false; then :
  3922   3928     $as_echo_n "(cached) " >&6
  3923   3929   else
  3924   3930     lt_cv_nm_interface="BSD nm"
  3925   3931     echo "int some_variable = 0;" > conftest.$ac_ext
  3926         -  (eval echo "\"\$as_me:3926: $ac_compile\"" >&5)
         3932  +  (eval echo "\"\$as_me:3932: $ac_compile\"" >&5)
  3927   3933     (eval "$ac_compile" 2>conftest.err)
  3928   3934     cat conftest.err >&5
  3929         -  (eval echo "\"\$as_me:3929: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
         3935  +  (eval echo "\"\$as_me:3935: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
  3930   3936     (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
  3931   3937     cat conftest.err >&5
  3932         -  (eval echo "\"\$as_me:3932: output\"" >&5)
         3938  +  (eval echo "\"\$as_me:3938: output\"" >&5)
  3933   3939     cat conftest.out >&5
  3934   3940     if $GREP 'External.*some_variable' conftest.out > /dev/null; then
  3935   3941       lt_cv_nm_interface="MS dumpbin"
  3936   3942     fi
  3937   3943     rm -f conftest*
  3938   3944   fi
  3939   3945   { $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_nm_interface" >&5
................................................................................
  5131   5137   	;;
  5132   5138       esac
  5133   5139     fi
  5134   5140     rm -rf conftest*
  5135   5141     ;;
  5136   5142   *-*-irix6*)
  5137   5143     # Find out which ABI we are using.
  5138         -  echo '#line 5138 "configure"' > conftest.$ac_ext
         5144  +  echo '#line 5144 "configure"' > conftest.$ac_ext
  5139   5145     if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
  5140   5146     (eval $ac_compile) 2>&5
  5141   5147     ac_status=$?
  5142   5148     $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
  5143   5149     test $ac_status = 0; }; then
  5144   5150       if test "$lt_cv_prog_gnu_ld" = yes; then
  5145   5151         case `/usr/bin/file conftest.$ac_objext` in
................................................................................
  6656   6662      # Note that $ac_compile itself does not contain backslashes and begins
  6657   6663      # with a dollar sign (not a hyphen), so the echo should work correctly.
  6658   6664      # The option is referenced via a variable to avoid confusing sed.
  6659   6665      lt_compile=`echo "$ac_compile" | $SED \
  6660   6666      -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
  6661   6667      -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
  6662   6668      -e 's:$: $lt_compiler_flag:'`
  6663         -   (eval echo "\"\$as_me:6663: $lt_compile\"" >&5)
         6669  +   (eval echo "\"\$as_me:6669: $lt_compile\"" >&5)
  6664   6670      (eval "$lt_compile" 2>conftest.err)
  6665   6671      ac_status=$?
  6666   6672      cat conftest.err >&5
  6667         -   echo "$as_me:6667: \$? = $ac_status" >&5
         6673  +   echo "$as_me:6673: \$? = $ac_status" >&5
  6668   6674      if (exit $ac_status) && test -s "$ac_outfile"; then
  6669   6675        # The compiler can only warn and ignore the option if not recognized
  6670   6676        # So say no if there are warnings other than the usual output.
  6671   6677        $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp
  6672   6678        $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2
  6673   6679        if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then
  6674   6680          lt_cv_prog_compiler_rtti_exceptions=yes
................................................................................
  6995   7001      # Note that $ac_compile itself does not contain backslashes and begins
  6996   7002      # with a dollar sign (not a hyphen), so the echo should work correctly.
  6997   7003      # The option is referenced via a variable to avoid confusing sed.
  6998   7004      lt_compile=`echo "$ac_compile" | $SED \
  6999   7005      -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
  7000   7006      -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
  7001   7007      -e 's:$: $lt_compiler_flag:'`
  7002         -   (eval echo "\"\$as_me:7002: $lt_compile\"" >&5)
         7008  +   (eval echo "\"\$as_me:7008: $lt_compile\"" >&5)
  7003   7009      (eval "$lt_compile" 2>conftest.err)
  7004   7010      ac_status=$?
  7005   7011      cat conftest.err >&5
  7006         -   echo "$as_me:7006: \$? = $ac_status" >&5
         7012  +   echo "$as_me:7012: \$? = $ac_status" >&5
  7007   7013      if (exit $ac_status) && test -s "$ac_outfile"; then
  7008   7014        # The compiler can only warn and ignore the option if not recognized
  7009   7015        # So say no if there are warnings other than the usual output.
  7010   7016        $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp
  7011   7017        $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2
  7012   7018        if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then
  7013   7019          lt_cv_prog_compiler_pic_works=yes
................................................................................
  7100   7106      # (2) before a word containing "conftest.", or (3) at the end.
  7101   7107      # Note that $ac_compile itself does not contain backslashes and begins
  7102   7108      # with a dollar sign (not a hyphen), so the echo should work correctly.
  7103   7109      lt_compile=`echo "$ac_compile" | $SED \
  7104   7110      -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
  7105   7111      -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
  7106   7112      -e 's:$: $lt_compiler_flag:'`
  7107         -   (eval echo "\"\$as_me:7107: $lt_compile\"" >&5)
         7113  +   (eval echo "\"\$as_me:7113: $lt_compile\"" >&5)
  7108   7114      (eval "$lt_compile" 2>out/conftest.err)
  7109   7115      ac_status=$?
  7110   7116      cat out/conftest.err >&5
  7111         -   echo "$as_me:7111: \$? = $ac_status" >&5
         7117  +   echo "$as_me:7117: \$? = $ac_status" >&5
  7112   7118      if (exit $ac_status) && test -s out/conftest2.$ac_objext
  7113   7119      then
  7114   7120        # The compiler can only warn and ignore the option if not recognized
  7115   7121        # So say no if there are warnings
  7116   7122        $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp
  7117   7123        $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2
  7118   7124        if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then
................................................................................
  7155   7161      # (2) before a word containing "conftest.", or (3) at the end.
  7156   7162      # Note that $ac_compile itself does not contain backslashes and begins
  7157   7163      # with a dollar sign (not a hyphen), so the echo should work correctly.
  7158   7164      lt_compile=`echo "$ac_compile" | $SED \
  7159   7165      -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
  7160   7166      -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
  7161   7167      -e 's:$: $lt_compiler_flag:'`
  7162         -   (eval echo "\"\$as_me:7162: $lt_compile\"" >&5)
         7168  +   (eval echo "\"\$as_me:7168: $lt_compile\"" >&5)
  7163   7169      (eval "$lt_compile" 2>out/conftest.err)
  7164   7170      ac_status=$?
  7165   7171      cat out/conftest.err >&5
  7166         -   echo "$as_me:7166: \$? = $ac_status" >&5
         7172  +   echo "$as_me:7172: \$? = $ac_status" >&5
  7167   7173      if (exit $ac_status) && test -s out/conftest2.$ac_objext
  7168   7174      then
  7169   7175        # The compiler can only warn and ignore the option if not recognized
  7170   7176        # So say no if there are warnings
  7171   7177        $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp
  7172   7178        $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2
  7173   7179        if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then
................................................................................
  9535   9541   else
  9536   9542     	  if test "$cross_compiling" = yes; then :
  9537   9543     lt_cv_dlopen_self=cross
  9538   9544   else
  9539   9545     lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  9540   9546     lt_status=$lt_dlunknown
  9541   9547     cat > conftest.$ac_ext <<_LT_EOF
  9542         -#line 9542 "configure"
         9548  +#line 9548 "configure"
  9543   9549   #include "confdefs.h"
  9544   9550   
  9545   9551   #if HAVE_DLFCN_H
  9546   9552   #include <dlfcn.h>
  9547   9553   #endif
  9548   9554   
  9549   9555   #include <stdio.h>
................................................................................
  9631   9637   else
  9632   9638     	  if test "$cross_compiling" = yes; then :
  9633   9639     lt_cv_dlopen_self_static=cross
  9634   9640   else
  9635   9641     lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  9636   9642     lt_status=$lt_dlunknown
  9637   9643     cat > conftest.$ac_ext <<_LT_EOF
  9638         -#line 9638 "configure"
         9644  +#line 9644 "configure"
  9639   9645   #include "confdefs.h"
  9640   9646   
  9641   9647   #if HAVE_DLFCN_H
  9642   9648   #include <dlfcn.h>
  9643   9649   #endif
  9644   9650   
  9645   9651   #include <stdio.h>
................................................................................
 10750  10756             if test -f "$i/tclConfig.sh" ; then
 10751  10757               ac_cv_c_tclconfig="$i"
 10752  10758               break
 10753  10759             fi
 10754  10760           done
 10755  10761         fi
 10756  10762       fi
        10763  +
        10764  +    # Recent versions of Xcode on Macs hid the tclConfig.sh file
        10765  +    # in a strange place.
        10766  +    if test x"${ac_cv_c_tclconfig}" = x ; then
        10767  +      if test x"$cross_compiling" = xno; then
        10768  +        for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
        10769  +        do
        10770  +          if test -f "$i/tclConfig.sh" ; then
        10771  +            ac_cv_c_tclconfig="$i"
        10772  +            break
        10773  +          fi
        10774  +        done
        10775  +      fi
        10776  +    fi
 10757  10777   
 10758  10778       # then check for a private Tcl installation
 10759  10779       if test x"${ac_cv_c_tclconfig}" = x ; then
 10760  10780         for i in \
 10761  10781               ../tcl \
 10762  10782               `ls -dr ../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
 10763  10783               `ls -dr ../tcl[8-9].[0-9] 2>/dev/null` \
................................................................................
 11228  11248   if test "${enable_debug+set}" = set; then :
 11229  11249     enableval=$enable_debug; use_debug=$enableval
 11230  11250   else
 11231  11251     use_debug=no
 11232  11252   fi
 11233  11253   
 11234  11254   if test "${use_debug}" = "yes" ; then
 11235         -  TARGET_DEBUG="-DSQLITE_DEBUG=1"
        11255  +  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE"
 11236  11256   else
 11237  11257     TARGET_DEBUG="-DNDEBUG"
 11238  11258   fi
 11239  11259   
 11240  11260   
 11241  11261   #########
 11242  11262   # See whether we should use the amalgamation to build
................................................................................
 11318  11338     test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
 11319  11339   
 11320  11340   fi
 11321  11341   
 11322  11342   else
 11323  11343     OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1"
 11324  11344   fi
        11345  +
        11346  +##########
        11347  +# Do we want to support memsys3 and/or memsys5
        11348  +#
        11349  +# Check whether --enable-memsys5 was given.
        11350  +if test "${enable_memsys5+set}" = set; then :
        11351  +  enableval=$enable_memsys5; enable_memsys5=yes
        11352  +else
        11353  +  enable_memsys5=no
        11354  +fi
        11355  +
        11356  +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS5" >&5
        11357  +$as_echo_n "checking whether to support MEMSYS5... " >&6; }
        11358  +if test "${enable_memsys5}" = "yes"; then
        11359  +  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS5"
        11360  +  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
        11361  +$as_echo "yes" >&6; }
        11362  +else
        11363  +  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
        11364  +$as_echo "no" >&6; }
        11365  +fi
        11366  +# Check whether --enable-memsys3 was given.
        11367  +if test "${enable_memsys3+set}" = set; then :
        11368  +  enableval=$enable_memsys3; enable_memsys3=yes
        11369  +else
        11370  +  enable_memsys3=no
        11371  +fi
        11372  +
        11373  +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS3" >&5
        11374  +$as_echo_n "checking whether to support MEMSYS3... " >&6; }
        11375  +if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
        11376  +  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS3"
        11377  +  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
        11378  +$as_echo "yes" >&6; }
        11379  +else
        11380  +  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
        11381  +$as_echo "no" >&6; }
        11382  +fi
 11325  11383   
 11326  11384   #########
 11327  11385   # See whether we should enable Full Text Search extensions
 11328  11386   # Check whether --enable-fts3 was given.
 11329  11387   if test "${enable_fts3+set}" = set; then :
 11330  11388     enableval=$enable_fts3; enable_fts3=yes
 11331  11389   else
................................................................................
 11489  11547   else
 11490  11548     enable_rtree=no
 11491  11549   fi
 11492  11550   
 11493  11551   if test "${enable_rtree}" = "yes" ; then
 11494  11552     OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE"
 11495  11553   fi
        11554  +
        11555  +#########
        11556  +# See whether we should enable the SESSION extension
        11557  +# Check whether --enable-session was given.
        11558  +if test "${enable_session+set}" = set; then :
        11559  +  enableval=$enable_session; enable_session=yes
        11560  +else
        11561  +  enable_session=no
        11562  +fi
        11563  +
        11564  +if test "${enable_session}" = "yes" ; then
        11565  +  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_SESSION"
        11566  +  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_PREUPDATE_HOOK"
        11567  +fi
 11496  11568   
 11497  11569   #########
 11498  11570   # attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter
 11499  11571   for option in $CFLAGS $CPPFLAGS
 11500  11572   do
 11501  11573     case $option in
 11502  11574       -DSQLITE_OMIT*) OPT_FEATURE_FLAGS="$OPT_FEATURE_FLAGS $option";;
................................................................................
 12075  12147   test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1
 12076  12148   
 12077  12149   cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 12078  12150   # Save the log message, to keep $0 and so on meaningful, and to
 12079  12151   # report actual input values of CONFIG_FILES etc. instead of their
 12080  12152   # values after options handling.
 12081  12153   ac_log="
 12082         -This file was extended by sqlite $as_me 3.12.0, which was
        12154  +This file was extended by sqlite $as_me 3.19.0, which was
 12083  12155   generated by GNU Autoconf 2.69.  Invocation command line was
 12084  12156   
 12085  12157     CONFIG_FILES    = $CONFIG_FILES
 12086  12158     CONFIG_HEADERS  = $CONFIG_HEADERS
 12087  12159     CONFIG_LINKS    = $CONFIG_LINKS
 12088  12160     CONFIG_COMMANDS = $CONFIG_COMMANDS
 12089  12161     $ $0 $@
................................................................................
 12141  12213   
 12142  12214   Report bugs to the package provider."
 12143  12215   
 12144  12216   _ACEOF
 12145  12217   cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 12146  12218   ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 12147  12219   ac_cs_version="\\
 12148         -sqlite config.status 3.12.0
        12220  +sqlite config.status 3.19.0
 12149  12221   configured by $0, generated by GNU Autoconf 2.69,
 12150  12222     with options \\"\$ac_cs_config\\"
 12151  12223   
 12152  12224   Copyright (C) 2012 Free Software Foundation, Inc.
 12153  12225   This config.status script is free software; the Free Software Foundation
 12154  12226   gives unlimited permission to copy, distribute and modify it."
 12155  12227   

Changes to configure.ac.

   329    329             if test -f "$i/tclConfig.sh" ; then
   330    330               ac_cv_c_tclconfig="$i"
   331    331               break
   332    332             fi
   333    333           done
   334    334         fi
   335    335       fi
          336  +
          337  +    # Recent versions of Xcode on Macs hid the tclConfig.sh file
          338  +    # in a strange place.
          339  +    if test x"${ac_cv_c_tclconfig}" = x ; then
          340  +      if test x"$cross_compiling" = xno; then
          341  +        for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
          342  +        do
          343  +          if test -f "$i/tclConfig.sh" ; then
          344  +            ac_cv_c_tclconfig="$i"
          345  +            break
          346  +          fi
          347  +        done
          348  +      fi
          349  +    fi
   336    350   
   337    351       # then check for a private Tcl installation
   338    352       if test x"${ac_cv_c_tclconfig}" = x ; then
   339    353         for i in \
   340    354               ../tcl \
   341    355               `ls -dr ../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
   342    356               `ls -dr ../tcl[[8-9]].[[0-9]] 2>/dev/null` \
................................................................................
   542    556   AC_SEARCH_LIBS(fdatasync, [rt])
   543    557   
   544    558   #########
   545    559   # check for debug enabled
   546    560   AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]),
   547    561         [use_debug=$enableval],[use_debug=no])
   548    562   if test "${use_debug}" = "yes" ; then
   549         -  TARGET_DEBUG="-DSQLITE_DEBUG=1"
          563  +  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE"
   550    564   else
   551    565     TARGET_DEBUG="-DNDEBUG"
   552    566   fi
   553    567   AC_SUBST(TARGET_DEBUG)
   554    568   
   555    569   #########
   556    570   # See whether we should use the amalgamation to build
................................................................................
   569    583         [use_loadextension=$enableval],[use_loadextension=yes])
   570    584   if test "${use_loadextension}" = "yes" ; then
   571    585     OPT_FEATURE_FLAGS=""
   572    586     AC_SEARCH_LIBS(dlopen, dl)
   573    587   else
   574    588     OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1"
   575    589   fi
          590  +
          591  +##########
          592  +# Do we want to support memsys3 and/or memsys5
          593  +#
          594  +AC_ARG_ENABLE(memsys5, 
          595  +  AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]),
          596  +  [enable_memsys5=yes],[enable_memsys5=no])
          597  +AC_MSG_CHECKING([whether to support MEMSYS5])
          598  +if test "${enable_memsys5}" = "yes"; then
          599  +  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS5"
          600  +  AC_MSG_RESULT([yes])
          601  +else
          602  +  AC_MSG_RESULT([no])
          603  +fi
          604  +AC_ARG_ENABLE(memsys3, 
          605  +  AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]),
          606  +  [enable_memsys3=yes],[enable_memsys3=no])
          607  +AC_MSG_CHECKING([whether to support MEMSYS3])
          608  +if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
          609  +  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS3"
          610  +  AC_MSG_RESULT([yes])
          611  +else
          612  +  AC_MSG_RESULT([no])
          613  +fi
   576    614   
   577    615   #########
   578    616   # See whether we should enable Full Text Search extensions
   579    617   AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3],
   580    618         [Enable the FTS3 extension]),
   581    619         [enable_fts3=yes],[enable_fts3=no])
   582    620   if test "${enable_fts3}" = "yes" ; then
................................................................................
   610    648   # See whether we should enable RTREE
   611    649   AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree],
   612    650         [Enable the RTREE extension]),
   613    651         [enable_rtree=yes],[enable_rtree=no])
   614    652   if test "${enable_rtree}" = "yes" ; then
   615    653     OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE"
   616    654   fi
          655  +
          656  +#########
          657  +# See whether we should enable the SESSION extension
          658  +AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session],
          659  +      [Enable the SESSION extension]),
          660  +      [enable_session=yes],[enable_session=no])
          661  +if test "${enable_session}" = "yes" ; then
          662  +  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_SESSION"
          663  +  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_PREUPDATE_HOOK"
          664  +fi
   617    665   
   618    666   #########
   619    667   # attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter
   620    668   for option in $CFLAGS $CPPFLAGS
   621    669   do
   622    670     case $option in
   623    671       -DSQLITE_OMIT*) OPT_FEATURE_FLAGS="$OPT_FEATURE_FLAGS $option";;

Changes to doc/lemon.html.

     1      1   <html>
     2      2   <head>
     3      3   <title>The Lemon Parser Generator</title>
     4      4   </head>
     5      5   <body bgcolor=white>
     6      6   <h1 align=center>The Lemon Parser Generator</h1>
     7      7   
     8         -<p>Lemon is an LALR(1) parser generator for C or C++.  
     9         -It does the same job as ``bison'' and ``yacc''.
    10         -But lemon is not another bison or yacc clone.  It
            8  +<p>Lemon is an LALR(1) parser generator for C.
            9  +It does the same job as "bison" and "yacc".
           10  +But lemon is not a bison or yacc clone.  Lemon
    11     11   uses a different grammar syntax which is designed to
    12         -reduce the number of coding errors.  Lemon also uses a more
    13         -sophisticated parsing engine that is faster than yacc and
    14         -bison and which is both reentrant and thread-safe.
    15         -Furthermore, Lemon implements features that can be used
           12  +reduce the number of coding errors.  Lemon also uses a
           13  +parsing engine that is faster than yacc and
           14  +bison and which is both reentrant and threadsafe.
           15  +(Update: Since the previous sentence was written, bison
           16  +has also been updated so that it too can generate a
           17  +reentrant and threadsafe parser.)
           18  +Lemon also implements features that can be used
    16     19   to eliminate resource leaks, making is suitable for use
    17     20   in long-running programs such as graphical user interfaces
    18     21   or embedded controllers.</p>
    19     22   
    20     23   <p>This document is an introduction to the Lemon
    21     24   parser generator.</p>
    22     25   
................................................................................
    40     43   <ul>
    41     44   <li>C code to implement the parser.
    42     45   <li>A header file defining an integer ID for each terminal symbol.
    43     46   <li>An information file that describes the states of the generated parser
    44     47       automaton.
    45     48   </ul>
    46     49   By default, all three of these output files are generated.
    47         -The header file is suppressed if the ``-m'' command-line option is
    48         -used and the report file is omitted when ``-q'' is selected.</p>
           50  +The header file is suppressed if the "-m" command-line option is
           51  +used and the report file is omitted when "-q" is selected.</p>
    49     52   
    50         -<p>The grammar specification file uses a ``.y'' suffix, by convention.
           53  +<p>The grammar specification file uses a ".y" suffix, by convention.
    51     54   In the examples used in this document, we'll assume the name of the
    52         -grammar file is ``gram.y''.  A typical use of Lemon would be the
           55  +grammar file is "gram.y".  A typical use of Lemon would be the
    53     56   following command:
    54     57   <pre>
    55     58      lemon gram.y
    56     59   </pre>
    57         -This command will generate three output files named ``gram.c'',
    58         -``gram.h'' and ``gram.out''.
           60  +This command will generate three output files named "gram.c",
           61  +"gram.h" and "gram.out".
    59     62   The first is C code to implement the parser.  The second
    60     63   is the header file that defines numerical values for all
    61     64   terminal symbols, and the last is the report that explains
    62     65   the states used by the parser automaton.</p>
    63     66   
    64     67   <h3>Command Line Options</h3>
    65     68   
................................................................................
    67     70   You can obtain a list of the available command-line options together
    68     71   with a brief explanation of what each does by typing
    69     72   <pre>
    70     73      lemon -?
    71     74   </pre>
    72     75   As of this writing, the following command-line options are supported:
    73     76   <ul>
    74         -<li><tt>-b</tt>
    75         -<li><tt>-c</tt>
    76         -<li><tt>-g</tt>
    77         -<li><tt>-m</tt>
    78         -<li><tt>-q</tt>
    79         -<li><tt>-s</tt>
    80         -<li><tt>-x</tt>
    81         -</ul>
    82         -The ``-b'' option reduces the amount of text in the report file by
    83         -printing only the basis of each parser state, rather than the full
    84         -configuration.
    85         -The ``-c'' option suppresses action table compression.  Using -c
    86         -will make the parser a little larger and slower but it will detect
    87         -syntax errors sooner.
    88         -The ``-g'' option causes no output files to be generated at all.
    89         -Instead, the input grammar file is printed on standard output but
    90         -with all comments, actions and other extraneous text deleted.  This
    91         -is a useful way to get a quick summary of a grammar.
    92         -The ``-m'' option causes the output C source file to be compatible
    93         -with the ``makeheaders'' program.
    94         -Makeheaders is a program that automatically generates header files
    95         -from C source code.  When the ``-m'' option is used, the header
    96         -file is not output since the makeheaders program will take care
    97         -of generated all header files automatically.
    98         -The ``-q'' option suppresses the report file.
    99         -Using ``-s'' causes a brief summary of parser statistics to be
   100         -printed.  Like this:
   101         -<pre>
   102         -   Parser statistics: 74 terminals, 70 nonterminals, 179 rules
   103         -                      340 states, 2026 parser table entries, 0 conflicts
   104         -</pre>
   105         -Finally, the ``-x'' option causes Lemon to print its version number
   106         -and then stops without attempting to read the grammar or generate a parser.</p>
           77  +<li><b>-b</b>
           78  +Show only the basis for each parser state in the report file.
           79  +<li><b>-c</b>
           80  +Do not compress the generated action tables.
           81  +<li><b>-D<i>name</i></b>
           82  +Define C preprocessor macro <i>name</i>.  This macro is useable by
           83  +"%ifdef" lines in the grammar file.
           84  +<li><b>-g</b>
           85  +Do not generate a parser.  Instead write the input grammar to standard
           86  +output with all comments, actions, and other extraneous text removed.
           87  +<li><b>-l</b>
           88  +Omit "#line" directives in the generated parser C code.
           89  +<li><b>-m</b>
           90  +Cause the output C source code to be compatible with the "makeheaders"
           91  +program. 
           92  +<li><b>-p</b>
           93  +Display all conflicts that are resolved by 
           94  +<a href='#precrules'>precedence rules</a>.
           95  +<li><b>-q</b>
           96  +Suppress generation of the report file.
           97  +<li><b>-r</b>
           98  +Do not sort or renumber the parser states as part of optimization.
           99  +<li><b>-s</b>
          100  +Show parser statistics before existing.
          101  +<li><b>-T<i>file</i></b>
          102  +Use <i>file</i> as the template for the generated C-code parser implementation.
          103  +<li><b>-x</b>
          104  +Print the Lemon version number.
          105  +</ul>
   107    106   
   108    107   <h3>The Parser Interface</h3>
   109    108   
   110    109   <p>Lemon doesn't generate a complete, working program.  It only generates
   111    110   a few subroutines that implement a parser.  This section describes
   112    111   the interface to those subroutines.  It is up to the programmer to
   113    112   call these subroutines in an appropriate way in order to produce a
................................................................................
   117    116   must first create the parser.
   118    117   A new parser is created as follows:
   119    118   <pre>
   120    119      void *pParser = ParseAlloc( malloc );
   121    120   </pre>
   122    121   The ParseAlloc() routine allocates and initializes a new parser and
   123    122   returns a pointer to it.
   124         -The actual data structure used to represent a parser is opaque --
          123  +The actual data structure used to represent a parser is opaque &mdash;
   125    124   its internal structure is not visible or usable by the calling routine.
   126    125   For this reason, the ParseAlloc() routine returns a pointer to void
   127    126   rather than a pointer to some particular structure.
   128    127   The sole argument to the ParseAlloc() routine is a pointer to the
   129         -subroutine used to allocate memory.  Typically this means ``malloc()''.</p>
          128  +subroutine used to allocate memory.  Typically this means malloc().</p>
   130    129   
   131    130   <p>After a program is finished using a parser, it can reclaim all
   132    131   memory allocated by that parser by calling
   133    132   <pre>
   134    133      ParseFree(pParser, free);
   135    134   </pre>
   136    135   The first argument is the same pointer returned by ParseAlloc().  The
................................................................................
   147    146   The first argument to the Parse() routine is the pointer returned by
   148    147   ParseAlloc().
   149    148   The second argument is a small positive integer that tells the parse the
   150    149   type of the next token in the data stream.
   151    150   There is one token type for each terminal symbol in the grammar.
   152    151   The gram.h file generated by Lemon contains #define statements that
   153    152   map symbolic terminal symbol names into appropriate integer values.
   154         -(A value of 0 for the second argument is a special flag to the
   155         -parser to indicate that the end of input has been reached.)
          153  +A value of 0 for the second argument is a special flag to the
          154  +parser to indicate that the end of input has been reached.
   156    155   The third argument is the value of the given token.  By default,
   157    156   the type of the third argument is integer, but the grammar will
   158    157   usually redefine this type to be some kind of structure.
   159    158   Typically the second argument will be a broad category of tokens
   160         -such as ``identifier'' or ``number'' and the third argument will
          159  +such as "identifier" or "number" and the third argument will
   161    160   be the name of the identifier or the value of the number.</p>
   162    161   
   163    162   <p>The Parse() function may have either three or four arguments,
   164    163   depending on the grammar.  If the grammar specification file requests
   165    164   it (via the <a href='#extraarg'><tt>extra_argument</tt> directive</a>),
   166    165   the Parse() function will have a fourth parameter that can be
   167    166   of any type chosen by the programmer.  The parser doesn't do anything
................................................................................
   189    188      15    ParseFree(pParser, free );
   190    189      16    TokenizerFree(pTokenizer);
   191    190      17    return sState.treeRoot;
   192    191      18 }
   193    192   </pre>
   194    193   This example shows a user-written routine that parses a file of
   195    194   text and returns a pointer to the parse tree.
   196         -(We've omitted all error-handling from this example to keep it
          195  +(All error-handling code is omitted from this example to keep it
   197    196   simple.)
   198    197   We assume the existence of some kind of tokenizer which is created
   199    198   using TokenizerCreate() on line 8 and deleted by TokenizerFree()
   200    199   on line 16.  The GetNextToken() function on line 11 retrieves the
   201    200   next token from the input file and puts its type in the 
   202    201   integer variable hTokenId.  The sToken variable is assumed to be
   203    202   some kind of structure that contains details about each token,
................................................................................
   283    282   declaration can occur at any point in the file.
   284    283   Lemon ignores whitespace (except where it is needed to separate
   285    284   tokens) and it honors the same commenting conventions as C and C++.</p>
   286    285   
   287    286   <h3>Terminals and Nonterminals</h3>
   288    287   
   289    288   <p>A terminal symbol (token) is any string of alphanumeric
   290         -and underscore characters
          289  +and/or underscore characters
   291    290   that begins with an upper case letter.
   292    291   A terminal can contain lowercase letters after the first character,
   293    292   but the usual convention is to make terminals all upper case.
   294    293   A nonterminal, on the other hand, is any string of alphanumeric
   295    294   and underscore characters than begins with a lower case letter.
   296    295   Again, the usual convention is to make nonterminals use all lower
   297    296   case letters.</p>
................................................................................
   310    309   must have alphanumeric names.</p>
   311    310   
   312    311   <h3>Grammar Rules</h3>
   313    312   
   314    313   <p>The main component of a Lemon grammar file is a sequence of grammar
   315    314   rules.
   316    315   Each grammar rule consists of a nonterminal symbol followed by
   317         -the special symbol ``::='' and then a list of terminals and/or nonterminals.
          316  +the special symbol "::=" and then a list of terminals and/or nonterminals.
   318    317   The rule is terminated by a period.
   319    318   The list of terminals and nonterminals on the right-hand side of the
   320    319   rule can be empty.
   321    320   Rules can occur in any order, except that the left-hand side of the
   322    321   first rule is assumed to be the start symbol for the grammar (unless
   323    322   specified otherwise using the <tt>%start</tt> directive described below.)
   324    323   A typical sequence of grammar rules might look something like this:
................................................................................
   326    325     expr ::= expr PLUS expr.
   327    326     expr ::= expr TIMES expr.
   328    327     expr ::= LPAREN expr RPAREN.
   329    328     expr ::= VALUE.
   330    329   </pre>
   331    330   </p>
   332    331   
   333         -<p>There is one non-terminal in this example, ``expr'', and five
   334         -terminal symbols or tokens: ``PLUS'', ``TIMES'', ``LPAREN'',
   335         -``RPAREN'' and ``VALUE''.</p>
          332  +<p>There is one non-terminal in this example, "expr", and five
          333  +terminal symbols or tokens: "PLUS", "TIMES", "LPAREN",
          334  +"RPAREN" and "VALUE".</p>
   336    335   
   337    336   <p>Like yacc and bison, Lemon allows the grammar to specify a block
   338    337   of C code that will be executed whenever a grammar rule is reduced
   339    338   by the parser.
   340    339   In Lemon, this action is specified by putting the C code (contained
   341    340   within curly braces <tt>{...}</tt>) immediately after the
   342    341   period that closes the rule.
................................................................................
   344    343   <pre>
   345    344     expr ::= expr PLUS expr.   { printf("Doing an addition...\n"); }
   346    345   </pre>
   347    346   </p>
   348    347   
   349    348   <p>In order to be useful, grammar actions must normally be linked to
   350    349   their associated grammar rules.
   351         -In yacc and bison, this is accomplished by embedding a ``$$'' in the
          350  +In yacc and bison, this is accomplished by embedding a "$$" in the
   352    351   action to stand for the value of the left-hand side of the rule and
   353         -symbols ``$1'', ``$2'', and so forth to stand for the value of
          352  +symbols "$1", "$2", and so forth to stand for the value of
   354    353   the terminal or nonterminal at position 1, 2 and so forth on the
   355    354   right-hand side of the rule.
   356    355   This idea is very powerful, but it is also very error-prone.  The
   357    356   single most common source of errors in a yacc or bison grammar is
   358    357   to miscount the number of symbols on the right-hand side of a grammar
   359         -rule and say ``$7'' when you really mean ``$8''.</p>
          358  +rule and say "$7" when you really mean "$8".</p>
   360    359   
   361    360   <p>Lemon avoids the need to count grammar symbols by assigning symbolic
   362    361   names to each symbol in a grammar rule and then using those symbolic
   363    362   names in the action.
   364    363   In yacc or bison, one would write this:
   365    364   <pre>
   366    365     expr -> expr PLUS expr  { $$ = $1 + $3; };
................................................................................
   382    381   includes a linking symbol in parentheses but that linking symbol
   383    382   is not actually used in the reduce action, then an error message
   384    383   is generated.
   385    384   For example, the rule
   386    385   <pre>
   387    386     expr(A) ::= expr(B) PLUS expr(C).  { A = B; }
   388    387   </pre>
   389         -will generate an error because the linking symbol ``C'' is used
          388  +will generate an error because the linking symbol "C" is used
   390    389   in the grammar rule but not in the reduce action.</p>
   391    390   
   392    391   <p>The Lemon notation for linking grammar rules to reduce actions
   393    392   also facilitates the use of destructors for reclaiming memory
   394    393   allocated by the values of terminals and nonterminals on the
   395    394   right-hand side of a rule.</p>
   396    395   
          396  +<a name='precrules'></a>
   397    397   <h3>Precedence Rules</h3>
   398    398   
   399    399   <p>Lemon resolves parsing ambiguities in exactly the same way as
   400    400   yacc and bison.  A shift-reduce conflict is resolved in favor
   401    401   of the shift, and a reduce-reduce conflict is resolved by reducing
   402    402   whichever rule comes first in the grammar file.</p>
   403    403   
   404    404   <p>Just like in
   405    405   yacc and bison, Lemon allows a measure of control 
   406    406   over the resolution of paring conflicts using precedence rules.
   407    407   A precedence value can be assigned to any terminal symbol
   408         -using the %left, %right or %nonassoc directives.  Terminal symbols
          408  +using the 
          409  +<a href='#pleft'>%left</a>,
          410  +<a href='#pright'>%right</a> or
          411  +<a href='#pnonassoc'>%nonassoc</a> directives.  Terminal symbols
   409    412   mentioned in earlier directives have a lower precedence that
   410    413   terminal symbols mentioned in later directives.  For example:</p>
   411    414   
   412    415   <p><pre>
   413    416      %left AND.
   414    417      %left OR.
   415    418      %nonassoc EQ NE GT GE LT LE.
................................................................................
   521    524   
   522    525   <p>Lemon supports the following special directives:
   523    526   <ul>
   524    527   <li><tt>%code</tt>
   525    528   <li><tt>%default_destructor</tt>
   526    529   <li><tt>%default_type</tt>
   527    530   <li><tt>%destructor</tt>
          531  +<li><tt>%endif</tt>
   528    532   <li><tt>%extra_argument</tt>
          533  +<li><tt>%fallback</tt>
          534  +<li><tt>%ifdef</tt>
          535  +<li><tt>%ifndef</tt>
   529    536   <li><tt>%include</tt>
   530    537   <li><tt>%left</tt>
   531    538   <li><tt>%name</tt>
   532    539   <li><tt>%nonassoc</tt>
   533    540   <li><tt>%parse_accept</tt>
   534    541   <li><tt>%parse_failure </tt>
   535    542   <li><tt>%right</tt>
   536    543   <li><tt>%stack_overflow</tt>
   537    544   <li><tt>%stack_size</tt>
   538    545   <li><tt>%start_symbol</tt>
   539    546   <li><tt>%syntax_error</tt>
          547  +<li><tt>%token_class</tt>
   540    548   <li><tt>%token_destructor</tt>
   541    549   <li><tt>%token_prefix</tt>
   542    550   <li><tt>%token_type</tt>
   543    551   <li><tt>%type</tt>
          552  +<li><tt>%wildcard</tt>
   544    553   </ul>
   545    554   Each of these directives will be described separately in the
   546    555   following sections:</p>
   547    556   
          557  +<a name='pcode'></a>
   548    558   <h4>The <tt>%code</tt> directive</h4>
   549    559   
   550         -<p>The %code directive is used to specify addition C/C++ code that
          560  +<p>The %code directive is used to specify addition C code that
   551    561   is added to the end of the main output file.  This is similar to
   552         -the %include directive except that %include is inserted at the
   553         -beginning of the main output file.</p>
          562  +the <a href='#pinclude'>%include</a> directive except that %include
          563  +is inserted at the beginning of the main output file.</p>
   554    564   
   555    565   <p>%code is typically used to include some action routines or perhaps
   556         -a tokenizer as part of the output file.</p>
          566  +a tokenizer or even the "main()" function 
          567  +as part of the output file.</p>
   557    568   
          569  +<a name='default_destructor'></a>
   558    570   <h4>The <tt>%default_destructor</tt> directive</h4>
   559    571   
   560    572   <p>The %default_destructor directive specifies a destructor to 
   561    573   use for non-terminals that do not have their own destructor
   562    574   specified by a separate %destructor directive.  See the documentation
   563         -on the %destructor directive below for additional information.</p>
          575  +on the <a name='#destructor'>%destructor</a> directive below for
          576  +additional information.</p>
   564    577   
   565    578   <p>In some grammers, many different non-terminal symbols have the
   566    579   same datatype and hence the same destructor.  This directive is
   567    580   a convenience way to specify the same destructor for all those
   568    581   non-terminals using a single statement.</p>
   569    582   
          583  +<a name='default_type'></a>
   570    584   <h4>The <tt>%default_type</tt> directive</h4>
   571    585   
   572    586   <p>The %default_type directive specifies the datatype of non-terminal
   573    587   symbols that do no have their own datatype defined using a separate
   574         -%type directive.  See the documentation on %type below for addition
   575         -information.</p>
          588  +<a href='#ptype'>%type</a> directive.  
          589  +</p>
   576    590   
          591  +<a name='destructor'></a>
   577    592   <h4>The <tt>%destructor</tt> directive</h4>
   578    593   
   579    594   <p>The %destructor directive is used to specify a destructor for
   580    595   a non-terminal symbol.
   581         -(See also the %token_destructor directive which is used to
   582         -specify a destructor for terminal symbols.)</p>
          596  +(See also the <a href='#token_destructor'>%token_destructor</a>
          597  +directive which is used to specify a destructor for terminal symbols.)</p>
   583    598   
   584    599   <p>A non-terminal's destructor is called to dispose of the
   585    600   non-terminal's value whenever the non-terminal is popped from
   586    601   the stack.  This includes all of the following circumstances:
   587    602   <ul>
   588    603   <li> When a rule reduces and the value of a non-terminal on
   589    604        the right-hand side is not linked to C code.
................................................................................
   598    613   <pre>
   599    614      %type nt {void*}
   600    615      %destructor nt { free($$); }
   601    616      nt(A) ::= ID NUM.   { A = malloc( 100 ); }
   602    617   </pre>
   603    618   This example is a bit contrived but it serves to illustrate how
   604    619   destructors work.  The example shows a non-terminal named
   605         -``nt'' that holds values of type ``void*''.  When the rule for
   606         -an ``nt'' reduces, it sets the value of the non-terminal to
          620  +"nt" that holds values of type "void*".  When the rule for
          621  +an "nt" reduces, it sets the value of the non-terminal to
   607    622   space obtained from malloc().  Later, when the nt non-terminal
   608    623   is popped from the stack, the destructor will fire and call
   609    624   free() on this malloced space, thus avoiding a memory leak.
   610         -(Note that the symbol ``$$'' in the destructor code is replaced
          625  +(Note that the symbol "$$" in the destructor code is replaced
   611    626   by the value of the non-terminal.)</p>
   612    627   
   613    628   <p>It is important to note that the value of a non-terminal is passed
   614    629   to the destructor whenever the non-terminal is removed from the
   615    630   stack, unless the non-terminal is used in a C-code action.  If
   616    631   the non-terminal is used by C-code, then it is assumed that the
   617         -C-code will take care of destroying it if it should really
   618         -be destroyed.  More commonly, the value is used to build some
          632  +C-code will take care of destroying it.
          633  +More commonly, the value is used to build some
   619    634   larger structure and we don't want to destroy it, which is why
   620    635   the destructor is not called in this circumstance.</p>
   621    636   
   622         -<p>By appropriate use of destructors, it is possible to
   623         -build a parser using Lemon that can be used within a long-running
   624         -program, such as a GUI, that will not leak memory or other resources.
          637  +<p>Destructors help avoid memory leaks by automatically freeing
          638  +allocated objects when they go out of scope.
   625    639   To do the same using yacc or bison is much more difficult.</p>
   626    640   
   627    641   <a name="extraarg"></a>
   628    642   <h4>The <tt>%extra_argument</tt> directive</h4>
   629    643   
   630    644   The %extra_argument directive instructs Lemon to add a 4th parameter
   631    645   to the parameter list of the Parse() function it generates.  Lemon
................................................................................
   634    648   and so forth.  For example, if the grammar file contains:</p>
   635    649   
   636    650   <p><pre>
   637    651       %extra_argument { MyStruct *pAbc }
   638    652   </pre></p>
   639    653   
   640    654   <p>Then the Parse() function generated will have an 4th parameter
   641         -of type ``MyStruct*'' and all action routines will have access to
   642         -a variable named ``pAbc'' that is the value of the 4th parameter
          655  +of type "MyStruct*" and all action routines will have access to
          656  +a variable named "pAbc" that is the value of the 4th parameter
   643    657   in the most recent call to Parse().</p>
   644    658   
          659  +<a name='pfallback'></a>
          660  +<h4>The <tt>%fallback</tt> directive</h4>
          661  +
          662  +<p>The %fallback directive specifies an alternative meaning for one
          663  +or more tokens.  The alternative meaning is tried if the original token
          664  +would have generated a syntax error.
          665  +
          666  +<p>The %fallback directive was added to support robust parsing of SQL
          667  +syntax in <a href="https://www.sqlite.org/">SQLite</a>.
          668  +The SQL language contains a large assortment of keywords, each of which
          669  +appears as a different token to the language parser.  SQL contains so
          670  +many keywords, that it can be difficult for programmers to keep up with
          671  +them all.  Programmers will, therefore, sometimes mistakenly use an
          672  +obscure language keyword for an identifier.  The %fallback directive
          673  +provides a mechanism to tell the parser:  "If you are unable to parse
          674  +this keyword, try treating it as an identifier instead."
          675  +
          676  +<p>The syntax of %fallback is as follows:
          677  +
          678  +<blockquote>
          679  +<tt>%fallback</tt>  <i>ID</i> <i>TOKEN...</i> <b>.</b>
          680  +</blockquote>
          681  +
          682  +<p>In words, the %fallback directive is followed by a list of token names
          683  +terminated by a period.  The first token name is the fallback token - the
          684  +token to which all the other tokens fall back to.  The second and subsequent
          685  +arguments are tokens which fall back to the token identified by the first
          686  +argument.
          687  +
          688  +<a name='pifdef'></a>
          689  +<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives.</h4>
          690  +
          691  +<p>The %ifdef, %ifndef, and %endif directives are similar to
          692  +#ifdef, #ifndef, and #endif in the C-preprocessor, just not as general.
          693  +Each of these directives must begin at the left margin.  No whitespace
          694  +is allowed between the "%" and the directive name.
          695  +
          696  +<p>Grammar text in between "%ifdef MACRO" and the next nested "%endif" is
          697  +ignored unless the "-DMACRO" command-line option is used.  Grammar text
          698  +betwen "%ifndef MACRO" and the next nested "%endif" is included except when
          699  +the "-DMACRO" command-line option is used.
          700  +
          701  +<p>Note that the argument to %ifdef and %ifndef must be a single 
          702  +preprocessor symbol name, not a general expression.  There is no "%else"
          703  +directive.
          704  +
          705  +
          706  +<a name='pinclude'></a>
   645    707   <h4>The <tt>%include</tt> directive</h4>
   646    708   
   647    709   <p>The %include directive specifies C code that is included at the
   648    710   top of the generated parser.  You can include any text you want --
   649    711   the Lemon parser generator copies it blindly.  If you have multiple
   650         -%include directives in your grammar file the value of the last
   651         -%include directive overwrites all the others.</p.
          712  +%include directives in your grammar file, their values are concatenated
          713  +so that all %include code ultimately appears near the top of the
          714  +generated parser, in the same order as it appeared in the grammer.</p>
   652    715   
   653    716   <p>The %include directive is very handy for getting some extra #include
   654    717   preprocessor statements at the beginning of the generated parser.
   655    718   For example:</p>
   656    719   
   657    720   <p><pre>
   658    721      %include {#include &lt;unistd.h&gt;}
   659    722   </pre></p>
   660    723   
   661    724   <p>This might be needed, for example, if some of the C actions in the
   662    725   grammar call functions that are prototyed in unistd.h.</p>
   663    726   
          727  +<a name='pleft'></a>
   664    728   <h4>The <tt>%left</tt> directive</h4>
   665    729   
   666         -The %left directive is used (along with the %right and
   667         -%nonassoc directives) to declare precedences of terminal
   668         -symbols.  Every terminal symbol whose name appears after
   669         -a %left directive but before the next period (``.'') is
          730  +The %left directive is used (along with the <a href='#pright'>%right</a> and
          731  +<a href='#pnonassoc'>%nonassoc</a> directives) to declare precedences of 
          732  +terminal symbols.  Every terminal symbol whose name appears after
          733  +a %left directive but before the next period (".") is
   670    734   given the same left-associative precedence value.  Subsequent
   671    735   %left directives have higher precedence.  For example:</p>
   672    736   
   673    737   <p><pre>
   674    738      %left AND.
   675    739      %left OR.
   676    740      %nonassoc EQ NE GT GE LT LE.
................................................................................
   683    747   directive.</p>
   684    748   
   685    749   <p>LALR(1) grammars can get into a situation where they require
   686    750   a large amount of stack space if you make heavy use or right-associative
   687    751   operators.  For this reason, it is recommended that you use %left
   688    752   rather than %right whenever possible.</p>
   689    753   
          754  +<a name='pname'></a>
   690    755   <h4>The <tt>%name</tt> directive</h4>
   691    756   
   692    757   <p>By default, the functions generated by Lemon all begin with the
   693         -five-character string ``Parse''.  You can change this string to something
          758  +five-character string "Parse".  You can change this string to something
   694    759   different using the %name directive.  For instance:</p>
   695    760   
   696    761   <p><pre>
   697    762      %name Abcde
   698    763   </pre></p>
   699    764   
   700    765   <p>Putting this directive in the grammar file will cause Lemon to generate
................................................................................
   705    770   <li> AbcdeTrace(), and
   706    771   <li> Abcde().
   707    772   </ul>
   708    773   The %name directive allows you to generator two or more different
   709    774   parsers and link them all into the same executable.
   710    775   </p>
   711    776   
          777  +<a name='pnonassoc'></a>
   712    778   <h4>The <tt>%nonassoc</tt> directive</h4>
   713    779   
   714    780   <p>This directive is used to assign non-associative precedence to
   715         -one or more terminal symbols.  See the section on precedence rules
   716         -or on the %left directive for additional information.</p>
          781  +one or more terminal symbols.  See the section on 
          782  +<a href='#precrules'>precedence rules</a>
          783  +or on the <a href='#pleft'>%left</a> directive for additional information.</p>
   717    784   
          785  +<a name='parse_accept'></a>
   718    786   <h4>The <tt>%parse_accept</tt> directive</h4>
   719    787   
   720    788   <p>The %parse_accept directive specifies a block of C code that is
   721         -executed whenever the parser accepts its input string.  To ``accept''
          789  +executed whenever the parser accepts its input string.  To "accept"
   722    790   an input string means that the parser was able to process all tokens
   723    791   without error.</p>
   724    792   
   725    793   <p>For example:</p>
   726    794   
   727    795   <p><pre>
   728    796      %parse_accept {
   729    797         printf("parsing complete!\n");
   730    798      }
   731    799   </pre></p>
   732    800   
   733         -
          801  +<a name='parse_failure'></a>
   734    802   <h4>The <tt>%parse_failure</tt> directive</h4>
   735    803   
   736    804   <p>The %parse_failure directive specifies a block of C code that
   737    805   is executed whenever the parser fails complete.  This code is not
   738    806   executed until the parser has tried and failed to resolve an input
   739    807   error using is usual error recovery strategy.  The routine is
   740    808   only invoked when parsing is unable to continue.</p>
................................................................................
   741    809   
   742    810   <p><pre>
   743    811      %parse_failure {
   744    812        fprintf(stderr,"Giving up.  Parser is hopelessly lost...\n");
   745    813      }
   746    814   </pre></p>
   747    815   
          816  +<a name='pright'></a>
   748    817   <h4>The <tt>%right</tt> directive</h4>
   749    818   
   750    819   <p>This directive is used to assign right-associative precedence to
   751         -one or more terminal symbols.  See the section on precedence rules
   752         -or on the %left directive for additional information.</p>
          820  +one or more terminal symbols.  See the section on 
          821  +<a href='#precrules'>precedence rules</a>
          822  +or on the <a href='#pleft'>%left</a> directive for additional information.</p>
   753    823   
          824  +<a name='stack_overflow'></a>
   754    825   <h4>The <tt>%stack_overflow</tt> directive</h4>
   755    826   
   756    827   <p>The %stack_overflow directive specifies a block of C code that
   757    828   is executed if the parser's internal stack ever overflows.  Typically
   758    829   this just prints an error message.  After a stack overflow, the parser
   759    830   will be unable to continue and must be reset.</p>
   760    831   
................................................................................
   775    846   </pre>
   776    847   Not like this:
   777    848   <pre>
   778    849      list ::= element list.      // right-recursion.  Bad!
   779    850      list ::= .
   780    851   </pre>
   781    852   
          853  +<a name='stack_size'></a>
   782    854   <h4>The <tt>%stack_size</tt> directive</h4>
   783    855   
   784    856   <p>If stack overflow is a problem and you can't resolve the trouble
   785    857   by using left-recursion, then you might want to increase the size
   786    858   of the parser's stack using this directive.  Put an positive integer
   787    859   after the %stack_size directive and Lemon will generate a parse
   788    860   with a stack of the requested size.  The default value is 100.</p>
   789    861   
   790    862   <p><pre>
   791    863      %stack_size 2000
   792    864   </pre></p>
   793    865   
          866  +<a name='start_symbol'></a>
   794    867   <h4>The <tt>%start_symbol</tt> directive</h4>
   795    868   
   796    869   <p>By default, the start-symbol for the grammar that Lemon generates
   797    870   is the first non-terminal that appears in the grammar file.  But you
   798    871   can choose a different start-symbol using the %start_symbol directive.</p>
   799    872   
   800    873   <p><pre>
   801    874      %start_symbol  prog
   802    875   </pre></p>
   803    876   
          877  +<a name='token_destructor'></a>
   804    878   <h4>The <tt>%token_destructor</tt> directive</h4>
   805    879   
   806    880   <p>The %destructor directive assigns a destructor to a non-terminal
   807    881   symbol.  (See the description of the %destructor directive above.)
   808    882   This directive does the same thing for all terminal symbols.</p>
   809    883   
   810    884   <p>Unlike non-terminal symbols which may each have a different data type
   811    885   for their values, terminals all use the same data type (defined by
   812    886   the %token_type directive) and so they use a common destructor.  Other
   813    887   than that, the token destructor works just like the non-terminal
   814    888   destructors.</p>
   815    889   
          890  +<a name='token_prefix'></a>
   816    891   <h4>The <tt>%token_prefix</tt> directive</h4>
   817    892   
   818    893   <p>Lemon generates #defines that assign small integer constants
   819    894   to each terminal symbol in the grammar.  If desired, Lemon will
   820    895   add a prefix specified by this directive
   821    896   to each of the #defines it generates.
   822    897   So if the default output of Lemon looked like this:
................................................................................
   834    909   <pre>
   835    910       #define TOKEN_AND        1
   836    911       #define TOKEN_MINUS      2
   837    912       #define TOKEN_OR         3
   838    913       #define TOKEN_PLUS       4
   839    914   </pre>
   840    915   
          916  +<a name='token_type'></a><a name='ptype'></a>
   841    917   <h4>The <tt>%token_type</tt> and <tt>%type</tt> directives</h4>
   842    918   
   843    919   <p>These directives are used to specify the data types for values
   844    920   on the parser's stack associated with terminal and non-terminal
   845    921   symbols.  The values of all terminal symbols must be of the same
   846    922   type.  This turns out to be the same data type as the 3rd parameter
   847    923   to the Parse() function generated by Lemon.  Typically, you will
................................................................................
   849    925   token structure.  Like this:</p>
   850    926   
   851    927   <p><pre>
   852    928      %token_type    {Token*}
   853    929   </pre></p>
   854    930   
   855    931   <p>If the data type of terminals is not specified, the default value
   856         -is ``int''.</p>
          932  +is "void*".</p>
   857    933   
   858    934   <p>Non-terminal symbols can each have their own data types.  Typically
   859    935   the data type  of a non-terminal is a pointer to the root of a parse-tree
   860    936   structure that contains all information about that non-terminal.
   861    937   For example:</p>
   862    938   
   863    939   <p><pre>
................................................................................
   870    946   on what the corresponding non-terminal or terminal symbol is.  But
   871    947   the grammar designer should keep in mind that the size of the union
   872    948   will be the size of its largest element.  So if you have a single
   873    949   non-terminal whose data type requires 1K of storage, then your 100
   874    950   entry parser stack will require 100K of heap space.  If you are willing
   875    951   and able to pay that price, fine.  You just need to know.</p>
   876    952   
          953  +<a name='pwildcard'></a>
          954  +<h4>The <tt>%wildcard</tt> directive</h4>
          955  +
          956  +<p>The %wildcard directive is followed by a single token name and a
          957  +period.  This directive specifies that the identified token should 
          958  +match any input token.
          959  +
          960  +<p>When the generated parser has the choice of matching an input against
          961  +the wildcard token and some other token, the other token is always used.
          962  +The wildcard token is only matched if there are no other alternatives.
          963  +
   877    964   <h3>Error Processing</h3>
   878    965   
   879    966   <p>After extensive experimentation over several years, it has been
   880    967   discovered that the error recovery strategy used by yacc is about
   881    968   as good as it gets.  And so that is what Lemon uses.</p>
   882    969   
   883    970   <p>When a Lemon-generated parser encounters a syntax error, it
   884    971   first invokes the code specified by the %syntax_error directive, if
   885    972   any.  It then enters its error recovery strategy.  The error recovery
   886    973   strategy is to begin popping the parsers stack until it enters a
   887    974   state where it is permitted to shift a special non-terminal symbol
   888         -named ``error''.  It then shifts this non-terminal and continues
          975  +named "error".  It then shifts this non-terminal and continues
   889    976   parsing.  But the %syntax_error routine will not be called again
   890    977   until at least three new tokens have been successfully shifted.</p>
   891    978   
   892    979   <p>If the parser pops its stack until the stack is empty, and it still
   893    980   is unable to shift the error symbol, then the %parse_failed routine
   894    981   is invoked and the parser resets itself to its start state, ready
   895    982   to begin parsing a new file.  This is what will happen at the very
   896    983   first syntax error, of course, if there are no instances of the 
   897         -``error'' non-terminal in your grammar.</p>
          984  +"error" non-terminal in your grammar.</p>
   898    985   
   899    986   </body>
   900    987   </html>

Name change from ext/README.txt to ext/README.md.

     1         -Version loadable extensions to SQLite are found in subfolders
     2         -of this folder.
            1  +## Loadable Extensions
            2  +
            3  +Various [loadable extensions](https://www.sqlite.org/loadext.html) for
            4  +SQLite are found in subfolders.
            5  +
            6  +Most subfolders are dedicated to a single loadable extension (for
            7  +example FTS5, or RTREE).  But the misc/ subfolder contains a collection
            8  +of smaller single-file extensions.

Changes to ext/fts2/fts2_tokenizer.c.

    95     95     }
    96     96   
    97     97     sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
    98     98   }
    99     99   
   100    100   #ifdef SQLITE_TEST
   101    101   
   102         -#include <tcl.h>
          102  +#if defined(INCLUDE_SQLITE_TCL_H)
          103  +#  include "sqlite_tcl.h"
          104  +#else
          105  +#  include "tcl.h"
          106  +#endif
   103    107   #include <string.h>
   104    108   
   105    109   /*
   106    110   ** Implementation of a special SQL scalar function for testing tokenizers 
   107    111   ** designed to be used in concert with the Tcl testing framework. This
   108    112   ** function must be called with two arguments:
   109    113   **

Changes to ext/fts3/fts3.c.

   345    345     if( (v & mask2)==0 ){ var = v; return ret; }
   346    346   
   347    347   /* 
   348    348   ** Read a 64-bit variable-length integer from memory starting at p[0].
   349    349   ** Return the number of bytes read, or 0 on error.
   350    350   ** The value is stored in *v.
   351    351   */
   352         -int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){
   353         -  const char *pStart = p;
          352  +int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){
          353  +  const unsigned char *p = (const unsigned char*)pBuf;
          354  +  const unsigned char *pStart = p;
   354    355     u32 a;
   355    356     u64 b;
   356    357     int shift;
   357    358   
   358    359     GETVARINT_INIT(a, p, 0,  0x00,     0x80, *v, 1);
   359    360     GETVARINT_STEP(a, p, 7,  0x7F,     0x4000, *v, 2);
   360    361     GETVARINT_STEP(a, p, 14, 0x3FFF,   0x200000, *v, 3);
................................................................................
   488    489     Fts3Table *p = (Fts3Table *)pVtab;
   489    490     int i;
   490    491   
   491    492     assert( p->nPendingData==0 );
   492    493     assert( p->pSegments==0 );
   493    494   
   494    495     /* Free any prepared statements held */
          496  +  sqlite3_finalize(p->pSeekStmt);
   495    497     for(i=0; i<SizeofArray(p->aStmt); i++){
   496    498       sqlite3_finalize(p->aStmt[i]);
   497    499     }
   498    500     sqlite3_free(p->zSegmentsTbl);
   499    501     sqlite3_free(p->zReadExprlist);
   500    502     sqlite3_free(p->zWriteExprlist);
   501    503     sqlite3_free(p->zContentTbl);
................................................................................
  1359   1361     p->db = db;
  1360   1362     p->nColumn = nCol;
  1361   1363     p->nPendingData = 0;
  1362   1364     p->azColumn = (char **)&p[1];
  1363   1365     p->pTokenizer = pTokenizer;
  1364   1366     p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
  1365   1367     p->bHasDocsize = (isFts4 && bNoDocsize==0);
  1366         -  p->bHasStat = isFts4;
  1367         -  p->bFts4 = isFts4;
  1368         -  p->bDescIdx = bDescIdx;
         1368  +  p->bHasStat = (u8)isFts4;
         1369  +  p->bFts4 = (u8)isFts4;
         1370  +  p->bDescIdx = (u8)bDescIdx;
  1369   1371     p->nAutoincrmerge = 0xff;   /* 0xff means setting unknown */
  1370   1372     p->zContentTbl = zContent;
  1371   1373     p->zLanguageid = zLanguageid;
  1372   1374     zContent = 0;
  1373   1375     zLanguageid = 0;
  1374   1376     TESTONLY( p->inTransaction = -1 );
  1375   1377     TESTONLY( p->mxSavepoint = -1 );
................................................................................
  1392   1394     zCsr += nDb;
  1393   1395   
  1394   1396     /* Fill in the azColumn array */
  1395   1397     for(iCol=0; iCol<nCol; iCol++){
  1396   1398       char *z; 
  1397   1399       int n = 0;
  1398   1400       z = (char *)sqlite3Fts3NextToken(aCol[iCol], &n);
  1399         -    memcpy(zCsr, z, n);
         1401  +    if( n>0 ){
         1402  +      memcpy(zCsr, z, n);
         1403  +    }
  1400   1404       zCsr[n] = '\0';
  1401   1405       sqlite3Fts3Dequote(zCsr);
  1402   1406       p->azColumn[iCol] = zCsr;
  1403   1407       zCsr += n+1;
  1404   1408       assert( zCsr <= &((char *)p)[nByte] );
  1405   1409     }
  1406   1410   
................................................................................
  1675   1679     *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor));
  1676   1680     if( !pCsr ){
  1677   1681       return SQLITE_NOMEM;
  1678   1682     }
  1679   1683     memset(pCsr, 0, sizeof(Fts3Cursor));
  1680   1684     return SQLITE_OK;
  1681   1685   }
         1686  +
         1687  +/*
         1688  +** Finalize the statement handle at pCsr->pStmt.
         1689  +**
         1690  +** Or, if that statement handle is one created by fts3CursorSeekStmt(),
         1691  +** and the Fts3Table.pSeekStmt slot is currently NULL, save the statement
         1692  +** pointer there instead of finalizing it.
         1693  +*/
         1694  +static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){
         1695  +  if( pCsr->bSeekStmt ){
         1696  +    Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
         1697  +    if( p->pSeekStmt==0 ){
         1698  +      p->pSeekStmt = pCsr->pStmt;
         1699  +      sqlite3_reset(pCsr->pStmt);
         1700  +      pCsr->pStmt = 0;
         1701  +    }
         1702  +    pCsr->bSeekStmt = 0;
         1703  +  }
         1704  +  sqlite3_finalize(pCsr->pStmt);
         1705  +}
  1682   1706   
  1683   1707   /*
  1684   1708   ** Close the cursor.  For additional information see the documentation
  1685   1709   ** on the xClose method of the virtual table interface.
  1686   1710   */
  1687   1711   static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
  1688   1712     Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  1689   1713     assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  1690         -  sqlite3_finalize(pCsr->pStmt);
         1714  +  fts3CursorFinalizeStmt(pCsr);
  1691   1715     sqlite3Fts3ExprFree(pCsr->pExpr);
  1692   1716     sqlite3Fts3FreeDeferredTokens(pCsr);
  1693   1717     sqlite3_free(pCsr->aDoclist);
  1694   1718     sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
  1695   1719     assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  1696   1720     sqlite3_free(pCsr);
  1697   1721     return SQLITE_OK;
................................................................................
  1701   1725   ** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then
  1702   1726   ** compose and prepare an SQL statement of the form:
  1703   1727   **
  1704   1728   **    "SELECT <columns> FROM %_content WHERE rowid = ?"
  1705   1729   **
  1706   1730   ** (or the equivalent for a content=xxx table) and set pCsr->pStmt to
  1707   1731   ** it. If an error occurs, return an SQLite error code.
  1708         -**
  1709         -** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK.
  1710   1732   */
  1711         -static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){
         1733  +static int fts3CursorSeekStmt(Fts3Cursor *pCsr){
  1712   1734     int rc = SQLITE_OK;
  1713   1735     if( pCsr->pStmt==0 ){
  1714   1736       Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
  1715   1737       char *zSql;
  1716         -    zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
  1717         -    if( !zSql ) return SQLITE_NOMEM;
  1718         -    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
  1719         -    sqlite3_free(zSql);
         1738  +    if( p->pSeekStmt ){
         1739  +      pCsr->pStmt = p->pSeekStmt;
         1740  +      p->pSeekStmt = 0;
         1741  +    }else{
         1742  +      zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
         1743  +      if( !zSql ) return SQLITE_NOMEM;
         1744  +      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
         1745  +      sqlite3_free(zSql);
         1746  +    }
         1747  +    if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1;
  1720   1748     }
  1721         -  *ppStmt = pCsr->pStmt;
  1722   1749     return rc;
  1723   1750   }
  1724   1751   
  1725   1752   /*
  1726   1753   ** Position the pCsr->pStmt statement so that it is on the row
  1727   1754   ** of the %_content table that contains the last match.  Return
  1728   1755   ** SQLITE_OK on success.  
  1729   1756   */
  1730   1757   static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
  1731   1758     int rc = SQLITE_OK;
  1732   1759     if( pCsr->isRequireSeek ){
  1733         -    sqlite3_stmt *pStmt = 0;
  1734         -
  1735         -    rc = fts3CursorSeekStmt(pCsr, &pStmt);
         1760  +    rc = fts3CursorSeekStmt(pCsr);
  1736   1761       if( rc==SQLITE_OK ){
  1737   1762         sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
  1738   1763         pCsr->isRequireSeek = 0;
  1739   1764         if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
  1740   1765           return SQLITE_OK;
  1741   1766         }else{
  1742   1767           rc = sqlite3_reset(pCsr->pStmt);
................................................................................
  3186   3211     if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++];
  3187   3212     if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++];
  3188   3213     if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++];
  3189   3214     if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++];
  3190   3215     assert( iIdx==nVal );
  3191   3216   
  3192   3217     /* In case the cursor has been used before, clear it now. */
  3193         -  sqlite3_finalize(pCsr->pStmt);
         3218  +  fts3CursorFinalizeStmt(pCsr);
  3194   3219     sqlite3_free(pCsr->aDoclist);
  3195   3220     sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
  3196   3221     sqlite3Fts3ExprFree(pCsr->pExpr);
  3197   3222     memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
  3198   3223   
  3199   3224     /* Set the lower and upper bounds on docids to return */
  3200   3225     pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64);
................................................................................
  3254   3279       if( zSql ){
  3255   3280         rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
  3256   3281         sqlite3_free(zSql);
  3257   3282       }else{
  3258   3283         rc = SQLITE_NOMEM;
  3259   3284       }
  3260   3285     }else if( eSearch==FTS3_DOCID_SEARCH ){
  3261         -    rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt);
         3286  +    rc = fts3CursorSeekStmt(pCsr);
  3262   3287       if( rc==SQLITE_OK ){
  3263   3288         rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons);
  3264   3289       }
  3265   3290     }
  3266   3291     if( rc!=SQLITE_OK ) return rc;
  3267   3292   
  3268   3293     return fts3NextMethod(pCursor);
................................................................................
  3382   3407     ** of blocks from the segments table. But this is not considered overhead
  3383   3408     ** as it would also be required by a crisis-merge that used the same input 
  3384   3409     ** segments.
  3385   3410     */
  3386   3411     const u32 nMinMerge = 64;       /* Minimum amount of incr-merge work to do */
  3387   3412   
  3388   3413     Fts3Table *p = (Fts3Table*)pVtab;
  3389         -  int rc = sqlite3Fts3PendingTermsFlush(p);
         3414  +  int rc;
         3415  +  i64 iLastRowid = sqlite3_last_insert_rowid(p->db);
  3390   3416   
         3417  +  rc = sqlite3Fts3PendingTermsFlush(p);
  3391   3418     if( rc==SQLITE_OK 
  3392   3419      && p->nLeafAdd>(nMinMerge/16) 
  3393   3420      && p->nAutoincrmerge && p->nAutoincrmerge!=0xff
  3394   3421     ){
  3395   3422       int mxLevel = 0;              /* Maximum relative level value in db */
  3396   3423       int A;                        /* Incr-merge parameter A */
  3397   3424   
................................................................................
  3398   3425       rc = sqlite3Fts3MaxLevel(p, &mxLevel);
  3399   3426       assert( rc==SQLITE_OK || mxLevel==0 );
  3400   3427       A = p->nLeafAdd * mxLevel;
  3401   3428       A += (A/2);
  3402   3429       if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge);
  3403   3430     }
  3404   3431     sqlite3Fts3SegmentsClose(p);
         3432  +  sqlite3_set_last_insert_rowid(p->db, iLastRowid);
  3405   3433     return rc;
  3406   3434   }
  3407   3435   
  3408   3436   /*
  3409   3437   ** If it is currently unknown whether or not the FTS table has an %_stat
  3410   3438   ** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat
  3411   3439   ** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code
................................................................................
  3418   3446       char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName);
  3419   3447       if( zSql ){
  3420   3448         sqlite3_stmt *pStmt = 0;
  3421   3449         rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
  3422   3450         if( rc==SQLITE_OK ){
  3423   3451           int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW);
  3424   3452           rc = sqlite3_finalize(pStmt);
  3425         -        if( rc==SQLITE_OK ) p->bHasStat = bHasStat;
         3453  +        if( rc==SQLITE_OK ) p->bHasStat = (u8)bHasStat;
  3426   3454         }
  3427   3455         sqlite3_free(zSql);
  3428   3456       }else{
  3429   3457         rc = SQLITE_NOMEM;
  3430   3458       }
  3431   3459     }
  3432   3460     return rc;

Changes to ext/fts3/fts3Int.h.

   226    226     int nAutoincrmerge;             /* Value configured by 'automerge' */
   227    227     u32 nLeafAdd;                   /* Number of leaf blocks added this trans */
   228    228   
   229    229     /* Precompiled statements used by the implementation. Each of these 
   230    230     ** statements is run and reset within a single virtual table API call. 
   231    231     */
   232    232     sqlite3_stmt *aStmt[40];
          233  +  sqlite3_stmt *pSeekStmt;        /* Cache for fts3CursorSeekStmt() */
   233    234   
   234    235     char *zReadExprlist;
   235    236     char *zWriteExprlist;
   236    237   
   237    238     int nNodeSize;                  /* Soft limit for node size */
   238    239     u8 bFts4;                       /* True for FTS4, false for FTS3 */
   239    240     u8 bHasStat;                    /* True if %_stat table exists (2==unknown) */
................................................................................
   295    296   ** the xOpen method. Cursors are destroyed using the xClose method.
   296    297   */
   297    298   struct Fts3Cursor {
   298    299     sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
   299    300     i16 eSearch;                    /* Search strategy (see below) */
   300    301     u8 isEof;                       /* True if at End Of Results */
   301    302     u8 isRequireSeek;               /* True if must seek pStmt to %_content row */
          303  +  u8 bSeekStmt;                   /* True if pStmt is a seek */
   302    304     sqlite3_stmt *pStmt;            /* Prepared statement in use by the cursor */
   303    305     Fts3Expr *pExpr;                /* Parsed MATCH query string */
   304    306     int iLangid;                    /* Language being queried for */
   305    307     int nPhrase;                    /* Number of matchable phrases in query */
   306    308     Fts3DeferredToken *pDeferred;   /* Deferred search tokens, if any */
   307    309     sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
   308    310     char *pNextId;                  /* Pointer into the body of aDoclist */

Changes to ext/fts3/fts3_test.c.

    14     14   ** testing. It contains a Tcl command that can be used to test if a document
    15     15   ** matches an FTS NEAR expression.
    16     16   **
    17     17   ** As of March 2012, it also contains a version 1 tokenizer used for testing
    18     18   ** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly.
    19     19   */
    20     20   
    21         -#include <tcl.h>
           21  +#if defined(INCLUDE_SQLITE_TCL_H)
           22  +#  include "sqlite_tcl.h"
           23  +#else
           24  +#  include "tcl.h"
           25  +#  ifndef SQLITE_TCLAPI
           26  +#    define SQLITE_TCLAPI
           27  +#  endif
           28  +#endif
    22     29   #include <string.h>
    23     30   #include <assert.h>
    24     31   
    25     32   #if defined(SQLITE_TEST)
    26     33   #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
    27     34   
    28     35   /* Required so that the "ifdef SQLITE_ENABLE_FTS3" below works */
................................................................................
   139    146   
   140    147     return nOcc;
   141    148   }
   142    149   
   143    150   /*
   144    151   ** Tclcmd: fts3_near_match DOCUMENT EXPR ?OPTIONS?
   145    152   */
   146         -static int fts3_near_match_cmd(
          153  +static int SQLITE_TCLAPI fts3_near_match_cmd(
   147    154     ClientData clientData,
   148    155     Tcl_Interp *interp,
   149    156     int objc,
   150    157     Tcl_Obj *CONST objv[]
   151    158   ){
   152    159     int nTotal = 0;
   153    160     int rc;
................................................................................
   274    281   **    set cfg [fts3_configure_incr_load $new_chunksize $new_threshold]
   275    282   **
   276    283   **    .... run tests ....
   277    284   **
   278    285   **    # Restore initial incr-load settings:
   279    286   **    eval fts3_configure_incr_load $cfg
   280    287   */
   281         -static int fts3_configure_incr_load_cmd(
          288  +static int SQLITE_TCLAPI fts3_configure_incr_load_cmd(
   282    289     ClientData clientData,
   283    290     Tcl_Interp *interp,
   284    291     int objc,
   285    292     Tcl_Obj *CONST objv[]
   286    293   ){
   287    294   #ifdef SQLITE_ENABLE_FTS3
   288    295     extern int test_fts3_node_chunksize;
................................................................................
   454    461         rc = SQLITE_NOMEM;
   455    462       }else{
   456    463         int i;
   457    464   
   458    465         if( pCsr->iLangid & 0x00000001 ){
   459    466           for(i=0; i<nToken; i++) pCsr->aBuffer[i] = pToken[i];
   460    467         }else{
   461         -        for(i=0; i<nToken; i++) pCsr->aBuffer[i] = testTolower(pToken[i]);
          468  +        for(i=0; i<nToken; i++) pCsr->aBuffer[i] = (char)testTolower(pToken[i]);
   462    469         }
   463    470         pCsr->iToken++;
   464    471         pCsr->iInput = (int)(p - pCsr->aInput);
   465    472   
   466    473         *ppToken = pCsr->aBuffer;
   467    474         *pnBytes = nToken;
   468    475         *piStartOffset = (int)(pToken - pCsr->aInput);
................................................................................
   484    491     if( pCsr->iLangid>=100 ){
   485    492       rc = SQLITE_ERROR;
   486    493     }
   487    494     return rc;
   488    495   }
   489    496   #endif
   490    497   
   491         -static int fts3_test_tokenizer_cmd(
          498  +static int SQLITE_TCLAPI fts3_test_tokenizer_cmd(
   492    499     ClientData clientData,
   493    500     Tcl_Interp *interp,
   494    501     int objc,
   495    502     Tcl_Obj *CONST objv[]
   496    503   ){
   497    504   #ifdef SQLITE_ENABLE_FTS3
   498    505     static const sqlite3_tokenizer_module testTokenizerModule = {
................................................................................
   513    520       (const unsigned char *)&pPtr, sizeof(sqlite3_tokenizer_module *)
   514    521     ));
   515    522   #endif
   516    523     UNUSED_PARAMETER(clientData);
   517    524     return TCL_OK;
   518    525   }
   519    526   
   520         -static int fts3_test_varint_cmd(
          527  +static int SQLITE_TCLAPI fts3_test_varint_cmd(
   521    528     ClientData clientData,
   522    529     Tcl_Interp *interp,
   523    530     int objc,
   524    531     Tcl_Obj *CONST objv[]
   525    532   ){
   526    533   #ifdef SQLITE_ENABLE_FTS3
   527    534     char aBuf[24];

Changes to ext/fts3/fts3_tokenizer.c.

   220    220     sqlite3_free(zCopy);
   221    221     return rc;
   222    222   }
   223    223   
   224    224   
   225    225   #ifdef SQLITE_TEST
   226    226   
   227         -#include <tcl.h>
          227  +#if defined(INCLUDE_SQLITE_TCL_H)
          228  +#  include "sqlite_tcl.h"
          229  +#else
          230  +#  include "tcl.h"
          231  +#endif
   228    232   #include <string.h>
   229    233   
   230    234   /*
   231    235   ** Implementation of a special SQL scalar function for testing tokenizers 
   232    236   ** designed to be used in concert with the Tcl testing framework. This
   233    237   ** function must be called with two or more arguments:
   234    238   **

Changes to ext/fts3/fts3_unicode.c.

   132    132     unicode_tokenizer *p,           /* Tokenizer to add exceptions to */
   133    133     int bAlnum,                     /* Replace Isalnum() return value with this */
   134    134     const char *zIn,                /* Array of characters to make exceptions */
   135    135     int nIn                         /* Length of z in bytes */
   136    136   ){
   137    137     const unsigned char *z = (const unsigned char *)zIn;
   138    138     const unsigned char *zTerm = &z[nIn];
   139         -  int iCode;
          139  +  unsigned int iCode;
   140    140     int nEntry = 0;
   141    141   
   142    142     assert( bAlnum==0 || bAlnum==1 );
   143    143   
   144    144     while( z<zTerm ){
   145    145       READ_UTF8(z, zTerm, iCode);
   146         -    assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
   147         -    if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum 
   148         -     && sqlite3FtsUnicodeIsdiacritic(iCode)==0 
          146  +    assert( (sqlite3FtsUnicodeIsalnum((int)iCode) & 0xFFFFFFFE)==0 );
          147  +    if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum 
          148  +     && sqlite3FtsUnicodeIsdiacritic((int)iCode)==0 
   149    149       ){
   150    150         nEntry++;
   151    151       }
   152    152     }
   153    153   
   154    154     if( nEntry ){
   155    155       int *aNew;                    /* New aiException[] array */
................................................................................
   158    158       aNew = sqlite3_realloc(p->aiException, (p->nException+nEntry)*sizeof(int));
   159    159       if( aNew==0 ) return SQLITE_NOMEM;
   160    160       nNew = p->nException;
   161    161   
   162    162       z = (const unsigned char *)zIn;
   163    163       while( z<zTerm ){
   164    164         READ_UTF8(z, zTerm, iCode);
   165         -      if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum 
   166         -       && sqlite3FtsUnicodeIsdiacritic(iCode)==0
          165  +      if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum 
          166  +       && sqlite3FtsUnicodeIsdiacritic((int)iCode)==0
   167    167         ){
   168    168           int i, j;
   169         -        for(i=0; i<nNew && aNew[i]<iCode; i++);
          169  +        for(i=0; i<nNew && aNew[i]<(int)iCode; i++);
   170    170           for(j=nNew; j>i; j--) aNew[j] = aNew[j-1];
   171         -        aNew[i] = iCode;
          171  +        aNew[i] = (int)iCode;
   172    172           nNew++;
   173    173         }
   174    174       }
   175    175       p->aiException = aNew;
   176    176       p->nException = nNew;
   177    177     }
   178    178   
................................................................................
   314    314     int *pnToken,                   /* OUT: Number of bytes at *paToken */
   315    315     int *piStart,                   /* OUT: Starting offset of token */
   316    316     int *piEnd,                     /* OUT: Ending offset of token */
   317    317     int *piPos                      /* OUT: Position integer of token */
   318    318   ){
   319    319     unicode_cursor *pCsr = (unicode_cursor *)pC;
   320    320     unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer);
   321         -  int iCode = 0;
          321  +  unsigned int iCode = 0;
   322    322     char *zOut;
   323    323     const unsigned char *z = &pCsr->aInput[pCsr->iOff];
   324    324     const unsigned char *zStart = z;
   325    325     const unsigned char *zEnd;
   326    326     const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput];
   327    327   
   328    328     /* Scan past any delimiter characters before the start of the next token.
   329    329     ** Return SQLITE_DONE early if this takes us all the way to the end of 
   330    330     ** the input.  */
   331    331     while( z<zTerm ){
   332    332       READ_UTF8(z, zTerm, iCode);
   333         -    if( unicodeIsAlnum(p, iCode) ) break;
          333  +    if( unicodeIsAlnum(p, (int)iCode) ) break;
   334    334       zStart = z;
   335    335     }
   336    336     if( zStart>=zTerm ) return SQLITE_DONE;
   337    337   
   338    338     zOut = pCsr->zToken;
   339    339     do {
   340    340       int iOut;
................................................................................
   346    346         zOut = &zNew[zOut - pCsr->zToken];
   347    347         pCsr->zToken = zNew;
   348    348         pCsr->nAlloc += 64;
   349    349       }
   350    350   
   351    351       /* Write the folded case of the last character read to the output */
   352    352       zEnd = z;
   353         -    iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic);
          353  +    iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
   354    354       if( iOut ){
   355    355         WRITE_UTF8(zOut, iOut);
   356    356       }
   357    357   
   358    358       /* If the cursor is not at EOF, read the next character */
   359    359       if( z>=zTerm ) break;
   360    360       READ_UTF8(z, zTerm, iCode);
   361         -  }while( unicodeIsAlnum(p, iCode) 
   362         -       || sqlite3FtsUnicodeIsdiacritic(iCode)
          361  +  }while( unicodeIsAlnum(p, (int)iCode) 
          362  +       || sqlite3FtsUnicodeIsdiacritic((int)iCode)
   363    363     );
   364    364   
   365    365     /* Set the output variables and return. */
   366    366     pCsr->iOff = (int)(z - pCsr->aInput);
   367    367     *paToken = pCsr->zToken;
   368    368     *pnToken = (int)(zOut - pCsr->zToken);
   369    369     *piStart = (int)(zStart - pCsr->aInput);

Changes to ext/fts3/fts3_unicode2.c.

   123    123       0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060,
   124    124       0x380400F0,
   125    125     };
   126    126     static const unsigned int aAscii[4] = {
   127    127       0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
   128    128     };
   129    129   
   130         -  if( c<128 ){
   131         -    return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
   132         -  }else if( c<(1<<22) ){
          130  +  if( (unsigned int)c<128 ){
          131  +    return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 );
          132  +  }else if( (unsigned int)c<(1<<22) ){
   133    133       unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
   134    134       int iRes = 0;
   135    135       int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
   136    136       int iLo = 0;
   137    137       while( iHi>=iLo ){
   138    138         int iTest = (iHi + iLo) / 2;
   139    139         if( key >= aEntry[iTest] ){
................................................................................
   318    318      65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, 
   319    319      65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, 
   320    320      65514, 65521, 65527, 65528, 65529, 
   321    321     };
   322    322   
   323    323     int ret = c;
   324    324   
   325         -  assert( c>=0 );
   326    325     assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );
   327    326   
   328    327     if( c<128 ){
   329    328       if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
   330    329     }else if( c<65536 ){
          330  +    const struct TableEntry *p;
   331    331       int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
   332    332       int iLo = 0;
   333    333       int iRes = -1;
   334    334   
          335  +    assert( c>aEntry[0].iCode );
   335    336       while( iHi>=iLo ){
   336    337         int iTest = (iHi + iLo) / 2;
   337    338         int cmp = (c - aEntry[iTest].iCode);
   338    339         if( cmp>=0 ){
   339    340           iRes = iTest;
   340    341           iLo = iTest+1;
   341    342         }else{
   342    343           iHi = iTest-1;
   343    344         }
   344    345       }
   345         -    assert( iRes<0 || c>=aEntry[iRes].iCode );
   346    346   
   347         -    if( iRes>=0 ){
   348         -      const struct TableEntry *p = &aEntry[iRes];
   349         -      if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
   350         -        ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
   351         -        assert( ret>0 );
   352         -      }
          347  +    assert( iRes>=0 && c>=aEntry[iRes].iCode );
          348  +    p = &aEntry[iRes];
          349  +    if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
          350  +      ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
          351  +      assert( ret>0 );
   353    352       }
   354    353   
   355    354       if( bRemoveDiacritic ) ret = remove_diacritic(ret);
   356    355     }
   357    356     
   358    357     else if( c>=66560 && c<66600 ){
   359    358       ret = c + 40;
   360    359     }
   361    360   
   362    361     return ret;
   363    362   }
   364    363   #endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */
   365    364   #endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */

Changes to ext/fts3/fts3_write.c.

  4952   4952     return rc;
  4953   4953   }
  4954   4954   
  4955   4955   /*
  4956   4956   ** Convert the text beginning at *pz into an integer and return
  4957   4957   ** its value.  Advance *pz to point to the first character past
  4958   4958   ** the integer.
         4959  +**
         4960  +** This function used for parameters to merge= and incrmerge=
         4961  +** commands. 
  4959   4962   */
  4960   4963   static int fts3Getint(const char **pz){
  4961   4964     const char *z = *pz;
  4962   4965     int i = 0;
  4963         -  while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0';
         4966  +  while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0';
  4964   4967     *pz = z;
  4965   4968     return i;
  4966   4969   }
  4967   4970   
  4968   4971   /*
  4969   4972   ** Process statements of the form:
  4970   4973   **

Added ext/fts3/tool/fts3cov.sh.

            1  +#!/bin/sh
            2  +
            3  +set -e
            4  +
            5  +srcdir=`dirname $(dirname $(dirname $(dirname $0)))`
            6  +./testfixture $srcdir/test/fts3.test --output=fts3cov-out.txt
            7  +
            8  +echo ""
            9  +
           10  +for f in `ls $srcdir/ext/fts3/*.c` 
           11  +do
           12  +  f=`basename $f`
           13  +  echo -ne "$f: "
           14  +  gcov -b $f | grep Taken | sed 's/Taken at least once://'
           15  +done
           16  +

Changes to ext/fts3/tool/fts3view.c.

   394    394              "         WHERE (a.blockid BETWEEN b.start_block"
   395    395                                          " AND b.leaves_end_block)"
   396    396              "           AND (b.level%%1024)==%d)",
   397    397              pgsz-45, zTab, zTab, i);
   398    398       if( sqlite3_step(pStmt)==SQLITE_ROW
   399    399        && (nLeaf = sqlite3_column_int(pStmt, 0))>0
   400    400       ){
   401         -      nIdx = sqlite3_column_int(pStmt, 5);
   402    401         sqlite3_int64 sz;
          402  +      nIdx = sqlite3_column_int(pStmt, 5);
   403    403         printf("For level %d:\n", i);
   404    404         printf("  Number of indexes...................... %9d\n", nIdx);
   405    405         printf("  Number of leaf segments................ %9d\n", nLeaf);
   406    406         if( nIdx>1 ){
   407    407           printf("  Average leaf segments per index........ %11.1f\n",
   408    408                  (double)nLeaf/(double)nIdx);
   409    409         }

Changes to ext/fts3/unicode/mkunicode.tcl.

   223    223     puts "** is less than zero."
   224    224     puts "*/"
   225    225     puts "int ${zFunc}\(int c)\{"
   226    226     an_print_range_array $lRange
   227    227     an_print_ascii_bitmap $lRange
   228    228     puts {
   229    229     if( (unsigned int)c<128 ){
   230         -    return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
          230  +    return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 );
   231    231     }else if( (unsigned int)c<(1<<22) ){
   232    232       unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
   233    233       int iRes = 0;
   234    234       int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
   235    235       int iLo = 0;
   236    236       while( iHi>=iLo ){
   237    237         int iTest = (iHi + iLo) / 2;

Changes to ext/fts5/fts5.h.

   139    139   ** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback):
   140    140   **   This API function is used to query the FTS table for phrase iPhrase
   141    141   **   of the current query. Specifically, a query equivalent to:
   142    142   **
   143    143   **       ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid
   144    144   **
   145    145   **   with $p set to a phrase equivalent to the phrase iPhrase of the
   146         -**   current query is executed. For each row visited, the callback function
   147         -**   passed as the fourth argument is invoked. The context and API objects 
   148         -**   passed to the callback function may be used to access the properties of
   149         -**   each matched row. Invoking Api.xUserData() returns a copy of the pointer
   150         -**   passed as the third argument to pUserData.
          146  +**   current query is executed. Any column filter that applies to
          147  +**   phrase iPhrase of the current query is included in $p. For each 
          148  +**   row visited, the callback function passed as the fourth argument 
          149  +**   is invoked. The context and API objects passed to the callback 
          150  +**   function may be used to access the properties of each matched row.
          151  +**   Invoking Api.xUserData() returns a copy of the pointer passed as 
          152  +**   the third argument to pUserData.
   151    153   **
   152    154   **   If the callback function returns any value other than SQLITE_OK, the
   153    155   **   query is abandoned and the xQueryPhrase function returns immediately.
   154    156   **   If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.
   155    157   **   Otherwise, the error code is propagated upwards.
   156    158   **
   157    159   **   If the query runs to completion without incident, SQLITE_OK is returned.
................................................................................
   312    314   ** Applications may also register custom tokenizer types. A tokenizer 
   313    315   ** is registered by providing fts5 with a populated instance of the 
   314    316   ** following structure. All structure methods must be defined, setting
   315    317   ** any member of the fts5_tokenizer struct to NULL leads to undefined
   316    318   ** behaviour. The structure methods are expected to function as follows:
   317    319   **
   318    320   ** xCreate:
   319         -**   This function is used to allocate and inititalize a tokenizer instance.
          321  +**   This function is used to allocate and initialize a tokenizer instance.
   320    322   **   A tokenizer instance is required to actually tokenize text.
   321    323   **
   322    324   **   The first argument passed to this function is a copy of the (void*)
   323    325   **   pointer provided by the application when the fts5_tokenizer object
   324    326   **   was registered with FTS5 (the third argument to xCreateTokenizer()). 
   325    327   **   The second and third arguments are an array of nul-terminated strings
   326    328   **   containing the tokenizer arguments, if any, specified following the
................................................................................
   571    573   *************************************************************************/
   572    574   
   573    575   #ifdef __cplusplus
   574    576   }  /* end of the 'extern "C"' block */
   575    577   #endif
   576    578   
   577    579   #endif /* _FTS5_H */
   578         -

Changes to ext/fts5/fts5Int.h.

    26     26   typedef unsigned char  u8;
    27     27   typedef unsigned int   u32;
    28     28   typedef unsigned short u16;
    29     29   typedef short i16;
    30     30   typedef sqlite3_int64 i64;
    31     31   typedef sqlite3_uint64 u64;
    32     32   
    33         -#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
           33  +#ifndef ArraySize
           34  +# define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
           35  +#endif
    34     36   
    35     37   #define testcase(x)
    36     38   #define ALWAYS(x) 1
    37     39   #define NEVER(x) 0
    38     40   
    39     41   #define MIN(x,y) (((x) < (y)) ? (x) : (y))
    40     42   #define MAX(x,y) (((x) > (y)) ? (x) : (y))
................................................................................
    43     45   ** Constants for the largest and smallest possible 64-bit signed integers.
    44     46   */
    45     47   # define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
    46     48   # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
    47     49   
    48     50   #endif
    49     51   
           52  +/* Truncate very long tokens to this many bytes. Hard limit is 
           53  +** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset
           54  +** field that occurs at the start of each leaf page (see fts5_index.c). */
           55  +#define FTS5_MAX_TOKEN_SIZE 32768
    50     56   
    51     57   /*
    52     58   ** Maximum number of prefix indexes on single FTS5 table. This must be
    53     59   ** less than 32. If it is set to anything large than that, an #error
    54     60   ** directive in fts5_index.c will cause the build to fail.
    55     61   */
    56     62   #define FTS5_MAX_PREFIX_INDEXES 31
................................................................................
   476    482   ** this connection since it was created.
   477    483   */
   478    484   int sqlite3Fts5IndexReads(Fts5Index *p);
   479    485   
   480    486   int sqlite3Fts5IndexReinit(Fts5Index *p);
   481    487   int sqlite3Fts5IndexOptimize(Fts5Index *p);
   482    488   int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
          489  +int sqlite3Fts5IndexReset(Fts5Index *p);
   483    490   
   484    491   int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
   485    492   
   486    493   /*
   487    494   ** End of interface to code in fts5_index.c.
   488    495   **************************************************************************/
   489    496   
................................................................................
   618    625       Fts5Storage *p, const char*, sqlite3_value*, int
   619    626   );
   620    627   
   621    628   int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
   622    629   int sqlite3Fts5StorageRebuild(Fts5Storage *p);
   623    630   int sqlite3Fts5StorageOptimize(Fts5Storage *p);
   624    631   int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
          632  +int sqlite3Fts5StorageReset(Fts5Storage *p);
   625    633   
   626    634   /*
   627    635   ** End of interface to code in fts5_storage.c.
   628    636   **************************************************************************/
   629    637   
   630    638   
   631    639   /**************************************************************************
................................................................................
   676    684   
   677    685   typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
   678    686   Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
   679    687   int sqlite3Fts5ExprPopulatePoslists(
   680    688       Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
   681    689   );
   682    690   void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
   683         -void sqlite3Fts5ExprClearEof(Fts5Expr*);
   684    691   
   685    692   int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
   686    693   
   687    694   int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
   688    695   
   689    696   /*******************************************
   690    697   ** The fts5_expr.c API above this point is used by the other hand-written
................................................................................
   728    735   
   729    736   void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
   730    737   void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
   731    738   void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
   732    739   
   733    740   void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
   734    741   void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
          742  +Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
   735    743   void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
   736    744   void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
   737    745   
   738    746   /*
   739    747   ** End of interface to code in fts5_expr.c.
   740    748   **************************************************************************/
   741    749   

Changes to ext/fts5/fts5_aux.c.

   185    185         rc = fts5CInstIterNext(&p->iter);
   186    186       }
   187    187     }
   188    188   
   189    189     if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){
   190    190       fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
   191    191       p->iOff = iEndOff;
   192         -    if( iPos<p->iter.iEnd ){
          192  +    if( iPos>=p->iter.iStart && iPos<p->iter.iEnd ){
   193    193         fts5HighlightAppend(&rc, p, p->zClose, -1);
   194    194       }
   195    195     }
   196    196   
   197    197     return rc;
   198    198   }
   199    199   
................................................................................
   241    241     if( rc!=SQLITE_OK ){
   242    242       sqlite3_result_error_code(pCtx, rc);
   243    243     }
   244    244   }
   245    245   /*
   246    246   ** End of highlight() implementation.
   247    247   **************************************************************************/
          248  +
          249  +/*
          250  +** Context object passed to the fts5SentenceFinderCb() function.
          251  +*/
          252  +typedef struct Fts5SFinder Fts5SFinder;
          253  +struct Fts5SFinder {
          254  +  int iPos;                       /* Current token position */
          255  +  int nFirstAlloc;                /* Allocated size of aFirst[] */
          256  +  int nFirst;                     /* Number of entries in aFirst[] */
          257  +  int *aFirst;                    /* Array of first token in each sentence */
          258  +  const char *zDoc;               /* Document being tokenized */
          259  +};
          260  +
          261  +/*
          262  +** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if
          263  +** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an
          264  +** error occurs.
          265  +*/
          266  +static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){
          267  +  if( p->nFirstAlloc==p->nFirst ){
          268  +    int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64;
          269  +    int *aNew;
          270  +
          271  +    aNew = (int*)sqlite3_realloc(p->aFirst, nNew*sizeof(int));
          272  +    if( aNew==0 ) return SQLITE_NOMEM;
          273  +    p->aFirst = aNew;
          274  +    p->nFirstAlloc = nNew;
          275  +  }
          276  +  p->aFirst[p->nFirst++] = iAdd;
          277  +  return SQLITE_OK;
          278  +}
          279  +
          280  +/*
          281  +** This function is an xTokenize() callback used by the auxiliary snippet()
          282  +** function. Its job is to identify tokens that are the first in a sentence.
          283  +** For each such token, an entry is added to the SFinder.aFirst[] array.
          284  +*/
          285  +static int fts5SentenceFinderCb(
          286  +  void *pContext,                 /* Pointer to HighlightContext object */
          287  +  int tflags,                     /* Mask of FTS5_TOKEN_* flags */
          288  +  const char *pToken,             /* Buffer containing token */
          289  +  int nToken,                     /* Size of token in bytes */
          290  +  int iStartOff,                  /* Start offset of token */
          291  +  int iEndOff                     /* End offset of token */
          292  +){
          293  +  int rc = SQLITE_OK;
          294  +
          295  +  UNUSED_PARAM2(pToken, nToken);
          296  +  UNUSED_PARAM(iEndOff);
          297  +
          298  +  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){
          299  +    Fts5SFinder *p = (Fts5SFinder*)pContext;
          300  +    if( p->iPos>0 ){
          301  +      int i;
          302  +      char c = 0;
          303  +      for(i=iStartOff-1; i>=0; i--){
          304  +        c = p->zDoc[i];
          305  +        if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break;
          306  +      }
          307  +      if( i!=iStartOff-1 && (c=='.' || c==':') ){
          308  +        rc = fts5SentenceFinderAdd(p, p->iPos);
          309  +      }
          310  +    }else{
          311  +      rc = fts5SentenceFinderAdd(p, 0);
          312  +    }
          313  +    p->iPos++;
          314  +  }
          315  +  return rc;
          316  +}
          317  +
          318  +static int fts5SnippetScore(
          319  +  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
          320  +  Fts5Context *pFts,              /* First arg to pass to pApi functions */
          321  +  int nDocsize,                   /* Size of column in tokens */
          322  +  unsigned char *aSeen,           /* Array with one element per query phrase */
          323  +  int iCol,                       /* Column to score */
          324  +  int iPos,                       /* Starting offset to score */
          325  +  int nToken,                     /* Max tokens per snippet */
          326  +  int *pnScore,                   /* OUT: Score */
          327  +  int *piPos                      /* OUT: Adjusted offset */
          328  +){
          329  +  int rc;
          330  +  int i;
          331  +  int ip = 0;
          332  +  int ic = 0;
          333  +  int iOff = 0;
          334  +  int iFirst = -1;
          335  +  int nInst;
          336  +  int nScore = 0;
          337  +  int iLast = 0;
          338  +
          339  +  rc = pApi->xInstCount(pFts, &nInst);
          340  +  for(i=0; i<nInst && rc==SQLITE_OK; i++){
          341  +    rc = pApi->xInst(pFts, i, &ip, &ic, &iOff);
          342  +    if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<(iPos+nToken) ){
          343  +      nScore += (aSeen[ip] ? 1 : 1000);
          344  +      aSeen[ip] = 1;
          345  +      if( iFirst<0 ) iFirst = iOff;
          346  +      iLast = iOff + pApi->xPhraseSize(pFts, ip);
          347  +    }
          348  +  }
          349  +
          350  +  *pnScore = nScore;
          351  +  if( piPos ){
          352  +    int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2;
          353  +    if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
          354  +    if( iAdj<0 ) iAdj = 0;
          355  +    *piPos = iAdj;
          356  +  }
          357  +
          358  +  return rc;
          359  +}
   248    360   
   249    361   /*
   250    362   ** Implementation of snippet() function.
   251    363   */
   252    364   static void fts5SnippetFunction(
   253    365     const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
   254    366     Fts5Context *pFts,              /* First arg to pass to pApi functions */
................................................................................
   263    375     int nToken;                     /* 5th argument to snippet() */
   264    376     int nInst = 0;                  /* Number of instance matches this row */
   265    377     int i;                          /* Used to iterate through instances */
   266    378     int nPhrase;                    /* Number of phrases in query */
   267    379     unsigned char *aSeen;           /* Array of "seen instance" flags */
   268    380     int iBestCol;                   /* Column containing best snippet */
   269    381     int iBestStart = 0;             /* First token of best snippet */
   270         -  int iBestLast;                  /* Last token of best snippet */
   271    382     int nBestScore = 0;             /* Score of best snippet */
   272    383     int nColSize = 0;               /* Total size of iBestCol in tokens */
          384  +  Fts5SFinder sFinder;            /* Used to find the beginnings of sentences */
          385  +  int nCol;
   273    386   
   274    387     if( nVal!=5 ){
   275    388       const char *zErr = "wrong number of arguments to function snippet()";
   276    389       sqlite3_result_error(pCtx, zErr, -1);
   277    390       return;
   278    391     }
   279    392   
          393  +  nCol = pApi->xColumnCount(pFts);
   280    394     memset(&ctx, 0, sizeof(HighlightContext));
   281    395     iCol = sqlite3_value_int(apVal[0]);
   282    396     ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
   283    397     ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
   284    398     zEllips = (const char*)sqlite3_value_text(apVal[3]);
   285    399     nToken = sqlite3_value_int(apVal[4]);
   286         -  iBestLast = nToken-1;
   287    400   
   288    401     iBestCol = (iCol>=0 ? iCol : 0);
   289    402     nPhrase = pApi->xPhraseCount(pFts);
   290    403     aSeen = sqlite3_malloc(nPhrase);
   291    404     if( aSeen==0 ){
   292    405       rc = SQLITE_NOMEM;
   293    406     }
   294         -
   295    407     if( rc==SQLITE_OK ){
   296    408       rc = pApi->xInstCount(pFts, &nInst);
   297    409     }
   298         -  for(i=0; rc==SQLITE_OK && i<nInst; i++){
   299         -    int ip, iSnippetCol, iStart;
   300         -    memset(aSeen, 0, nPhrase);
   301         -    rc = pApi->xInst(pFts, i, &ip, &iSnippetCol, &iStart);
   302         -    if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){
   303         -      int nScore = 1000;
   304         -      int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip);
   305         -      int j;
   306         -      aSeen[ip] = 1;
   307         -
   308         -      for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
   309         -        int ic; int io; int iFinal;
   310         -        rc = pApi->xInst(pFts, j, &ip, &ic, &io);
   311         -        iFinal = io + pApi->xPhraseSize(pFts, ip) - 1;
   312         -        if( rc==SQLITE_OK && ic==iSnippetCol && iLast<iStart+nToken ){
   313         -          nScore += aSeen[ip] ? 1000 : 1;
   314         -          aSeen[ip] = 1;
   315         -          if( iFinal>iLast ) iLast = iFinal;
          410  +
          411  +  memset(&sFinder, 0, sizeof(Fts5SFinder));
          412  +  for(i=0; i<nCol; i++){
          413  +    if( iCol<0 || iCol==i ){
          414  +      int nDoc;
          415  +      int nDocsize;
          416  +      int ii;
          417  +      sFinder.iPos = 0;
          418  +      sFinder.nFirst = 0;
          419  +      rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
          420  +      if( rc!=SQLITE_OK ) break;
          421  +      rc = pApi->xTokenize(pFts, 
          422  +          sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb
          423  +      );
          424  +      if( rc!=SQLITE_OK ) break;
          425  +      rc = pApi->xColumnSize(pFts, i, &nDocsize);
          426  +      if( rc!=SQLITE_OK ) break;
          427  +
          428  +      for(ii=0; rc==SQLITE_OK && ii<nInst; ii++){
          429  +        int ip, ic, io;
          430  +        int iAdj;
          431  +        int nScore;
          432  +        int jj;
          433  +
          434  +        rc = pApi->xInst(pFts, ii, &ip, &ic, &io);
          435  +        if( ic!=i || rc!=SQLITE_OK ) continue;
          436  +        memset(aSeen, 0, nPhrase);
          437  +        rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
          438  +            io, nToken, &nScore, &iAdj
          439  +        );
          440  +        if( rc==SQLITE_OK && nScore>nBestScore ){
          441  +          nBestScore = nScore;
          442  +          iBestCol = i;
          443  +          iBestStart = iAdj;
          444  +          nColSize = nDocsize;
   316    445           }
   317         -      }
   318    446   
   319         -      if( rc==SQLITE_OK && nScore>nBestScore ){
   320         -        iBestCol = iSnippetCol;
   321         -        iBestStart = iStart;
   322         -        iBestLast = iLast;
   323         -        nBestScore = nScore;
          447  +        if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){
          448  +          for(jj=0; jj<(sFinder.nFirst-1); jj++){
          449  +            if( sFinder.aFirst[jj+1]>io ) break;
          450  +          }
          451  +
          452  +          if( sFinder.aFirst[jj]<io ){
          453  +            memset(aSeen, 0, nPhrase);
          454  +            rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, 
          455  +              sFinder.aFirst[jj], nToken, &nScore, 0
          456  +            );
          457  +
          458  +            nScore += (sFinder.aFirst[jj]==0 ? 120 : 100);
          459  +            if( rc==SQLITE_OK && nScore>nBestScore ){
          460  +              nBestScore = nScore;
          461  +              iBestCol = i;
          462  +              iBestStart = sFinder.aFirst[jj];
          463  +              nColSize = nDocsize;
          464  +            }
          465  +          }
          466  +        }
   324    467         }
   325    468       }
   326    469     }
   327    470   
   328    471     if( rc==SQLITE_OK ){
   329         -    rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
          472  +    rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
   330    473     }
   331         -  if( rc==SQLITE_OK ){
   332         -    rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
          474  +  if( rc==SQLITE_OK && nColSize==0 ){
          475  +    rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
   333    476     }
   334    477     if( ctx.zIn ){
   335    478       if( rc==SQLITE_OK ){
   336    479         rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
   337    480       }
   338    481   
   339         -    if( (iBestStart+nToken-1)>iBestLast ){
   340         -      iBestStart -= (iBestStart+nToken-1-iBestLast) / 2;
   341         -    }
   342         -    if( iBestStart+nToken>nColSize ){
   343         -      iBestStart = nColSize - nToken;
   344         -    }
   345         -    if( iBestStart<0 ) iBestStart = 0;
   346         -
   347    482       ctx.iRangeStart = iBestStart;
   348    483       ctx.iRangeEnd = iBestStart + nToken - 1;
   349    484   
   350    485       if( iBestStart>0 ){
   351    486         fts5HighlightAppend(&rc, &ctx, zEllips, -1);
   352    487       }
          488  +
          489  +    /* Advance iterator ctx.iter so that it points to the first coalesced
          490  +    ** phrase instance at or following position iBestStart. */
          491  +    while( ctx.iter.iStart>=0 && ctx.iter.iStart<iBestStart && rc==SQLITE_OK ){
          492  +      rc = fts5CInstIterNext(&ctx.iter);
          493  +    }
          494  +
   353    495       if( rc==SQLITE_OK ){
   354    496         rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
   355    497       }
   356    498       if( ctx.iRangeEnd>=(nColSize-1) ){
   357    499         fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
   358    500       }else{
   359    501         fts5HighlightAppend(&rc, &ctx, zEllips, -1);
   360    502       }
   361         -
   362         -    if( rc==SQLITE_OK ){
   363         -      sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
   364         -    }else{
   365         -      sqlite3_result_error_code(pCtx, rc);
   366         -    }
   367         -    sqlite3_free(ctx.zOut);
          503  +  }
          504  +  if( rc==SQLITE_OK ){
          505  +    sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
          506  +  }else{
          507  +    sqlite3_result_error_code(pCtx, rc);
   368    508     }
          509  +  sqlite3_free(ctx.zOut);
   369    510     sqlite3_free(aSeen);
          511  +  sqlite3_free(sFinder.aFirst);
   370    512   }
   371    513   
   372    514   /************************************************************************/
   373    515   
   374    516   /*
   375    517   ** The first time the bm25() function is called for a query, an instance
   376    518   ** of the following structure is allocated and populated.

Changes to ext/fts5/fts5_expr.c.

   163    163       case ')':  tok = FTS5_RP;    break;
   164    164       case '{':  tok = FTS5_LCP;   break;
   165    165       case '}':  tok = FTS5_RCP;   break;
   166    166       case ':':  tok = FTS5_COLON; break;
   167    167       case ',':  tok = FTS5_COMMA; break;
   168    168       case '+':  tok = FTS5_PLUS;  break;
   169    169       case '*':  tok = FTS5_STAR;  break;
          170  +    case '-':  tok = FTS5_MINUS; break;
   170    171       case '\0': tok = FTS5_EOF;   break;
   171    172   
   172    173       case '"': {
   173    174         const char *z2;
   174    175         tok = FTS5_STRING;
   175    176   
   176    177         for(z2=&z[1]; 1; z2++){
................................................................................
   741    742   }
   742    743   
   743    744   
   744    745   /*
   745    746   ** Initialize all term iterators in the pNear object. If any term is found
   746    747   ** to match no documents at all, return immediately without initializing any
   747    748   ** further iterators.
          749  +**
          750  +** If an error occurs, return an SQLite error code. Otherwise, return
          751  +** SQLITE_OK. It is not considered an error if some term matches zero
          752  +** documents.
   748    753   */
   749    754   static int fts5ExprNearInitAll(
   750    755     Fts5Expr *pExpr,
   751    756     Fts5ExprNode *pNode
   752    757   ){
   753    758     Fts5ExprNearset *pNear = pNode->pNear;
   754         -  int i, j;
   755         -  int rc = SQLITE_OK;
          759  +  int i;
   756    760   
   757    761     assert( pNode->bNomatch==0 );
   758         -  for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
          762  +  for(i=0; i<pNear->nPhrase; i++){
   759    763       Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
   760         -    for(j=0; j<pPhrase->nTerm; j++){
   761         -      Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
   762         -      Fts5ExprTerm *p;
   763         -      int bEof = 1;
   764         -
   765         -      for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){
   766         -        if( p->pIter ){
   767         -          sqlite3Fts5IterClose(p->pIter);
   768         -          p->pIter = 0;
   769         -        }
   770         -        rc = sqlite3Fts5IndexQuery(
   771         -            pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
   772         -            (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
   773         -            (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
   774         -            pNear->pColset,
   775         -            &p->pIter
   776         -        );
   777         -        assert( rc==SQLITE_OK || p->pIter==0 );
   778         -        if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){
   779         -          bEof = 0;
   780         -        }
   781         -      }
   782         -
   783         -      if( bEof ){
   784         -        pNode->bEof = 1;
   785         -        return rc;
   786         -      }
   787         -    }
   788         -  }
   789         -
   790         -  return rc;
          764  +    if( pPhrase->nTerm==0 ){
          765  +      pNode->bEof = 1;
          766  +      return SQLITE_OK;
          767  +    }else{
          768  +      int j;
          769  +      for(j=0; j<pPhrase->nTerm; j++){
          770  +        Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
          771  +        Fts5ExprTerm *p;
          772  +        int bHit = 0;
          773  +
          774  +        for(p=pTerm; p; p=p->pSynonym){
          775  +          int rc;
          776  +          if( p->pIter ){
          777  +            sqlite3Fts5IterClose(p->pIter);
          778  +            p->pIter = 0;
          779  +          }
          780  +          rc = sqlite3Fts5IndexQuery(
          781  +              pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
          782  +              (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
          783  +              (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
          784  +              pNear->pColset,
          785  +              &p->pIter
          786  +          );
          787  +          assert( (rc==SQLITE_OK)==(p->pIter!=0) );
          788  +          if( rc!=SQLITE_OK ) return rc;
          789  +          if( 0==sqlite3Fts5IterEof(p->pIter) ){
          790  +            bHit = 1;
          791  +          }
          792  +        }
          793  +
          794  +        if( bHit==0 ){
          795  +          pNode->bEof = 1;
          796  +          return SQLITE_OK;
          797  +        }
          798  +      }
          799  +    }
          800  +  }
          801  +
          802  +  pNode->bEof = 0;
          803  +  return SQLITE_OK;
   791    804   }
   792    805   
   793    806   /*
   794    807   ** If pExpr is an ASC iterator, this function returns a value with the
   795    808   ** same sign as:
   796    809   **
   797    810   **   (iLhs - iRhs)
................................................................................
   916    929             if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){
   917    930               pNode->bNomatch = 0;
   918    931               pNode->bEof = 1;
   919    932               return rc;
   920    933             }
   921    934           }else{
   922    935             Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
   923         -          if( pIter->iRowid==iLast ) continue;
          936  +          if( pIter->iRowid==iLast || pIter->bEof ) continue;
   924    937             bMatch = 0;
   925    938             if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
   926    939               return rc;
   927    940             }
   928    941           }
   929    942         }
   930    943       }
................................................................................
  1093   1106       Fts5ExprNode *p1 = pNode->apChild[i];
  1094   1107       assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 );
  1095   1108       if( p1->bEof==0 ){
  1096   1109         if( (p1->iRowid==iLast) 
  1097   1110          || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0)
  1098   1111         ){
  1099   1112           int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom);
  1100         -        if( rc!=SQLITE_OK ) return rc;
         1113  +        if( rc!=SQLITE_OK ){
         1114  +          pNode->bNomatch = 0;
         1115  +          return rc;
         1116  +        }
  1101   1117         }
  1102   1118       }
  1103   1119     }
  1104   1120   
  1105   1121     fts5ExprNodeTest_OR(pExpr, pNode);
  1106   1122     return SQLITE_OK;
  1107   1123   }
................................................................................
  1124   1140       bMatch = 1;
  1125   1141       for(iChild=0; iChild<pAnd->nChild; iChild++){
  1126   1142         Fts5ExprNode *pChild = pAnd->apChild[iChild];
  1127   1143         int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid);
  1128   1144         if( cmp>0 ){
  1129   1145           /* Advance pChild until it points to iLast or laster */
  1130   1146           rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast);
  1131         -        if( rc!=SQLITE_OK ) return rc;
         1147  +        if( rc!=SQLITE_OK ){
         1148  +          pAnd->bNomatch = 0;
         1149  +          return rc;
         1150  +        }
  1132   1151         }
  1133   1152   
  1134   1153         /* If the child node is now at EOF, so is the parent AND node. Otherwise,
  1135   1154         ** the child node is guaranteed to have advanced at least as far as
  1136   1155         ** rowid iLast. So if it is not at exactly iLast, pChild->iRowid is the
  1137   1156         ** new lastest rowid seen so far.  */
  1138   1157         assert( pChild->bEof || fts5RowidCmp(pExpr, iLast, pChild->iRowid)<=0 );
................................................................................
  1163   1182     Fts5ExprNode *pNode,
  1164   1183     int bFromValid,
  1165   1184     i64 iFrom
  1166   1185   ){
  1167   1186     int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom);
  1168   1187     if( rc==SQLITE_OK ){
  1169   1188       rc = fts5ExprNodeTest_AND(pExpr, pNode);
         1189  +  }else{
         1190  +    pNode->bNomatch = 0;
  1170   1191     }
  1171   1192     return rc;
  1172   1193   }
  1173   1194   
  1174   1195   static int fts5ExprNodeTest_NOT(
  1175   1196     Fts5Expr *pExpr,                /* Expression pPhrase belongs to */
  1176   1197     Fts5ExprNode *pNode             /* FTS5_NOT node to advance */
................................................................................
  1205   1226     int bFromValid,
  1206   1227     i64 iFrom
  1207   1228   ){
  1208   1229     int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom);
  1209   1230     if( rc==SQLITE_OK ){
  1210   1231       rc = fts5ExprNodeTest_NOT(pExpr, pNode);
  1211   1232     }
         1233  +  if( rc!=SQLITE_OK ){
         1234  +    pNode->bNomatch = 0;
         1235  +  }
  1212   1236     return rc;
  1213   1237   }
  1214   1238   
  1215   1239   /*
  1216   1240   ** If pNode currently points to a match, this function returns SQLITE_OK
  1217   1241   ** without modifying it. Otherwise, pNode is advanced until it does point
  1218   1242   ** to a match or EOF is reached.
................................................................................
  1327   1351   
  1328   1352     p->pIndex = pIdx;
  1329   1353     p->bDesc = bDesc;
  1330   1354     rc = fts5ExprNodeFirst(p, pRoot);
  1331   1355   
  1332   1356     /* If not at EOF but the current rowid occurs earlier than iFirst in
  1333   1357     ** the iteration order, move to document iFirst or later. */
  1334         -  if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){
         1358  +  if( rc==SQLITE_OK 
         1359  +   && 0==pRoot->bEof 
         1360  +   && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 
         1361  +  ){
  1335   1362       rc = fts5ExprNodeNext(p, pRoot, 1, iFirst);
  1336   1363     }
  1337   1364   
  1338   1365     /* If the iterator is not at a real match, skip forward until it is. */
  1339   1366     while( pRoot->bNomatch ){
  1340   1367       assert( pRoot->bEof==0 && rc==SQLITE_OK );
  1341   1368       rc = fts5ExprNodeNext(p, pRoot, 0, 0);
................................................................................
  1489   1516     TokenCtx *pCtx = (TokenCtx*)pContext;
  1490   1517     Fts5ExprPhrase *pPhrase = pCtx->pPhrase;
  1491   1518   
  1492   1519     UNUSED_PARAM2(iUnused1, iUnused2);
  1493   1520   
  1494   1521     /* If an error has already occurred, this is a no-op */
  1495   1522     if( pCtx->rc!=SQLITE_OK ) return pCtx->rc;
         1523  +  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
  1496   1524   
  1497   1525     if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){
  1498   1526       Fts5ExprTerm *pSyn;
  1499   1527       int nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1;
  1500   1528       pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte);
  1501   1529       if( pSyn==0 ){
  1502   1530         rc = SQLITE_NOMEM;
................................................................................
  1580   1608     char *z = 0;
  1581   1609   
  1582   1610     memset(&sCtx, 0, sizeof(TokenCtx));
  1583   1611     sCtx.pPhrase = pAppend;
  1584   1612   
  1585   1613     rc = fts5ParseStringFromToken(pToken, &z);
  1586   1614     if( rc==SQLITE_OK ){
  1587         -    int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0);
         1615  +    int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0);
  1588   1616       int n;
  1589   1617       sqlite3Fts5Dequote(z);
  1590   1618       n = (int)strlen(z);
  1591   1619       rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize);
  1592   1620     }
  1593   1621     sqlite3_free(z);
  1594   1622     if( rc || (rc = sCtx.rc) ){
................................................................................
  1632   1660   int sqlite3Fts5ExprClonePhrase(
  1633   1661     Fts5Expr *pExpr, 
  1634   1662     int iPhrase, 
  1635   1663     Fts5Expr **ppNew
  1636   1664   ){
  1637   1665     int rc = SQLITE_OK;             /* Return code */
  1638   1666     Fts5ExprPhrase *pOrig;          /* The phrase extracted from pExpr */
  1639         -  int i;                          /* Used to iterate through phrase terms */
  1640   1667     Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
  1641   1668     TokenCtx sCtx = {0,0};          /* Context object for fts5ParseTokenize */
  1642   1669   
  1643   1670     pOrig = pExpr->apExprPhrase[iPhrase];
  1644   1671     pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
  1645   1672     if( rc==SQLITE_OK ){
  1646   1673       pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, 
................................................................................
  1650   1677       pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, 
  1651   1678           sizeof(Fts5ExprNode));
  1652   1679     }
  1653   1680     if( rc==SQLITE_OK ){
  1654   1681       pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, 
  1655   1682           sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*));
  1656   1683     }
  1657         -
  1658         -  for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
  1659         -    int tflags = 0;
  1660         -    Fts5ExprTerm *p;
  1661         -    for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
  1662         -      const char *zTerm = p->zTerm;
  1663         -      rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
  1664         -          0, 0);
  1665         -      tflags = FTS5_TOKEN_COLOCATED;
  1666         -    }
  1667         -    if( rc==SQLITE_OK ){
  1668         -      sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
  1669         -    }
         1684  +  if( rc==SQLITE_OK ){
         1685  +    Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset;
         1686  +    if( pColsetOrig ){
         1687  +      int nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int);
         1688  +      Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte);
         1689  +      if( pColset ){ 
         1690  +        memcpy(pColset, pColsetOrig, nByte);
         1691  +      }
         1692  +      pNew->pRoot->pNear->pColset = pColset;
         1693  +    }
         1694  +  }
         1695  +
         1696  +  if( pOrig->nTerm ){
         1697  +    int i;                          /* Used to iterate through phrase terms */
         1698  +    for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
         1699  +      int tflags = 0;
         1700  +      Fts5ExprTerm *p;
         1701  +      for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
         1702  +        const char *zTerm = p->zTerm;
         1703  +        rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
         1704  +            0, 0);
         1705  +        tflags = FTS5_TOKEN_COLOCATED;
         1706  +      }
         1707  +      if( rc==SQLITE_OK ){
         1708  +        sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
         1709  +      }
         1710  +    }
         1711  +  }else{
         1712  +    /* This happens when parsing a token or quoted phrase that contains
         1713  +    ** no token characters at all. (e.g ... MATCH '""'). */
         1714  +    sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase));
  1670   1715     }
  1671   1716   
  1672   1717     if( rc==SQLITE_OK ){
  1673   1718       /* All the allocations succeeded. Put the expression object together. */
  1674   1719       pNew->pIndex = pExpr->pIndex;
  1675   1720       pNew->pConfig = pExpr->pConfig;
  1676   1721       pNew->nPhrase = 1;
................................................................................
  1776   1821       /* Check that the array is in order and contains no duplicate entries. */
  1777   1822       for(i=1; i<pNew->nCol; i++) assert( pNew->aiCol[i]>pNew->aiCol[i-1] );
  1778   1823   #endif
  1779   1824     }
  1780   1825   
  1781   1826     return pNew;
  1782   1827   }
         1828  +
         1829  +/*
         1830  +** Allocate and return an Fts5Colset object specifying the inverse of
         1831  +** the colset passed as the second argument. Free the colset passed
         1832  +** as the second argument before returning.
         1833  +*/
         1834  +Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){
         1835  +  Fts5Colset *pRet;
         1836  +  int nCol = pParse->pConfig->nCol;
         1837  +
         1838  +  pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc, 
         1839  +      sizeof(Fts5Colset) + sizeof(int)*nCol
         1840  +  );
         1841  +  if( pRet ){
         1842  +    int i;
         1843  +    int iOld = 0;
         1844  +    for(i=0; i<nCol; i++){
         1845  +      if( iOld>=p->nCol || p->aiCol[iOld]!=i ){
         1846  +        pRet->aiCol[pRet->nCol++] = i;
         1847  +      }else{
         1848  +        iOld++;
         1849  +      }
         1850  +    }
         1851  +  }
         1852  +
         1853  +  sqlite3_free(p);
         1854  +  return pRet;
         1855  +}
  1783   1856   
  1784   1857   Fts5Colset *sqlite3Fts5ParseColset(
  1785   1858     Fts5Parse *pParse,              /* Store SQLITE_NOMEM here if required */
  1786   1859     Fts5Colset *pColset,            /* Existing colset object */
  1787   1860     Fts5Token *p
  1788   1861   ){
  1789   1862     Fts5Colset *pRet = 0;
................................................................................
  2491   2564   ){
  2492   2565     Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
  2493   2566     Fts5Expr *pExpr = p->pExpr;
  2494   2567     int i;
  2495   2568   
  2496   2569     UNUSED_PARAM2(iUnused1, iUnused2);
  2497   2570   
         2571  +  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
  2498   2572     if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
  2499   2573     for(i=0; i<pExpr->nPhrase; i++){
  2500   2574       Fts5ExprTerm *pTerm;
  2501   2575       if( p->aPopulator[i].bOk==0 ) continue;
  2502   2576       for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
  2503         -      int nTerm = strlen(pTerm->zTerm);
         2577  +      int nTerm = (int)strlen(pTerm->zTerm);
  2504   2578         if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
  2505   2579          && memcmp(pTerm->zTerm, pToken, nTerm)==0
  2506   2580         ){
  2507   2581           int rc = sqlite3Fts5PoslistWriterAppend(
  2508   2582               &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
  2509   2583           );
  2510   2584           if( rc ) return rc;
................................................................................
  2600   2674     return 1;
  2601   2675   }
  2602   2676   
  2603   2677   void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
  2604   2678     fts5ExprCheckPoslists(pExpr->pRoot, iRowid);
  2605   2679   }
  2606   2680   
  2607         -static void fts5ExprClearEof(Fts5ExprNode *pNode){
  2608         -  int i;
  2609         -  for(i=0; i<pNode->nChild; i++){
  2610         -    fts5ExprClearEof(pNode->apChild[i]);
  2611         -  }
  2612         -  pNode->bEof = 0;
  2613         -}
  2614         -void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){
  2615         -  fts5ExprClearEof(pExpr->pRoot);
  2616         -}
  2617         -
  2618   2681   /*
  2619   2682   ** This function is only called for detail=columns tables. 
  2620   2683   */
  2621   2684   int sqlite3Fts5ExprPhraseCollist(
  2622   2685     Fts5Expr *pExpr, 
  2623   2686     int iPhrase, 
  2624   2687     const u8 **ppCollist, 

Changes to ext/fts5/fts5_hash.c.

   341    341       }else{
   342    342         /* Append a new column value, if necessary */
   343    343         assert( iCol>=p->iCol );
   344    344         if( iCol!=p->iCol ){
   345    345           if( pHash->eDetail==FTS5_DETAIL_FULL ){
   346    346             pPtr[p->nData++] = 0x01;
   347    347             p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
   348         -          p->iCol = iCol;
          348  +          p->iCol = (i16)iCol;
   349    349             p->iPos = 0;
   350    350           }else{
   351    351             bNew = 1;
   352         -          p->iCol = iPos = iCol;
          352  +          p->iCol = (i16)(iPos = iCol);
   353    353           }
   354    354         }
   355    355   
   356    356         /* Append the new position offset, if necessary */
   357    357         if( bNew ){
   358    358           p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
   359    359           p->iPos = iPos;

Changes to ext/fts5/fts5_index.c.

   300    300     sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
   301    301     sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
   302    302     sqlite3_stmt *pDeleter;         /* "DELETE FROM %_data ... id>=? AND id<=?" */
   303    303     sqlite3_stmt *pIdxWriter;       /* "INSERT ... %_idx VALUES(?,?,?,?)" */
   304    304     sqlite3_stmt *pIdxDeleter;      /* "DELETE FROM %_idx WHERE segid=? */
   305    305     sqlite3_stmt *pIdxSelect;
   306    306     int nRead;                      /* Total number of blocks read */
          307  +
          308  +  sqlite3_stmt *pDataVersion;
          309  +  i64 iStructVersion;             /* data_version when pStruct read */
          310  +  Fts5Structure *pStruct;         /* Current db structure (or NULL) */
   307    311   };
   308    312   
   309    313   struct Fts5DoclistIter {
   310    314     u8 *aEof;                       /* Pointer to 1 byte past end of doclist */
   311    315   
   312    316     /* Output variables. aPoslist==0 at EOF */
   313    317     i64 iRowid;
................................................................................
   694    698       p->nRead++;
   695    699     }
   696    700   
   697    701     assert( (pRet==0)==(p->rc!=SQLITE_OK) );
   698    702     return pRet;
   699    703   }
   700    704   
   701         -
   702    705   /*
   703    706   ** Release a reference to data record returned by an earlier call to
   704    707   ** fts5DataRead().
   705    708   */
   706    709   static void fts5DataRelease(Fts5Data *pData){
   707    710     sqlite3_free(pData);
   708    711   }
          712  +
          713  +static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){
          714  +  Fts5Data *pRet = fts5DataRead(p, iRowid);
          715  +  if( pRet ){
          716  +    if( pRet->szLeaf>pRet->nn ){
          717  +      p->rc = FTS5_CORRUPT;
          718  +      fts5DataRelease(pRet);
          719  +      pRet = 0;
          720  +    }
          721  +  }
          722  +  return pRet;
          723  +}
   709    724   
   710    725   static int fts5IndexPrepareStmt(
   711    726     Fts5Index *p,
   712    727     sqlite3_stmt **ppStmt,
   713    728     char *zSql
   714    729   ){
   715    730     if( p->rc==SQLITE_OK ){
................................................................................
   954    969         }
   955    970         pLvl->aSeg = aNew;
   956    971       }else{
   957    972         *pRc = SQLITE_NOMEM;
   958    973       }
   959    974     }
   960    975   }
          976  +
          977  +static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){
          978  +  Fts5Structure *pRet = 0;
          979  +  Fts5Config *pConfig = p->pConfig;
          980  +  int iCookie;                    /* Configuration cookie */
          981  +  Fts5Data *pData;
          982  +
          983  +  pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
          984  +  if( p->rc==SQLITE_OK ){
          985  +    /* TODO: Do we need this if the leaf-index is appended? Probably... */
          986  +    memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
          987  +    p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
          988  +    if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
          989  +      p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
          990  +    }
          991  +    fts5DataRelease(pData);
          992  +    if( p->rc!=SQLITE_OK ){
          993  +      fts5StructureRelease(pRet);
          994  +      pRet = 0;
          995  +    }
          996  +  }
          997  +
          998  +  return pRet;
          999  +}
         1000  +
         1001  +static i64 fts5IndexDataVersion(Fts5Index *p){
         1002  +  i64 iVersion = 0;
         1003  +
         1004  +  if( p->rc==SQLITE_OK ){
         1005  +    if( p->pDataVersion==0 ){
         1006  +      p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, 
         1007  +          sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb)
         1008  +          );
         1009  +      if( p->rc ) return 0;
         1010  +    }
         1011  +
         1012  +    if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){
         1013  +      iVersion = sqlite3_column_int64(p->pDataVersion, 0);
         1014  +    }
         1015  +    p->rc = sqlite3_reset(p->pDataVersion);
         1016  +  }
         1017  +
         1018  +  return iVersion;
         1019  +}
   961   1020   
   962   1021   /*
   963   1022   ** Read, deserialize and return the structure record.
   964   1023   **
   965   1024   ** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array
   966   1025   ** are over-allocated as described for function fts5StructureDecode() 
   967   1026   ** above.
   968   1027   **
   969   1028   ** If an error occurs, NULL is returned and an error code left in the
   970   1029   ** Fts5Index handle. If an error has already occurred when this function
   971   1030   ** is called, it is a no-op.
   972   1031   */
   973   1032   static Fts5Structure *fts5StructureRead(Fts5Index *p){
   974         -  Fts5Config *pConfig = p->pConfig;
   975         -  Fts5Structure *pRet = 0;        /* Object to return */
   976         -  int iCookie;                    /* Configuration cookie */
   977         -  Fts5Data *pData;
   978         -
   979         -  pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
   980         -  if( p->rc ) return 0;
   981         -  /* TODO: Do we need this if the leaf-index is appended? Probably... */
   982         -  memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
   983         -  p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
   984         -  if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
   985         -    p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
   986         -  }
   987         -
   988         -  fts5DataRelease(pData);
   989         -  if( p->rc!=SQLITE_OK ){
   990         -    fts5StructureRelease(pRet);
   991         -    pRet = 0;
   992         -  }
   993         -  return pRet;
         1033  +
         1034  +  if( p->pStruct==0 ){
         1035  +    p->iStructVersion = fts5IndexDataVersion(p);
         1036  +    if( p->rc==SQLITE_OK ){
         1037  +      p->pStruct = fts5StructureReadUncached(p);
         1038  +    }
         1039  +  }
         1040  +
         1041  +#if 0
         1042  +  else{
         1043  +    Fts5Structure *pTest = fts5StructureReadUncached(p);
         1044  +    if( pTest ){
         1045  +      int i, j;
         1046  +      assert_nc( p->pStruct->nSegment==pTest->nSegment );
         1047  +      assert_nc( p->pStruct->nLevel==pTest->nLevel );
         1048  +      for(i=0; i<pTest->nLevel; i++){
         1049  +        assert_nc( p->pStruct->aLevel[i].nMerge==pTest->aLevel[i].nMerge );
         1050  +        assert_nc( p->pStruct->aLevel[i].nSeg==pTest->aLevel[i].nSeg );
         1051  +        for(j=0; j<pTest->aLevel[i].nSeg; j++){
         1052  +          Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j];
         1053  +          Fts5StructureSegment *p2 = &p->pStruct->aLevel[i].aSeg[j];
         1054  +          assert_nc( p1->iSegid==p2->iSegid );
         1055  +          assert_nc( p1->pgnoFirst==p2->pgnoFirst );
         1056  +          assert_nc( p1->pgnoLast==p2->pgnoLast );
         1057  +        }
         1058  +      }
         1059  +      fts5StructureRelease(pTest);
         1060  +    }
         1061  +  }
         1062  +#endif
         1063  +
         1064  +  if( p->rc!=SQLITE_OK ) return 0;
         1065  +  assert( p->iStructVersion!=0 );
         1066  +  assert( p->pStruct!=0 );
         1067  +  fts5StructureRef(p->pStruct);
         1068  +  return p->pStruct;
         1069  +}
         1070  +
         1071  +static void fts5StructureInvalidate(Fts5Index *p){
         1072  +  if( p->pStruct ){
         1073  +    fts5StructureRelease(p->pStruct);
         1074  +    p->pStruct = 0;
         1075  +  }
   994   1076   }
   995   1077   
   996   1078   /*
   997   1079   ** Return the total number of segments in index structure pStruct. This
   998   1080   ** function is only ever used as part of assert() conditions.
   999   1081   */
  1000   1082   #ifdef SQLITE_DEBUG
................................................................................
  1444   1526     Fts5StructureSegment *pSeg = pIter->pSeg;
  1445   1527     fts5DataRelease(pIter->pLeaf);
  1446   1528     pIter->iLeafPgno++;
  1447   1529     if( pIter->pNextLeaf ){
  1448   1530       pIter->pLeaf = pIter->pNextLeaf;
  1449   1531       pIter->pNextLeaf = 0;
  1450   1532     }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){
  1451         -    pIter->pLeaf = fts5DataRead(p, 
         1533  +    pIter->pLeaf = fts5LeafRead(p, 
  1452   1534           FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno)
  1453   1535       );
  1454   1536     }else{
  1455   1537       pIter->pLeaf = 0;
  1456   1538     }
  1457   1539     pLeaf = pIter->pLeaf;
  1458   1540   
................................................................................
  1947   2029         if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOff<pLeaf->szLeaf ){
  1948   2030           iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid);
  1949   2031           pIter->iLeafOffset = iOff;
  1950   2032   
  1951   2033           if( pLeaf->nn>pLeaf->szLeaf ){
  1952   2034             pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
  1953   2035                 &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist
  1954         -              );
         2036  +          );
  1955   2037           }
  1956         -
  1957   2038         }
  1958   2039         else if( pLeaf->nn>pLeaf->szLeaf ){
  1959   2040           pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
  1960   2041               &pLeaf->p[pLeaf->szLeaf], iOff
  1961         -            );
         2042  +        );
  1962   2043           pIter->iLeafOffset = iOff;
  1963   2044           pIter->iEndofDoclist = iOff;
  1964   2045           bNewTerm = 1;
  1965   2046         }
  1966   2047         assert_nc( iOff<pLeaf->szLeaf );
  1967   2048         if( iOff>pLeaf->szLeaf ){
  1968   2049           p->rc = FTS5_CORRUPT;
................................................................................
  1988   2069         ** code is inlined. 
  1989   2070         **
  1990   2071         ** Later: Switched back to fts5SegIterLoadNPos() because it supports
  1991   2072         ** detail=none mode. Not ideal.
  1992   2073         */
  1993   2074         int nSz;
  1994   2075         assert( p->rc==SQLITE_OK );
         2076  +      assert( pIter->iLeafOffset<=pIter->pLeaf->nn );
  1995   2077         fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz);
  1996   2078         pIter->bDel = (nSz & 0x0001);
  1997   2079         pIter->nPos = nSz>>1;
  1998   2080         assert_nc( pIter->nPos>=0 );
  1999   2081       }
  2000   2082     }
  2001   2083   }
................................................................................
  2193   2275         bEndOfPage = 1;
  2194   2276         break;
  2195   2277       }
  2196   2278   
  2197   2279       iPgidx += fts5GetVarint32(&a[iPgidx], nKeep);
  2198   2280       iTermOff += nKeep;
  2199   2281       iOff = iTermOff;
         2282  +
         2283  +    if( iOff>=n ){
         2284  +      p->rc = FTS5_CORRUPT;
         2285  +      return;
         2286  +    }
  2200   2287   
  2201   2288       /* Read the nKeep field of the next term. */
  2202   2289       fts5FastGetVarint32(a, iOff, nKeep);
  2203   2290     }
  2204   2291   
  2205   2292    search_failed:
  2206   2293     if( bGe==0 ){
................................................................................
  2245   2332       pIter->iEndofDoclist = iTermOff + nExtra;
  2246   2333     }
  2247   2334     pIter->iPgidxOff = iPgidx;
  2248   2335   
  2249   2336     fts5SegIterLoadRowid(p, pIter);
  2250   2337     fts5SegIterLoadNPos(p, pIter);
  2251   2338   }
         2339  +
         2340  +static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){
         2341  +  if( p->pIdxSelect==0 ){
         2342  +    Fts5Config *pConfig = p->pConfig;
         2343  +    fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf(
         2344  +          "SELECT pgno FROM '%q'.'%q_idx' WHERE "
         2345  +          "segid=? AND term<=? ORDER BY term DESC LIMIT 1",
         2346  +          pConfig->zDb, pConfig->zName
         2347  +    ));
         2348  +  }
         2349  +  return p->pIdxSelect;
         2350  +}
  2252   2351   
  2253   2352   /*
  2254   2353   ** Initialize the object pIter to point to term pTerm/nTerm within segment
  2255   2354   ** pSeg. If there is no such term in the index, the iterator is set to EOF.
  2256   2355   **
  2257   2356   ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If 
  2258   2357   ** an error has already occurred when this function is called, it is a no-op.
................................................................................
  2263   2362     int flags,                      /* Mask of FTS5INDEX_XXX flags */
  2264   2363     Fts5StructureSegment *pSeg,     /* Description of segment */
  2265   2364     Fts5SegIter *pIter              /* Object to populate */
  2266   2365   ){
  2267   2366     int iPg = 1;
  2268   2367     int bGe = (flags & FTS5INDEX_QUERY_SCAN);
  2269   2368     int bDlidx = 0;                 /* True if there is a doclist-index */
         2369  +  sqlite3_stmt *pIdxSelect = 0;
  2270   2370   
  2271   2371     assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 );
  2272   2372     assert( pTerm && nTerm );
  2273   2373     memset(pIter, 0, sizeof(*pIter));
  2274   2374     pIter->pSeg = pSeg;
  2275   2375   
  2276   2376     /* This block sets stack variable iPg to the leaf page number that may
  2277   2377     ** contain term (pTerm/nTerm), if it is present in the segment. */
  2278         -  if( p->pIdxSelect==0 ){
  2279         -    Fts5Config *pConfig = p->pConfig;
  2280         -    fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf(
  2281         -          "SELECT pgno FROM '%q'.'%q_idx' WHERE "
  2282         -          "segid=? AND term<=? ORDER BY term DESC LIMIT 1",
  2283         -          pConfig->zDb, pConfig->zName
  2284         -    ));
  2285         -  }
         2378  +  pIdxSelect = fts5IdxSelectStmt(p);
  2286   2379     if( p->rc ) return;
  2287         -  sqlite3_bind_int(p->pIdxSelect, 1, pSeg->iSegid);
  2288         -  sqlite3_bind_blob(p->pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC);
  2289         -  if( SQLITE_ROW==sqlite3_step(p->pIdxSelect) ){
  2290         -    i64 val = sqlite3_column_int(p->pIdxSelect, 0);
         2380  +  sqlite3_bind_int(pIdxSelect, 1, pSeg->iSegid);
         2381  +  sqlite3_bind_blob(pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC);
         2382  +  if( SQLITE_ROW==sqlite3_step(pIdxSelect) ){
         2383  +    i64 val = sqlite3_column_int(pIdxSelect, 0);
  2291   2384       iPg = (int)(val>>1);
  2292   2385       bDlidx = (val & 0x0001);
  2293   2386     }
  2294         -  p->rc = sqlite3_reset(p->pIdxSelect);
         2387  +  p->rc = sqlite3_reset(pIdxSelect);
  2295   2388   
  2296   2389     if( iPg<pSeg->pgnoFirst ){
  2297   2390       iPg = pSeg->pgnoFirst;
  2298   2391       bDlidx = 0;
  2299   2392     }
  2300   2393   
  2301   2394     pIter->iLeafPgno = iPg - 1;
................................................................................
  2744   2837   static void fts5MultiIterNext(
  2745   2838     Fts5Index *p, 
  2746   2839     Fts5Iter *pIter,
  2747   2840     int bFrom,                      /* True if argument iFrom is valid */
  2748   2841     i64 iFrom                       /* Advance at least as far as this */
  2749   2842   ){
  2750   2843     int bUseFrom = bFrom;
         2844  +  assert( pIter->base.bEof==0 );
  2751   2845     while( p->rc==SQLITE_OK ){
  2752   2846       int iFirst = pIter->aFirst[1].iFirst;
  2753   2847       int bNewTerm = 0;
  2754   2848       Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
  2755   2849       assert( p->rc==SQLITE_OK );
  2756   2850       if( bUseFrom && pSeg->pDlidx ){
  2757   2851         fts5SegIterNextFrom(p, pSeg, iFrom);
................................................................................
  2970   3064       xChunk(p, pCtx, pChunk, nChunk);
  2971   3065       nRem -= nChunk;
  2972   3066       fts5DataRelease(pData);
  2973   3067       if( nRem<=0 ){
  2974   3068         break;
  2975   3069       }else{
  2976   3070         pgno++;
  2977         -      pData = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno));
         3071  +      pData = fts5LeafRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno));
  2978   3072         if( pData==0 ) break;
  2979   3073         pChunk = &pData->p[4];
  2980   3074         nChunk = MIN(nRem, pData->szLeaf - 4);
  2981   3075         if( pgno==pgnoSave ){
  2982   3076           assert( pSeg->pNextLeaf==0 );
  2983   3077           pSeg->pNextLeaf = pData;
  2984   3078           pData = 0;
................................................................................
  3113   3207       ** Fts5Iter.poslist buffer and then set the output pointer to point
  3114   3208       ** to this buffer.  */
  3115   3209       fts5BufferZero(&pIter->poslist);
  3116   3210       fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist);
  3117   3211       pIter->base.pData = pIter->poslist.p;
  3118   3212     }
  3119   3213   }
         3214  +
         3215  +/*
         3216  +** xSetOutputs callback used when the Fts5Colset object has nCol==0 (match
         3217  +** against no columns at all).
         3218  +*/
         3219  +static void fts5IterSetOutputs_ZeroColset(Fts5Iter *pIter, Fts5SegIter *pSeg){
         3220  +  UNUSED_PARAM(pSeg);
         3221  +  pIter->base.nData = 0;
         3222  +}
  3120   3223   
  3121   3224   /*
  3122   3225   ** xSetOutputs callback used by detail=col when there is a column filter
  3123   3226   ** and there are 100 or more columns. Also called as a fallback from
  3124   3227   ** fts5IterSetOutputs_Col100 if the column-list spans more than one page.
  3125   3228   */
  3126   3229   static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){
................................................................................
  3163   3266       while( a<pEnd ){
  3164   3267         iPrev += (int)a++[0] - 2;
  3165   3268         while( *aiCol<iPrev ){
  3166   3269           aiCol++;
  3167   3270           if( aiCol==aiColEnd ) goto setoutputs_col_out;
  3168   3271         }
  3169   3272         if( *aiCol==iPrev ){
  3170         -        *aOut++ = (iPrev - iPrevOut) + 2;
         3273  +        *aOut++ = (u8)((iPrev - iPrevOut) + 2);
  3171   3274           iPrevOut = iPrev;
  3172   3275         }
  3173   3276       }
  3174   3277   
  3175   3278   setoutputs_col_out:
  3176   3279       pIter->base.pData = pIter->poslist.p;
  3177   3280       pIter->base.nData = aOut - pIter->poslist.p;
................................................................................
  3218   3321       if( pConfig->eDetail==FTS5_DETAIL_NONE ){
  3219   3322         pIter->xSetOutputs = fts5IterSetOutputs_None;
  3220   3323       }
  3221   3324   
  3222   3325       else if( pIter->pColset==0 ){
  3223   3326         pIter->xSetOutputs = fts5IterSetOutputs_Nocolset;
  3224   3327       }
         3328  +
         3329  +    else if( pIter->pColset->nCol==0 ){
         3330  +      pIter->xSetOutputs = fts5IterSetOutputs_ZeroColset;
         3331  +    }
  3225   3332   
  3226   3333       else if( pConfig->eDetail==FTS5_DETAIL_FULL ){
  3227   3334         pIter->xSetOutputs = fts5IterSetOutputs_Full;
  3228   3335       }
  3229   3336   
  3230   3337       else{
  3231   3338         assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS );
................................................................................
  3449   3556   static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
  3450   3557     int iSegid = 0;
  3451   3558   
  3452   3559     if( p->rc==SQLITE_OK ){
  3453   3560       if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){
  3454   3561         p->rc = SQLITE_FULL;
  3455   3562       }else{
  3456         -      while( iSegid==0 ){
  3457         -        int iLvl, iSeg;
  3458         -        sqlite3_randomness(sizeof(u32), (void*)&iSegid);
  3459         -        iSegid = iSegid & ((1 << FTS5_DATA_ID_B)-1);
  3460         -        for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
  3461         -          for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
  3462         -            if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){
  3463         -              iSegid = 0;
  3464         -            }
         3563  +      /* FTS5_MAX_SEGMENT is currently defined as 2000. So the following
         3564  +      ** array is 63 elements, or 252 bytes, in size.  */
         3565  +      u32 aUsed[(FTS5_MAX_SEGMENT+31) / 32];
         3566  +      int iLvl, iSeg;
         3567  +      int i;
         3568  +      u32 mask;
         3569  +      memset(aUsed, 0, sizeof(aUsed));
         3570  +      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
         3571  +        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
         3572  +          int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid;
         3573  +          if( iId<=FTS5_MAX_SEGMENT ){
         3574  +            aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32);
  3465   3575             }
  3466   3576           }
  3467   3577         }
         3578  +
         3579  +      for(i=0; aUsed[i]==0xFFFFFFFF; i++);
         3580  +      mask = aUsed[i];
         3581  +      for(iSegid=0; mask & (1 << iSegid); iSegid++);
         3582  +      iSegid += 1 + i*32;
         3583  +
         3584  +#ifdef SQLITE_DEBUG
         3585  +      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
         3586  +        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
         3587  +          assert( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid );
         3588  +        }
         3589  +      }
         3590  +      assert( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT );
         3591  +
         3592  +      {
         3593  +        sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p);
         3594  +        if( p->rc==SQLITE_OK ){
         3595  +          u8 aBlob[2] = {0xff, 0xff};
         3596  +          sqlite3_bind_int(pIdxSelect, 1, iSegid);
         3597  +          sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC);
         3598  +          assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW );
         3599  +          p->rc = sqlite3_reset(pIdxSelect);
         3600  +        }
         3601  +      }
         3602  +#endif
  3468   3603       }
  3469   3604     }
  3470   3605   
  3471   3606     return iSegid;
  3472   3607   }
  3473   3608   
  3474   3609   /*
................................................................................
  3705   3840     }
  3706   3841   }
  3707   3842   
  3708   3843   static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){
  3709   3844     static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 };
  3710   3845     Fts5PageWriter *pPage = &pWriter->writer;
  3711   3846     i64 iRowid;
         3847  +
         3848  +static int nCall = 0;
         3849  +nCall++;
  3712   3850   
  3713   3851     assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) );
  3714   3852   
  3715   3853     /* Set the szLeaf header field. */
  3716   3854     assert( 0==fts5GetU16(&pPage->buf.p[2]) );
  3717   3855     fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n);
  3718   3856   
................................................................................
  3905   4043     Fts5PageWriter *pLeaf = &pWriter->writer;
  3906   4044     if( p->rc==SQLITE_OK ){
  3907   4045       assert( pLeaf->pgno>=1 );
  3908   4046       if( pLeaf->buf.n>4 ){
  3909   4047         fts5WriteFlushLeaf(p, pWriter);
  3910   4048       }
  3911   4049       *pnLeaf = pLeaf->pgno-1;
  3912         -    fts5WriteFlushBtree(p, pWriter);
         4050  +    if( pLeaf->pgno>1 ){
         4051  +      fts5WriteFlushBtree(p, pWriter);
         4052  +    }
  3913   4053     }
  3914   4054     fts5BufferFree(&pLeaf->term);
  3915   4055     fts5BufferFree(&pLeaf->buf);
  3916   4056     fts5BufferFree(&pLeaf->pgidx);
  3917   4057     fts5BufferFree(&pWriter->btterm);
  3918   4058   
  3919   4059     for(i=0; i<pWriter->nDlidx; i++){
................................................................................
  4324   4464     int iSegid;
  4325   4465     int pgnoLast = 0;                 /* Last leaf page number in segment */
  4326   4466   
  4327   4467     /* Obtain a reference to the index structure and allocate a new segment-id
  4328   4468     ** for the new level-0 segment.  */
  4329   4469     pStruct = fts5StructureRead(p);
  4330   4470     iSegid = fts5AllocateSegid(p, pStruct);
         4471  +  fts5StructureInvalidate(p);
  4331   4472   
  4332   4473     if( iSegid ){
  4333   4474       const int pgsz = p->pConfig->pgsz;
  4334   4475       int eDetail = p->pConfig->eDetail;
  4335   4476       Fts5StructureSegment *pSeg;   /* New segment within pStruct */
  4336   4477       Fts5Buffer *pBuf;             /* Buffer in which to assemble leaf page */
  4337   4478       Fts5Buffer *pPgidx;           /* Buffer in which to assemble pgidx */
................................................................................
  4508   4649     }
  4509   4650   
  4510   4651     nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
  4511   4652     pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
  4512   4653   
  4513   4654     if( pNew ){
  4514   4655       Fts5StructureLevel *pLvl;
  4515         -    int nByte = nSeg * sizeof(Fts5StructureSegment);
         4656  +    nByte = nSeg * sizeof(Fts5StructureSegment);
  4516   4657       pNew->nLevel = pStruct->nLevel+1;
  4517   4658       pNew->nRef = 1;
  4518   4659       pNew->nWriteCounter = pStruct->nWriteCounter;
  4519   4660       pLvl = &pNew->aLevel[pStruct->nLevel];
  4520   4661       pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
  4521   4662       if( pLvl->aSeg ){
  4522   4663         int iLvl, iSeg;
................................................................................
  4543   4684   int sqlite3Fts5IndexOptimize(Fts5Index *p){
  4544   4685     Fts5Structure *pStruct;
  4545   4686     Fts5Structure *pNew = 0;
  4546   4687   
  4547   4688     assert( p->rc==SQLITE_OK );
  4548   4689     fts5IndexFlush(p);
  4549   4690     pStruct = fts5StructureRead(p);
         4691  +  fts5StructureInvalidate(p);
  4550   4692   
  4551   4693     if( pStruct ){
  4552   4694       pNew = fts5IndexOptimizeStruct(p, pStruct);
  4553   4695     }
  4554   4696     fts5StructureRelease(pStruct);
  4555   4697   
  4556   4698     assert( pNew==0 || pNew->nSegment>0 );
................................................................................
  4573   4715   ** This is called to implement the special "VALUES('merge', $nMerge)"
  4574   4716   ** INSERT command.
  4575   4717   */
  4576   4718   int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
  4577   4719     Fts5Structure *pStruct = fts5StructureRead(p);
  4578   4720     if( pStruct ){
  4579   4721       int nMin = p->pConfig->nUsermerge;
         4722  +    fts5StructureInvalidate(p);
  4580   4723       if( nMerge<0 ){
  4581   4724         Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct);
  4582   4725         fts5StructureRelease(pStruct);
  4583   4726         pStruct = pNew;
  4584   4727         nMin = 2;
  4585   4728         nMerge = nMerge*-1;
  4586   4729       }
................................................................................
  5000   5143   ** to the database. Additionally, assume that the contents of the %_data
  5001   5144   ** table may have changed on disk. So any in-memory caches of %_data 
  5002   5145   ** records must be invalidated.
  5003   5146   */
  5004   5147   int sqlite3Fts5IndexRollback(Fts5Index *p){
  5005   5148     fts5CloseReader(p);
  5006   5149     fts5IndexDiscardData(p);
         5150  +  fts5StructureInvalidate(p);
  5007   5151     /* assert( p->rc==SQLITE_OK ); */
  5008   5152     return SQLITE_OK;
  5009   5153   }
  5010   5154   
  5011   5155   /*
  5012   5156   ** The %_data table is completely empty when this function is called. This
  5013   5157   ** function populates it with the initial structure objects for each index,
  5014   5158   ** and the initial version of the "averages" record (a zero-byte blob).
  5015   5159   */
  5016   5160   int sqlite3Fts5IndexReinit(Fts5Index *p){
  5017   5161     Fts5Structure s;
         5162  +  fts5StructureInvalidate(p);
  5018   5163     memset(&s, 0, sizeof(Fts5Structure));
  5019   5164     fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0);
  5020   5165     fts5StructureWrite(p, &s);
  5021   5166     return fts5IndexReturn(p);
  5022   5167   }
  5023   5168   
  5024   5169   /*
................................................................................
  5069   5214   /*
  5070   5215   ** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen().
  5071   5216   */
  5072   5217   int sqlite3Fts5IndexClose(Fts5Index *p){
  5073   5218     int rc = SQLITE_OK;
  5074   5219     if( p ){
  5075   5220       assert( p->pReader==0 );
         5221  +    fts5StructureInvalidate(p);
  5076   5222       sqlite3_finalize(p->pWriter);
  5077   5223       sqlite3_finalize(p->pDeleter);
  5078   5224       sqlite3_finalize(p->pIdxWriter);
  5079   5225       sqlite3_finalize(p->pIdxDeleter);
  5080   5226       sqlite3_finalize(p->pIdxSelect);
         5227  +    sqlite3_finalize(p->pDataVersion);
  5081   5228       sqlite3Fts5HashFree(p->pHash);
  5082   5229       sqlite3_free(p->zDataTbl);
  5083   5230       sqlite3_free(p);
  5084   5231     }
  5085   5232     return rc;
  5086   5233   }
  5087   5234   
................................................................................
  5679   5826       int iIdxLeaf = sqlite3_column_int(pStmt, 2);
  5680   5827       int bIdxDlidx = sqlite3_column_int(pStmt, 3);
  5681   5828   
  5682   5829       /* If the leaf in question has already been trimmed from the segment, 
  5683   5830       ** ignore this b-tree entry. Otherwise, load it into memory. */
  5684   5831       if( iIdxLeaf<pSeg->pgnoFirst ) continue;
  5685   5832       iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, iIdxLeaf);
  5686         -    pLeaf = fts5DataRead(p, iRow);
         5833  +    pLeaf = fts5LeafRead(p, iRow);
  5687   5834       if( pLeaf==0 ) break;
  5688   5835   
  5689   5836       /* Check that the leaf contains at least one term, and that it is equal
  5690   5837       ** to or larger than the split-key in zIdxTerm.  Also check that if there
  5691   5838       ** is also a rowid pointer within the leaf page header, it points to a
  5692   5839       ** location before the term.  */
  5693   5840       if( pLeaf->nn<=pLeaf->szLeaf ){
................................................................................
  6329   6476     if( rc==SQLITE_OK ){
  6330   6477       rc = sqlite3_create_function(
  6331   6478           db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
  6332   6479       );
  6333   6480     }
  6334   6481     return rc;
  6335   6482   }
         6483  +
         6484  +
         6485  +int sqlite3Fts5IndexReset(Fts5Index *p){
         6486  +  assert( p->pStruct==0 || p->iStructVersion!=0 );
         6487  +  if( fts5IndexDataVersion(p)!=p->iStructVersion ){
         6488  +    fts5StructureInvalidate(p);
         6489  +  }
         6490  +  return fts5IndexReturn(p);
         6491  +}

Changes to ext/fts5/fts5_main.c.

   592    592         pInfo->aConstraintUsage[pC->iConsIndex].omit = (unsigned char)pC->omit;
   593    593       }
   594    594     }
   595    595   
   596    596     pInfo->idxNum = idxFlags;
   597    597     return SQLITE_OK;
   598    598   }
          599  +
          600  +static int fts5NewTransaction(Fts5Table *pTab){
          601  +  Fts5Cursor *pCsr;
          602  +  for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
          603  +    if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK;
          604  +  }
          605  +  return sqlite3Fts5StorageReset(pTab->pStorage);
          606  +}
   599    607   
   600    608   /*
   601    609   ** Implementation of xOpen method.
   602    610   */
   603    611   static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
   604    612     Fts5Table *pTab = (Fts5Table*)pVTab;
   605    613     Fts5Config *pConfig = pTab->pConfig;
   606         -  Fts5Cursor *pCsr;               /* New cursor object */
          614  +  Fts5Cursor *pCsr = 0;           /* New cursor object */
   607    615     int nByte;                      /* Bytes of space to allocate */
   608         -  int rc = SQLITE_OK;             /* Return code */
          616  +  int rc;                         /* Return code */
   609    617   
   610         -  nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
   611         -  pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
   612         -  if( pCsr ){
   613         -    Fts5Global *pGlobal = pTab->pGlobal;
   614         -    memset(pCsr, 0, nByte);
   615         -    pCsr->aColumnSize = (int*)&pCsr[1];
   616         -    pCsr->pNext = pGlobal->pCsr;
   617         -    pGlobal->pCsr = pCsr;
   618         -    pCsr->iCsrId = ++pGlobal->iNextId;
   619         -  }else{
   620         -    rc = SQLITE_NOMEM;
          618  +  rc = fts5NewTransaction(pTab);
          619  +  if( rc==SQLITE_OK ){
          620  +    nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
          621  +    pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
          622  +    if( pCsr ){
          623  +      Fts5Global *pGlobal = pTab->pGlobal;
          624  +      memset(pCsr, 0, nByte);
          625  +      pCsr->aColumnSize = (int*)&pCsr[1];
          626  +      pCsr->pNext = pGlobal->pCsr;
          627  +      pGlobal->pCsr = pCsr;
          628  +      pCsr->iCsrId = ++pGlobal->iNextId;
          629  +    }else{
          630  +      rc = SQLITE_NOMEM;
          631  +    }
   621    632     }
   622    633     *ppCsr = (sqlite3_vtab_cursor*)pCsr;
   623    634     return rc;
   624    635   }
   625    636   
   626    637   static int fts5StmtType(Fts5Cursor *pCsr){
   627    638     if( pCsr->ePlan==FTS5_PLAN_SCAN ){
................................................................................
  1171   1182       assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 );
  1172   1183       assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 );
  1173   1184       assert( pCsr->iLastRowid==LARGEST_INT64 );
  1174   1185       assert( pCsr->iFirstRowid==SMALLEST_INT64 );
  1175   1186       pCsr->ePlan = FTS5_PLAN_SOURCE;
  1176   1187       pCsr->pExpr = pTab->pSortCsr->pExpr;
  1177   1188       rc = fts5CursorFirst(pTab, pCsr, bDesc);
  1178         -    sqlite3Fts5ExprClearEof(pCsr->pExpr);
  1179   1189     }else if( pMatch ){
  1180   1190       const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
  1181   1191       if( zExpr==0 ) zExpr = "";
  1182   1192   
  1183   1193       rc = fts5CursorParseRank(pConfig, pCsr, pRank);
  1184   1194       if( rc==SQLITE_OK ){
  1185   1195         if( zExpr[0]=='*' ){
................................................................................
  1574   1584     return rc;
  1575   1585   }
  1576   1586   
  1577   1587   /*
  1578   1588   ** Implementation of xBegin() method. 
  1579   1589   */
  1580   1590   static int fts5BeginMethod(sqlite3_vtab *pVtab){
  1581         -  UNUSED_PARAM(pVtab);  /* Call below is a no-op for NDEBUG builds */
  1582   1591     fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0);
         1592  +  fts5NewTransaction((Fts5Table*)pVtab);
  1583   1593     return SQLITE_OK;
  1584   1594   }
  1585   1595   
  1586   1596   /*
  1587   1597   ** Implementation of xCommit() method. This is a no-op. The contents of
  1588   1598   ** the pending-terms hash-table have already been flushed into the database
  1589   1599   ** by fts5SyncMethod().

Changes to ext/fts5/fts5_storage.c.

   141    141         if( rc!=SQLITE_OK && pzErrMsg ){
   142    142           *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db));
   143    143         }
   144    144       }
   145    145     }
   146    146   
   147    147     *ppStmt = p->aStmt[eStmt];
          148  +  sqlite3_reset(*ppStmt);
   148    149     return rc;
   149    150   }
   150    151   
   151    152   
   152    153   static int fts5ExecPrintf(
   153    154     sqlite3 *db,
   154    155     char **pzErr,
................................................................................
   242    243     int bWithout,                   /* True for without rowid */
   243    244     char **pzErr                    /* OUT: Error message */
   244    245   ){
   245    246     int rc;
   246    247     char *zErr = 0;
   247    248   
   248    249     rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s",
   249         -      pConfig->zDb, pConfig->zName, zPost, zDefn, bWithout?" WITHOUT ROWID":""
          250  +      pConfig->zDb, pConfig->zName, zPost, zDefn, 
          251  +#ifndef SQLITE_FTS5_NO_WITHOUT_ROWID
          252  +      bWithout?" WITHOUT ROWID":
          253  +#endif
          254  +      ""
   250    255     );
   251    256     if( zErr ){
   252    257       *pzErr = sqlite3_mprintf(
   253    258           "fts5: error creating shadow table %q_%s: %s", 
   254    259           pConfig->zName, zPost, zErr
   255    260       );
   256    261       sqlite3_free(zErr);
................................................................................
   364    369     int nToken,                     /* Size of token in bytes */
   365    370     int iUnused1,                   /* Start offset of token */
   366    371     int iUnused2                    /* End offset of token */
   367    372   ){
   368    373     Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
   369    374     Fts5Index *pIdx = pCtx->pStorage->pIndex;
   370    375     UNUSED_PARAM2(iUnused1, iUnused2);
          376  +  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
   371    377     if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
   372    378       pCtx->szCol++;
   373    379     }
   374    380     return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
   375    381   }
   376    382   
   377    383   /*
................................................................................
   535    541       if( rc==SQLITE_OK ){
   536    542         sqlite3_bind_int64(pDel, 1, iDel);
   537    543         sqlite3_step(pDel);
   538    544         rc = sqlite3_reset(pDel);
   539    545       }
   540    546     }
   541    547   
   542         -  /* Write the averages record */
   543         -  if( rc==SQLITE_OK ){
   544         -    rc = fts5StorageSaveTotals(p);
   545         -  }
   546         -
   547    548     return rc;
   548    549   }
   549    550   
   550    551   /*
   551    552   ** Delete all entries in the FTS5 index.
   552    553   */
   553    554   int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){
................................................................................
   634    635   int sqlite3Fts5StorageOptimize(Fts5Storage *p){
   635    636     return sqlite3Fts5IndexOptimize(p->pIndex);
   636    637   }
   637    638   
   638    639   int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){
   639    640     return sqlite3Fts5IndexMerge(p->pIndex, nMerge);
   640    641   }
          642  +
          643  +int sqlite3Fts5StorageReset(Fts5Storage *p){
          644  +  return sqlite3Fts5IndexReset(p->pIndex);
          645  +}
   641    646   
   642    647   /*
   643    648   ** Allocate a new rowid. This is used for "external content" tables when
   644    649   ** a NULL value is inserted into the rowid column. The new rowid is allocated
   645    650   ** by inserting a dummy row into the %_docsize table. The dummy will be
   646    651   ** overwritten later.
   647    652   **
................................................................................
   739    744   
   740    745     /* Write the %_docsize record */
   741    746     if( rc==SQLITE_OK ){
   742    747       rc = fts5StorageInsertDocsize(p, iRowid, &buf);
   743    748     }
   744    749     sqlite3_free(buf.p);
   745    750   
   746         -  /* Write the averages record */
   747         -  if( rc==SQLITE_OK ){
   748         -    rc = fts5StorageSaveTotals(p);
   749         -  }
   750         -
   751    751     return rc;
   752    752   }
   753    753   
   754    754   static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){
   755    755     Fts5Config *pConfig = p->pConfig;
   756    756     char *zSql;
   757    757     int rc;
................................................................................
   806    806     int bPresent;
   807    807     int ii;
   808    808     int rc = SQLITE_OK;
   809    809     int iPos;
   810    810     int iCol;
   811    811   
   812    812     UNUSED_PARAM2(iUnused1, iUnused2);
          813  +  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
   813    814   
   814    815     if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
   815    816       pCtx->szCol++;
   816    817     }
   817    818   
   818    819     switch( pCtx->pConfig->eDetail ){
   819    820       case FTS5_DETAIL_FULL:
................................................................................
  1077   1078     return rc;
  1078   1079   }
  1079   1080   
  1080   1081   /*
  1081   1082   ** Flush any data currently held in-memory to disk.
  1082   1083   */
  1083   1084   int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit){
  1084         -  if( bCommit && p->bTotalsValid ){
  1085         -    int rc = fts5StorageSaveTotals(p);
  1086         -    p->bTotalsValid = 0;
  1087         -    if( rc!=SQLITE_OK ) return rc;
         1085  +  int rc = SQLITE_OK;
         1086  +  i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db);
         1087  +  if( p->bTotalsValid ){
         1088  +    rc = fts5StorageSaveTotals(p);
         1089  +    if( bCommit ) p->bTotalsValid = 0;
         1090  +  }
         1091  +  if( rc==SQLITE_OK ){
         1092  +    rc = sqlite3Fts5IndexSync(p->pIndex, bCommit);
  1088   1093     }
  1089         -  return sqlite3Fts5IndexSync(p->pIndex, bCommit);
         1094  +  sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid);
         1095  +  return rc;
  1090   1096   }
  1091   1097   
  1092   1098   int sqlite3Fts5StorageRollback(Fts5Storage *p){
  1093   1099     p->bTotalsValid = 0;
  1094   1100     return sqlite3Fts5IndexRollback(p->pIndex);
  1095   1101   }
  1096   1102   
................................................................................
  1117   1123       rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);
  1118   1124       if( rc==SQLITE_OK ){
  1119   1125         p->pConfig->iCookie = iNew;
  1120   1126       }
  1121   1127     }
  1122   1128     return rc;
  1123   1129   }
  1124         -
  1125         -

Changes to ext/fts5/fts5_tcl.c.

    10     10   **
    11     11   ******************************************************************************
    12     12   **
    13     13   */
    14     14   
    15     15   
    16     16   #ifdef SQLITE_TEST
    17         -#include <tcl.h>
           17  +#if defined(INCLUDE_SQLITE_TCL_H)
           18  +#  include "sqlite_tcl.h"
           19  +#else
           20  +#  include "tcl.h"
           21  +#  ifndef SQLITE_TCLAPI
           22  +#    define SQLITE_TCLAPI
           23  +#  endif
           24  +#endif
    18     25   
    19     26   #ifdef SQLITE_ENABLE_FTS5
    20     27   
    21     28   #include "fts5.h"
    22     29   #include <string.h>
    23     30   #include <assert.h>
    24     31   
................................................................................
    74     81         return aErr[i].rc;
    75     82       }
    76     83     }
    77     84   
    78     85     return SQLITE_ERROR;
    79     86   }
    80     87   
    81         -static int f5tDbAndApi(
           88  +static int SQLITE_TCLAPI f5tDbAndApi(
    82     89     Tcl_Interp *interp, 
    83     90     Tcl_Obj *pObj, 
    84     91     sqlite3 **ppDb, 
    85     92     fts5_api **ppApi
    86     93   ){
    87     94     sqlite3 *db = 0;
    88     95     int rc = f5tDbPointer(interp, pObj, &db);
................................................................................
   160    167     if( rc==TCL_OK ){
   161    168       rc = f5tResultToErrorCode(Tcl_GetStringResult(p->interp));
   162    169     }
   163    170   
   164    171     return rc;
   165    172   }
   166    173   
   167         -static int xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);
          174  +static int SQLITE_TCLAPI xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);
   168    175   
   169    176   static int xQueryPhraseCb(
   170    177     const Fts5ExtensionApi *pApi, 
   171    178     Fts5Context *pFts, 
   172    179     void *pCtx
   173    180   ){
   174    181     F5tFunction *p = (F5tFunction*)pCtx;
................................................................................
   205    212   }
   206    213   
   207    214   /*
   208    215   **      api sub-command...
   209    216   **
   210    217   ** Description...
   211    218   */
   212         -static int xF5tApi(
          219  +static int SQLITE_TCLAPI xF5tApi(
   213    220     void * clientData,
   214    221     Tcl_Interp *interp,
   215    222     int objc,
   216    223     Tcl_Obj *CONST objv[]
   217    224   ){
   218    225     struct Sub {
   219    226       const char *zName;
................................................................................
   598    605   }
   599    606   
   600    607   /*
   601    608   **      sqlite3_fts5_create_function DB NAME SCRIPT
   602    609   **
   603    610   ** Description...
   604    611   */
   605         -static int f5tCreateFunction(
          612  +static int SQLITE_TCLAPI f5tCreateFunction(
   606    613     void * clientData,
   607    614     Tcl_Interp *interp,
   608    615     int objc,
   609    616     Tcl_Obj *CONST objv[]
   610    617   ){
   611    618     char *zName;
   612    619     Tcl_Obj *pScript;
................................................................................
   668    675   
   669    676   
   670    677   /*
   671    678   **      sqlite3_fts5_tokenize DB TOKENIZER TEXT
   672    679   **
   673    680   ** Description...
   674    681   */
   675         -static int f5tTokenize(
          682  +static int SQLITE_TCLAPI f5tTokenize(
   676    683     void * clientData,
   677    684     Tcl_Interp *interp,
   678    685     int objc,
   679    686     Tcl_Obj *CONST objv[]
   680    687   ){
   681    688     char *zText;
   682    689     int nText;
................................................................................
   874    881     pInst->pContext->xToken = xOldToken;
   875    882     return rc;
   876    883   }
   877    884   
   878    885   /*
   879    886   ** sqlite3_fts5_token ?-colocated? TEXT START END
   880    887   */
   881         -static int f5tTokenizerReturn(
          888  +static int SQLITE_TCLAPI f5tTokenizerReturn(
   882    889     void * clientData,
   883    890     Tcl_Interp *interp,
   884    891     int objc,
   885    892     Tcl_Obj *CONST objv[]
   886    893   ){
   887    894     F5tTokenizerContext *p = (F5tTokenizerContext*)clientData;
   888    895     int iStart;
................................................................................
   945    952   ** tokenizer instance "returned" by SCRIPT. Specifically, to tokenize
   946    953   ** text SCRIPT2 is invoked with a single argument appended to it - the
   947    954   ** text to tokenize.
   948    955   **
   949    956   ** SCRIPT2 should invoke the [sqlite3_fts5_token] command once for each
   950    957   ** token within the tokenized text.
   951    958   */
   952         -static int f5tCreateTokenizer(
          959  +static int SQLITE_TCLAPI f5tCreateTokenizer(
   953    960     ClientData clientData,
   954    961     Tcl_Interp *interp,
   955    962     int objc,
   956    963     Tcl_Obj *CONST objv[]
   957    964   ){
   958    965     F5tTokenizerContext *pContext = (F5tTokenizerContext*)clientData;
   959    966     sqlite3 *db;
................................................................................
   988    995       Tcl_AppendResult(interp, "error in fts5_api.xCreateTokenizer()", 0);
   989    996       return TCL_ERROR;
   990    997     }
   991    998   
   992    999     return TCL_OK;
   993   1000   }
   994   1001   
   995         -static void xF5tFree(ClientData clientData){
         1002  +static void SQLITE_TCLAPI xF5tFree(ClientData clientData){
   996   1003     ckfree(clientData);
   997   1004   }
   998   1005   
   999   1006   /*
  1000   1007   **      sqlite3_fts5_may_be_corrupt BOOLEAN
  1001   1008   **
  1002   1009   ** Set or clear the global "may-be-corrupt" flag. Return the old value.
  1003   1010   */
  1004         -static int f5tMayBeCorrupt(
         1011  +static int SQLITE_TCLAPI f5tMayBeCorrupt(
  1005   1012     void * clientData,
  1006   1013     Tcl_Interp *interp,
  1007   1014     int objc,
  1008   1015     Tcl_Obj *CONST objv[]
  1009   1016   ){
  1010   1017     int bOld = sqlite3_fts5_may_be_corrupt;
  1011   1018   
................................................................................
  1029   1036     unsigned int h = 13;
  1030   1037     for(i=n-1; i>=0; i--){
  1031   1038       h = (h << 3) ^ h ^ p[i];
  1032   1039     }
  1033   1040     return (h % nSlot);
  1034   1041   }
  1035   1042   
  1036         -static int f5tTokenHash(
         1043  +static int SQLITE_TCLAPI f5tTokenHash(
  1037   1044     void * clientData,
  1038   1045     Tcl_Interp *interp,
  1039   1046     int objc,
  1040   1047     Tcl_Obj *CONST objv[]
  1041   1048   ){
  1042   1049     char *z;
  1043   1050     int n;
................................................................................
  1054   1061     z = Tcl_GetStringFromObj(objv[2], &n);
  1055   1062   
  1056   1063     iVal = f5t_fts5HashKey(nSlot, z, n);
  1057   1064     Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
  1058   1065     return TCL_OK;
  1059   1066   }
  1060   1067   
  1061         -static int f5tRegisterMatchinfo(
         1068  +static int SQLITE_TCLAPI f5tRegisterMatchinfo(
  1062   1069     void * clientData,
  1063   1070     Tcl_Interp *interp,
  1064   1071     int objc,
  1065   1072     Tcl_Obj *CONST objv[]
  1066   1073   ){
  1067   1074     int rc;
  1068   1075     sqlite3 *db = 0;
................................................................................
  1079   1086     if( rc!=SQLITE_OK ){
  1080   1087       Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
  1081   1088       return TCL_ERROR;
  1082   1089     }
  1083   1090     return TCL_OK;
  1084   1091   }
  1085   1092   
  1086         -static int f5tRegisterTok(
         1093  +static int SQLITE_TCLAPI f5tRegisterTok(
  1087   1094     void * clientData,
  1088   1095     Tcl_Interp *interp,
  1089   1096     int objc,
  1090   1097     Tcl_Obj *CONST objv[]
  1091   1098   ){
  1092   1099     int rc;
  1093   1100     sqlite3 *db = 0;

Changes to ext/fts5/fts5_vocab.c.

   274    274       }
   275    275       if( iTermLe>=0 ){
   276    276         idxNum |= FTS5_VOCAB_TERM_LE;
   277    277         pInfo->aConstraintUsage[iTermLe].argvIndex = ++nArg;
   278    278         pInfo->estimatedCost = pInfo->estimatedCost / 2;
   279    279       }
   280    280     }
          281  +
          282  +  /* This virtual table always delivers results in ascending order of
          283  +  ** the "term" column (column 0). So if the user has requested this
          284  +  ** specifically - "ORDER BY term" or "ORDER BY term ASC" - set the
          285  +  ** sqlite3_index_info.orderByConsumed flag to tell the core the results
          286  +  ** are already in sorted order.  */
          287  +  if( pInfo->nOrderBy==1 
          288  +   && pInfo->aOrderBy[0].iColumn==0 
          289  +   && pInfo->aOrderBy[0].desc==0
          290  +  ){
          291  +    pInfo->orderByConsumed = 1;
          292  +  }
   281    293   
   282    294     pInfo->idxNum = idxNum;
   283         -
   284    295     return SQLITE_OK;
   285    296   }
   286    297   
   287    298   /*
   288    299   ** Implementation of xOpen method.
   289    300   */
   290    301   static int fts5VocabOpenMethod(

Changes to ext/fts5/fts5parse.y.

   116    116   }
   117    117   
   118    118   %type colset {Fts5Colset*}
   119    119   %destructor colset { sqlite3_free($$); }
   120    120   %type colsetlist {Fts5Colset*}
   121    121   %destructor colsetlist { sqlite3_free($$); }
   122    122   
          123  +colset(A) ::= MINUS LCP colsetlist(X) RCP. { 
          124  +    A = sqlite3Fts5ParseColsetInvert(pParse, X);
          125  +}
   123    126   colset(A) ::= LCP colsetlist(X) RCP. { A = X; }
   124    127   colset(A) ::= STRING(X). {
   125    128     A = sqlite3Fts5ParseColset(pParse, 0, &X);
   126    129   }
          130  +colset(A) ::= MINUS STRING(X). {
          131  +  A = sqlite3Fts5ParseColset(pParse, 0, &X);
          132  +  A = sqlite3Fts5ParseColsetInvert(pParse, A);
          133  +}
   127    134   
   128    135   colsetlist(A) ::= colsetlist(Y) STRING(X). { 
   129    136     A = sqlite3Fts5ParseColset(pParse, Y, &X); }
   130    137   colsetlist(A) ::= STRING(X). { 
   131    138     A = sqlite3Fts5ParseColset(pParse, 0, &X); 
   132    139   }
   133         -
   134    140   
   135    141   %type nearset     {Fts5ExprNearset*}
   136    142   %type nearphrases {Fts5ExprNearset*}
   137    143   %destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
   138    144   %destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }
   139    145   
   140    146   nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }

Changes to ext/fts5/test/fts5aa.test.

   428    428     set fd [db incrblob main n1_data block 10]
   429    429     fconfigure $fd -encoding binary -translation binary
   430    430     puts -nonewline $fd "\x44\x45"
   431    431     close $fd
   432    432   }
   433    433   db func funk funk
   434    434   
          435  +# This test case corrupts the structure record within the first invocation
          436  +# of function funk(). Which used to cause the bm25() function to throw an
          437  +# exception. But since bm25() can now used the cached structure record,
          438  +# it never sees the corruption introduced by funk() and so the following 
          439  +# statement no longer fails.
          440  +#
   435    441   do_catchsql_test 16.2 {
   436    442     SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
   437         -} {1 {SQL logic error or missing database}}
          443  +} {0 {{} -1e-06 {}}}
          444  +# {1 {SQL logic error or missing database}}
   438    445   
   439    446   #-------------------------------------------------------------------------
   440    447   #
   441    448   reset_db
   442    449   do_execsql_test 17.1 {
   443    450     CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
   444    451     INSERT INTO b2 VALUES('a');

Changes to ext/fts5/test/fts5af.test.

    68     68     1.6 {o o o o o X o} {o o o o o [X] o}
    69     69     1.7 {o o o o o o X} {o o o o o o [X]}
    70     70   
    71     71     2.1 {X o o o o o o o} {[X] o o o o o o...}
    72     72     2.2 {o X o o o o o o} {o [X] o o o o o...}
    73     73     2.3 {o o X o o o o o} {o o [X] o o o o...}
    74     74     2.4 {o o o X o o o o} {o o o [X] o o o...}
    75         -  2.5 {o o o o X o o o} {...o o o [X] o o o}
    76         -  2.6 {o o o o o X o o} {...o o o o [X] o o}
    77         -  2.7 {o o o o o o X o} {...o o o o o [X] o}
           75  +  2.5 {o o o o X o o o} {o o o o [X] o o...}
           76  +  2.6 {o o o o o X o o} {o o o o o [X] o...}
           77  +  2.7 {o o o o o o X o} {o o o o o o [X]...}
    78     78     2.8 {o o o o o o o X} {...o o o o o o [X]}
           79  +
           80  +  2.9  {o o o o o o o X o}       {...o o o o o [X] o}
           81  +  2.10 {o o o o o o o X o o}     {...o o o o [X] o o}
           82  +  2.11 {o o o o o o o X o o o}   {...o o o [X] o o o}
           83  +  2.12 {o o o o o o o X o o o o} {...o o o [X] o o o...}
           84  +
    79     85   
    80     86     3.1 {X o o o o o o o o} {[X] o o o o o o...}
    81     87     3.2 {o X o o o o o o o} {o [X] o o o o o...}
    82     88     3.3 {o o X o o o o o o} {o o [X] o o o o...}
    83     89     3.4 {o o o X o o o o o} {o o o [X] o o o...}
    84         -  3.5 {o o o o X o o o o} {...o o o [X] o o o...}
    85         -  3.6 {o o o o o X o o o} {...o o o [X] o o o}
    86         -  3.7 {o o o o o o X o o} {...o o o o [X] o o}
    87         -  3.8 {o o o o o o o X o} {...o o o o o [X] o}
    88         -  3.9 {o o o o o o o o X} {...o o o o o o [X]}
           90  +
           91  +  3.5 {o o o o o o o X o o o o} {...o o o [X] o o o...}
           92  +  3.6 {o o o o o o o o X o o o} {...o o o [X] o o o}
           93  +  3.7 {o o o o o o o o o X o o} {...o o o o [X] o o}
           94  +  3.8 {o o o o o o o o o o X o} {...o o o o o [X] o}
           95  +  3.9 {o o o o o o o o o o o X} {...o o o o o o [X]}
    89     96   
    90     97     4.1 {X o o o o o X o o} {[X] o o o o o [X]...}
    91         -  4.2 {o X o o o o o X o} {...[X] o o o o o [X]...}
    92         -  4.3 {o o X o o o o o X} {...[X] o o o o o [X]}
           98  +  4.2 {o o o o o o o X o o o o o X o} {...[X] o o o o o [X]...}
           99  +  4.3 {o o o o o o o o X o o o o o X} {...[X] o o o o o [X]}
    93    100   
    94    101     5.1 {X o o o o X o o o} {[X] o o o o [X] o...}
    95         -  5.2 {o X o o o o X o o} {...[X] o o o o [X] o...}
    96         -  5.3 {o o X o o o o X o} {...[X] o o o o [X] o}
    97         -  5.4 {o o o X o o o o X} {...o [X] o o o o [X]}
          102  +  5.2 {o o o o o o o X o o o o X o o} {...[X] o o o o [X] o...}
          103  +  5.3 {o o o o o o o o X o o o o X o} {...[X] o o o o [X] o}
          104  +  5.4 {o o o o o o o o o X o o o o X} {...o [X] o o o o [X]}
    98    105   
    99    106     6.1 {X o o o X o o o} {[X] o o o [X] o o...}
   100    107     6.2 {o X o o o X o o o} {o [X] o o o [X] o...}
   101         -  6.3 {o o X o o o X o o} {...o [X] o o o [X] o...}
   102         -  6.4 {o o o X o o o X o} {...o [X] o o o [X] o}
   103         -  6.5 {o o o o X o o o X} {...o o [X] o o o [X]}
          108  +  6.3 {o o o o o o o X o o o X o o} {...o [X] o o o [X] o...}
          109  +  6.4 {o o o o o o o o X o o o X o} {...o [X] o o o [X] o}
          110  +  6.5 {o o o o o o o o o X o o o X} {...o o [X] o o o [X]}
   104    111   
   105    112     7.1 {X o o X o o o o o} {[X] o o [X] o o o...}
   106    113     7.2 {o X o o X o o o o} {o [X] o o [X] o o...}
   107         -  7.3 {o o X o o X o o o} {...o [X] o o [X] o o...}
   108         -  7.4 {o o o X o o X o o} {...o [X] o o [X] o o}
   109         -  7.5 {o o o o X o o X o} {...o o [X] o o [X] o}
   110         -  7.6 {o o o o o X o o X} {...o o o [X] o o [X]}
          114  +  7.3 {o o o o o o o X o o X o o o} {...o [X] o o [X] o o...}
          115  +  7.4 {o o o o o o o o X o o X o o} {...o [X] o o [X] o o}
          116  +  7.5 {o o o o o o o o o X o o X o} {...o o [X] o o [X] o}
          117  +  7.6 {o o o o o o o o o o X o o X} {...o o o [X] o o [X]}
          118  +
          119  +  8.1 {o o o o o o o o o X o o o o o o o o o o o o o o o o X X X o o o}
          120  +      {...o o [X] [X] [X] o o...}
          121  +  8.2 {o o o o o o o. o o X o o o o o o o o o o o o o o o o X X X o o o} 
          122  +      {...o o [X] o o o o...}
          123  +  8.3 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o} 
          124  +      {o o o o [X] o o...}
   111    125   } {
   112    126     do_snippet_test 1.$tn $doc X $res
   113    127   }
   114    128   
   115    129   if {[detail_is_full]} {
   116    130     foreach {tn doc res} {
   117    131       1.1 {X Y o o o o o} {[X Y] o o o o o}
................................................................................
   120    134       1.4 {o o o X Y o o} {o o o [X Y] o o}
   121    135       1.5 {o o o o X Y o} {o o o o [X Y] o}
   122    136       1.6 {o o o o o X Y} {o o o o o [X Y]}
   123    137   
   124    138       2.1 {X Y o o o o o o} {[X Y] o o o o o...}
   125    139       2.2 {o X Y o o o o o} {o [X Y] o o o o...}
   126    140       2.3 {o o X Y o o o o} {o o [X Y] o o o...}
   127         -    2.4 {o o o X Y o o o} {...o o [X Y] o o o}
   128         -    2.5 {o o o o X Y o o} {...o o o [X Y] o o}
   129         -    2.6 {o o o o o X Y o} {...o o o o [X Y] o}
   130         -    2.7 {o o o o o o X Y} {...o o o o o [X Y]}
          141  +    2.4 {o o o o o o o X Y o o o} {...o o [X Y] o o o}
          142  +    2.5 {o o o o o o o o X Y o o} {...o o o [X Y] o o}
          143  +    2.6 {o o o o o o o o o X Y o} {...o o o o [X Y] o}
          144  +    2.7 {o o o o o o o o o o X Y} {...o o o o o [X Y]}
   131    145   
   132    146       3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
   133    147       3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
   134    148       3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
   135         -    3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
   136         -    3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
   137         -    3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
   138         -    3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
   139         -    3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
          149  +    3.4 {o o o o o o o X Y o o o o} {...o o [X Y] o o o...}
          150  +    3.5 {o o o o o o o o X Y o o o} {...o o [X Y] o o o}
          151  +    3.6 {o o o o o o o o o X Y o o} {...o o o [X Y] o o}
          152  +    3.7 {o o o o o o o o o o X Y o} {...o o o o [X Y] o}
          153  +    3.8 {o o o o o o o o o o o X Y} {...o o o o o [X Y]}
   140    154     } {
   141    155       do_snippet_test 2.$tn $doc "X + Y" $res
   142    156     }
   143    157   }
   144    158   
          159  +do_execsql_test 4.0 {
          160  +  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
          161  +  INSERT INTO x1 VALUES('xyz', '1 2 3 4 5 6 7 8 9 10 11 12 13');
          162  +  SELECT snippet(x1, 1, '[', ']', '...', 5) FROM x1('xyz');
          163  +} {
          164  +  {1 2 3 4 5...}
          165  +}
          166  +
          167  +do_execsql_test 5.0 {
          168  +  CREATE VIRTUAL TABLE p1 USING fts5(a, b);
          169  +  INSERT INTO p1 VALUES(
          170  +    'x a a a a a a a a a a',
          171  +    'a a a a a a a a a a a a a a a a a a a x'
          172  +  );
          173  +}
          174  +do_execsql_test 5.1 {
          175  +  SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
          176  +} {{[x] a a a a a...}}
          177  +
   145    178   } ;# foreach_detail_mode 
   146    179   
   147    180   finish_test
   148    181   

Changes to ext/fts5/test/fts5aux.test.

   241    241     execsql { DELETE FROM x1 }
   242    242     foreach row $lRow { execsql { INSERT INTO x1 VALUES($row) } }
   243    243     breakpoint
   244    244     do_execsql_test 8.$tn {
   245    245       SELECT highlight(x1, 0, '[', ']') FROM x1 WHERE x1 MATCH 'a OR (b AND d)';
   246    246     } $res
   247    247   }
          248  +
          249  +#-------------------------------------------------------------------------
          250  +# Test the built-in bm25() demo.
          251  +#
          252  +reset_db
          253  +do_execsql_test 9.1 {
          254  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b);
          255  +  INSERT INTO t1 VALUES('a',   NULL);           -- 1
          256  +  INSERT INTO t1 VALUES('a',   NULL);           -- 2
          257  +  INSERT INTO t1 VALUES('a',   NULL);           -- 3
          258  +  INSERT INTO t1 VALUES('a',   NULL);           -- 4
          259  +  INSERT INTO t1 VALUES('a',   NULL);           -- 5
          260  +  INSERT INTO t1 VALUES('a',   NULL);           -- 6
          261  +  INSERT INTO t1 VALUES('a',   NULL);           -- 7
          262  +  INSERT INTO t1 VALUES('a',   NULL);           -- 8
          263  +  INSERT INTO t1 VALUES(NULL,  'a a b');        -- 9
          264  +  INSERT INTO t1 VALUES(NULL,  'b b a');        -- 10
          265  +}
          266  +
          267  +do_execsql_test 9.2 {
          268  +  SELECT rowid FROM t1('a AND b') ORDER BY rank;
          269  +} {
          270  +  10 9
          271  +}
          272  +
          273  +do_execsql_test 9.3 {
          274  +  SELECT rowid FROM t1('b:a AND b:b') ORDER BY rank;
          275  +} {
          276  +  9 10
          277  +}
          278  +
          279  +
   248    280   
   249    281   finish_test
   250    282   

Added ext/fts5/test/fts5colset.test.

            1  +# 2016 August 10
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5colset
           17  +
           18  +# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
           19  +ifcapable !fts5 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +foreach_detail_mode $::testprefix {
           25  +  if {[detail_is_none]} continue
           26  +
           27  +  do_execsql_test 1.0 {
           28  +    CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d, detail=%DETAIL%);
           29  +    INSERT INTO t1 VALUES('a', 'b', 'c', 'd');  -- 1
           30  +    INSERT INTO t1 VALUES('d', 'a', 'b', 'c');  -- 2
           31  +    INSERT INTO t1 VALUES('c', 'd', 'a', 'b');  -- 3
           32  +    INSERT INTO t1 VALUES('b', 'c', 'd', 'a');  -- 4
           33  +  }
           34  +
           35  +  foreach {tn q res} {
           36  +    1 "a"          {1 2 3 4}
           37  +    2 "{a}   : a"  {1}
           38  +    3 "-{a}   : a" {2 3 4}
           39  +    4 "- {a c} : a" {2 4}
           40  +    5 " - {d d c} : a" {1 2}
           41  +    6 "- {d c b a} : a" {}
           42  +    7 "-{\"a\"} : b" {1 2 3}
           43  +    8 "- c : a" {1 2 4}
           44  +    9 "-c : a"  {1 2 4}
           45  +    10 "-\"c\" : a"  {1 2 4}
           46  +  } {
           47  +  breakpoint
           48  +    do_execsql_test 1.$tn {
           49  +      SELECT rowid FROM t1($q)
           50  +    } $res
           51  +  }
           52  +
           53  +
           54  +}
           55  +
           56  +
           57  +finish_test
           58  +
           59  +

Changes to ext/fts5/test/fts5corrupt2.test.

    33     33     CREATE VIRTUAL TABLE t1 USING fts5(x);
    34     34     INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    35     35     WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
    36     36     INSERT INTO t1 SELECT rnddoc(10) FROM ii;
    37     37   }
    38     38   set mask [expr 31 << 31]
    39     39   
    40         -if 1 {
           40  +if 0 {
    41     41   
    42     42   # Test 1:
    43     43   #
    44     44   #   For each page in the t1_data table, open a transaction and DELETE
    45     45   #   the t1_data entry. Then run:
    46     46   #
    47     47   #     * an integrity-check, and
................................................................................
    78     78       }
    79     79     
    80     80       do_execsql_test 1.$tno.$tn.3.$rowid {
    81     81         ROLLBACK;
    82     82         INSERT INTO t1(t1) VALUES('integrity-check');
    83     83       } {}
    84     84     }
           85  +}
           86  +
    85     87   }
    86     88   
    87     89   # Using the same database as the 1.* tests.
    88     90   #
    89     91   # Run N-1 tests, where N is the number of bytes in the rightmost leaf page
    90     92   # of the fts index. For test $i, truncate the rightmost leafpage to $i
    91     93   # bytes. Then test both the integrity-check detects the corruption.
................................................................................
   206    208         set {} 1
   207    209       } {1}
   208    210   
   209    211       execsql ROLLBACK
   210    212     }
   211    213   
   212    214     # do_test 4.$tn.x { expr $nCorrupt>0 } 1
   213         -}
   214         -
   215    215   }
   216    216   
   217    217   set doc [string repeat "A B C " 1000]
   218    218   do_execsql_test 5.0 {
   219    219     CREATE VIRTUAL TABLE x5 USING fts5(tt);
   220    220     INSERT INTO x5(x5, rank) VALUES('pgsz', 32);
   221    221     WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10) 

Changes to ext/fts5/test/fts5corrupt3.test.

   175    175   for {set i 1} {1} {incr i} {
   176    176     set struct [db one {SELECT block FROM t1_data WHERE id=10}]
   177    177     binary scan $struct c* var
   178    178     set end [lindex $var end]
   179    179     if {$end<=$i} break
   180    180     lset var end [expr $end - $i]
   181    181     set struct [binary format c* $var]
          182  +
          183  +  db close
          184  +  sqlite3 db test.db
          185  +
   182    186     db eval {
   183    187       BEGIN;
   184    188       UPDATE t1_data SET block = $struct WHERE id=10;
   185    189     }
   186    190     do_test 4.1.$i {
   187    191       incr nErr [catch { db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } }]
   188    192       set {} {}

Added ext/fts5/test/fts5determin.test.

            1  +# 2016 March 21
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +# Specifically, that the fts5 module is deterministic. At one point, when
           15  +# segment ids were allocated using sqlite3_randomness(), this was not the
           16  +# case.
           17  +#
           18  +
           19  +source [file join [file dirname [info script]] fts5_common.tcl]
           20  +set testprefix fts5aa
           21  +return_if_no_fts5 
           22  +
           23  +proc do_determin_test {tn} {
           24  +  uplevel [list
           25  +    do_execsql_test $tn {
           26  +      SELECT (SELECT md5sum(id, block) FROM t1_data)==
           27  +             (SELECT md5sum(id, block) FROM t2_data),
           28  +             (SELECT md5sum(id, block) FROM t1_data)==
           29  +             (SELECT md5sum(id, block) FROM t3_data)
           30  +    } {1 1}
           31  +  ]
           32  +}
           33  +
           34  +foreach_detail_mode $::testprefix {
           35  +  do_execsql_test 1.0 {
           36  +    CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1 2", detail=%DETAIL%);
           37  +    CREATE VIRTUAL TABLE t2 USING fts5(a, b, prefix="1 2", detail=%DETAIL%);
           38  +    CREATE VIRTUAL TABLE t3 USING fts5(a, b, prefix="1 2", detail=%DETAIL%);
           39  +  }
           40  +
           41  +  do_test 1.1 {
           42  +    foreach t {t1 t2 t3} {
           43  +      execsql [string map [list TBL $t] {
           44  +        INSERT INTO TBL VALUES('a b c', 'd e f');
           45  +        INSERT INTO TBL VALUES('c1 c2 c3', 'c1 c2 c3');
           46  +        INSERT INTO TBL VALUES('xyzxyzxyz', 'xyzxyzxyz');
           47  +      }]
           48  +    }
           49  +  } {}
           50  +
           51  +  do_determin_test 1.2
           52  +
           53  +  do_test 1.3 {
           54  +    foreach t {t1 t2 t3} {
           55  +      execsql [string map [list TBL $t] {
           56  +        INSERT INTO TBL(TBL) VALUES('optimize');
           57  +      }]
           58  +    }
           59  +  } {}
           60  +
           61  +  do_determin_test 1.4
           62  +}
           63  +
           64  +
           65  +finish_test
           66  +
           67  +

Changes to ext/fts5/test/fts5dlidx.test.

   174    174   
   175    175   do_execsql_test 3.2 {
   176    176     SELECT rowid FROM abc WHERE abc 
   177    177     MATCH 'IteratorpItercurrentlypointstothefirstrowidofadoclist' 
   178    178     ORDER BY rowid DESC;
   179    179   } {16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1}
   180    180   
   181         -do_execsql_test 3.2 {
          181  +do_execsql_test 3.3 {
   182    182     INSERT INTO abc(abc) VALUES('integrity-check');
   183    183     INSERT INTO abc(abc) VALUES('optimize');
   184    184     INSERT INTO abc(abc) VALUES('integrity-check');
   185    185   }
   186    186   
   187    187   set v [lindex $vocab 0]
   188    188   set i 0
   189    189   foreach v $vocab {
   190         -  do_execsql_test 3.3.[incr i] {
          190  +  do_execsql_test 3.4.[incr i] {
   191    191       SELECT rowid FROM abc WHERE abc MATCH $v
   192    192     } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
   193    193   }
   194    194   
   195    195   } ;# foreach_detail_mode
   196    196   
   197    197   
   198    198   
   199    199   finish_test
   200    200   

Changes to ext/fts5/test/fts5eb.test.

    58     58   do_catchsql_test 2.1 {
    59     59     SELECT fts5_expr()
    60     60   } {1 {wrong number of arguments to function fts5_expr}}
    61     61   
    62     62   do_catchsql_test 2.1 {
    63     63     SELECT fts5_expr_tcl()
    64     64   } {1 {wrong number of arguments to function fts5_expr_tcl}}
           65  +
           66  +
           67  +do_execsql_test 3.0 {
           68  +  CREATE VIRTUAL TABLE e1 USING fts5(text, tokenize = 'porter unicode61');
           69  +  INSERT INTO e1 VALUES ("just a few words with a / inside");
           70  +}
           71  +do_execsql_test 3.1 {
           72  +  SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank;
           73  +} {1 -1e-06}
           74  +do_execsql_test 3.2 {
           75  +  SELECT rowid FROM e1 WHERE e1 MATCH '"/" OR "just"'
           76  +} 1
           77  +do_execsql_test 3.3 {
           78  +  SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank;
           79  +} {1 -1e-06}
           80  +
           81  +
    65     82   
    66     83   finish_test
    67     84   
    68     85   
    69     86   

Changes to ext/fts5/test/fts5fault4.test.

    82     82   }
    83     83   
    84     84   set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}]
    85     85   
    86     86   do_faultsim_test 4 -faults oom-* -body {
    87     87     db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
    88     88   } -test {
    89         -  faultsim_test_result {0 {0 {} 4}}
           89  +  faultsim_test_result {0 {0 {} 3}}
    90     90   }
    91     91   
    92     92   #-------------------------------------------------------------------------
    93     93   # An OOM within a query that uses a custom rank function.
    94     94   #
    95     95   reset_db
    96     96   do_execsql_test 5.0 {

Changes to ext/fts5/test/fts5faultB.test.

    74     74   
    75     75   do_faultsim_test 2.4 -faults oom* -body {
    76     76     execsql { SELECT mit(matchinfo(t1, 's')) FROM t1('a b c') }
    77     77   } -test {
    78     78     faultsim_test_result {0 {{3 2} {2 3}}} 
    79     79   }
    80     80   
           81  +#-------------------------------------------------------------------------
           82  +#
           83  +reset_db 
           84  +do_execsql_test 3.0 {
           85  +  CREATE VIRTUAL TABLE x1 USING fts5(z);
           86  +}
           87  +
           88  +do_faultsim_test 3.1 -faults oom* -body {
           89  +  execsql {
           90  +    SELECT rowid FROM x1('c') WHERE rowid>1;
           91  +  }
           92  +} -test {
           93  +  faultsim_test_result {0 {}}
           94  +}
           95  +
           96  +do_execsql_test 3.2 {
           97  +  INSERT INTO x1 VALUES('a b c');
           98  +  INSERT INTO x1 VALUES('b c d');
           99  +  INSERT INTO x1 VALUES('c d e');
          100  +  INSERT INTO x1 VALUES('d e f');
          101  +}
          102  +do_faultsim_test 3.3 -faults oom* -body {
          103  +  execsql {
          104  +    SELECT rowid FROM x1('c') WHERE rowid>1;
          105  +  }
          106  +} -test {
          107  +  faultsim_test_result {0 {2 3}}
          108  +}
    81    109   
    82    110   finish_test
    83    111   

Added ext/fts5/test/fts5faultD.test.

            1  +# 2016 February 2
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +# This file is focused on OOM errors.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +source $testdir/malloc_common.tcl
           17  +set testprefix fts5faultA
           18  +
           19  +# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           20  +ifcapable !fts5 {
           21  +  finish_test
           22  +  return
           23  +}
           24  +
           25  +foreach_detail_mode $testprefix {
           26  +  if {"%DETAIL%"=="none"} continue
           27  +
           28  +  do_execsql_test 1.0 {
           29  +    CREATE VIRTUAL TABLE o1 USING fts5(a, b, c, detail=%DETAIL%);
           30  +    INSERT INTO o1(o1, rank) VALUES('pgsz', 32);
           31  +
           32  +    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 )
           33  +    INSERT INTO o1 SELECT 'A', 'B', 'C' FROM s;
           34  +
           35  +    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 )
           36  +    INSERT INTO o1 SELECT 'C', 'A', 'B' FROM s;
           37  +
           38  +    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 )
           39  +    INSERT INTO o1 SELECT 'B', 'C', 'A' FROM s;
           40  +  }
           41  +  
           42  +  do_faultsim_test 1 -faults int* -prep {
           43  +    sqlite3 db test.db
           44  +  } -body {
           45  +    execsql { SELECT count(*) FROM o1('a') }
           46  +  } -test {
           47  +    faultsim_test_result {0 180} {1 {vtable constructor failed: o1}}
           48  +  }
           49  +
           50  +  do_faultsim_test 2 -faults int* -prep {
           51  +    sqlite3 db test.db
           52  +  } -body {
           53  +    execsql { SELECT * FROM o1('a:a AND {b c}:b') ORDER BY rank }
           54  +    expr 1
           55  +  } -test {
           56  +    faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
           57  +  }
           58  +
           59  +  do_faultsim_test 3 -faults int* -prep {
           60  +    sqlite3 db test.db
           61  +  } -body {
           62  +    execsql { SELECT * FROM o1('{b c}:b NOT a:a') ORDER BY rank }
           63  +    expr 1
           64  +  } -test {
           65  +    faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
           66  +  }
           67  +
           68  +  do_faultsim_test 4 -faults int* -prep {
           69  +    sqlite3 db test.db
           70  +  } -body {
           71  +    execsql { SELECT * FROM o1('b:b OR a:a') }
           72  +    expr 1
           73  +  } -test {
           74  +    faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
           75  +  }
           76  +
           77  +  do_faultsim_test 5 -faults int* -prep {
           78  +    sqlite3 db test.db
           79  +  } -body {
           80  +    execsql { SELECT count(*) FROM o1('c:b') }
           81  +    expr 1
           82  +  } -test {
           83  +    faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
           84  +  }
           85  +}
           86  +
           87  +finish_test

Added ext/fts5/test/fts5lastrowid.test.

            1  +# 2017 Feb 27
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# Tests of the last_insert_rowid functionality with fts5.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5lastrowid
           17  +
           18  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           19  +ifcapable !fts5 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +do_execsql_test 1.0 {
           25  +  CREATE VIRTUAL TABLE t1 USING fts5(str);
           26  +}
           27  +
           28  +do_execsql_test 1.1 {
           29  +  INSERT INTO t1 VALUES('one string');
           30  +  INSERT INTO t1 VALUES('two string');
           31  +  INSERT INTO t1 VALUES('three string');
           32  +  SELECT last_insert_rowid();
           33  +} {3}
           34  +
           35  +do_execsql_test 1.2 {
           36  +  BEGIN;
           37  +    INSERT INTO t1 VALUES('one string');
           38  +    INSERT INTO t1 VALUES('two string');
           39  +    INSERT INTO t1 VALUES('three string');
           40  +  COMMIT;
           41  +  SELECT last_insert_rowid();
           42  +} {6}
           43  +
           44  +do_execsql_test 1.3 {
           45  +  INSERT INTO t1(rowid, str) VALUES(-22, 'some more text');
           46  +  SELECT last_insert_rowid();
           47  +} {-22}
           48  +
           49  +do_execsql_test 1.4 {
           50  +  BEGIN;
           51  +    INSERT INTO t1(rowid, str) VALUES(45, 'some more text');
           52  +    INSERT INTO t1(rowid, str) VALUES(46, 'some more text');
           53  +    INSERT INTO t1(rowid, str) VALUES(222, 'some more text');
           54  +    SELECT last_insert_rowid();
           55  +  COMMIT;
           56  +  SELECT last_insert_rowid();
           57  +} {222 222}
           58  +
           59  +do_execsql_test 1.5 {
           60  +  CREATE TABLE x1(x);
           61  +  INSERT INTO x1 VALUES('john'), ('paul'), ('george'), ('ringo');
           62  +  INSERT INTO t1 SELECT x FROM x1;
           63  +  SELECT last_insert_rowid();
           64  +} {226}
           65  +
           66  +do_execsql_test 1.6 {
           67  +  INSERT INTO t1(rowid, str) SELECT rowid+10, x FROM x1;
           68  +  SELECT last_insert_rowid();
           69  +} {14}
           70  +
           71  +
           72  +finish_test
           73  +

Added ext/fts5/test/fts5multiclient.test.

            1  +# 2016 March 17
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +
           13  +source [file join [file dirname [info script]] fts5_common.tcl]
           14  +source $testdir/lock_common.tcl
           15  +
           16  +set testprefix fts5multiclient
           17  +return_if_no_fts5
           18  +
           19  +foreach_detail_mode $testprefix {
           20  +
           21  +do_multiclient_test tn {
           22  +
           23  +  do_test 1.$tn.1 {
           24  +    sql1 { CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%) }
           25  +    sql1 { INSERT INTO t1 VALUES('a b c') }
           26  +    sql2 { SELECT rowid FROM t1('b') }
           27  +  } {1}
           28  +
           29  +  do_test 1.$tn.2 {
           30  +    sql2 { INSERT INTO t1 VALUES('a b c') }
           31  +    sql1 { SELECT rowid FROM t1('b') }
           32  +  } {1 2}
           33  +
           34  +  do_test 1.$tn.3 {
           35  +    sql2 { INSERT INTO t1 VALUES('a b c') }
           36  +    sql1 { SELECT rowid FROM t1('b') }
           37  +  } {1 2 3}
           38  +
           39  +  do_test 1.$tn.4 {
           40  +    sql2 { INSERT INTO t1 VALUES('a b c') }
           41  +    sql1 { INSERT INTO t1 VALUES('a b c') }
           42  +    sql3 { INSERT INTO t1(t1) VALUES('integrity-check') }
           43  +  } {}
           44  +
           45  +};# do_multiclient_test
           46  +};# foreach_detail_mode
           47  +finish_test
           48  +

Changes to ext/fts5/test/fts5prefix.test.

     5      5   #
     6      6   #    May you do good and not evil.
     7      7   #    May you find forgiveness for yourself and forgive others.
     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #***********************************************************************
    11     11   #
    12         -# This file containst tests focused on prefix indexes.
           12  +# This file contains tests focused on prefix indexes.
    13     13   #
    14     14   
    15     15   source [file join [file dirname [info script]] fts5_common.tcl]
    16     16   set testprefix fts5prefix
    17     17   
    18     18   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    19     19   ifcapable !fts5 {

Changes to ext/fts5/test/fts5rank.test.

    87     87   } {1 3 2}
    88     88   
    89     89   do_test 2.7 {
    90     90     execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db
    91     91   } {1 3 2}
    92     92   
    93     93   
           94  +#--------------------------------------------------------------------------
           95  +# At one point there was a problem with queries such as:
           96  +#
           97  +#   ... MATCH 'x OR y' ORDER BY rank;
           98  +#
           99  +# if there were zero occurrences of token 'y' in the dataset. The
          100  +# following tests verify that that problem has been addressed.
          101  +#
          102  +foreach_detail_mode $::testprefix {
          103  +  do_execsql_test 3.1.0 {
          104  +    CREATE VIRTUAL TABLE y1 USING fts5(z, detail=%DETAIL%);
          105  +    INSERT INTO y1 VALUES('test xyz');
          106  +    INSERT INTO y1 VALUES('test test xyz test');
          107  +    INSERT INTO y1 VALUES('test test xyz');
          108  +  }
          109  +
          110  +  do_execsql_test 3.1.1 {
          111  +    SELECT rowid FROM y1('test OR tset');
          112  +  } {1 2 3}
          113  +
          114  +  do_execsql_test 3.1.2 {
          115  +    SELECT rowid FROM y1('test OR tset') ORDER BY bm25(y1)
          116  +  } {2 3 1}
          117  +
          118  +  do_execsql_test 3.1.3 {
          119  +    SELECT rowid FROM y1('test OR tset') ORDER BY +rank
          120  +  } {2 3 1}
          121  +
          122  +  do_execsql_test 3.1.4 {
          123  +    SELECT rowid FROM y1('test OR tset') ORDER BY rank
          124  +  } {2 3 1}
          125  +
          126  +  do_execsql_test 3.1.5 {
          127  +    SELECT rowid FROM y1('test OR xyz') ORDER BY rank
          128  +  } {3 2 1}
          129  +
          130  +
          131  +  do_execsql_test 3.2.1 {
          132  +    CREATE VIRTUAL TABLE z1 USING fts5(a, detail=%DETAIL%);
          133  +    INSERT INTO z1 VALUES('wrinkle in time');
          134  +    SELECT * FROM z1 WHERE z1 MATCH 'wrinkle in time OR a wrinkle in time';
          135  +  } {{wrinkle in time}}
          136  +}
          137  +
          138  +do_execsql_test 4.1 {
          139  +  DROP TABLE IF EXISTS VTest;
          140  +  CREATE virtual TABLE VTest USING FTS5(
          141  +    Title, AUthor, tokenize ='porter unicode61 remove_diacritics 1', 
          142  +    columnsize='1', detail=full
          143  +  );
          144  +  INSERT INTO VTest (Title, Author) VALUES ('wrinkle in time', 'Bill Smith');
    94    145   
          146  +  SELECT * FROM VTest WHERE 
          147  +  VTest MATCH 'wrinkle in time OR a wrinkle in time' ORDER BY rank;
          148  +} {{wrinkle in time} {Bill Smith}}
    95    149   
    96    150   
    97    151   
    98    152   
    99    153   finish_test
   100    154   

Changes to ext/fts5/test/fts5simple.test.

   261    261   # Test that character 0x1A is allowed in fts5 barewords.
   262    262   #
   263    263   do_test 11.0 {
   264    264     execsql "CREATE VIRTUAL TABLE t4 USING fts5(x, tokenize=\"ascii tokenchars '\x1A'\")"
   265    265     execsql "
   266    266       INSERT INTO t4 VALUES('a b c \x1A');
   267    267       INSERT INTO t4 VALUES('a b c d\x1A');
   268         -    INSERT INTO t4 VALUES('a b c \x1Ad');
          268  +    INSERT INTO t4 VALUES('a b c \x1Ag');
   269    269       INSERT INTO t4 VALUES('a b c d');
   270    270     "
   271    271   } {}
   272    272   
   273    273   do_test 11.1 {
   274    274     execsql "SELECT rowid FROM t4('\x1A')"
   275    275   } {1}
................................................................................
   336    336   }
   337    337   do_test 14.2 { 
   338    338     fts5_level_segs ttt 
   339    339   } {1}
   340    340   
   341    341   #-------------------------------------------------------------------------
   342    342   db func rnddoc fts5_rnddoc
   343         -do_execsql_test 4.0 {
          343  +do_execsql_test 14.3 {
   344    344     CREATE VIRTUAL TABLE x1 USING fts5(x);
   345    345     INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
   346    346   
   347    347     WITH ii(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10 )
   348    348     INSERT INTO x1 SELECT rnddoc(5) FROM ii;
   349    349   }
   350    350   
   351         -do_execsql_test 4.1 {
          351  +do_execsql_test 14.4 {
   352    352     SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'
   353         -} {0 {} 4}
          353  +} {0 {} 3}
   354    354   
   355    355   #-------------------------------------------------------------------------
   356    356   reset_db
   357    357   do_execsql_test 15.0 {
   358    358     CREATE VIRTUAL TABLE x2 USING fts5(x, prefix=1);
   359    359     INSERT INTO x2 VALUES('ab');
   360    360   }
................................................................................
   443    443     execsql { INSERT INTO x1(x1) VALUES('optimize'); }
   444    444     execsql { DELETE FROM x1 WHERE rowid = 4; }
   445    445   } {}
   446    446   do_execsql_test 20.2 {
   447    447     INSERT INTO x1(x1) VALUES('optimize');
   448    448     INSERT INTO x1(x1) VALUES('integrity-check');
   449    449   } {}
          450  +
          451  +#-------------------------------------------------------------------------
          452  +reset_db
          453  +set doc "a b [string repeat x 100000]"
          454  +do_execsql_test 21.0 {
          455  +  CREATE VIRTUAL TABLE x1 USING fts5(x);
          456  +  INSERT INTO x1(rowid, x) VALUES(11111, $doc);
          457  +  INSERT INTO x1(rowid, x) VALUES(11112, $doc);
          458  +}
          459  +do_execsql_test 21.1 {
          460  +  INSERT INTO x1(x1) VALUES('integrity-check');
          461  +}
          462  +do_execsql_test 21.2 {
          463  +  SELECT rowid FROM x1($doc);
          464  +} {11111 11112}
          465  +do_execsql_test 21.3 {
          466  +  DELETE FROM x1 WHERE rowid=11111;
          467  +  INSERT INTO x1(x1) VALUES('integrity-check');
          468  +  SELECT rowid FROM x1($doc);
          469  +} {11112}
   450    470   
   451    471   finish_test

Changes to ext/fts5/test/fts5simple2.test.

   327    327       INSERT INTO t2(rowid, x) VALUES(1, 'a b c');
   328    328       INSERT INTO t2(rowid, x) VALUES(456, 'a b c');
   329    329       INSERT INTO t2(rowid, x) VALUES(1000, 'a b c');
   330    330     COMMIT;
   331    331     UPDATE t2 SET x=x;
   332    332     DELETE FROM t2;
   333    333   }
          334  +
          335  +#-------------------------------------------------------------------------
          336  +#
          337  +reset_db
          338  +do_execsql_test 17.0 {
          339  +  CREATE VIRTUAL TABLE t2 USING fts5(x, y);
          340  +  BEGIN;
          341  +    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
          342  +    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
          343  +    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
          344  +  COMMIT;
          345  +}
          346  +do_execsql_test 17.1 { SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 }
          347  +do_execsql_test 17.2 {
          348  +  BEGIN;
          349  +    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
          350  +    SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 ;
          351  +}
          352  +do_execsql_test 17.3 {
          353  +  COMMIT
          354  +}
          355  +
          356  +reset_db
          357  +do_execsql_test 17.4 {
          358  +  CREATE VIRTUAL TABLE t2 USING fts5(x, y);
          359  +  BEGIN;
          360  +    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
          361  +    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
          362  +    SELECT * FROM t2('y:a*') WHERE rowid>66;
          363  +}
          364  +do_execsql_test 17.5 { SELECT * FROM t2('x:b* OR y:a*') }
          365  +do_execsql_test 17.5 { COMMIT ; SELECT * FROM t2('x:b* OR y:a*') }
          366  +do_execsql_test 17.6 { 
          367  +  SELECT * FROM t2('x:b* OR y:a*') WHERE rowid>55
          368  +}
   334    369   
   335    370   #db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM t2_data} {puts $r}
   336    371     
   337    372   finish_test
   338    373   

Changes to ext/fts5/test/fts5simple3.test.

    75     75   do_execsql_test 3.0 {
    76     76     CREATE VIRTUAL TABLE x3 USING fts5(one);
    77     77     INSERT INTO x3 VALUES('a b c');
    78     78     INSERT INTO x3 VALUES('c b a');
    79     79     INSERT INTO x3 VALUES('o t t');
    80     80     SELECT * FROM x3('x OR y OR z');
    81     81   }
           82  +
           83  +#-------------------------------------------------------------------------
           84  +# Test that a crash occuring when the second or subsequent tokens in a
           85  +# phrase matched zero rows has been fixed.
           86  +#
           87  +do_execsql_test 4.0 {
           88  +  CREATE VIRTUAL TABLE t1 USING fts5(x);
           89  +  INSERT INTO t1 VALUES('ab');
           90  +  INSERT INTO t1 VALUES('cd');
           91  +  INSERT INTO t1 VALUES('ab cd');
           92  +  INSERT INTO t1 VALUES('ab cdXXX');
           93  +  INSERT INTO t1 VALUES('abXXX cd');
           94  +}
           95  +do_execsql_test 4.1 {
           96  +  SELECT * FROM t1('"ab cd" OR "ab cd" *');
           97  +} {{ab cd} {ab cdXXX}}
           98  +do_execsql_test 4.2 {
           99  +  SELECT * FROM t1('"xy zz" OR "ab cd" *');
          100  +} {{ab cd} {ab cdXXX}}
          101  +do_execsql_test 4.3 {
          102  +  SELECT * FROM t1('"xy zz" OR "xy zz" *');
          103  +}
          104  +do_execsql_test 4.4 {
          105  +  SELECT * FROM t1('"ab cd" OR "xy zz" *');
          106  +} {{ab cd}}
          107  +do_execsql_test 4.5 {
          108  +  CREATE VIRTUAL TABLE t2 USING fts5(x);
          109  +  INSERT INTO t2 VALUES('ab');
          110  +  INSERT INTO t2 VALUES('cd');
          111  +  INSERT INTO t2 VALUES('ef');
          112  +} 
          113  +do_execsql_test 4.6 {
          114  +  SELECT * FROM t2('ab + xyz');
          115  +}
    82    116   
    83    117   
    84    118   finish_test
    85    119   

Changes to ext/fts5/test/fts5synonym.test.

   148    148   reset_db
   149    149   fts5_tclnum_register db
   150    150   
   151    151   foreach {tn expr res} {
   152    152     1  {abc}                           {"abc"}
   153    153     2  {one}                           {"one"|"i"|"1"}
   154    154     3  {3}                             {"3"|"iii"|"three"}
   155         -  4  {3*}                            {"3"|"iii"|"three" *}
          155  +  4  {3*}                            {"3" *}
   156    156   } {
   157    157     do_execsql_test 4.1.$tn {
   158    158       SELECT fts5_expr($expr, 'tokenize=tclnum')
   159    159     } [list $res]
   160    160   }
   161    161   
   162    162   do_execsql_test 4.2.1 {

Changes to ext/fts5/test/fts5tokenizer.test.

   257    257     INSERT INTO e6 VALUES('theAquickBbrownCfoxDjumpedWoverXtheYlazyZdog');
   258    258     CREATE VIRTUAL TABLE e7 USING fts5vocab(e6, 'row');
   259    259     SELECT term FROM e7;
   260    260     ROLLBACK;
   261    261   } {
   262    262     brown dog fox jump lazi over quick the
   263    263   }
          264  +
          265  +#-------------------------------------------------------------------------
          266  +# Check that the FTS5_TOKENIZE_PREFIX flag is passed to the tokenizer
          267  +# implementation.
          268  +#
          269  +reset_db
          270  +proc tcl_create {args} { return "tcl_tokenize" }
          271  +sqlite3_fts5_create_tokenizer db tcl tcl_create
          272  +set ::flags [list]
          273  +proc tcl_tokenize {tflags text} {
          274  +  lappend ::flags $tflags
          275  +  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
          276  +    sqlite3_fts5_token $w $iStart $iEnd
          277  +  }
          278  +}
          279  +
          280  +do_execsql_test 9.1.1 {
          281  +  CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl);
          282  +  INSERT INTO t1 VALUES('abc');
          283  +  INSERT INTO t1 VALUES('xyz');
          284  +} {}
          285  +do_test 9.1.2 { set ::flags } {document document}
          286  +
          287  +set ::flags [list]
          288  +do_execsql_test 9.2.1 { SELECT * FROM t1('abc'); } {abc}
          289  +do_test 9.2.2 { set ::flags } {query}
          290  +
          291  +set ::flags [list]
          292  +do_execsql_test 9.3.1 { SELECT * FROM t1('ab*'); } {abc}
          293  +do_test 9.3.2 { set ::flags } {prefixquery}
          294  +
          295  +set ::flags [list]
          296  +do_execsql_test 9.4.1 { SELECT * FROM t1('"abc xyz" *'); } {}
          297  +do_test 9.4.2 { set ::flags } {prefixquery}
          298  +
          299  +set ::flags [list]
          300  +do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {}
          301  +do_test 9.5.2 { set ::flags } {query}
          302  +
   264    303   
   265    304   finish_test
   266    305   

Changes to ext/fts5/test/fts5unicode2.test.

   156    156        the maximum x value.
   157    157     }
   158    158     3 "ROW" {
   159    159        ...returns the value of y on the same [row] that contains 
   160    160        the maximum x value.
   161    161     }
   162    162     4 "rollback" {
   163         -     ...[ROLLBACK]. Instead, the pending statement
   164         -     will return SQLITE_ABORT upon next access after the [ROLLBACK].
          163  +     Pending statements no longer block [ROLLBACK]. Instead, the pending
          164  +     statement will return SQLITE_ABORT upon...
   165    165     }
   166    166     5 "rOllback" {
   167         -     ...[ROLLBACK]. Instead, the pending statement
   168         -     will return SQLITE_ABORT upon next access after the [ROLLBACK].
          167  +     Pending statements no longer block [ROLLBACK]. Instead, the pending
          168  +     statement will return SQLITE_ABORT upon...
   169    169     }
   170    170     6 "lang*" {
   171    171        Added support for the FTS4 [languageid] option.
   172    172     }
   173    173   } {
   174    174     do_test 2.$tn {
   175    175       set q [mapdoc $query]

Changes to ext/fts5/test/fts5vocab.test.

   438    438   } else {
   439    439     do_catchsql_test 8.2.2 { 
   440    440       SELECT * FROM x1_c 
   441    441     } {1 {database disk image is malformed}}
   442    442   }
   443    443   
   444    444   sqlite3_fts5_may_be_corrupt 0
          445  +}
          446  +
          447  +#-------------------------------------------------------------------------
          448  +# Test that both "ORDER BY term" and "ORDER BY term DESC" work.
          449  +#
          450  +reset_db
          451  +do_execsql_test 9.1 {
          452  +  CREATE VIRTUAL TABLE x1 USING fts5(x);
          453  +  INSERT INTO x1 VALUES('def ABC ghi');
          454  +  INSERT INTO x1 VALUES('DEF abc GHI');
          455  +}
   445    456   
          457  +do_execsql_test 9.2 {
          458  +  CREATE VIRTUAL TABLE rrr USING fts5vocab(x1, row);
          459  +  SELECT * FROM rrr
          460  +} {
          461  +  abc 2 2 def 2 2 ghi 2 2
          462  +}
          463  +do_execsql_test 9.3 {
          464  +  SELECT * FROM rrr ORDER BY term ASC
          465  +} {
          466  +  abc 2 2 def 2 2 ghi 2 2
          467  +}
          468  +do_execsql_test 9.4 {
          469  +  SELECT * FROM rrr ORDER BY term DESC
          470  +} {
          471  +  ghi 2 2 def 2 2 abc 2 2 
   446    472   }
          473  +do_test 9.5 {
          474  +  set e2 [db eval { EXPLAIN SELECT * FROM rrr ORDER BY term ASC }]
          475  +  expr [lsearch $e2 SorterSort]<0
          476  +} 1
          477  +do_test 9.6 {
          478  +  set e2 [db eval { EXPLAIN SELECT * FROM rrr ORDER BY term DESC }]
          479  +  expr [lsearch $e2 SorterSort]<0
          480  +} 0
          481  +
          482  +
   447    483   
   448    484   finish_test
   449    485   

Changes to ext/fts5/tool/fts5txt2db.tcl.

    13     13       {fts5                 "use fts5 (this is the default)"}
    14     14       {fts4                 "use fts4"}
    15     15       {colsize   "10 10 10" "list of column sizes"}
    16     16       {tblname   "t1"       "table name to create"}
    17     17       {detail    "full"     "Fts5 detail mode to use"}
    18     18       {repeat    1          "Load each file this many times"}
    19     19       {prefix    ""         "Fts prefix= option"}
           20  +    {trans     1          "True to use a transaction"}
    20     21       database
    21     22       file...
    22     23     } {
    23     24     This script is designed to create fts4/5 tables with more than one column.
    24     25     The -colsize option should be set to a Tcl list of integer values, one for
    25     26     each column in the table. Each value is the number of tokens that will be
    26     27     inserted into the column value for each row. For example, setting the -colsize
................................................................................
   210    211   set cols [create_table]
   211    212   set sql "INSERT INTO $A(tblname) VALUES(\$R([lindex $cols 0])"
   212    213   foreach c [lrange $cols 1 end] {
   213    214     append sql ", \$R($c)"
   214    215   }
   215    216   append sql ")"
   216    217   
   217         -db eval BEGIN
          218  +if {$A(trans)} { db eval BEGIN }
   218    219     while {$i < $N} {
   219    220       foreach c $cols s $A(colsize) {
   220    221         set R($c) [lrange $tokens $i [expr $i+$s-1]]
   221    222         incr i $s
   222    223       }
   223    224       db eval $sql
   224    225     }
   225         -db eval COMMIT
          226  +if {$A(trans)} { db eval COMMIT }
   226    227   
   227    228   
   228    229   

Changes to ext/icu/icu.c.

   345    345   ** To access ICU "language specific" case mapping, upper() or lower()
   346    346   ** should be invoked with two arguments. The second argument is the name
   347    347   ** of the locale to use. Passing an empty string ("") or SQL NULL value
   348    348   ** as the second argument is the same as invoking the 1 argument version
   349    349   ** of upper() or lower().
   350    350   **
   351    351   **     lower('I', 'en_us') -> 'i'
   352         -**     lower('I', 'tr_tr') -> 'ı' (small dotless i)
          352  +**     lower('I', 'tr_tr') -> '\u131' (small dotless i)
   353    353   **
   354    354   ** http://www.icu-project.org/userguide/posix.html#case_mappings
   355    355   */
   356    356   static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){
   357         -  const UChar *zInput;
   358         -  UChar *zOutput = 0;
   359         -  int nInput;
   360         -  int nOut;
          357  +  const UChar *zInput;            /* Pointer to input string */
          358  +  UChar *zOutput = 0;             /* Pointer to output buffer */
          359  +  int nInput;                     /* Size of utf-16 input string in bytes */
          360  +  int nOut;                       /* Size of output buffer in bytes */
   361    361     int cnt;
          362  +  int bToUpper;                   /* True for toupper(), false for tolower() */
   362    363     UErrorCode status;
   363    364     const char *zLocale = 0;
   364    365   
   365    366     assert(nArg==1 || nArg==2);
          367  +  bToUpper = (sqlite3_user_data(p)!=0);
   366    368     if( nArg==2 ){
   367    369       zLocale = (const char *)sqlite3_value_text(apArg[1]);
   368    370     }
   369    371   
   370    372     zInput = sqlite3_value_text16(apArg[0]);
   371    373     if( !zInput ){
   372    374       return;
................................................................................
   382    384       if( zNew==0 ){
   383    385         sqlite3_free(zOutput);
   384    386         sqlite3_result_error_nomem(p);
   385    387         return;
   386    388       }
   387    389       zOutput = zNew;
   388    390       status = U_ZERO_ERROR;
   389         -    if( sqlite3_user_data(p) ){
          391  +    if( bToUpper ){
   390    392         nOut = 2*u_strToUpper(zOutput,nOut/2,zInput,nInput/2,zLocale,&status);
   391    393       }else{
   392    394         nOut = 2*u_strToLower(zOutput,nOut/2,zInput,nInput/2,zLocale,&status);
   393    395       }
   394         -    if( !U_SUCCESS(status) ){
   395         -      if( status==U_BUFFER_OVERFLOW_ERROR ) continue;
   396         -      icuFunctionError(p,
   397         -          sqlite3_user_data(p) ? "u_strToUpper" : "u_strToLower", status);
   398         -      return;
          396  +
          397  +    if( U_SUCCESS(status) ){
          398  +      sqlite3_result_text16(p, zOutput, nOut, xFree);
          399  +    }else if( status==U_BUFFER_OVERFLOW_ERROR ){
          400  +      assert( cnt==0 );
          401  +      continue;
          402  +    }else{
          403  +      icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status);
   399    404       }
          405  +    return;
   400    406     }
   401         -  sqlite3_result_text16(p, zOutput, nOut, xFree);
          407  +  assert( 0 );     /* Unreachable */
   402    408   }
   403    409   
   404    410   /*
   405    411   ** Collation sequence destructor function. The pCtx argument points to
   406    412   ** a UCollator structure previously allocated using ucol_open().
   407    413   */
   408    414   static void icuCollationDel(void *pCtx){
................................................................................
   483    489     }
   484    490   }
   485    491   
   486    492   /*
   487    493   ** Register the ICU extension functions with database db.
   488    494   */
   489    495   int sqlite3IcuInit(sqlite3 *db){
   490         -  struct IcuScalar {
          496  +  static const struct IcuScalar {
   491    497       const char *zName;                        /* Function name */
   492         -    int nArg;                                 /* Number of arguments */
   493         -    int enc;                                  /* Optimal text encoding */
   494         -    void *pContext;                           /* sqlite3_user_data() context */
          498  +    unsigned char nArg;                       /* Number of arguments */
          499  +    unsigned short enc;                       /* Optimal text encoding */
          500  +    unsigned char iContext;                   /* sqlite3_user_data() context */
   495    501       void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
   496    502     } scalars[] = {
   497         -    {"regexp", 2, SQLITE_ANY,          0, icuRegexpFunc},
   498         -
   499         -    {"lower",  1, SQLITE_UTF16,        0, icuCaseFunc16},
   500         -    {"lower",  2, SQLITE_UTF16,        0, icuCaseFunc16},
   501         -    {"upper",  1, SQLITE_UTF16, (void*)1, icuCaseFunc16},
   502         -    {"upper",  2, SQLITE_UTF16, (void*)1, icuCaseFunc16},
   503         -
   504         -    {"lower",  1, SQLITE_UTF8,         0, icuCaseFunc16},
   505         -    {"lower",  2, SQLITE_UTF8,         0, icuCaseFunc16},
   506         -    {"upper",  1, SQLITE_UTF8,  (void*)1, icuCaseFunc16},
   507         -    {"upper",  2, SQLITE_UTF8,  (void*)1, icuCaseFunc16},
   508         -
   509         -    {"like",   2, SQLITE_UTF8,         0, icuLikeFunc},
   510         -    {"like",   3, SQLITE_UTF8,         0, icuLikeFunc},
   511         -
   512         -    {"icu_load_collation",  2, SQLITE_UTF8, (void*)db, icuLoadCollation},
          503  +    {"icu_load_collation",  2, SQLITE_UTF8,                1, icuLoadCollation},
          504  +    {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC,         0, icuRegexpFunc},
          505  +    {"lower",  1, SQLITE_UTF16|SQLITE_DETERMINISTIC,       0, icuCaseFunc16},
          506  +    {"lower",  2, SQLITE_UTF16|SQLITE_DETERMINISTIC,       0, icuCaseFunc16},
          507  +    {"upper",  1, SQLITE_UTF16|SQLITE_DETERMINISTIC,       1, icuCaseFunc16},
          508  +    {"upper",  2, SQLITE_UTF16|SQLITE_DETERMINISTIC,       1, icuCaseFunc16},
          509  +    {"lower",  1, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuCaseFunc16},
          510  +    {"lower",  2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuCaseFunc16},
          511  +    {"upper",  1, SQLITE_UTF8|SQLITE_DETERMINISTIC,        1, icuCaseFunc16},
          512  +    {"upper",  2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        1, icuCaseFunc16},
          513  +    {"like",   2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuLikeFunc},
          514  +    {"like",   3, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuLikeFunc},
   513    515     };
   514         -
   515    516     int rc = SQLITE_OK;
   516    517     int i;
   517    518   
          519  +  
   518    520     for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){
   519         -    struct IcuScalar *p = &scalars[i];
          521  +    const struct IcuScalar *p = &scalars[i];
   520    522       rc = sqlite3_create_function(
   521         -        db, p->zName, p->nArg, p->enc, p->pContext, p->xFunc, 0, 0
          523  +        db, p->zName, p->nArg, p->enc, 
          524  +        p->iContext ? (void*)db : (void*)0,
          525  +        p->xFunc, 0, 0
   522    526       );
   523    527     }
   524    528   
   525    529     return rc;
   526    530   }
   527    531   
   528    532   #if !SQLITE_CORE

Added ext/misc/README.md.

            1  +## Miscellaneous Extensions
            2  +
            3  +This folder contains a collection of smaller loadable extensions.
            4  +See <https://www.sqlite.org/loadext.html> for instructions on how
            5  +to compile and use loadable extensions.
            6  +Each extension in this folder is implemented in a single file of C code.
            7  +
            8  +Each source file contains a description in its header comment.  See the
            9  +header comments for details about each extension.  Additional notes are
           10  +as follows:
           11  +
           12  +  *  **carray.c** &mdash;  This module implements the
           13  +     [carray](https://www.sqlite.org/carray.html) table-valued function.
           14  +     It is a good example of how to go about implementing a custom
           15  +     [table-valued function](https://www.sqlite.org/vtab.html#tabfunc2).
           16  +
           17  +  *  **dbdump.c** &mdash;  This is not actually a loadable extension, but
           18  +     rather a library that implements an approximate equivalent to the
           19  +     ".dump" command of the
           20  +     [command-line shell](https://www.sqlite.org/cli.html).
           21  +
           22  +  *  **memvfs.c** &mdash;  This file implements a custom
           23  +     [VFS](https://www.sqlite.org/vfs.html) that stores an entire database
           24  +     file in a single block of RAM.  It serves as a good example of how
           25  +     to implement a simple custom VFS.
           26  +
           27  +  *  **rot13.c** &mdash;  This file implements the very simple rot13()
           28  +     substitution function.  This file makes a good template for implementing
           29  +     new custom SQL functions for SQLite.
           30  +
           31  +  *  **series.c** &mdash;  This is an implementation of the
           32  +     "generate_series" [virtual table](https://www.sqlite.org/vtab.html).
           33  +     It can make a good template for new custom virtual table implementations.
           34  +
           35  +  *  **shathree.c** &mdash;  An implementation of the sha3() and
           36  +     sha3_query() SQL functions.  The file is named "shathree.c" instead
           37  +     of "sha3.c" because the default entry point names in SQLite are based
           38  +     on the source filename with digits removed, so if we used the name
           39  +     "sha3.c" then the entry point would conflict with the prior "sha1.c"
           40  +     extension.

Changes to ext/misc/amatch.c.

   621    621     {
   622    622       pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo );
   623    623       if( pRule==0 ){
   624    624         rc = SQLITE_NOMEM;
   625    625       }else{
   626    626         memset(pRule, 0, sizeof(*pRule));
   627    627         pRule->zFrom = &pRule->zTo[nTo+1];
   628         -      pRule->nFrom = nFrom;
          628  +      pRule->nFrom = (amatch_len)nFrom;
   629    629         memcpy(pRule->zFrom, zFrom, nFrom+1);
   630    630         memcpy(pRule->zTo, zTo, nTo+1);
   631         -      pRule->nTo = nTo;
          631  +      pRule->nTo = (amatch_len)nTo;
   632    632         pRule->rCost = rCost;
   633    633         pRule->iLang = (int)iLang;
   634    634       }
   635    635     }
   636    636   
   637    637     *ppRule = pRule;
   638    638     return rc;
................................................................................
  1077   1077     }
  1078   1078     pWord = sqlite3_malloc( sizeof(*pWord) + nBase + nTail - 1 );
  1079   1079     if( pWord==0 ) return;
  1080   1080     memset(pWord, 0, sizeof(*pWord));
  1081   1081     pWord->rCost = rCost;
  1082   1082     pWord->iSeq = pCur->nWord++;
  1083   1083     amatchWriteCost(pWord);
  1084         -  pWord->nMatch = nMatch;
         1084  +  pWord->nMatch = (short)nMatch;
  1085   1085     pWord->pNext = pCur->pAllWords;
  1086   1086     pCur->pAllWords = pWord;
  1087   1087     pWord->sCost.zKey = pWord->zCost;
  1088   1088     pWord->sCost.pWord = pWord;
  1089   1089     pOther = amatchAvlInsert(&pCur->pCost, &pWord->sCost);
  1090   1090     assert( pOther==0 ); (void)pOther;
  1091   1091     pWord->sWord.zKey = pWord->zWord;
................................................................................
  1158   1158   #ifdef AMATCH_TRACE_1
  1159   1159       printf("PROCESS [%s][%.*s^%s] %d (\"%s\" \"%s\")\n",
  1160   1160          pWord->zWord+2, pWord->nMatch, pCur->zInput, pCur->zInput+pWord->nMatch,
  1161   1161          pWord->rCost, pWord->zWord, pWord->zCost);
  1162   1162   #endif
  1163   1163       nWord = (int)strlen(pWord->zWord+2);
  1164   1164       if( nWord+20>nBuf ){
  1165         -      nBuf = nWord+100;
         1165  +      nBuf = (char)(nWord+100);
  1166   1166         zBuf = sqlite3_realloc(zBuf, nBuf);
  1167   1167         if( zBuf==0 ) return SQLITE_NOMEM;
  1168   1168       }
  1169   1169       amatchStrcpy(zBuf, pWord->zWord+2);
  1170   1170       zNext[0] = 0;
  1171   1171       zNextIn[0] = pCur->zInput[pWord->nMatch];
  1172   1172       if( zNextIn[0] ){

Added ext/misc/carray.c.

            1  +/*
            2  +** 2016-06-29
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +**
           13  +** This file demonstrates how to create a table-valued-function that
           14  +** returns the values in a C-language array.
           15  +** Examples:
           16  +**
           17  +**      SELECT * FROM carray($ptr,5)
           18  +**
           19  +** The query above returns 5 integers contained in a C-language array
           20  +** at the address $ptr.  $ptr is a pointer to the array of integers that
           21  +** has been cast to an integer.
           22  +**
           23  +** There is an optional third parameter to determine the datatype of
           24  +** the C-language array.  Allowed values of the third parameter are
           25  +** 'int32', 'int64', 'double', 'char*'.  Example:
           26  +**
           27  +**      SELECT * FROM carray($ptr,10,'char*');
           28  +**
           29  +** HOW IT WORKS
           30  +**
           31  +** The carray "function" is really a virtual table with the
           32  +** following schema:
           33  +**
           34  +**     CREATE TABLE carray(
           35  +**       value,
           36  +**       pointer HIDDEN,
           37  +**       count HIDDEN,
           38  +**       ctype TEXT HIDDEN
           39  +**     );
           40  +**
           41  +** If the hidden columns "pointer" and "count" are unconstrained, then 
           42  +** the virtual table has no rows.  Otherwise, the virtual table interprets
           43  +** the integer value of "pointer" as a pointer to the array and "count"
           44  +** as the number of elements in the array.  The virtual table steps through
           45  +** the array, element by element.
           46  +*/
           47  +#include "sqlite3ext.h"
           48  +SQLITE_EXTENSION_INIT1
           49  +#include <assert.h>
           50  +#include <string.h>
           51  +
           52  +#ifndef SQLITE_OMIT_VIRTUALTABLE
           53  +
           54  +/*
           55  +** Allowed datatypes
           56  +*/
           57  +#define CARRAY_INT32    0
           58  +#define CARRAY_INT64    1
           59  +#define CARRAY_DOUBLE   2
           60  +#define CARRAY_TEXT     3
           61  +
           62  +/*
           63  +** Names of types
           64  +*/
           65  +static const char *azType[] = { "int32", "int64", "double", "char*" };
           66  +
           67  +
           68  +/* carray_cursor is a subclass of sqlite3_vtab_cursor which will
           69  +** serve as the underlying representation of a cursor that scans
           70  +** over rows of the result
           71  +*/
           72  +typedef struct carray_cursor carray_cursor;
           73  +struct carray_cursor {
           74  +  sqlite3_vtab_cursor base;  /* Base class - must be first */
           75  +  sqlite3_int64 iRowid;      /* The rowid */
           76  +  sqlite3_int64 iPtr;        /* Pointer to array of values */
           77  +  sqlite3_int64 iCnt;        /* Number of integers in the array */
           78  +  unsigned char eType;       /* One of the CARRAY_type values */
           79  +};
           80  +
           81  +/*
           82  +** The carrayConnect() method is invoked to create a new
           83  +** carray_vtab that describes the carray virtual table.
           84  +**
           85  +** Think of this routine as the constructor for carray_vtab objects.
           86  +**
           87  +** All this routine needs to do is:
           88  +**
           89  +**    (1) Allocate the carray_vtab object and initialize all fields.
           90  +**
           91  +**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
           92  +**        result set of queries against carray will look like.
           93  +*/
           94  +static int carrayConnect(
           95  +  sqlite3 *db,
           96  +  void *pAux,
           97  +  int argc, const char *const*argv,
           98  +  sqlite3_vtab **ppVtab,
           99  +  char **pzErr
          100  +){
          101  +  sqlite3_vtab *pNew;
          102  +  int rc;
          103  +
          104  +/* Column numbers */
          105  +#define CARRAY_COLUMN_VALUE   0
          106  +#define CARRAY_COLUMN_POINTER 1
          107  +#define CARRAY_COLUMN_COUNT   2
          108  +#define CARRAY_COLUMN_CTYPE   3
          109  +
          110  +  rc = sqlite3_declare_vtab(db,
          111  +     "CREATE TABLE x(value,pointer hidden,count hidden,ctype hidden)");
          112  +  if( rc==SQLITE_OK ){
          113  +    pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) );
          114  +    if( pNew==0 ) return SQLITE_NOMEM;
          115  +    memset(pNew, 0, sizeof(*pNew));
          116  +  }
          117  +  return rc;
          118  +}
          119  +
          120  +/*
          121  +** This method is the destructor for carray_cursor objects.
          122  +*/
          123  +static int carrayDisconnect(sqlite3_vtab *pVtab){
          124  +  sqlite3_free(pVtab);
          125  +  return SQLITE_OK;
          126  +}
          127  +
          128  +/*
          129  +** Constructor for a new carray_cursor object.
          130  +*/
          131  +static int carrayOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
          132  +  carray_cursor *pCur;
          133  +  pCur = sqlite3_malloc( sizeof(*pCur) );
          134  +  if( pCur==0 ) return SQLITE_NOMEM;
          135  +  memset(pCur, 0, sizeof(*pCur));
          136  +  *ppCursor = &pCur->base;
          137  +  return SQLITE_OK;
          138  +}
          139  +
          140  +/*
          141  +** Destructor for a carray_cursor.
          142  +*/
          143  +static int carrayClose(sqlite3_vtab_cursor *cur){
          144  +  sqlite3_free(cur);
          145  +  return SQLITE_OK;
          146  +}
          147  +
          148  +
          149  +/*
          150  +** Advance a carray_cursor to its next row of output.
          151  +*/
          152  +static int carrayNext(sqlite3_vtab_cursor *cur){
          153  +  carray_cursor *pCur = (carray_cursor*)cur;
          154  +  pCur->iRowid++;
          155  +  return SQLITE_OK;
          156  +}
          157  +
          158  +/*
          159  +** Return values of columns for the row at which the carray_cursor
          160  +** is currently pointing.
          161  +*/
          162  +static int carrayColumn(
          163  +  sqlite3_vtab_cursor *cur,   /* The cursor */
          164  +  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
          165  +  int i                       /* Which column to return */
          166  +){
          167  +  carray_cursor *pCur = (carray_cursor*)cur;
          168  +  sqlite3_int64 x = 0;
          169  +  switch( i ){
          170  +    case CARRAY_COLUMN_POINTER:   x = pCur->iPtr;   break;
          171  +    case CARRAY_COLUMN_COUNT:     x = pCur->iCnt;   break;
          172  +    case CARRAY_COLUMN_CTYPE: {
          173  +      sqlite3_result_text(ctx, azType[pCur->eType], -1, SQLITE_STATIC);
          174  +      return SQLITE_OK;
          175  +    }
          176  +    default: {
          177  +      switch( pCur->eType ){
          178  +        case CARRAY_INT32: {
          179  +          int *p = (int*)pCur->iPtr;
          180  +          sqlite3_result_int(ctx, p[pCur->iRowid-1]);
          181  +          return SQLITE_OK;
          182  +        }
          183  +        case CARRAY_INT64: {
          184  +          sqlite3_int64 *p = (sqlite3_int64*)pCur->iPtr;
          185  +          sqlite3_result_int64(ctx, p[pCur->iRowid-1]);
          186  +          return SQLITE_OK;
          187  +        }
          188  +        case CARRAY_DOUBLE: {
          189  +          double *p = (double*)pCur->iPtr;
          190  +          sqlite3_result_double(ctx, p[pCur->iRowid-1]);
          191  +          return SQLITE_OK;
          192  +        }
          193  +        case CARRAY_TEXT: {
          194  +          const char **p = (const char**)pCur->iPtr;
          195  +          sqlite3_result_text(ctx, p[pCur->iRowid-1], -1, SQLITE_TRANSIENT);
          196  +          return SQLITE_OK;
          197  +        }
          198  +      }
          199  +    }
          200  +  }
          201  +  sqlite3_result_int64(ctx, x);
          202  +  return SQLITE_OK;
          203  +}
          204  +
          205  +/*
          206  +** Return the rowid for the current row.  In this implementation, the
          207  +** rowid is the same as the output value.
          208  +*/
          209  +static int carrayRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
          210  +  carray_cursor *pCur = (carray_cursor*)cur;
          211  +  *pRowid = pCur->iRowid;
          212  +  return SQLITE_OK;
          213  +}
          214  +
          215  +/*
          216  +** Return TRUE if the cursor has been moved off of the last
          217  +** row of output.
          218  +*/
          219  +static int carrayEof(sqlite3_vtab_cursor *cur){
          220  +  carray_cursor *pCur = (carray_cursor*)cur;
          221  +  return pCur->iRowid>pCur->iCnt;
          222  +}
          223  +
          224  +/*
          225  +** This method is called to "rewind" the carray_cursor object back
          226  +** to the first row of output.
          227  +*/
          228  +static int carrayFilter(
          229  +  sqlite3_vtab_cursor *pVtabCursor, 
          230  +  int idxNum, const char *idxStr,
          231  +  int argc, sqlite3_value **argv
          232  +){
          233  +  carray_cursor *pCur = (carray_cursor *)pVtabCursor;
          234  +  if( idxNum ){
          235  +    pCur->iPtr = sqlite3_value_int64(argv[0]);
          236  +    pCur->iCnt = sqlite3_value_int64(argv[1]);
          237  +    if( idxNum<3 ){
          238  +      pCur->eType = CARRAY_INT32;
          239  +    }else{
          240  +      unsigned char i;
          241  +      const char *zType = (const char*)sqlite3_value_text(argv[2]);
          242  +      for(i=0; i<sizeof(azType)/sizeof(azType[0]); i++){
          243  +        if( sqlite3_stricmp(zType, azType[i])==0 ) break;
          244  +      }
          245  +      if( i>=sizeof(azType)/sizeof(azType[0]) ){
          246  +        pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf(
          247  +          "unknown datatype: %Q", zType);
          248  +        return SQLITE_ERROR;
          249  +      }else{
          250  +        pCur->eType = i;
          251  +      }
          252  +    }
          253  +  }else{
          254  +    pCur->iPtr = 0;
          255  +    pCur->iCnt = 0;
          256  +  }
          257  +  pCur->iRowid = 1;
          258  +  return SQLITE_OK;
          259  +}
          260  +
          261  +/*
          262  +** SQLite will invoke this method one or more times while planning a query
          263  +** that uses the carray virtual table.  This routine needs to create
          264  +** a query plan for each invocation and compute an estimated cost for that
          265  +** plan.
          266  +**
          267  +** In this implementation idxNum is used to represent the
          268  +** query plan.  idxStr is unused.
          269  +**
          270  +** idxNum is 2 if the pointer= and count= constraints exist,
          271  +** 3 if the ctype= constraint also exists, and is 0 otherwise.
          272  +** If idxNum is 0, then carray becomes an empty table.
          273  +*/
          274  +static int carrayBestIndex(
          275  +  sqlite3_vtab *tab,
          276  +  sqlite3_index_info *pIdxInfo
          277  +){
          278  +  int i;                 /* Loop over constraints */
          279  +  int ptrIdx = -1;       /* Index of the pointer= constraint, or -1 if none */
          280  +  int cntIdx = -1;       /* Index of the count= constraint, or -1 if none */
          281  +  int ctypeIdx = -1;     /* Index of the ctype= constraint, or -1 if none */
          282  +
          283  +  const struct sqlite3_index_constraint *pConstraint;
          284  +  pConstraint = pIdxInfo->aConstraint;
          285  +  for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
          286  +    if( pConstraint->usable==0 ) continue;
          287  +    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
          288  +    switch( pConstraint->iColumn ){
          289  +      case CARRAY_COLUMN_POINTER:
          290  +        ptrIdx = i;
          291  +        break;
          292  +      case CARRAY_COLUMN_COUNT:
          293  +        cntIdx = i;
          294  +        break;
          295  +      case CARRAY_COLUMN_CTYPE:
          296  +        ctypeIdx = i;
          297  +        break;
          298  +    }
          299  +  }
          300  +  if( ptrIdx>=0 && cntIdx>=0 ){
          301  +    pIdxInfo->aConstraintUsage[ptrIdx].argvIndex = 1;
          302  +    pIdxInfo->aConstraintUsage[ptrIdx].omit = 1;
          303  +    pIdxInfo->aConstraintUsage[cntIdx].argvIndex = 2;
          304  +    pIdxInfo->aConstraintUsage[cntIdx].omit = 1;
          305  +    pIdxInfo->estimatedCost = (double)1;
          306  +    pIdxInfo->estimatedRows = 100;
          307  +    pIdxInfo->idxNum = 2;
          308  +    if( ctypeIdx>=0 ){
          309  +      pIdxInfo->aConstraintUsage[ctypeIdx].argvIndex = 3;
          310  +      pIdxInfo->aConstraintUsage[ctypeIdx].omit = 1;
          311  +      pIdxInfo->idxNum = 3;
          312  +    }
          313  +  }else{
          314  +    pIdxInfo->estimatedCost = (double)2147483647;
          315  +    pIdxInfo->estimatedRows = 2147483647;
          316  +    pIdxInfo->idxNum = 0;
          317  +  }
          318  +  return SQLITE_OK;
          319  +}
          320  +
          321  +/*
          322  +** This following structure defines all the methods for the 
          323  +** carray virtual table.
          324  +*/
          325  +static sqlite3_module carrayModule = {
          326  +  0,                         /* iVersion */
          327  +  0,                         /* xCreate */
          328  +  carrayConnect,             /* xConnect */
          329  +  carrayBestIndex,           /* xBestIndex */
          330  +  carrayDisconnect,          /* xDisconnect */
          331  +  0,                         /* xDestroy */
          332  +  carrayOpen,                /* xOpen - open a cursor */
          333  +  carrayClose,               /* xClose - close a cursor */
          334  +  carrayFilter,              /* xFilter - configure scan constraints */
          335  +  carrayNext,                /* xNext - advance a cursor */
          336  +  carrayEof,                 /* xEof - check for end of scan */
          337  +  carrayColumn,              /* xColumn - read data */
          338  +  carrayRowid,               /* xRowid - read data */
          339  +  0,                         /* xUpdate */
          340  +  0,                         /* xBegin */
          341  +  0,                         /* xSync */
          342  +  0,                         /* xCommit */
          343  +  0,                         /* xRollback */
          344  +  0,                         /* xFindMethod */
          345  +  0,                         /* xRename */
          346  +};
          347  +
          348  +#endif /* SQLITE_OMIT_VIRTUALTABLE */
          349  +
          350  +#ifdef _WIN32
          351  +__declspec(dllexport)
          352  +#endif
          353  +int sqlite3_carray_init(
          354  +  sqlite3 *db, 
          355  +  char **pzErrMsg, 
          356  +  const sqlite3_api_routines *pApi
          357  +){
          358  +  int rc = SQLITE_OK;
          359  +  SQLITE_EXTENSION_INIT2(pApi);
          360  +#ifndef SQLITE_OMIT_VIRTUALTABLE
          361  +  rc = sqlite3_create_module(db, "carray", &carrayModule, 0);
          362  +#endif
          363  +  return rc;
          364  +}

Added ext/misc/csv.c.

            1  +/*
            2  +** 2016-05-28
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** This file contains the implementation of an SQLite virtual table for
           14  +** reading CSV files.
           15  +**
           16  +** Usage:
           17  +**
           18  +**    .load ./csv
           19  +**    CREATE VIRTUAL TABLE temp.csv USING csv(filename=FILENAME);
           20  +**    SELECT * FROM csv;
           21  +**
           22  +** The columns are named "c1", "c2", "c3", ... by default.  But the
           23  +** application can define its own CREATE TABLE statement as an additional
           24  +** parameter.  For example:
           25  +**
           26  +**    CREATE VIRTUAL TABLE temp.csv2 USING csv(
           27  +**       filename = "../http.log",
           28  +**       schema = "CREATE TABLE x(date,ipaddr,url,referrer,userAgent)"
           29  +**    );
           30  +**
           31  +** Instead of specifying a file, the text of the CSV can be loaded using
           32  +** the data= parameter.
           33  +**
           34  +** If the columns=N parameter is supplied, then the CSV file is assumed to have
           35  +** N columns.  If the columns parameter is omitted, the CSV file is opened
           36  +** as soon as the virtual table is constructed and the first row of the CSV
           37  +** is read in order to count the tables.
           38  +**
           39  +** Some extra debugging features (used for testing virtual tables) are available
           40  +** if this module is compiled with -DSQLITE_TEST.
           41  +*/
           42  +#include <sqlite3ext.h>
           43  +SQLITE_EXTENSION_INIT1
           44  +#include <string.h>
           45  +#include <stdlib.h>
           46  +#include <assert.h>
           47  +#include <stdarg.h>
           48  +#include <ctype.h>
           49  +#include <stdio.h>
           50  +
           51  +#ifndef SQLITE_OMIT_VIRTUALTABLE
           52  +
           53  +/*
           54  +** A macro to hint to the compiler that a function should not be
           55  +** inlined.
           56  +*/
           57  +#if defined(__GNUC__)
           58  +#  define CSV_NOINLINE  __attribute__((noinline))
           59  +#elif defined(_MSC_VER) && _MSC_VER>=1310
           60  +#  define CSV_NOINLINE  __declspec(noinline)
           61  +#else
           62  +#  define CSV_NOINLINE
           63  +#endif
           64  +
           65  +
           66  +/* Max size of the error message in a CsvReader */
           67  +#define CSV_MXERR 200
           68  +
           69  +/* Size of the CsvReader input buffer */
           70  +#define CSV_INBUFSZ 1024
           71  +
           72  +/* A context object used when read a CSV file. */
           73  +typedef struct CsvReader CsvReader;
           74  +struct CsvReader {
           75  +  FILE *in;              /* Read the CSV text from this input stream */
           76  +  char *z;               /* Accumulated text for a field */
           77  +  int n;                 /* Number of bytes in z */
           78  +  int nAlloc;            /* Space allocated for z[] */
           79  +  int nLine;             /* Current line number */
           80  +  char cTerm;            /* Character that terminated the most recent field */
           81  +  size_t iIn;            /* Next unread character in the input buffer */
           82  +  size_t nIn;            /* Number of characters in the input buffer */
           83  +  char *zIn;             /* The input buffer */
           84  +  char zErr[CSV_MXERR];  /* Error message */
           85  +};
           86  +
           87  +/* Initialize a CsvReader object */
           88  +static void csv_reader_init(CsvReader *p){
           89  +  p->in = 0;
           90  +  p->z = 0;
           91  +  p->n = 0;
           92  +  p->nAlloc = 0;
           93  +  p->nLine = 0;
           94  +  p->nIn = 0;
           95  +  p->zIn = 0;
           96  +  p->zErr[0] = 0;
           97  +}
           98  +
           99  +/* Close and reset a CsvReader object */
          100  +static void csv_reader_reset(CsvReader *p){
          101  +  if( p->in ){
          102  +    fclose(p->in);
          103  +    sqlite3_free(p->zIn);
          104  +  }
          105  +  sqlite3_free(p->z);
          106  +  csv_reader_init(p);
          107  +}
          108  +
          109  +/* Report an error on a CsvReader */
          110  +static void csv_errmsg(CsvReader *p, const char *zFormat, ...){
          111  +  va_list ap;
          112  +  va_start(ap, zFormat);
          113  +  sqlite3_vsnprintf(CSV_MXERR, p->zErr, zFormat, ap);
          114  +  va_end(ap);
          115  +}
          116  +
          117  +/* Open the file associated with a CsvReader
          118  +** Return the number of errors.
          119  +*/
          120  +static int csv_reader_open(
          121  +  CsvReader *p,               /* The reader to open */
          122  +  const char *zFilename,      /* Read from this filename */
          123  +  const char *zData           /*  ... or use this data */
          124  +){
          125  +  if( zFilename ){
          126  +    p->zIn = sqlite3_malloc( CSV_INBUFSZ );
          127  +    if( p->zIn==0 ){
          128  +      csv_errmsg(p, "out of memory");
          129  +      return 1;
          130  +    }
          131  +    p->in = fopen(zFilename, "rb");
          132  +    if( p->in==0 ){
          133  +      csv_reader_reset(p);
          134  +      csv_errmsg(p, "cannot open '%s' for reading", zFilename);
          135  +      return 1;
          136  +    }
          137  +  }else{
          138  +    assert( p->in==0 );
          139  +    p->zIn = (char*)zData;
          140  +    p->nIn = strlen(zData);
          141  +  }
          142  +  return 0;
          143  +}
          144  +
          145  +/* The input buffer has overflowed.  Refill the input buffer, then
          146  +** return the next character
          147  +*/
          148  +static CSV_NOINLINE int csv_getc_refill(CsvReader *p){
          149  +  size_t got;
          150  +
          151  +  assert( p->iIn>=p->nIn );  /* Only called on an empty input buffer */
          152  +  assert( p->in!=0 );        /* Only called if reading froma file */
          153  +
          154  +  got = fread(p->zIn, 1, CSV_INBUFSZ, p->in);
          155  +  if( got==0 ) return EOF;
          156  +  p->nIn = got;
          157  +  p->iIn = 1;
          158  +  return p->zIn[0];
          159  +}
          160  +
          161  +/* Return the next character of input.  Return EOF at end of input. */
          162  +static int csv_getc(CsvReader *p){
          163  +  if( p->iIn >= p->nIn ){
          164  +    if( p->in!=0 ) return csv_getc_refill(p);
          165  +    return EOF;
          166  +  }
          167  +  return p->zIn[p->iIn++];
          168  +}
          169  +
          170  +/* Increase the size of p->z and append character c to the end. 
          171  +** Return 0 on success and non-zero if there is an OOM error */
          172  +static CSV_NOINLINE int csv_resize_and_append(CsvReader *p, char c){
          173  +  char *zNew;
          174  +  int nNew = p->nAlloc*2 + 100;
          175  +  zNew = sqlite3_realloc64(p->z, nNew);
          176  +  if( zNew ){
          177  +    p->z = zNew;
          178  +    p->nAlloc = nNew;
          179  +    p->z[p->n++] = c;
          180  +    return 0;
          181  +  }else{
          182  +    csv_errmsg(p, "out of memory");
          183  +    return 1;
          184  +  }
          185  +}
          186  +
          187  +/* Append a single character to the CsvReader.z[] array.
          188  +** Return 0 on success and non-zero if there is an OOM error */
          189  +static int csv_append(CsvReader *p, char c){
          190  +  if( p->n>=p->nAlloc-1 ) return csv_resize_and_append(p, c);
          191  +  p->z[p->n++] = c;
          192  +  return 0;
          193  +}
          194  +
          195  +/* Read a single field of CSV text.  Compatible with rfc4180 and extended
          196  +** with the option of having a separator other than ",".
          197  +**
          198  +**   +  Input comes from p->in.
          199  +**   +  Store results in p->z of length p->n.  Space to hold p->z comes
          200  +**      from sqlite3_malloc64().
          201  +**   +  Keep track of the line number in p->nLine.
          202  +**   +  Store the character that terminates the field in p->cTerm.  Store
          203  +**      EOF on end-of-file.
          204  +**
          205  +** Return "" at EOF.  Return 0 on an OOM error.
          206  +*/
          207  +static char *csv_read_one_field(CsvReader *p){
          208  +  int c;
          209  +  p->n = 0;
          210  +  c = csv_getc(p);
          211  +  if( c==EOF ){
          212  +    p->cTerm = EOF;
          213  +    return "";
          214  +  }
          215  +  if( c=='"' ){
          216  +    int pc, ppc;
          217  +    int startLine = p->nLine;
          218  +    pc = ppc = 0;
          219  +    while( 1 ){
          220  +      c = csv_getc(p);
          221  +      if( c<='"' || pc=='"' ){
          222  +        if( c=='\n' ) p->nLine++;
          223  +        if( c=='"' ){
          224  +          if( pc=='"' ){
          225  +            pc = 0;
          226  +            continue;
          227  +          }
          228  +        }
          229  +        if( (c==',' && pc=='"')
          230  +         || (c=='\n' && pc=='"')
          231  +         || (c=='\n' && pc=='\r' && ppc=='"')
          232  +         || (c==EOF && pc=='"')
          233  +        ){
          234  +          do{ p->n--; }while( p->z[p->n]!='"' );
          235  +          p->cTerm = (char)c;
          236  +          break;
          237  +        }
          238  +        if( pc=='"' && c!='\r' ){
          239  +          csv_errmsg(p, "line %d: unescaped %c character", p->nLine, '"');
          240  +          break;
          241  +        }
          242  +        if( c==EOF ){
          243  +          csv_errmsg(p, "line %d: unterminated %c-quoted field\n",
          244  +                     startLine, '"');
          245  +          p->cTerm = (char)c;
          246  +          break;
          247  +        }
          248  +      }
          249  +      if( csv_append(p, (char)c) ) return 0;
          250  +      ppc = pc;
          251  +      pc = c;
          252  +    }
          253  +  }else{
          254  +    while( c>',' || (c!=EOF && c!=',' && c!='\n') ){
          255  +      if( csv_append(p, (char)c) ) return 0;
          256  +      c = csv_getc(p);
          257  +    }
          258  +    if( c=='\n' ){
          259  +      p->nLine++;
          260  +      if( p->n>0 && p->z[p->n-1]=='\r' ) p->n--;
          261  +    }
          262  +    p->cTerm = (char)c;
          263  +  }
          264  +  if( p->z ) p->z[p->n] = 0;
          265  +  return p->z;
          266  +}
          267  +
          268  +
          269  +/* Forward references to the various virtual table methods implemented
          270  +** in this file. */
          271  +static int csvtabCreate(sqlite3*, void*, int, const char*const*, 
          272  +                           sqlite3_vtab**,char**);
          273  +static int csvtabConnect(sqlite3*, void*, int, const char*const*, 
          274  +                           sqlite3_vtab**,char**);
          275  +static int csvtabBestIndex(sqlite3_vtab*,sqlite3_index_info*);
          276  +static int csvtabDisconnect(sqlite3_vtab*);
          277  +static int csvtabOpen(sqlite3_vtab*, sqlite3_vtab_cursor**);
          278  +static int csvtabClose(sqlite3_vtab_cursor*);
          279  +static int csvtabFilter(sqlite3_vtab_cursor*, int idxNum, const char *idxStr,
          280  +                          int argc, sqlite3_value **argv);
          281  +static int csvtabNext(sqlite3_vtab_cursor*);
          282  +static int csvtabEof(sqlite3_vtab_cursor*);
          283  +static int csvtabColumn(sqlite3_vtab_cursor*,sqlite3_context*,int);
          284  +static int csvtabRowid(sqlite3_vtab_cursor*,sqlite3_int64*);
          285  +
          286  +/* An instance of the CSV virtual table */
          287  +typedef struct CsvTable {
          288  +  sqlite3_vtab base;              /* Base class.  Must be first */
          289  +  char *zFilename;                /* Name of the CSV file */
          290  +  char *zData;                    /* Raw CSV data in lieu of zFilename */
          291  +  long iStart;                    /* Offset to start of data in zFilename */
          292  +  int nCol;                       /* Number of columns in the CSV file */
          293  +  unsigned int tstFlags;          /* Bit values used for testing */
          294  +} CsvTable;
          295  +
          296  +/* Allowed values for tstFlags */
          297  +#define CSVTEST_FIDX  0x0001      /* Pretend that constrained searchs cost less*/
          298  +
          299  +/* A cursor for the CSV virtual table */
          300  +typedef struct CsvCursor {
          301  +  sqlite3_vtab_cursor base;       /* Base class.  Must be first */
          302  +  CsvReader rdr;                  /* The CsvReader object */
          303  +  char **azVal;                   /* Value of the current row */
          304  +  int *aLen;                      /* Length of each entry */
          305  +  sqlite3_int64 iRowid;           /* The current rowid.  Negative for EOF */
          306  +} CsvCursor;
          307  +
          308  +/* Transfer error message text from a reader into a CsvTable */
          309  +static void csv_xfer_error(CsvTable *pTab, CsvReader *pRdr){
          310  +  sqlite3_free(pTab->base.zErrMsg);
          311  +  pTab->base.zErrMsg = sqlite3_mprintf("%s", pRdr->zErr);
          312  +}
          313  +
          314  +/*
          315  +** This method is the destructor fo a CsvTable object.
          316  +*/
          317  +static int csvtabDisconnect(sqlite3_vtab *pVtab){
          318  +  CsvTable *p = (CsvTable*)pVtab;
          319  +  sqlite3_free(p->zFilename);
          320  +  sqlite3_free(p->zData);
          321  +  sqlite3_free(p);
          322  +  return SQLITE_OK;
          323  +}
          324  +
          325  +/* Skip leading whitespace.  Return a pointer to the first non-whitespace
          326  +** character, or to the zero terminator if the string has only whitespace */
          327  +static const char *csv_skip_whitespace(const char *z){
          328  +  while( isspace((unsigned char)z[0]) ) z++;
          329  +  return z;
          330  +}
          331  +
          332  +/* Remove trailing whitespace from the end of string z[] */
          333  +static void csv_trim_whitespace(char *z){
          334  +  size_t n = strlen(z);
          335  +  while( n>0 && isspace((unsigned char)z[n]) ) n--;
          336  +  z[n] = 0;
          337  +}
          338  +
          339  +/* Dequote the string */
          340  +static void csv_dequote(char *z){
          341  +  int j;
          342  +  char cQuote = z[0];
          343  +  size_t i, n;
          344  +
          345  +  if( cQuote!='\'' && cQuote!='"' ) return;
          346  +  n = strlen(z);
          347  +  if( n<2 || z[n-1]!=z[0] ) return;
          348  +  for(i=1, j=0; i<n-1; i++){
          349  +    if( z[i]==cQuote && z[i+1]==cQuote ) i++;
          350  +    z[j++] = z[i];
          351  +  }
          352  +  z[j] = 0;
          353  +}
          354  +
          355  +/* Check to see if the string is of the form:  "TAG = VALUE" with optional
          356  +** whitespace before and around tokens.  If it is, return a pointer to the
          357  +** first character of VALUE.  If it is not, return NULL.
          358  +*/
          359  +static const char *csv_parameter(const char *zTag, int nTag, const char *z){
          360  +  z = csv_skip_whitespace(z);
          361  +  if( strncmp(zTag, z, nTag)!=0 ) return 0;
          362  +  z = csv_skip_whitespace(z+nTag);
          363  +  if( z[0]!='=' ) return 0;
          364  +  return csv_skip_whitespace(z+1);
          365  +}
          366  +
          367  +/* Decode a parameter that requires a dequoted string.
          368  +**
          369  +** Return 1 if the parameter is seen, or 0 if not.  1 is returned
          370  +** even if there is an error.  If an error occurs, then an error message
          371  +** is left in p->zErr.  If there are no errors, p->zErr[0]==0.
          372  +*/
          373  +static int csv_string_parameter(
          374  +  CsvReader *p,            /* Leave the error message here, if there is one */
          375  +  const char *zParam,      /* Parameter we are checking for */
          376  +  const char *zArg,        /* Raw text of the virtual table argment */
          377  +  char **pzVal             /* Write the dequoted string value here */
          378  +){
          379  +  const char *zValue;
          380  +  zValue = csv_parameter(zParam,(int)strlen(zParam),zArg);
          381  +  if( zValue==0 ) return 0;
          382  +  p->zErr[0] = 0;
          383  +  if( *pzVal ){
          384  +    csv_errmsg(p, "more than one '%s' parameter", zParam);
          385  +    return 1;
          386  +  }
          387  +  *pzVal = sqlite3_mprintf("%s", zValue);
          388  +  if( *pzVal==0 ){
          389  +    csv_errmsg(p, "out of memory");
          390  +    return 1;
          391  +  }
          392  +  csv_trim_whitespace(*pzVal);
          393  +  csv_dequote(*pzVal);
          394  +  return 1;
          395  +}
          396  +
          397  +
          398  +/* Return 0 if the argument is false and 1 if it is true.  Return -1 if
          399  +** we cannot really tell.
          400  +*/
          401  +static int csv_boolean(const char *z){
          402  +  if( sqlite3_stricmp("yes",z)==0
          403  +   || sqlite3_stricmp("on",z)==0
          404  +   || sqlite3_stricmp("true",z)==0
          405  +   || (z[0]=='1' && z[1]==0)
          406  +  ){
          407  +    return 1;
          408  +  }
          409  +  if( sqlite3_stricmp("no",z)==0
          410  +   || sqlite3_stricmp("off",z)==0
          411  +   || sqlite3_stricmp("false",z)==0
          412  +   || (z[0]=='0' && z[1]==0)
          413  +  ){
          414  +    return 0;
          415  +  }
          416  +  return -1;
          417  +}
          418  +
          419  +
          420  +/*
          421  +** Parameters:
          422  +**    filename=FILENAME          Name of file containing CSV content
          423  +**    data=TEXT                  Direct CSV content.
          424  +**    schema=SCHEMA              Alternative CSV schema.
          425  +**    header=YES|NO              First row of CSV defines the names of
          426  +**                               columns if "yes".  Default "no".
          427  +**    columns=N                  Assume the CSV file contains N columns.
          428  +**
          429  +** Only available if compiled with SQLITE_TEST:
          430  +**    
          431  +**    testflags=N                Bitmask of test flags.  Optional
          432  +**
          433  +** If schema= is omitted, then the columns are named "c0", "c1", "c2",
          434  +** and so forth.  If columns=N is omitted, then the file is opened and
          435  +** the number of columns in the first row is counted to determine the
          436  +** column count.  If header=YES, then the first row is skipped.
          437  +*/
          438  +static int csvtabConnect(
          439  +  sqlite3 *db,
          440  +  void *pAux,
          441  +  int argc, const char *const*argv,
          442  +  sqlite3_vtab **ppVtab,
          443  +  char **pzErr
          444  +){
          445  +  CsvTable *pNew = 0;        /* The CsvTable object to construct */
          446  +  int bHeader = -1;          /* header= flags.  -1 means not seen yet */
          447  +  int rc = SQLITE_OK;        /* Result code from this routine */
          448  +  int i, j;                  /* Loop counters */
          449  +#ifdef SQLITE_TEST
          450  +  int tstFlags = 0;          /* Value for testflags=N parameter */
          451  +#endif
          452  +  int nCol = -99;            /* Value of the columns= parameter */
          453  +  CsvReader sRdr;            /* A CSV file reader used to store an error
          454  +                             ** message and/or to count the number of columns */
          455  +  static const char *azParam[] = {
          456  +     "filename", "data", "schema", 
          457  +  };
          458  +  char *azPValue[3];         /* Parameter values */
          459  +# define CSV_FILENAME (azPValue[0])
          460  +# define CSV_DATA     (azPValue[1])
          461  +# define CSV_SCHEMA   (azPValue[2])
          462  +
          463  +
          464  +  assert( sizeof(azPValue)==sizeof(azParam) );
          465  +  memset(&sRdr, 0, sizeof(sRdr));
          466  +  memset(azPValue, 0, sizeof(azPValue));
          467  +  for(i=3; i<argc; i++){
          468  +    const char *z = argv[i];
          469  +    const char *zValue;
          470  +    for(j=0; j<sizeof(azParam)/sizeof(azParam[0]); j++){
          471  +      if( csv_string_parameter(&sRdr, azParam[j], z, &azPValue[j]) ) break;
          472  +    }
          473  +    if( j<sizeof(azParam)/sizeof(azParam[0]) ){
          474  +      if( sRdr.zErr[0] ) goto csvtab_connect_error;
          475  +    }else
          476  +    if( (zValue = csv_parameter("header",6,z))!=0 ){
          477  +      int x;
          478  +      if( bHeader>=0 ){
          479  +        csv_errmsg(&sRdr, "more than one 'header' parameter");
          480  +        goto csvtab_connect_error;
          481  +      }
          482  +      x = csv_boolean(zValue);
          483  +      if( x==1 ){
          484  +        bHeader = 1;
          485  +      }else if( x==0 ){
          486  +        bHeader = 0;
          487  +      }else{
          488  +        csv_errmsg(&sRdr, "unrecognized argument to 'header': %s", zValue);
          489  +        goto csvtab_connect_error;
          490  +      }
          491  +    }else
          492  +#ifdef SQLITE_TEST
          493  +    if( (zValue = csv_parameter("testflags",9,z))!=0 ){
          494  +      tstFlags = (unsigned int)atoi(zValue);
          495  +    }else
          496  +#endif
          497  +    if( (zValue = csv_parameter("columns",7,z))!=0 ){
          498  +      if( nCol>0 ){
          499  +        csv_errmsg(&sRdr, "more than one 'columns' parameter");
          500  +        goto csvtab_connect_error;
          501  +      }
          502  +      nCol = atoi(zValue);
          503  +      if( nCol<=0 ){
          504  +        csv_errmsg(&sRdr, "must have at least one column");
          505  +        goto csvtab_connect_error;
          506  +      }
          507  +    }else
          508  +    {
          509  +      csv_errmsg(&sRdr, "unrecognized parameter '%s'", z);
          510  +      goto csvtab_connect_error;
          511  +    }
          512  +  }
          513  +  if( (CSV_FILENAME==0)==(CSV_DATA==0) ){
          514  +    csv_errmsg(&sRdr, "must either filename= or data= but not both");
          515  +    goto csvtab_connect_error;
          516  +  }
          517  +  if( nCol<=0 && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA) ){
          518  +    goto csvtab_connect_error;
          519  +  }
          520  +  pNew = sqlite3_malloc( sizeof(*pNew) );
          521  +  *ppVtab = (sqlite3_vtab*)pNew;
          522  +  if( pNew==0 ) goto csvtab_connect_oom;
          523  +  memset(pNew, 0, sizeof(*pNew));
          524  +  if( nCol>0 ){
          525  +    pNew->nCol = nCol;
          526  +  }else{
          527  +    do{
          528  +      const char *z = csv_read_one_field(&sRdr);
          529  +      if( z==0 ) goto csvtab_connect_oom;
          530  +      pNew->nCol++;
          531  +    }while( sRdr.cTerm==',' );
          532  +  }
          533  +  pNew->zFilename = CSV_FILENAME;  CSV_FILENAME = 0;
          534  +  pNew->zData = CSV_DATA;          CSV_DATA = 0;
          535  +#ifdef SQLITE_TEST
          536  +  pNew->tstFlags = tstFlags;
          537  +#endif
          538  +  pNew->iStart = bHeader==1 ? ftell(sRdr.in) : 0;
          539  +  csv_reader_reset(&sRdr);
          540  +  if( CSV_SCHEMA==0 ){
          541  +    char *zSep = "";
          542  +    CSV_SCHEMA = sqlite3_mprintf("CREATE TABLE x(");
          543  +    if( CSV_SCHEMA==0 ) goto csvtab_connect_oom;
          544  +    for(i=0; i<pNew->nCol; i++){
          545  +      CSV_SCHEMA = sqlite3_mprintf("%z%sc%d TEXT",CSV_SCHEMA, zSep, i);
          546  +      zSep = ",";
          547  +    }
          548  +    CSV_SCHEMA = sqlite3_mprintf("%z);", CSV_SCHEMA);
          549  +  }
          550  +  rc = sqlite3_declare_vtab(db, CSV_SCHEMA);
          551  +  if( rc ) goto csvtab_connect_error;
          552  +  for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){
          553  +    sqlite3_free(azPValue[i]);
          554  +  }
          555  +  return SQLITE_OK;
          556  +
          557  +csvtab_connect_oom:
          558  +  rc = SQLITE_NOMEM;
          559  +  csv_errmsg(&sRdr, "out of memory");
          560  +
          561  +csvtab_connect_error:
          562  +  if( pNew ) csvtabDisconnect(&pNew->base);
          563  +  for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){
          564  +    sqlite3_free(azPValue[i]);
          565  +  }
          566  +  if( sRdr.zErr[0] ){
          567  +    sqlite3_free(*pzErr);
          568  +    *pzErr = sqlite3_mprintf("%s", sRdr.zErr);
          569  +  }
          570  +  csv_reader_reset(&sRdr);
          571  +  if( rc==SQLITE_OK ) rc = SQLITE_ERROR;
          572  +  return rc;
          573  +}
          574  +
          575  +/*
          576  +** Reset the current row content held by a CsvCursor.
          577  +*/
          578  +static void csvtabCursorRowReset(CsvCursor *pCur){
          579  +  CsvTable *pTab = (CsvTable*)pCur->base.pVtab;
          580  +  int i;
          581  +  for(i=0; i<pTab->nCol; i++){
          582  +    sqlite3_free(pCur->azVal[i]);
          583  +    pCur->azVal[i] = 0;
          584  +    pCur->aLen[i] = 0;
          585  +  }
          586  +}
          587  +
          588  +/*
          589  +** The xConnect and xCreate methods do the same thing, but they must be
          590  +** different so that the virtual table is not an eponymous virtual table.
          591  +*/
          592  +static int csvtabCreate(
          593  +  sqlite3 *db,
          594  +  void *pAux,
          595  +  int argc, const char *const*argv,
          596  +  sqlite3_vtab **ppVtab,
          597  +  char **pzErr
          598  +){
          599  + return csvtabConnect(db, pAux, argc, argv, ppVtab, pzErr);
          600  +}
          601  +
          602  +/*
          603  +** Destructor for a CsvCursor.
          604  +*/
          605  +static int csvtabClose(sqlite3_vtab_cursor *cur){
          606  +  CsvCursor *pCur = (CsvCursor*)cur;
          607  +  csvtabCursorRowReset(pCur);
          608  +  csv_reader_reset(&pCur->rdr);
          609  +  sqlite3_free(cur);
          610  +  return SQLITE_OK;
          611  +}
          612  +
          613  +/*
          614  +** Constructor for a new CsvTable cursor object.
          615  +*/
          616  +static int csvtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
          617  +  CsvTable *pTab = (CsvTable*)p;
          618  +  CsvCursor *pCur;
          619  +  size_t nByte;
          620  +  nByte = sizeof(*pCur) + (sizeof(char*)+sizeof(int))*pTab->nCol;
          621  +  pCur = sqlite3_malloc64( nByte );
          622  +  if( pCur==0 ) return SQLITE_NOMEM;
          623  +  memset(pCur, 0, nByte);
          624  +  pCur->azVal = (char**)&pCur[1];
          625  +  pCur->aLen = (int*)&pCur->azVal[pTab->nCol];
          626  +  *ppCursor = &pCur->base;
          627  +  if( csv_reader_open(&pCur->rdr, pTab->zFilename, pTab->zData) ){
          628  +    csv_xfer_error(pTab, &pCur->rdr);
          629  +    return SQLITE_ERROR;
          630  +  }
          631  +  return SQLITE_OK;
          632  +}
          633  +
          634  +
          635  +/*
          636  +** Advance a CsvCursor to its next row of input.
          637  +** Set the EOF marker if we reach the end of input.
          638  +*/
          639  +static int csvtabNext(sqlite3_vtab_cursor *cur){
          640  +  CsvCursor *pCur = (CsvCursor*)cur;
          641  +  CsvTable *pTab = (CsvTable*)cur->pVtab;
          642  +  int i = 0;
          643  +  char *z;
          644  +  do{
          645  +    z = csv_read_one_field(&pCur->rdr);
          646  +    if( z==0 ){
          647  +      csv_xfer_error(pTab, &pCur->rdr);
          648  +      break;
          649  +    }
          650  +    if( i<pTab->nCol ){
          651  +      if( pCur->aLen[i] < pCur->rdr.n+1 ){
          652  +        char *zNew = sqlite3_realloc64(pCur->azVal[i], pCur->rdr.n+1);
          653  +        if( zNew==0 ){
          654  +          csv_errmsg(&pCur->rdr, "out of memory");
          655  +          csv_xfer_error(pTab, &pCur->rdr);
          656  +          break;
          657  +        }
          658  +        pCur->azVal[i] = zNew;
          659  +        pCur->aLen[i] = pCur->rdr.n+1;
          660  +      }
          661  +      memcpy(pCur->azVal[i], z, pCur->rdr.n+1);
          662  +      i++;
          663  +    }
          664  +  }while( pCur->rdr.cTerm==',' );
          665  +  while( i<pTab->nCol ){
          666  +    sqlite3_free(pCur->azVal[i]);
          667  +    pCur->azVal[i] = 0;
          668  +    pCur->aLen[i] = 0;
          669  +    i++;
          670  +  }
          671  +  if( z==0 || pCur->rdr.cTerm==EOF ){
          672  +    pCur->iRowid = -1;
          673  +  }else{
          674  +    pCur->iRowid++;
          675  +  }
          676  +  return SQLITE_OK;
          677  +}
          678  +
          679  +/*
          680  +** Return values of columns for the row at which the CsvCursor
          681  +** is currently pointing.
          682  +*/
          683  +static int csvtabColumn(
          684  +  sqlite3_vtab_cursor *cur,   /* The cursor */
          685  +  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
          686  +  int i                       /* Which column to return */
          687  +){
          688  +  CsvCursor *pCur = (CsvCursor*)cur;
          689  +  CsvTable *pTab = (CsvTable*)cur->pVtab;
          690  +  if( i>=0 && i<pTab->nCol && pCur->azVal[i]!=0 ){
          691  +    sqlite3_result_text(ctx, pCur->azVal[i], -1, SQLITE_STATIC);
          692  +  }
          693  +  return SQLITE_OK;
          694  +}
          695  +
          696  +/*
          697  +** Return the rowid for the current row.
          698  +*/
          699  +static int csvtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
          700  +  CsvCursor *pCur = (CsvCursor*)cur;
          701  +  *pRowid = pCur->iRowid;
          702  +  return SQLITE_OK;
          703  +}
          704  +
          705  +/*
          706  +** Return TRUE if the cursor has been moved off of the last
          707  +** row of output.
          708  +*/
          709  +static int csvtabEof(sqlite3_vtab_cursor *cur){
          710  +  CsvCursor *pCur = (CsvCursor*)cur;
          711  +  return pCur->iRowid<0;
          712  +}
          713  +
          714  +/*
          715  +** Only a full table scan is supported.  So xFilter simply rewinds to
          716  +** the beginning.
          717  +*/
          718  +static int csvtabFilter(
          719  +  sqlite3_vtab_cursor *pVtabCursor, 
          720  +  int idxNum, const char *idxStr,
          721  +  int argc, sqlite3_value **argv
          722  +){
          723  +  CsvCursor *pCur = (CsvCursor*)pVtabCursor;
          724  +  CsvTable *pTab = (CsvTable*)pVtabCursor->pVtab;
          725  +  pCur->iRowid = 0;
          726  +  if( pCur->rdr.in==0 ){
          727  +    assert( pCur->rdr.zIn==pTab->zData );
          728  +    assert( pTab->iStart>=0 );
          729  +    assert( (size_t)pTab->iStart<=pCur->rdr.nIn );
          730  +    pCur->rdr.iIn = pTab->iStart;
          731  +  }else{
          732  +    fseek(pCur->rdr.in, pTab->iStart, SEEK_SET);
          733  +    pCur->rdr.iIn = 0;
          734  +    pCur->rdr.nIn = 0;
          735  +  }
          736  +  return csvtabNext(pVtabCursor);
          737  +}
          738  +
          739  +/*
          740  +** Only a forward full table scan is supported.  xBestIndex is mostly
          741  +** a no-op.  If CSVTEST_FIDX is set, then the presence of equality
          742  +** constraints lowers the estimated cost, which is fiction, but is useful
          743  +** for testing certain kinds of virtual table behavior.
          744  +*/
          745  +static int csvtabBestIndex(
          746  +  sqlite3_vtab *tab,
          747  +  sqlite3_index_info *pIdxInfo
          748  +){
          749  +  pIdxInfo->estimatedCost = 1000000;
          750  +#ifdef SQLITE_TEST
          751  +  if( (((CsvTable*)tab)->tstFlags & CSVTEST_FIDX)!=0 ){
          752  +    /* The usual (and sensible) case is to always do a full table scan.
          753  +    ** The code in this branch only runs when testflags=1.  This code
          754  +    ** generates an artifical and unrealistic plan which is useful
          755  +    ** for testing virtual table logic but is not helpful to real applications.
          756  +    **
          757  +    ** Any ==, LIKE, or GLOB constraint is marked as usable by the virtual
          758  +    ** table (even though it is not) and the cost of running the virtual table
          759  +    ** is reduced from 1 million to just 10.  The constraints are *not* marked
          760  +    ** as omittable, however, so the query planner should still generate a
          761  +    ** plan that gives a correct answer, even if they plan is not optimal.
          762  +    */
          763  +    int i;
          764  +    int nConst = 0;
          765  +    for(i=0; i<pIdxInfo->nConstraint; i++){
          766  +      unsigned char op;
          767  +      if( pIdxInfo->aConstraint[i].usable==0 ) continue;
          768  +      op = pIdxInfo->aConstraint[i].op;
          769  +      if( op==SQLITE_INDEX_CONSTRAINT_EQ 
          770  +       || op==SQLITE_INDEX_CONSTRAINT_LIKE
          771  +       || op==SQLITE_INDEX_CONSTRAINT_GLOB
          772  +      ){
          773  +        pIdxInfo->estimatedCost = 10;
          774  +        pIdxInfo->aConstraintUsage[nConst].argvIndex = nConst+1;
          775  +        nConst++;
          776  +      }
          777  +    }
          778  +  }
          779  +#endif
          780  +  return SQLITE_OK;
          781  +}
          782  +
          783  +
          784  +static sqlite3_module CsvModule = {
          785  +  0,                       /* iVersion */
          786  +  csvtabCreate,            /* xCreate */
          787  +  csvtabConnect,           /* xConnect */
          788  +  csvtabBestIndex,         /* xBestIndex */
          789  +  csvtabDisconnect,        /* xDisconnect */
          790  +  csvtabDisconnect,        /* xDestroy */
          791  +  csvtabOpen,              /* xOpen - open a cursor */
          792  +  csvtabClose,             /* xClose - close a cursor */
          793  +  csvtabFilter,            /* xFilter - configure scan constraints */
          794  +  csvtabNext,              /* xNext - advance a cursor */
          795  +  csvtabEof,               /* xEof - check for end of scan */
          796  +  csvtabColumn,            /* xColumn - read data */
          797  +  csvtabRowid,             /* xRowid - read data */
          798  +  0,                       /* xUpdate */
          799  +  0,                       /* xBegin */
          800  +  0,                       /* xSync */
          801  +  0,                       /* xCommit */
          802  +  0,                       /* xRollback */
          803  +  0,                       /* xFindMethod */
          804  +  0,                       /* xRename */
          805  +};
          806  +
          807  +#ifdef SQLITE_TEST
          808  +/*
          809  +** For virtual table testing, make a version of the CSV virtual table
          810  +** available that has an xUpdate function.  But the xUpdate always returns
          811  +** SQLITE_READONLY since the CSV file is not really writable.
          812  +*/
          813  +static int csvtabUpdate(sqlite3_vtab *p,int n,sqlite3_value**v,sqlite3_int64*x){
          814  +  return SQLITE_READONLY;
          815  +}
          816  +static sqlite3_module CsvModuleFauxWrite = {
          817  +  0,                       /* iVersion */
          818  +  csvtabCreate,            /* xCreate */
          819  +  csvtabConnect,           /* xConnect */
          820  +  csvtabBestIndex,         /* xBestIndex */
          821  +  csvtabDisconnect,        /* xDisconnect */
          822  +  csvtabDisconnect,        /* xDestroy */
          823  +  csvtabOpen,              /* xOpen - open a cursor */
          824  +  csvtabClose,             /* xClose - close a cursor */
          825  +  csvtabFilter,            /* xFilter - configure scan constraints */
          826  +  csvtabNext,              /* xNext - advance a cursor */
          827  +  csvtabEof,               /* xEof - check for end of scan */
          828  +  csvtabColumn,            /* xColumn - read data */
          829  +  csvtabRowid,             /* xRowid - read data */
          830  +  csvtabUpdate,            /* xUpdate */
          831  +  0,                       /* xBegin */
          832  +  0,                       /* xSync */
          833  +  0,                       /* xCommit */
          834  +  0,                       /* xRollback */
          835  +  0,                       /* xFindMethod */
          836  +  0,                       /* xRename */
          837  +};
          838  +#endif /* SQLITE_TEST */
          839  +
          840  +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */
          841  +
          842  +
          843  +#ifdef _WIN32
          844  +__declspec(dllexport)
          845  +#endif
          846  +/* 
          847  +** This routine is called when the extension is loaded.  The new
          848  +** CSV virtual table module is registered with the calling database
          849  +** connection.
          850  +*/
          851  +int sqlite3_csv_init(
          852  +  sqlite3 *db, 
          853  +  char **pzErrMsg, 
          854  +  const sqlite3_api_routines *pApi
          855  +){
          856  +#ifndef SQLITE_OMIT_VIRTUALTABLE	
          857  +  int rc;
          858  +  SQLITE_EXTENSION_INIT2(pApi);
          859  +  rc = sqlite3_create_module(db, "csv", &CsvModule, 0);
          860  +#ifdef SQLITE_TEST
          861  +  if( rc==SQLITE_OK ){
          862  +    rc = sqlite3_create_module(db, "csv_wr", &CsvModuleFauxWrite, 0);
          863  +  }
          864  +#endif
          865  +  return rc;
          866  +#else
          867  +  return SQLITE_OK;
          868  +#endif
          869  +}

Added ext/misc/dbdump.c.

            1  +/*
            2  +** 2016-03-13
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** This file implements a C-language subroutine that converts the content
           14  +** of an SQLite database into UTF-8 text SQL statements that can be used
           15  +** to exactly recreate the original database.  ROWID values are preserved.
           16  +**
           17  +** A prototype of the implemented subroutine is this:
           18  +**
           19  +**   int sqlite3_db_dump(
           20  +**          sqlite3 *db,
           21  +**          const char *zSchema,
           22  +**          const char *zTable,
           23  +**          void (*xCallback)(void*, const char*),
           24  +**          void *pArg
           25  +**   );
           26  +**
           27  +** The db parameter is the database connection.  zSchema is the schema within
           28  +** that database which is to be dumped.  Usually the zSchema is "main" but
           29  +** can also be "temp" or any ATTACH-ed database.  If zTable is not NULL, then
           30  +** only the content of that one table is dumped.  If zTable is NULL, then all
           31  +** tables are dumped.
           32  +**
           33  +** The generate text is passed to xCallback() in multiple calls.  The second
           34  +** argument to xCallback() is a copy of the pArg parameter.  The first
           35  +** argument is some of the output text that this routine generates.  The
           36  +** signature to xCallback() is designed to make it compatible with fputs().
           37  +**
           38  +** The sqlite3_db_dump() subroutine returns SQLITE_OK on success or some error
           39  +** code if it encounters a problem.
           40  +**
           41  +** If this file is compiled with -DDBDUMP_STANDALONE then a "main()" routine
           42  +** is included so that this routine becomes a command-line utility.  The
           43  +** command-line utility takes two or three arguments which are the name
           44  +** of the database file, the schema, and optionally the table, forming the
           45  +** first three arguments of a single call to the library routine.
           46  +*/
           47  +#include "sqlite3.h"
           48  +#include <stdarg.h>
           49  +#include <string.h>
           50  +#include <ctype.h>
           51  +
           52  +/*
           53  +** The state of the dump process.
           54  +*/
           55  +typedef struct DState DState;
           56  +struct DState {
           57  +  sqlite3 *db;                /* The database connection */
           58  +  int nErr;                   /* Number of errors seen so far */
           59  +  int rc;                     /* Error code */
           60  +  int writableSchema;                    /* True if in writable_schema mode */
           61  +  int (*xCallback)(const char*,void*);   /* Send output here */
           62  +  void *pArg;                            /* Argument to xCallback() */
           63  +};
           64  +
           65  +/*
           66  +** A variable length string to which one can append text.
           67  +*/
           68  +typedef struct DText DText;
           69  +struct DText {
           70  +  char *z;           /* The text */
           71  +  int n;             /* Number of bytes of content in z[] */
           72  +  int nAlloc;        /* Number of bytes allocated to z[] */
           73  +};
           74  +
           75  +/*
           76  +** Initialize and destroy a DText object
           77  +*/
           78  +static void initText(DText *p){
           79  +  memset(p, 0, sizeof(*p));
           80  +}
           81  +static void freeText(DText *p){
           82  +  sqlite3_free(p->z);
           83  +  initText(p);
           84  +}
           85  +
           86  +/* zIn is either a pointer to a NULL-terminated string in memory obtained
           87  +** from malloc(), or a NULL pointer. The string pointed to by zAppend is
           88  +** added to zIn, and the result returned in memory obtained from malloc().
           89  +** zIn, if it was not NULL, is freed.
           90  +**
           91  +** If the third argument, quote, is not '\0', then it is used as a
           92  +** quote character for zAppend.
           93  +*/
           94  +static void appendText(DText *p, char const *zAppend, char quote){
           95  +  int len;
           96  +  int i;
           97  +  int nAppend = (int)(strlen(zAppend) & 0x3fffffff);
           98  +
           99  +  len = nAppend+p->n+1;
          100  +  if( quote ){
          101  +    len += 2;
          102  +    for(i=0; i<nAppend; i++){
          103  +      if( zAppend[i]==quote ) len++;
          104  +    }
          105  +  }
          106  +
          107  +  if( p->n+len>=p->nAlloc ){
          108  +    char *zNew;
          109  +    p->nAlloc = p->nAlloc*2 + len + 20;
          110  +    zNew = sqlite3_realloc(p->z, p->nAlloc);
          111  +    if( zNew==0 ){
          112  +      freeText(p);
          113  +      return;
          114  +    }
          115  +    p->z = zNew;
          116  +  }
          117  +
          118  +  if( quote ){
          119  +    char *zCsr = p->z+p->n;
          120  +    *zCsr++ = quote;
          121  +    for(i=0; i<nAppend; i++){
          122  +      *zCsr++ = zAppend[i];
          123  +      if( zAppend[i]==quote ) *zCsr++ = quote;
          124  +    }
          125  +    *zCsr++ = quote;
          126  +    p->n = (int)(zCsr - p->z);
          127  +    *zCsr = '\0';
          128  +  }else{
          129  +    memcpy(p->z+p->n, zAppend, nAppend);
          130  +    p->n += nAppend;
          131  +    p->z[p->n] = '\0';
          132  +  }
          133  +}
          134  +
          135  +/*
          136  +** Attempt to determine if identifier zName needs to be quoted, either
          137  +** because it contains non-alphanumeric characters, or because it is an
          138  +** SQLite keyword.  Be conservative in this estimate:  When in doubt assume
          139  +** that quoting is required.
          140  +**
          141  +** Return '"' if quoting is required.  Return 0 if no quoting is required.
          142  +*/
          143  +static char quoteChar(const char *zName){
          144  +  /* All SQLite keywords, in alphabetical order */
          145  +  static const char *azKeywords[] = {
          146  +    "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
          147  +    "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
          148  +    "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
          149  +    "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
          150  +    "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
          151  +    "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
          152  +    "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
          153  +    "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
          154  +    "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
          155  +    "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
          156  +    "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
          157  +    "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
          158  +    "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
          159  +    "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
          160  +    "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
          161  +    "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
          162  +    "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
          163  +    "WITH", "WITHOUT",
          164  +  };
          165  +  int i, lwr, upr, mid, c;
          166  +  if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"';
          167  +  for(i=0; zName[i]; i++){
          168  +    if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"';
          169  +  }
          170  +  lwr = 0;
          171  +  upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1;
          172  +  while( lwr<=upr ){
          173  +    mid = (lwr+upr)/2;
          174  +    c = sqlite3_stricmp(azKeywords[mid], zName);
          175  +    if( c==0 ) return '"';
          176  +    if( c<0 ){
          177  +      lwr = mid+1;
          178  +    }else{
          179  +      upr = mid-1;
          180  +    }
          181  +  }
          182  +  return 0;
          183  +}
          184  +
          185  +
          186  +/*
          187  +** Release memory previously allocated by tableColumnList().
          188  +*/
          189  +static void freeColumnList(char **azCol){
          190  +  int i;
          191  +  for(i=1; azCol[i]; i++){
          192  +    sqlite3_free(azCol[i]);
          193  +  }
          194  +  /* azCol[0] is a static string */
          195  +  sqlite3_free(azCol);
          196  +}
          197  +
          198  +/*
          199  +** Return a list of pointers to strings which are the names of all
          200  +** columns in table zTab.   The memory to hold the names is dynamically
          201  +** allocated and must be released by the caller using a subsequent call
          202  +** to freeColumnList().
          203  +**
          204  +** The azCol[0] entry is usually NULL.  However, if zTab contains a rowid
          205  +** value that needs to be preserved, then azCol[0] is filled in with the
          206  +** name of the rowid column.
          207  +**
          208  +** The first regular column in the table is azCol[1].  The list is terminated
          209  +** by an entry with azCol[i]==0.
          210  +*/
          211  +static char **tableColumnList(DState *p, const char *zTab){
          212  +  char **azCol = 0;
          213  +  sqlite3_stmt *pStmt = 0;
          214  +  char *zSql;
          215  +  int nCol = 0;
          216  +  int nAlloc = 0;
          217  +  int nPK = 0;       /* Number of PRIMARY KEY columns seen */
          218  +  int isIPK = 0;     /* True if one PRIMARY KEY column of type INTEGER */
          219  +  int preserveRowid = 1;
          220  +  int rc;
          221  +
          222  +  zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab);
          223  +  if( zSql==0 ) return 0;
          224  +  rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
          225  +  sqlite3_free(zSql);
          226  +  if( rc ) return 0;
          227  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          228  +    if( nCol>=nAlloc-2 ){
          229  +      char **azNew;
          230  +      nAlloc = nAlloc*2 + nCol + 10;
          231  +      azNew = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0]));
          232  +      if( azNew==0 ) goto col_oom;
          233  +      azCol = azNew;
          234  +      azCol[0] = 0;
          235  +    }
          236  +    azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1));
          237  +    if( azCol[nCol]==0 ) goto col_oom;
          238  +    if( sqlite3_column_int(pStmt, 5) ){
          239  +      nPK++;
          240  +      if( nPK==1
          241  +       && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2),
          242  +                          "INTEGER")==0 
          243  +      ){
          244  +        isIPK = 1;
          245  +      }else{
          246  +        isIPK = 0;
          247  +      }
          248  +    }
          249  +  }
          250  +  sqlite3_finalize(pStmt);
          251  +  pStmt = 0;
          252  +  azCol[nCol+1] = 0;
          253  +
          254  +  /* The decision of whether or not a rowid really needs to be preserved
          255  +  ** is tricky.  We never need to preserve a rowid for a WITHOUT ROWID table
          256  +  ** or a table with an INTEGER PRIMARY KEY.  We are unable to preserve
          257  +  ** rowids on tables where the rowid is inaccessible because there are other
          258  +  ** columns in the table named "rowid", "_rowid_", and "oid".
          259  +  */
          260  +  if( isIPK ){
          261  +    /* If a single PRIMARY KEY column with type INTEGER was seen, then it
          262  +    ** might be an alise for the ROWID.  But it might also be a WITHOUT ROWID
          263  +    ** table or a INTEGER PRIMARY KEY DESC column, neither of which are
          264  +    ** ROWID aliases.  To distinguish these cases, check to see if
          265  +    ** there is a "pk" entry in "PRAGMA index_list".  There will be
          266  +    ** no "pk" index if the PRIMARY KEY really is an alias for the ROWID.
          267  +    */
          268  +    zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)"
          269  +                           " WHERE origin='pk'", zTab);
          270  +    if( zSql==0 ) goto col_oom;
          271  +    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
          272  +    sqlite3_free(zSql);
          273  +    if( rc ){
          274  +      freeColumnList(azCol);
          275  +      return 0;
          276  +    }
          277  +    rc = sqlite3_step(pStmt);
          278  +    sqlite3_finalize(pStmt);
          279  +    pStmt = 0;
          280  +    preserveRowid = rc==SQLITE_ROW;
          281  +  }
          282  +  if( preserveRowid ){
          283  +    /* Only preserve the rowid if we can find a name to use for the
          284  +    ** rowid */
          285  +    static char *azRowid[] = { "rowid", "_rowid_", "oid" };
          286  +    int i, j;
          287  +    for(j=0; j<3; j++){
          288  +      for(i=1; i<=nCol; i++){
          289  +        if( sqlite3_stricmp(azRowid[j],azCol[i])==0 ) break;
          290  +      }
          291  +      if( i>nCol ){
          292  +        /* At this point, we know that azRowid[j] is not the name of any
          293  +        ** ordinary column in the table.  Verify that azRowid[j] is a valid
          294  +        ** name for the rowid before adding it to azCol[0].  WITHOUT ROWID
          295  +        ** tables will fail this last check */
          296  +        int rc;
          297  +        rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0);
          298  +        if( rc==SQLITE_OK ) azCol[0] = azRowid[j];
          299  +        break;
          300  +      }
          301  +    }
          302  +  }
          303  +  return azCol;
          304  +
          305  +col_oom:
          306  +  sqlite3_finalize(pStmt);
          307  +  freeColumnList(azCol);
          308  +  p->nErr++;
          309  +  p->rc = SQLITE_NOMEM;
          310  +  return 0;
          311  +}
          312  +
          313  +/*
          314  +** Send mprintf-formatted content to the output callback.
          315  +*/
          316  +static void output_formatted(DState *p, const char *zFormat, ...){
          317  +  va_list ap;
          318  +  char *z;
          319  +  va_start(ap, zFormat);
          320  +  z = sqlite3_vmprintf(zFormat, ap);
          321  +  va_end(ap);
          322  +  p->xCallback(z, p->pArg);
          323  +  sqlite3_free(z);
          324  +}
          325  +
          326  +/*
          327  +** Output the given string as a quoted string using SQL quoting conventions.
          328  +**
          329  +** The "\n" and "\r" characters are converted to char(10) and char(13)
          330  +** to prevent them from being transformed by end-of-line translators.
          331  +*/
          332  +static void output_quoted_string(DState *p, const unsigned char *z){
          333  +  int i;
          334  +  char c;
          335  +  int inQuote = 0;
          336  +  int bStarted = 0;
          337  +
          338  +  for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){}
          339  +  if( c==0 ){
          340  +    output_formatted(p, "'%s'", z);
          341  +    return;
          342  +  }
          343  +  while( *z ){
          344  +    for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){}
          345  +    if( c=='\'' ) i++;
          346  +    if( i ){
          347  +      if( !inQuote ){
          348  +        if( bStarted ) p->xCallback("||", p->pArg);
          349  +        p->xCallback("'", p->pArg);
          350  +        inQuote = 1;
          351  +      }
          352  +      output_formatted(p, "%.*s", i, z);
          353  +      z += i;
          354  +      bStarted = 1;
          355  +    }
          356  +    if( c=='\'' ){
          357  +      p->xCallback("'", p->pArg);
          358  +      continue;
          359  +    }
          360  +    if( inQuote ){
          361  +      p->xCallback("'", p->pArg);
          362  +      inQuote = 0;
          363  +    }
          364  +    if( c==0 ){
          365  +      break;
          366  +    }
          367  +    for(i=0; (c = z[i])=='\r' || c=='\n'; i++){
          368  +      if( bStarted ) p->xCallback("||", p->pArg);
          369  +      output_formatted(p, "char(%d)", c);
          370  +      bStarted = 1;
          371  +    }
          372  +    z += i;
          373  +  }
          374  +  if( inQuote ) p->xCallback("'", p->pArg);
          375  +}
          376  +
          377  +/*
          378  +** This is an sqlite3_exec callback routine used for dumping the database.
          379  +** Each row received by this callback consists of a table name,
          380  +** the table type ("index" or "table") and SQL to create the table.
          381  +** This routine should print text sufficient to recreate the table.
          382  +*/
          383  +static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){
          384  +  int rc;
          385  +  const char *zTable;
          386  +  const char *zType;
          387  +  const char *zSql;
          388  +  DState *p = (DState*)pArg;
          389  +  sqlite3_stmt *pStmt;
          390  +
          391  +  (void)azCol;
          392  +  if( nArg!=3 ) return 1;
          393  +  zTable = azArg[0];
          394  +  zType = azArg[1];
          395  +  zSql = azArg[2];
          396  +
          397  +  if( strcmp(zTable, "sqlite_sequence")==0 ){
          398  +    p->xCallback("DELETE FROM sqlite_sequence;\n", p->pArg);
          399  +  }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 ){
          400  +    p->xCallback("ANALYZE sqlite_master;\n", p->pArg);
          401  +  }else if( strncmp(zTable, "sqlite_", 7)==0 ){
          402  +    return 0;
          403  +  }else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){
          404  +    if( !p->writableSchema ){
          405  +      p->xCallback("PRAGMA writable_schema=ON;\n", p->pArg);
          406  +      p->writableSchema = 1;
          407  +    }
          408  +    output_formatted(p,
          409  +       "INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)"
          410  +       "VALUES('table','%q','%q',0,'%q');",
          411  +       zTable, zTable, zSql);
          412  +    return 0;
          413  +  }else{
          414  +    if( sqlite3_strglob("CREATE TABLE ['\"]*", zSql)==0 ){
          415  +      p->xCallback("CREATE TABLE IF NOT EXISTS ", p->pArg);
          416  +      p->xCallback(zSql+13, p->pArg);
          417  +    }else{
          418  +      p->xCallback(zSql, p->pArg);
          419  +    }
          420  +    p->xCallback(";\n", p->pArg);
          421  +  }
          422  +
          423  +  if( strcmp(zType, "table")==0 ){
          424  +    DText sSelect;
          425  +    DText sTable;
          426  +    char **azCol;
          427  +    int i;
          428  +    int nCol;
          429  +
          430  +    azCol = tableColumnList(p, zTable);
          431  +    if( azCol==0 ) return 0;
          432  +
          433  +    initText(&sTable);
          434  +    appendText(&sTable, "INSERT INTO ", 0);
          435  +
          436  +    /* Always quote the table name, even if it appears to be pure ascii,
          437  +    ** in case it is a keyword. Ex:  INSERT INTO "table" ... */
          438  +    appendText(&sTable, zTable, quoteChar(zTable));
          439  +
          440  +    /* If preserving the rowid, add a column list after the table name.
          441  +    ** In other words:  "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)"
          442  +    ** instead of the usual "INSERT INTO tab VALUES(...)".
          443  +    */
          444  +    if( azCol[0] ){
          445  +      appendText(&sTable, "(", 0);
          446  +      appendText(&sTable, azCol[0], 0);
          447  +      for(i=1; azCol[i]; i++){
          448  +        appendText(&sTable, ",", 0);
          449  +        appendText(&sTable, azCol[i], quoteChar(azCol[i]));
          450  +      }
          451  +      appendText(&sTable, ")", 0);
          452  +    }
          453  +    appendText(&sTable, " VALUES(", 0);
          454  +
          455  +    /* Build an appropriate SELECT statement */
          456  +    initText(&sSelect);
          457  +    appendText(&sSelect, "SELECT ", 0);
          458  +    if( azCol[0] ){
          459  +      appendText(&sSelect, azCol[0], 0);
          460  +      appendText(&sSelect, ",", 0);
          461  +    }
          462  +    for(i=1; azCol[i]; i++){
          463  +      appendText(&sSelect, azCol[i], quoteChar(azCol[i]));
          464  +      if( azCol[i+1] ){
          465  +        appendText(&sSelect, ",", 0);
          466  +      }
          467  +    }
          468  +    nCol = i;
          469  +    if( azCol[0]==0 ) nCol--;
          470  +    freeColumnList(azCol);
          471  +    appendText(&sSelect, " FROM ", 0);
          472  +    appendText(&sSelect, zTable, quoteChar(zTable));
          473  +
          474  +    rc = sqlite3_prepare_v2(p->db, sSelect.z, -1, &pStmt, 0);
          475  +    if( rc!=SQLITE_OK ){
          476  +      p->nErr++;
          477  +      if( p->rc==SQLITE_OK ) p->rc = rc;
          478  +    }else{
          479  +      while( SQLITE_ROW==sqlite3_step(pStmt) ){
          480  +        p->xCallback(sTable.z, p->pArg);
          481  +        for(i=0; i<nCol; i++){
          482  +          if( i ) p->xCallback(",", p->pArg);
          483  +          switch( sqlite3_column_type(pStmt,i) ){
          484  +            case SQLITE_INTEGER: {
          485  +              output_formatted(p, "%lld", sqlite3_column_int64(pStmt,i));
          486  +              break;
          487  +            }
          488  +            case SQLITE_FLOAT: {
          489  +              double r = sqlite3_column_double(pStmt,i);
          490  +              output_formatted(p, "%!.20g", r);
          491  +              break;
          492  +            }
          493  +            case SQLITE_NULL: {
          494  +              p->xCallback("NULL", p->pArg);
          495  +              break;
          496  +            }
          497  +            case SQLITE_TEXT: {
          498  +              output_quoted_string(p, sqlite3_column_text(pStmt,i));
          499  +              break;
          500  +            }
          501  +            case SQLITE_BLOB: {
          502  +              int nByte = sqlite3_column_bytes(pStmt,i);
          503  +              unsigned char *a = (unsigned char*)sqlite3_column_blob(pStmt,i);
          504  +              int j;
          505  +              p->xCallback("x'", p->pArg);
          506  +              for(j=0; j<nByte; j++){
          507  +                char zWord[3];
          508  +                zWord[0] = "0123456789abcdef"[(a[j]>>4)&15];
          509  +                zWord[1] = "0123456789abcdef"[a[j]&15];
          510  +                zWord[2] = 0;
          511  +                p->xCallback(zWord, p->pArg);
          512  +              }
          513  +              p->xCallback("'", p->pArg);
          514  +              break;
          515  +            }
          516  +          }
          517  +        }
          518  +        p->xCallback(");\n", p->pArg);
          519  +      }
          520  +    }
          521  +    sqlite3_finalize(pStmt);
          522  +    freeText(&sTable);
          523  +    freeText(&sSelect);
          524  +  }
          525  +  return 0;
          526  +}
          527  +
          528  +
          529  +/*
          530  +** Execute a query statement that will generate SQL output.  Print
          531  +** the result columns, comma-separated, on a line and then add a
          532  +** semicolon terminator to the end of that line.
          533  +**
          534  +** If the number of columns is 1 and that column contains text "--"
          535  +** then write the semicolon on a separate line.  That way, if a
          536  +** "--" comment occurs at the end of the statement, the comment
          537  +** won't consume the semicolon terminator.
          538  +*/
          539  +static void output_sql_from_query(
          540  +  DState *p,               /* Query context */
          541  +  const char *zSelect,     /* SELECT statement to extract content */
          542  +  ...
          543  +){
          544  +  sqlite3_stmt *pSelect;
          545  +  int rc;
          546  +  int nResult;
          547  +  int i;
          548  +  const char *z;
          549  +  char *zSql;
          550  +  va_list ap;
          551  +  va_start(ap, zSelect);
          552  +  zSql = sqlite3_vmprintf(zSelect, ap);
          553  +  va_end(ap);
          554  +  if( zSql==0 ){
          555  +    p->rc = SQLITE_NOMEM;
          556  +    p->nErr++;
          557  +    return;
          558  +  }
          559  +  rc = sqlite3_prepare_v2(p->db, zSql, -1, &pSelect, 0);
          560  +  sqlite3_free(zSql);
          561  +  if( rc!=SQLITE_OK || !pSelect ){
          562  +    output_formatted(p, "/**** ERROR: (%d) %s *****/\n", rc,
          563  +                sqlite3_errmsg(p->db));
          564  +    p->nErr++;
          565  +    return;
          566  +  }
          567  +  rc = sqlite3_step(pSelect);
          568  +  nResult = sqlite3_column_count(pSelect);
          569  +  while( rc==SQLITE_ROW ){
          570  +    z = (const char*)sqlite3_column_text(pSelect, 0);
          571  +    p->xCallback(z, p->pArg);
          572  +    for(i=1; i<nResult; i++){
          573  +      p->xCallback(",", p->pArg);
          574  +      p->xCallback((const char*)sqlite3_column_text(pSelect,i), p->pArg);
          575  +    }
          576  +    if( z==0 ) z = "";
          577  +    while( z[0] && (z[0]!='-' || z[1]!='-') ) z++;
          578  +    if( z[0] ){
          579  +      p->xCallback("\n;\n", p->pArg);
          580  +    }else{
          581  +      p->xCallback(";\n", p->pArg);
          582  +    }
          583  +    rc = sqlite3_step(pSelect);
          584  +  }
          585  +  rc = sqlite3_finalize(pSelect);
          586  +  if( rc!=SQLITE_OK ){
          587  +    output_formatted(p, "/**** ERROR: (%d) %s *****/\n", rc,
          588  +                     sqlite3_errmsg(p->db));
          589  +    if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++;
          590  +  }
          591  +}
          592  +
          593  +/*
          594  +** Run zQuery.  Use dump_callback() as the callback routine so that
          595  +** the contents of the query are output as SQL statements.
          596  +**
          597  +** If we get a SQLITE_CORRUPT error, rerun the query after appending
          598  +** "ORDER BY rowid DESC" to the end.
          599  +*/
          600  +static void run_schema_dump_query(
          601  +  DState *p,
          602  +  const char *zQuery,
          603  +  ...
          604  +){
          605  +  char *zErr = 0;
          606  +  char *z;
          607  +  va_list ap;
          608  +  va_start(ap, zQuery);
          609  +  z = sqlite3_vmprintf(zQuery, ap);
          610  +  va_end(ap); 
          611  +  sqlite3_exec(p->db, z, dump_callback, p, &zErr);
          612  +  sqlite3_free(z);
          613  +  if( zErr ){
          614  +    output_formatted(p, "/****** %s ******/\n", zErr);
          615  +    sqlite3_free(zErr);
          616  +    p->nErr++;
          617  +    zErr = 0;
          618  +  }
          619  +}
          620  +
          621  +/*
          622  +** Convert an SQLite database into SQL statements that will recreate that
          623  +** database.
          624  +*/
          625  +int sqlite3_db_dump(
          626  +  sqlite3 *db,               /* The database connection */
          627  +  const char *zSchema,       /* Which schema to dump.  Usually "main". */
          628  +  const char *zTable,        /* Which table to dump.  NULL means everything. */
          629  +  int (*xCallback)(const char*,void*),   /* Output sent to this callback */
          630  +  void *pArg                             /* Second argument of the callback */
          631  +){
          632  +  DState x;
          633  +  memset(&x, 0, sizeof(x));
          634  +  x.rc = sqlite3_exec(db, "BEGIN", 0, 0, 0);
          635  +  if( x.rc ) return x.rc;
          636  +  x.db = db;
          637  +  x.xCallback = xCallback;
          638  +  x.pArg = pArg;
          639  +  xCallback("PRAGMA foreign_keys=OFF;\nBEGIN TRANSACTION;\n", pArg);
          640  +  if( zTable==0 ){
          641  +    run_schema_dump_query(&x,
          642  +      "SELECT name, type, sql FROM \"%w\".sqlite_master "
          643  +      "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'",
          644  +      zSchema
          645  +    );
          646  +    run_schema_dump_query(&x,
          647  +      "SELECT name, type, sql FROM \"%w\".sqlite_master "
          648  +      "WHERE name=='sqlite_sequence'", zSchema
          649  +    );
          650  +    output_sql_from_query(&x,
          651  +      "SELECT sql FROM sqlite_master "
          652  +      "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0
          653  +    );
          654  +  }else{
          655  +    run_schema_dump_query(&x,
          656  +      "SELECT name, type, sql FROM \"%w\".sqlite_master "
          657  +      "WHERE tbl_name=%Q COLLATE nocase AND type=='table'"
          658  +      "  AND sql NOT NULL",
          659  +      zSchema, zTable
          660  +    );
          661  +    output_sql_from_query(&x,
          662  +      "SELECT sql FROM \"%w\".sqlite_master "
          663  +      "WHERE sql NOT NULL"
          664  +      "  AND type IN ('index','trigger','view')"
          665  +      "  AND tbl_name=%Q COLLATE nocase",
          666  +      zSchema, zTable
          667  +    ); 
          668  +  }
          669  +  if( x.writableSchema ){
          670  +    xCallback("PRAGMA writable_schema=OFF;\n", pArg);
          671  +  }
          672  +  xCallback(x.nErr ? "ROLLBACK; -- due to errors\n" : "COMMIT;\n", pArg);
          673  +  sqlite3_exec(db, "COMMIT", 0, 0, 0);
          674  +  return x.rc;
          675  +}
          676  +
          677  +
          678  +
          679  +/* The generic subroutine is above.  The code the follows implements
          680  +** the command-line interface.
          681  +*/
          682  +#ifdef DBDUMP_STANDALONE
          683  +#include <stdio.h>
          684  +
          685  +/*
          686  +** Command-line interface
          687  +*/
          688  +int main(int argc, char **argv){
          689  +  sqlite3 *db;
          690  +  const char *zDb;
          691  +  const char *zSchema;
          692  +  const char *zTable = 0;
          693  +  int rc;
          694  +
          695  +  if( argc<2 || argc>4 ){
          696  +    fprintf(stderr, "Usage: %s DATABASE ?SCHEMA? ?TABLE?\n", argv[0]);
          697  +    return 1;
          698  +  }
          699  +  zDb = argv[1];
          700  +  zSchema = argc>=3 ? argv[2] : "main";
          701  +  zTable = argc==4 ? argv[3] : 0;
          702  +
          703  +  rc = sqlite3_open(zDb, &db);
          704  +  if( rc ){
          705  +    fprintf(stderr, "Cannot open \"%s\": %s\n", zDb, sqlite3_errmsg(db));
          706  +    sqlite3_close(db);
          707  +    return 1;
          708  +  }
          709  +  rc = sqlite3_db_dump(db, zSchema, zTable, 
          710  +          (int(*)(const char*,void*))fputs, (void*)stdout);
          711  +  if( rc ){
          712  +    fprintf(stderr, "Error: sqlite3_db_dump() returns %d\n", rc);
          713  +  }
          714  +  sqlite3_close(db);
          715  +  return rc!=SQLITE_OK;  
          716  +}
          717  +#endif /* DBDUMP_STANDALONE */

Changes to ext/misc/fuzzer.c.

   340    340       pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo );
   341    341       if( pRule==0 ){
   342    342         rc = SQLITE_NOMEM;
   343    343       }else{
   344    344         memset(pRule, 0, sizeof(*pRule));
   345    345         pRule->zFrom = pRule->zTo;
   346    346         pRule->zFrom += nTo + 1;
   347         -      pRule->nFrom = nFrom;
          347  +      pRule->nFrom = (fuzzer_len)nFrom;
   348    348         memcpy(pRule->zFrom, zFrom, nFrom+1);
   349    349         memcpy(pRule->zTo, zTo, nTo+1);
   350         -      pRule->nTo = nTo;
          350  +      pRule->nTo = (fuzzer_len)nTo;
   351    351         pRule->rCost = nCost;
   352    352         pRule->iRuleset = (int)iRuleset;
   353    353       }
   354    354     }
   355    355   
   356    356     *ppRule = pRule;
   357    357     return rc;

Changes to ext/misc/json1.c.

    18     18   ** For the time being, all JSON is stored as pure text.  (We might add
    19     19   ** a JSONB type in the future which stores a binary encoding of JSON in
    20     20   ** a BLOB, but there is no support for JSONB in the current implementation.
    21     21   ** This implementation parses JSON text at 250 MB/s, so it is hard to see
    22     22   ** how JSONB might improve on that.)
    23     23   */
    24     24   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1)
    25         -#if !defined(_SQLITEINT_H_)
           25  +#if !defined(SQLITEINT_H)
    26     26   #include "sqlite3ext.h"
    27     27   #endif
    28     28   SQLITE_EXTENSION_INIT1
    29     29   #include <assert.h>
    30     30   #include <string.h>
    31     31   #include <stdlib.h>
    32     32   #include <stdarg.h>
................................................................................
    45     45   /*
    46     46   ** Versions of isspace(), isalnum() and isdigit() to which it is safe
    47     47   ** to pass signed char values.
    48     48   */
    49     49   #ifdef sqlite3Isdigit
    50     50      /* Use the SQLite core versions if this routine is part of the
    51     51      ** SQLite amalgamation */
    52         -#  define safe_isdigit(x) sqlite3Isdigit(x)
    53         -#  define safe_isalnum(x) sqlite3Isalnum(x)
           52  +#  define safe_isdigit(x)  sqlite3Isdigit(x)
           53  +#  define safe_isalnum(x)  sqlite3Isalnum(x)
           54  +#  define safe_isxdigit(x) sqlite3Isxdigit(x)
    54     55   #else
    55     56      /* Use the standard library for separate compilation */
    56     57   #include <ctype.h>  /* amalgamator: keep */
    57         -#  define safe_isdigit(x) isdigit((unsigned char)(x))
    58         -#  define safe_isalnum(x) isalnum((unsigned char)(x))
           58  +#  define safe_isdigit(x)  isdigit((unsigned char)(x))
           59  +#  define safe_isalnum(x)  isalnum((unsigned char)(x))
           60  +#  define safe_isxdigit(x) isxdigit((unsigned char)(x))
    59     61   #endif
    60     62   
    61     63   /*
    62     64   ** Growing our own isspace() routine this way is twice as fast as
    63     65   ** the library isspace() function, resulting in a 7% overall performance
    64     66   ** increase for the parser.  (Ubuntu14.10 gcc 4.8.4 x64 with -Os).
    65     67   */
................................................................................
   132    134   };
   133    135   
   134    136   /* Bit values for the JsonNode.jnFlag field
   135    137   */
   136    138   #define JNODE_RAW     0x01         /* Content is raw, not JSON encoded */
   137    139   #define JNODE_ESCAPE  0x02         /* Content is text with \ escapes */
   138    140   #define JNODE_REMOVE  0x04         /* Do not output */
   139         -#define JNODE_REPLACE 0x08         /* Replace with JsonNode.iVal */
   140         -#define JNODE_APPEND  0x10         /* More ARRAY/OBJECT entries at u.iAppend */
   141         -#define JNODE_LABEL   0x20         /* Is a label of an object */
          141  +#define JNODE_REPLACE 0x08         /* Replace with JsonNode.u.iReplace */
          142  +#define JNODE_PATCH   0x10         /* Patch with JsonNode.u.pPatch */
          143  +#define JNODE_APPEND  0x20         /* More ARRAY/OBJECT entries at u.iAppend */
          144  +#define JNODE_LABEL   0x40         /* Is a label of an object */
   142    145   
   143    146   
   144    147   /* A single node of parsed JSON
   145    148   */
   146    149   struct JsonNode {
   147    150     u8 eType;              /* One of the JSON_ type values */
   148    151     u8 jnFlags;            /* JNODE flags */
   149         -  u8 iVal;               /* Replacement value when JNODE_REPLACE */
   150    152     u32 n;                 /* Bytes of content, or number of sub-nodes */
   151    153     union {
   152    154       const char *zJContent; /* Content for INT, REAL, and STRING */
   153    155       u32 iAppend;           /* More terms for ARRAY and OBJECT */
   154    156       u32 iKey;              /* Key for ARRAY objects in json_tree() */
          157  +    u32 iReplace;          /* Replacement content for JNODE_REPLACE */
          158  +    JsonNode *pPatch;      /* Node chain of patch for JNODE_PATCH */
   155    159     } u;
   156    160   };
   157    161   
   158    162   /* A completely parsed JSON string
   159    163   */
   160    164   struct JsonParse {
   161    165     u32 nNode;         /* Number of slots of aNode[] used */
................................................................................
   404    408   ** the number of JsonNode objects that are encoded.
   405    409   */
   406    410   static void jsonRenderNode(
   407    411     JsonNode *pNode,               /* The node to render */
   408    412     JsonString *pOut,              /* Write JSON here */
   409    413     sqlite3_value **aReplace       /* Replacement values */
   410    414   ){
          415  +  if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){
          416  +    if( pNode->jnFlags & JNODE_REPLACE ){
          417  +      jsonAppendValue(pOut, aReplace[pNode->u.iReplace]);
          418  +      return;
          419  +    }
          420  +    pNode = pNode->u.pPatch;
          421  +  }
   411    422     switch( pNode->eType ){
   412    423       default: {
   413    424         assert( pNode->eType==JSON_NULL );
   414    425         jsonAppendRaw(pOut, "null", 4);
   415    426         break;
   416    427       }
   417    428       case JSON_TRUE: {
................................................................................
   435    446         break;
   436    447       }
   437    448       case JSON_ARRAY: {
   438    449         u32 j = 1;
   439    450         jsonAppendChar(pOut, '[');
   440    451         for(;;){
   441    452           while( j<=pNode->n ){
   442         -          if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){
   443         -            if( pNode[j].jnFlags & JNODE_REPLACE ){
   444         -              jsonAppendSeparator(pOut);
   445         -              jsonAppendValue(pOut, aReplace[pNode[j].iVal]);
   446         -            }
   447         -          }else{
          453  +          if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){
   448    454               jsonAppendSeparator(pOut);
   449    455               jsonRenderNode(&pNode[j], pOut, aReplace);
   450    456             }
   451    457             j += jsonNodeSize(&pNode[j]);
   452    458           }
   453    459           if( (pNode->jnFlags & JNODE_APPEND)==0 ) break;
   454    460           pNode = &pNode[pNode->u.iAppend];
................................................................................
   462    468         jsonAppendChar(pOut, '{');
   463    469         for(;;){
   464    470           while( j<=pNode->n ){
   465    471             if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){
   466    472               jsonAppendSeparator(pOut);
   467    473               jsonRenderNode(&pNode[j], pOut, aReplace);
   468    474               jsonAppendChar(pOut, ':');
   469         -            if( pNode[j+1].jnFlags & JNODE_REPLACE ){
   470         -              jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]);
   471         -            }else{
   472         -              jsonRenderNode(&pNode[j+1], pOut, aReplace);
   473         -            }
          475  +            jsonRenderNode(&pNode[j+1], pOut, aReplace);
   474    476             }
   475    477             j += 1 + jsonNodeSize(&pNode[j+1]);
   476    478           }
   477    479           if( (pNode->jnFlags & JNODE_APPEND)==0 ) break;
   478    480           pNode = &pNode[pNode->u.iAppend];
   479    481           j = 1;
   480    482         }
................................................................................
   589    591             char c = z[i];
   590    592             if( c!='\\' ){
   591    593               zOut[j++] = c;
   592    594             }else{
   593    595               c = z[++i];
   594    596               if( c=='u' ){
   595    597                 u32 v = 0, k;
   596         -              for(k=0; k<4 && i<n-2; i++, k++){
          598  +              for(k=0; k<4; i++, k++){
          599  +                assert( i<n-2 );
   597    600                   c = z[i+1];
   598         -                if( c>='0' && c<='9' ) v = v*16 + c - '0';
   599         -                else if( c>='A' && c<='F' ) v = v*16 + c - 'A' + 10;
   600         -                else if( c>='a' && c<='f' ) v = v*16 + c - 'a' + 10;
   601         -                else break;
          601  +                assert( safe_isxdigit(c) );
          602  +                if( c<='9' ) v = v*16 + c - '0';
          603  +                else if( c<='F' ) v = v*16 + c - 'A' + 10;
          604  +                else v = v*16 + c - 'a' + 10;
   602    605                 }
   603    606                 if( v==0 ) break;
   604    607                 if( v<=0x7f ){
   605    608                   zOut[j++] = (char)v;
   606    609                 }else if( v<=0x7ff ){
   607    610                   zOut[j++] = (char)(0xc0 | (v>>6));
   608    611                   zOut[j++] = 0x80 | (v&0x3f);
................................................................................
   692    695     JsonNode *p;
   693    696     if( pParse->nNode>=pParse->nAlloc ){
   694    697       return jsonParseAddNodeExpand(pParse, eType, n, zContent);
   695    698     }
   696    699     p = &pParse->aNode[pParse->nNode];
   697    700     p->eType = (u8)eType;
   698    701     p->jnFlags = 0;
   699         -  p->iVal = 0;
   700    702     p->n = n;
   701    703     p->u.zJContent = zContent;
   702    704     return pParse->nNode++;
   703    705   }
          706  +
          707  +/*
          708  +** Return true if z[] begins with 4 (or more) hexadecimal digits
          709  +*/
          710  +static int jsonIs4Hex(const char *z){
          711  +  int i;
          712  +  for(i=0; i<4; i++) if( !safe_isxdigit(z[i]) ) return 0;
          713  +  return 1;
          714  +}
   704    715   
   705    716   /*
   706    717   ** Parse a single JSON value which begins at pParse->zJson[i].  Return the
   707    718   ** index of the first character past the end of the value parsed.
   708    719   **
   709    720   ** Return negative for a syntax error.  Special cases:  return -2 if the
   710    721   ** first non-whitespace character is '}' and return -3 if the first
................................................................................
   772    783       u8 jnFlags = 0;
   773    784       j = i+1;
   774    785       for(;;){
   775    786         c = pParse->zJson[j];
   776    787         if( c==0 ) return -1;
   777    788         if( c=='\\' ){
   778    789           c = pParse->zJson[++j];
   779         -        if( c==0 ) return -1;
   780         -        jnFlags = JNODE_ESCAPE;
          790  +        if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f'
          791  +           || c=='n' || c=='r' || c=='t'
          792  +           || (c=='u' && jsonIs4Hex(pParse->zJson+j+1)) ){
          793  +          jnFlags = JNODE_ESCAPE;
          794  +        }else{
          795  +          return -1;
          796  +        }
   781    797         }else if( c=='"' ){
   782    798           break;
   783    799         }
   784    800         j++;
   785    801       }
   786    802       jsonParseAddNode(pParse, JSON_STRING, j+1-i, &pParse->zJson[i]);
   787    803       if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags;
................................................................................
  1144   1160   ){
  1145   1161     char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments",
  1146   1162                                  zFuncName);
  1147   1163     sqlite3_result_error(pCtx, zMsg, -1);
  1148   1164     sqlite3_free(zMsg);     
  1149   1165   }
  1150   1166   
         1167  +/*
         1168  +** Mark all NULL entries in the Object passed in as JNODE_REMOVE.
         1169  +*/
         1170  +static void jsonRemoveAllNulls(JsonNode *pNode){
         1171  +  int i, n;
         1172  +  assert( pNode->eType==JSON_OBJECT );
         1173  +  n = pNode->n;
         1174  +  for(i=2; i<=n; i += jsonNodeSize(&pNode[i])+1){
         1175  +    switch( pNode[i].eType ){
         1176  +      case JSON_NULL:
         1177  +        pNode[i].jnFlags |= JNODE_REMOVE;
         1178  +        break;
         1179  +      case JSON_OBJECT:
         1180  +        jsonRemoveAllNulls(&pNode[i]);
         1181  +        break;
         1182  +    }
         1183  +  }
         1184  +}
         1185  +
  1151   1186   
  1152   1187   /****************************************************************************
  1153   1188   ** SQL functions used for testing and debugging
  1154   1189   ****************************************************************************/
  1155   1190   
  1156   1191   #ifdef SQLITE_DEBUG
  1157   1192   /*
................................................................................
  1206   1241     sqlite3_result_int(ctx, sqlite3_value_subtype(argv[0])==JSON_SUBTYPE);
  1207   1242   }
  1208   1243   #endif /* SQLITE_DEBUG */
  1209   1244   
  1210   1245   /****************************************************************************
  1211   1246   ** Scalar SQL function implementations
  1212   1247   ****************************************************************************/
         1248  +
         1249  +/*
         1250  +** Implementation of the json_QUOTE(VALUE) function.  Return a JSON value
         1251  +** corresponding to the SQL value input.  Mostly this means putting 
         1252  +** double-quotes around strings and returning the unquoted string "null"
         1253  +** when given a NULL input.
         1254  +*/
         1255  +static void jsonQuoteFunc(
         1256  +  sqlite3_context *ctx,
         1257  +  int argc,
         1258  +  sqlite3_value **argv
         1259  +){
         1260  +  JsonString jx;
         1261  +  UNUSED_PARAM(argc);
         1262  +
         1263  +  jsonInit(&jx, ctx);
         1264  +  jsonAppendValue(&jx, argv[0]);
         1265  +  jsonResult(&jx);
         1266  +  sqlite3_result_subtype(ctx, JSON_SUBTYPE);
         1267  +}
  1213   1268   
  1214   1269   /*
  1215   1270   ** Implementation of the json_array(VALUE,...) function.  Return a JSON
  1216   1271   ** array that contains all values given in arguments.  Or if any argument
  1217   1272   ** is a BLOB, throw an error.
  1218   1273   */
  1219   1274   static void jsonArrayFunc(
................................................................................
  1315   1370       jsonAppendChar(&jx, ']');
  1316   1371       jsonResult(&jx);
  1317   1372       sqlite3_result_subtype(ctx, JSON_SUBTYPE);
  1318   1373     }
  1319   1374     jsonReset(&jx);
  1320   1375     jsonParseReset(&x);
  1321   1376   }
         1377  +
         1378  +/* This is the RFC 7396 MergePatch algorithm.
         1379  +*/
         1380  +static JsonNode *jsonMergePatch(
         1381  +  JsonParse *pParse,   /* The JSON parser that contains the TARGET */
         1382  +  int iTarget,         /* Node of the TARGET in pParse */
         1383  +  JsonNode *pPatch     /* The PATCH */
         1384  +){
         1385  +  u32 i, j;
         1386  +  u32 iRoot;
         1387  +  JsonNode *pTarget;
         1388  +  if( pPatch->eType!=JSON_OBJECT ){
         1389  +    return pPatch;
         1390  +  }
         1391  +  assert( iTarget>=0 && iTarget<pParse->nNode );
         1392  +  pTarget = &pParse->aNode[iTarget];
         1393  +  assert( (pPatch->jnFlags & JNODE_APPEND)==0 );
         1394  +  if( pTarget->eType!=JSON_OBJECT ){
         1395  +    jsonRemoveAllNulls(pPatch);
         1396  +    return pPatch;
         1397  +  }
         1398  +  iRoot = iTarget;
         1399  +  for(i=1; i<pPatch->n; i += jsonNodeSize(&pPatch[i+1])+1){
         1400  +    u32 nKey;
         1401  +    const char *zKey;
         1402  +    assert( pPatch[i].eType==JSON_STRING );
         1403  +    assert( pPatch[i].jnFlags & JNODE_LABEL );
         1404  +    nKey = pPatch[i].n;
         1405  +    zKey = pPatch[i].u.zJContent;
         1406  +    assert( (pPatch[i].jnFlags & JNODE_RAW)==0 );
         1407  +    for(j=1; j<pTarget->n; j += jsonNodeSize(&pTarget[j+1])+1 ){
         1408  +      assert( pTarget[j].eType==JSON_STRING );
         1409  +      assert( pTarget[j].jnFlags & JNODE_LABEL );
         1410  +      assert( (pPatch[i].jnFlags & JNODE_RAW)==0 );
         1411  +      if( pTarget[j].n==nKey && strncmp(pTarget[j].u.zJContent,zKey,nKey)==0 ){
         1412  +        if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break;
         1413  +        if( pPatch[i+1].eType==JSON_NULL ){
         1414  +          pTarget[j+1].jnFlags |= JNODE_REMOVE;
         1415  +        }else{
         1416  +          JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]);
         1417  +          if( pNew==0 ) return 0;
         1418  +          pTarget = &pParse->aNode[iTarget];
         1419  +          if( pNew!=&pTarget[j+1] ){
         1420  +            pTarget[j+1].u.pPatch = pNew;
         1421  +            pTarget[j+1].jnFlags |= JNODE_PATCH;
         1422  +          }
         1423  +        }
         1424  +        break;
         1425  +      }
         1426  +    }
         1427  +    if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){
         1428  +      int iStart, iPatch;
         1429  +      iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0);
         1430  +      jsonParseAddNode(pParse, JSON_STRING, nKey, zKey);
         1431  +      iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0);
         1432  +      if( pParse->oom ) return 0;
         1433  +      jsonRemoveAllNulls(pPatch);
         1434  +      pTarget = &pParse->aNode[iTarget];
         1435  +      pParse->aNode[iRoot].jnFlags |= JNODE_APPEND;
         1436  +      pParse->aNode[iRoot].u.iAppend = iStart - iRoot;
         1437  +      iRoot = iStart;
         1438  +      pParse->aNode[iPatch].jnFlags |= JNODE_PATCH;
         1439  +      pParse->aNode[iPatch].u.pPatch = &pPatch[i+1];
         1440  +    }
         1441  +  }
         1442  +  return pTarget;
         1443  +}
         1444  +
         1445  +/*
         1446  +** Implementation of the json_mergepatch(JSON1,JSON2) function.  Return a JSON
         1447  +** object that is the result of running the RFC 7396 MergePatch() algorithm
         1448  +** on the two arguments.
         1449  +*/
         1450  +static void jsonPatchFunc(
         1451  +  sqlite3_context *ctx,
         1452  +  int argc,
         1453  +  sqlite3_value **argv
         1454  +){
         1455  +  JsonParse x;     /* The JSON that is being patched */
         1456  +  JsonParse y;     /* The patch */
         1457  +  JsonNode *pResult;   /* The result of the merge */
         1458  +
         1459  +  UNUSED_PARAM(argc);
         1460  +  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
         1461  +  if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){
         1462  +    jsonParseReset(&x);
         1463  +    return;
         1464  +  }
         1465  +  pResult = jsonMergePatch(&x, 0, y.aNode);
         1466  +  assert( pResult!=0 || x.oom );
         1467  +  if( pResult ){
         1468  +    jsonReturnJson(pResult, ctx, 0);
         1469  +  }else{
         1470  +    sqlite3_result_error_nomem(ctx);
         1471  +  }
         1472  +  jsonParseReset(&x);
         1473  +  jsonParseReset(&y);
         1474  +}
         1475  +
  1322   1476   
  1323   1477   /*
  1324   1478   ** Implementation of the json_object(NAME,VALUE,...) function.  Return a JSON
  1325   1479   ** object that contains all name/value given in arguments.  Or if any name
  1326   1480   ** is not a string or if any value is a BLOB, throw an error.
  1327   1481   */
  1328   1482   static void jsonObjectFunc(
................................................................................
  1419   1573     assert( x.nNode );
  1420   1574     for(i=1; i<(u32)argc; i+=2){
  1421   1575       zPath = (const char*)sqlite3_value_text(argv[i]);
  1422   1576       pNode = jsonLookup(&x, zPath, 0, ctx);
  1423   1577       if( x.nErr ) goto replace_err;
  1424   1578       if( pNode ){
  1425   1579         pNode->jnFlags |= (u8)JNODE_REPLACE;
  1426         -      pNode->iVal = (u8)(i+1);
         1580  +      pNode->u.iReplace = i + 1;
  1427   1581       }
  1428   1582     }
  1429   1583     if( x.aNode[0].jnFlags & JNODE_REPLACE ){
  1430         -    sqlite3_result_value(ctx, argv[x.aNode[0].iVal]);
         1584  +    sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]);
  1431   1585     }else{
  1432   1586       jsonReturnJson(x.aNode, ctx, argv);
  1433   1587     }
  1434   1588   replace_err:
  1435   1589     jsonParseReset(&x);
  1436   1590   }
  1437   1591   
................................................................................
  1473   1627       if( x.oom ){
  1474   1628         sqlite3_result_error_nomem(ctx);
  1475   1629         goto jsonSetDone;
  1476   1630       }else if( x.nErr ){
  1477   1631         goto jsonSetDone;
  1478   1632       }else if( pNode && (bApnd || bIsSet) ){
  1479   1633         pNode->jnFlags |= (u8)JNODE_REPLACE;
  1480         -      pNode->iVal = (u8)(i+1);
         1634  +      pNode->u.iReplace = i + 1;
  1481   1635       }
  1482   1636     }
  1483   1637     if( x.aNode[0].jnFlags & JNODE_REPLACE ){
  1484         -    sqlite3_result_value(ctx, argv[x.aNode[0].iVal]);
         1638  +    sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]);
  1485   1639     }else{
  1486   1640       jsonReturnJson(x.aNode, ctx, argv);
  1487   1641     }
  1488   1642   jsonSetDone:
  1489   1643     jsonParseReset(&x);
  1490   1644   }
  1491   1645   
................................................................................
  1621   1775   }
  1622   1776   static void jsonObjectFinal(sqlite3_context *ctx){
  1623   1777     JsonString *pStr;
  1624   1778     pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
  1625   1779     if( pStr ){
  1626   1780       jsonAppendChar(pStr, '}');
  1627   1781       if( pStr->bErr ){
  1628         -      if( pStr->bErr==0 ) sqlite3_result_error_nomem(ctx);
         1782  +      if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx);
  1629   1783         assert( pStr->bStatic );
  1630   1784       }else{
  1631   1785         sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed,
  1632   1786                             pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free);
  1633   1787         pStr->bStatic = 1;
  1634   1788       }
  1635   1789     }else{
................................................................................
  1899   2053           jsonEachComputePath(p, &x, p->sParse.aUp[p->i]);
  1900   2054           jsonResult(&x);
  1901   2055           break;
  1902   2056         }
  1903   2057         /* For json_each() path and root are the same so fall through
  1904   2058         ** into the root case */
  1905   2059       }
  1906         -    case JEACH_ROOT: {
         2060  +    default: {
  1907   2061         const char *zRoot = p->zRoot;
  1908         -       if( zRoot==0 ) zRoot = "$";
         2062  +      if( zRoot==0 ) zRoot = "$";
  1909   2063         sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC);
  1910   2064         break;
  1911   2065       }
  1912   2066       case JEACH_JSON: {
  1913   2067         assert( i==JEACH_JSON );
  1914   2068         sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC);
  1915   2069         break;
................................................................................
  2120   2274       { "json",                 1, 0,   jsonRemoveFunc        },
  2121   2275       { "json_array",          -1, 0,   jsonArrayFunc         },
  2122   2276       { "json_array_length",    1, 0,   jsonArrayLengthFunc   },
  2123   2277       { "json_array_length",    2, 0,   jsonArrayLengthFunc   },
  2124   2278       { "json_extract",        -1, 0,   jsonExtractFunc       },
  2125   2279       { "json_insert",         -1, 0,   jsonSetFunc           },
  2126   2280       { "json_object",         -1, 0,   jsonObjectFunc        },
         2281  +    { "json_patch",           2, 0,   jsonPatchFunc         },
         2282  +    { "json_quote",           1, 0,   jsonQuoteFunc         },
  2127   2283       { "json_remove",         -1, 0,   jsonRemoveFunc        },
  2128   2284       { "json_replace",        -1, 0,   jsonReplaceFunc       },
  2129   2285       { "json_set",            -1, 1,   jsonSetFunc           },
  2130   2286       { "json_type",            1, 0,   jsonTypeFunc          },
  2131   2287       { "json_type",            2, 0,   jsonTypeFunc          },
  2132   2288       { "json_valid",           1, 0,   jsonValidFunc         },
  2133   2289   

Added ext/misc/memvfs.c.

            1  +/*
            2  +** 2016-09-07
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** This is an in-memory read-only VFS implementation.  The application
           14  +** supplies a block of memory which is the database file, and this VFS
           15  +** uses that block of memory.
           16  +**
           17  +** Because there is no place to store journals and no good way to lock
           18  +** the "file", this VFS is read-only.
           19  +**
           20  +** USAGE:
           21  +**
           22  +**    sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336", &db,
           23  +**                    SQLITE_OPEN_READONLY | SQLITE_OPEN_URI,
           24  +**                    "memvfs");
           25  +**
           26  +** The ptr= and sz= query parameters are required or the open will fail.
           27  +** The ptr= parameter gives the memory address of the buffer holding the
           28  +** read-only database and sz= gives the size of the database.  The parameter
           29  +** values may be in hexadecimal or decimal.  The filename is ignored.
           30  +*/
           31  +#include <sqlite3ext.h>
           32  +SQLITE_EXTENSION_INIT1
           33  +#include <string.h>
           34  +#include <assert.h>
           35  +
           36  +
           37  +/*
           38  +** Forward declaration of objects used by this utility
           39  +*/
           40  +typedef struct sqlite3_vfs MemVfs;
           41  +typedef struct MemFile MemFile;
           42  +
           43  +/* Access to a lower-level VFS that (might) implement dynamic loading,
           44  +** access to randomness, etc.
           45  +*/
           46  +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData))
           47  +
           48  +/* An open file */
           49  +struct MemFile {
           50  +  sqlite3_file base;              /* IO methods */
           51  +  sqlite3_int64 sz;               /* Size of the file */
           52  +  unsigned char *aData;           /* content of the file */
           53  +};
           54  +
           55  +/*
           56  +** Methods for MemFile
           57  +*/
           58  +static int memClose(sqlite3_file*);
           59  +static int memRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
           60  +static int memWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
           61  +static int memTruncate(sqlite3_file*, sqlite3_int64 size);
           62  +static int memSync(sqlite3_file*, int flags);
           63  +static int memFileSize(sqlite3_file*, sqlite3_int64 *pSize);
           64  +static int memLock(sqlite3_file*, int);
           65  +static int memUnlock(sqlite3_file*, int);
           66  +static int memCheckReservedLock(sqlite3_file*, int *pResOut);
           67  +static int memFileControl(sqlite3_file*, int op, void *pArg);
           68  +static int memSectorSize(sqlite3_file*);
           69  +static int memDeviceCharacteristics(sqlite3_file*);
           70  +static int memShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**);
           71  +static int memShmLock(sqlite3_file*, int offset, int n, int flags);
           72  +static void memShmBarrier(sqlite3_file*);
           73  +static int memShmUnmap(sqlite3_file*, int deleteFlag);
           74  +static int memFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp);
           75  +static int memUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
           76  +
           77  +/*
           78  +** Methods for MemVfs
           79  +*/
           80  +static int memOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
           81  +static int memDelete(sqlite3_vfs*, const char *zName, int syncDir);
           82  +static int memAccess(sqlite3_vfs*, const char *zName, int flags, int *);
           83  +static int memFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut);
           84  +static void *memDlOpen(sqlite3_vfs*, const char *zFilename);
           85  +static void memDlError(sqlite3_vfs*, int nByte, char *zErrMsg);
           86  +static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void);
           87  +static void memDlClose(sqlite3_vfs*, void*);
           88  +static int memRandomness(sqlite3_vfs*, int nByte, char *zOut);
           89  +static int memSleep(sqlite3_vfs*, int microseconds);
           90  +static int memCurrentTime(sqlite3_vfs*, double*);
           91  +static int memGetLastError(sqlite3_vfs*, int, char *);
           92  +static int memCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
           93  +
           94  +static sqlite3_vfs mem_vfs = {
           95  +  2,                           /* iVersion */
           96  +  0,                           /* szOsFile (set when registered) */
           97  +  1024,                        /* mxPathname */
           98  +  0,                           /* pNext */
           99  +  "memvfs",                    /* zName */
          100  +  0,                           /* pAppData (set when registered) */ 
          101  +  memOpen,                     /* xOpen */
          102  +  memDelete,                   /* xDelete */
          103  +  memAccess,                   /* xAccess */
          104  +  memFullPathname,             /* xFullPathname */
          105  +  memDlOpen,                   /* xDlOpen */
          106  +  memDlError,                  /* xDlError */
          107  +  memDlSym,                    /* xDlSym */
          108  +  memDlClose,                  /* xDlClose */
          109  +  memRandomness,               /* xRandomness */
          110  +  memSleep,                    /* xSleep */
          111  +  memCurrentTime,              /* xCurrentTime */
          112  +  memGetLastError,             /* xGetLastError */
          113  +  memCurrentTimeInt64          /* xCurrentTimeInt64 */
          114  +};
          115  +
          116  +static const sqlite3_io_methods mem_io_methods = {
          117  +  3,                              /* iVersion */
          118  +  memClose,                      /* xClose */
          119  +  memRead,                       /* xRead */
          120  +  memWrite,                      /* xWrite */
          121  +  memTruncate,                   /* xTruncate */
          122  +  memSync,                       /* xSync */
          123  +  memFileSize,                   /* xFileSize */
          124  +  memLock,                       /* xLock */
          125  +  memUnlock,                     /* xUnlock */
          126  +  memCheckReservedLock,          /* xCheckReservedLock */
          127  +  memFileControl,                /* xFileControl */
          128  +  memSectorSize,                 /* xSectorSize */
          129  +  memDeviceCharacteristics,      /* xDeviceCharacteristics */
          130  +  memShmMap,                     /* xShmMap */
          131  +  memShmLock,                    /* xShmLock */
          132  +  memShmBarrier,                 /* xShmBarrier */
          133  +  memShmUnmap,                   /* xShmUnmap */
          134  +  memFetch,                      /* xFetch */
          135  +  memUnfetch                     /* xUnfetch */
          136  +};
          137  +
          138  +
          139  +
          140  +/*
          141  +** Close an mem-file.
          142  +**
          143  +** The pData pointer is owned by the application, so there is nothing
          144  +** to free.
          145  +*/
          146  +static int memClose(sqlite3_file *pFile){
          147  +  return SQLITE_OK;
          148  +}
          149  +
          150  +/*
          151  +** Read data from an mem-file.
          152  +*/
          153  +static int memRead(
          154  +  sqlite3_file *pFile, 
          155  +  void *zBuf, 
          156  +  int iAmt, 
          157  +  sqlite_int64 iOfst
          158  +){
          159  +  MemFile *p = (MemFile *)pFile;
          160  +  memcpy(zBuf, p->aData+iOfst, iAmt);
          161  +  return SQLITE_OK;
          162  +}
          163  +
          164  +/*
          165  +** Write data to an mem-file.
          166  +*/
          167  +static int memWrite(
          168  +  sqlite3_file *pFile,
          169  +  const void *z,
          170  +  int iAmt,
          171  +  sqlite_int64 iOfst
          172  +){
          173  +  return SQLITE_READONLY;
          174  +}
          175  +
          176  +/*
          177  +** Truncate an mem-file.
          178  +*/
          179  +static int memTruncate(sqlite3_file *pFile, sqlite_int64 size){
          180  +  return SQLITE_READONLY;
          181  +}
          182  +
          183  +/*
          184  +** Sync an mem-file.
          185  +*/
          186  +static int memSync(sqlite3_file *pFile, int flags){
          187  +  return SQLITE_READONLY;
          188  +}
          189  +
          190  +/*
          191  +** Return the current file-size of an mem-file.
          192  +*/
          193  +static int memFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
          194  +  MemFile *p = (MemFile *)pFile;
          195  +  *pSize = p->sz;
          196  +  return SQLITE_OK;
          197  +}
          198  +
          199  +/*
          200  +** Lock an mem-file.
          201  +*/
          202  +static int memLock(sqlite3_file *pFile, int eLock){
          203  +  return SQLITE_READONLY;
          204  +}
          205  +
          206  +/*
          207  +** Unlock an mem-file.
          208  +*/
          209  +static int memUnlock(sqlite3_file *pFile, int eLock){
          210  +  return SQLITE_OK;
          211  +}
          212  +
          213  +/*
          214  +** Check if another file-handle holds a RESERVED lock on an mem-file.
          215  +*/
          216  +static int memCheckReservedLock(sqlite3_file *pFile, int *pResOut){
          217  +  *pResOut = 0;
          218  +  return SQLITE_OK;
          219  +}
          220  +
          221  +/*
          222  +** File control method. For custom operations on an mem-file.
          223  +*/
          224  +static int memFileControl(sqlite3_file *pFile, int op, void *pArg){
          225  +  MemFile *p = (MemFile *)pFile;
          226  +  int rc = SQLITE_NOTFOUND;
          227  +  if( op==SQLITE_FCNTL_VFSNAME ){
          228  +    *(char**)pArg = sqlite3_mprintf("mem(%p,%lld)", p->aData, p->sz);
          229  +    rc = SQLITE_OK;
          230  +  }
          231  +  return rc;
          232  +}
          233  +
          234  +/*
          235  +** Return the sector-size in bytes for an mem-file.
          236  +*/
          237  +static int memSectorSize(sqlite3_file *pFile){
          238  +  return 1024;
          239  +}
          240  +
          241  +/*
          242  +** Return the device characteristic flags supported by an mem-file.
          243  +*/
          244  +static int memDeviceCharacteristics(sqlite3_file *pFile){
          245  +  return SQLITE_IOCAP_IMMUTABLE;
          246  +}
          247  +
          248  +/* Create a shared memory file mapping */
          249  +static int memShmMap(
          250  +  sqlite3_file *pFile,
          251  +  int iPg,
          252  +  int pgsz,
          253  +  int bExtend,
          254  +  void volatile **pp
          255  +){
          256  +  return SQLITE_READONLY;
          257  +}
          258  +
          259  +/* Perform locking on a shared-memory segment */
          260  +static int memShmLock(sqlite3_file *pFile, int offset, int n, int flags){
          261  +  return SQLITE_READONLY;
          262  +}
          263  +
          264  +/* Memory barrier operation on shared memory */
          265  +static void memShmBarrier(sqlite3_file *pFile){
          266  +  return;
          267  +}
          268  +
          269  +/* Unmap a shared memory segment */
          270  +static int memShmUnmap(sqlite3_file *pFile, int deleteFlag){
          271  +  return SQLITE_OK;
          272  +}
          273  +
          274  +/* Fetch a page of a memory-mapped file */
          275  +static int memFetch(
          276  +  sqlite3_file *pFile,
          277  +  sqlite3_int64 iOfst,
          278  +  int iAmt,
          279  +  void **pp
          280  +){
          281  +  MemFile *p = (MemFile *)pFile;
          282  +  *pp = (void*)(p->aData + iOfst);
          283  +  return SQLITE_OK;
          284  +}
          285  +
          286  +/* Release a memory-mapped page */
          287  +static int memUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
          288  +  return SQLITE_OK;
          289  +}
          290  +
          291  +/*
          292  +** Open an mem file handle.
          293  +*/
          294  +static int memOpen(
          295  +  sqlite3_vfs *pVfs,
          296  +  const char *zName,
          297  +  sqlite3_file *pFile,
          298  +  int flags,
          299  +  int *pOutFlags
          300  +){
          301  +  MemFile *p = (MemFile*)pFile;
          302  +  memset(p, 0, sizeof(*p));
          303  +  if( (flags & SQLITE_OPEN_MAIN_DB)==0 ) return SQLITE_CANTOPEN;
          304  +  p->aData = (unsigned char*)sqlite3_uri_int64(zName,"ptr",0);
          305  +  if( p->aData==0 ) return SQLITE_CANTOPEN;
          306  +  p->sz = sqlite3_uri_int64(zName,"sz",0);
          307  +  if( p->sz<0 ) return SQLITE_CANTOPEN;
          308  +  pFile->pMethods = &mem_io_methods;
          309  +  return SQLITE_OK;
          310  +}
          311  +
          312  +/*
          313  +** Delete the file located at zPath. If the dirSync argument is true,
          314  +** ensure the file-system modifications are synced to disk before
          315  +** returning.
          316  +*/
          317  +static int memDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
          318  +  return SQLITE_READONLY;
          319  +}
          320  +
          321  +/*
          322  +** Test for access permissions. Return true if the requested permission
          323  +** is available, or false otherwise.
          324  +*/
          325  +static int memAccess(
          326  +  sqlite3_vfs *pVfs, 
          327  +  const char *zPath, 
          328  +  int flags, 
          329  +  int *pResOut
          330  +){
          331  +  /* The spec says there are three possible values for flags.  But only
          332  +  ** two of them are actually used */
          333  +  assert( flags==SQLITE_ACCESS_EXISTS || flags==SQLITE_ACCESS_READWRITE );
          334  +  if( flags==SQLITE_ACCESS_READWRITE ){
          335  +    *pResOut = 0;
          336  +  }else{
          337  +    *pResOut = 1;
          338  +  }
          339  +  return SQLITE_OK;
          340  +}
          341  +
          342  +/*
          343  +** Populate buffer zOut with the full canonical pathname corresponding
          344  +** to the pathname in zPath. zOut is guaranteed to point to a buffer
          345  +** of at least (INST_MAX_PATHNAME+1) bytes.
          346  +*/
          347  +static int memFullPathname(
          348  +  sqlite3_vfs *pVfs, 
          349  +  const char *zPath, 
          350  +  int nOut, 
          351  +  char *zOut
          352  +){
          353  +  sqlite3_snprintf(nOut, zOut, "%s", zPath);
          354  +  return SQLITE_OK;
          355  +}
          356  +
          357  +/*
          358  +** Open the dynamic library located at zPath and return a handle.
          359  +*/
          360  +static void *memDlOpen(sqlite3_vfs *pVfs, const char *zPath){
          361  +  return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath);
          362  +}
          363  +
          364  +/*
          365  +** Populate the buffer zErrMsg (size nByte bytes) with a human readable
          366  +** utf-8 string describing the most recent error encountered associated 
          367  +** with dynamic libraries.
          368  +*/
          369  +static void memDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
          370  +  ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg);
          371  +}
          372  +
          373  +/*
          374  +** Return a pointer to the symbol zSymbol in the dynamic library pHandle.
          375  +*/
          376  +static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){
          377  +  return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym);
          378  +}
          379  +
          380  +/*
          381  +** Close the dynamic library handle pHandle.
          382  +*/
          383  +static void memDlClose(sqlite3_vfs *pVfs, void *pHandle){
          384  +  ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle);
          385  +}
          386  +
          387  +/*
          388  +** Populate the buffer pointed to by zBufOut with nByte bytes of 
          389  +** random data.
          390  +*/
          391  +static int memRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
          392  +  return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut);
          393  +}
          394  +
          395  +/*
          396  +** Sleep for nMicro microseconds. Return the number of microseconds 
          397  +** actually slept.
          398  +*/
          399  +static int memSleep(sqlite3_vfs *pVfs, int nMicro){
          400  +  return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro);
          401  +}
          402  +
          403  +/*
          404  +** Return the current time as a Julian Day number in *pTimeOut.
          405  +*/
          406  +static int memCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
          407  +  return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut);
          408  +}
          409  +
          410  +static int memGetLastError(sqlite3_vfs *pVfs, int a, char *b){
          411  +  return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b);
          412  +}
          413  +static int memCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
          414  +  return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p);
          415  +}
          416  +
          417  +#ifdef MEMVFS_TEST
          418  +/*
          419  +**       memload(FILENAME)
          420  +**
          421  +** This an SQL function used to help in testing the memvfs VFS.  The
          422  +** function reads the content of a file into memory and then returns
          423  +** a string that gives the locate and size of the in-memory buffer.
          424  +*/
          425  +#include <stdio.h>
          426  +static void memvfsMemloadFunc(
          427  +  sqlite3_context *context,
          428  +  int argc,
          429  +  sqlite3_value **argv
          430  +){
          431  +  unsigned char *p;
          432  +  sqlite3_int64 sz;
          433  +  FILE *in;
          434  +  const char *zFilename = (const char*)sqlite3_value_text(argv[0]);
          435  +  char zReturn[100];
          436  +
          437  +  if( zFilename==0 ) return;
          438  +  in = fopen(zFilename, "rb");
          439  +  if( in==0 ) return;
          440  +  fseek(in, 0, SEEK_END);
          441  +  sz = ftell(in);
          442  +  rewind(in);
          443  +  p = sqlite3_malloc( sz );
          444  +  if( p==0 ){
          445  +    fclose(in);
          446  +    sqlite3_result_error_nomem(context);
          447  +    return;
          448  +  }
          449  +  fread(p, sz, 1, in);
          450  +  fclose(in);
          451  +  sqlite3_snprintf(sizeof(zReturn),zReturn,"ptr=%lld&sz=%lld",
          452  +                   (sqlite3_int64)p, sz);
          453  +  sqlite3_result_text(context, zReturn, -1, SQLITE_TRANSIENT);
          454  +}
          455  +/* Called for each new database connection */
          456  +static int memvfsRegister(
          457  +  sqlite3 *db,
          458  +  const char **pzErrMsg,
          459  +  const struct sqlite3_api_routines *pThunk
          460  +){
          461  +  return sqlite3_create_function(db, "memload", 1, SQLITE_UTF8, 0,
          462  +                                 memvfsMemloadFunc, 0, 0);
          463  +}
          464  +#endif /* MEMVFS_TEST */
          465  +
          466  +  
          467  +#ifdef _WIN32
          468  +__declspec(dllexport)
          469  +#endif
          470  +/* 
          471  +** This routine is called when the extension is loaded.
          472  +** Register the new VFS.
          473  +*/
          474  +int sqlite3_memvfs_init(
          475  +  sqlite3 *db, 
          476  +  char **pzErrMsg, 
          477  +  const sqlite3_api_routines *pApi
          478  +){
          479  +  int rc = SQLITE_OK;
          480  +  SQLITE_EXTENSION_INIT2(pApi);
          481  +  mem_vfs.pAppData = sqlite3_vfs_find(0);
          482  +  mem_vfs.szOsFile = sizeof(MemFile);
          483  +  rc = sqlite3_vfs_register(&mem_vfs, 1);
          484  +#ifdef MEMVFS_TEST
          485  +  if( rc==SQLITE_OK ){
          486  +    rc = sqlite3_auto_extension((void(*)(void))memvfsRegister);
          487  +  }
          488  +#endif
          489  +  if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
          490  +  return rc;
          491  +}

Changes to ext/misc/percentile.c.

   163    163     }
   164    164     p->a[p->nUsed++] = y;
   165    165   }
   166    166   
   167    167   /*
   168    168   ** Compare to doubles for sorting using qsort()
   169    169   */
   170         -static int doubleCmp(const void *pA, const void *pB){
          170  +static int SQLITE_CDECL doubleCmp(const void *pA, const void *pB){
   171    171     double a = *(double*)pA;
   172    172     double b = *(double*)pB;
   173    173     if( a==b ) return 0;
   174    174     if( a<b ) return -1;
   175    175     return +1;
   176    176   }
   177    177   

Changes to ext/misc/regexp.c.

   132    132     unsigned nAlloc;            /* Slots allocated for aOp[] and aArg[] */
   133    133   };
   134    134   
   135    135   /* Add a state to the given state set if it is not already there */
   136    136   static void re_add_state(ReStateSet *pSet, int newState){
   137    137     unsigned i;
   138    138     for(i=0; i<pSet->nState; i++) if( pSet->aState[i]==newState ) return;
   139         -  pSet->aState[pSet->nState++] = newState;
          139  +  pSet->aState[pSet->nState++] = (ReStateNumber)newState;
   140    140   }
   141    141   
   142    142   /* Extract the next unicode character from *pzIn and return it.  Advance
   143    143   ** *pzIn to the first byte past the end of the character returned.  To
   144    144   ** be clear:  this routine converts utf8 to unicode.  This routine is 
   145    145   ** optimized for the common case where the next character is a single byte.
   146    146   */
................................................................................
   354    354     int i;
   355    355     if( p->nAlloc<=p->nState && re_resize(p, p->nAlloc*2) ) return 0;
   356    356     for(i=p->nState; i>iBefore; i--){
   357    357       p->aOp[i] = p->aOp[i-1];
   358    358       p->aArg[i] = p->aArg[i-1];
   359    359     }
   360    360     p->nState++;
   361         -  p->aOp[iBefore] = op;
          361  +  p->aOp[iBefore] = (char)op;
   362    362     p->aArg[iBefore] = arg;
   363    363     return iBefore;
   364    364   }
   365    365   
   366    366   /* Append a new opcode and argument to the end of the RE under construction.
   367    367   */
   368    368   static int re_append(ReCompiled *p, int op, int arg){
................................................................................
   673    673     ** regex engine over the string.  Do not worry able trying to match
   674    674     ** unicode characters beyond plane 0 - those are very rare and this is
   675    675     ** just an optimization. */
   676    676     if( pRe->aOp[0]==RE_OP_ANYSTAR ){
   677    677       for(j=0, i=1; j<sizeof(pRe->zInit)-2 && pRe->aOp[i]==RE_OP_MATCH; i++){
   678    678         unsigned x = pRe->aArg[i];
   679    679         if( x<=127 ){
   680         -        pRe->zInit[j++] = x;
          680  +        pRe->zInit[j++] = (unsigned char)x;
   681    681         }else if( x<=0xfff ){
   682         -        pRe->zInit[j++] = 0xc0 | (x>>6);
          682  +        pRe->zInit[j++] = (unsigned char)(0xc0 | (x>>6));
   683    683           pRe->zInit[j++] = 0x80 | (x&0x3f);
   684    684         }else if( x<=0xffff ){
   685         -        pRe->zInit[j++] = 0xd0 | (x>>12);
          685  +        pRe->zInit[j++] = (unsigned char)(0xd0 | (x>>12));
   686    686           pRe->zInit[j++] = 0x80 | ((x>>6)&0x3f);
   687    687           pRe->zInit[j++] = 0x80 | (x&0x3f);
   688    688         }else{
   689    689           break;
   690    690         }
   691    691       }
   692    692       if( j>0 && pRe->zInit[j-1]==0 ) j--;

Added ext/misc/remember.c.

            1  +/*
            2  +** 2016-08-09
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +**
           13  +** This file demonstrates how to create an SQL function that is a pass-through
           14  +** for integer values (it returns a copy of its argument) but also saves the
           15  +** value that is passed through into a C-language variable.  The address of
           16  +** the C-language variable is supplied as the second argument.
           17  +**
           18  +** This allows, for example, a counter to incremented and the original
           19  +** value retrieved, atomically, using a single statement:
           20  +**
           21  +**    UPDATE counterTab SET cnt=remember(cnt,$PTR)+1 WHERE id=$ID
           22  +**
           23  +** Prepare the above statement once.  Then to use it, bind the address
           24  +** of the output variable to $PTR and the id of the counter to $ID and
           25  +** run the prepared statement.
           26  +**
           27  +** One can imagine doing similar things with floating-point values and
           28  +** strings, but this demonstration extension will stick to using just
           29  +** integers.
           30  +*/
           31  +#include "sqlite3ext.h"
           32  +SQLITE_EXTENSION_INIT1
           33  +#include <assert.h>
           34  +
           35  +/*
           36  +**      remember(V,PTR)
           37  +**
           38  +** Return the integer value V.  Also save the value of V in a
           39  +** C-language variable whose address is PTR.
           40  +*/
           41  +static void rememberFunc(
           42  +  sqlite3_context *pCtx,
           43  +  int argc,
           44  +  sqlite3_value **argv
           45  +){
           46  +  sqlite3_int64 v;
           47  +  sqlite3_int64 ptr;
           48  +  assert( argc==2 );
           49  +  v = sqlite3_value_int64(argv[0]);
           50  +  ptr = sqlite3_value_int64(argv[1]);
           51  +  *((sqlite3_int64*)ptr) = v;
           52  +  sqlite3_result_int64(pCtx, v);
           53  +}
           54  +
           55  +#ifdef _WIN32
           56  +__declspec(dllexport)
           57  +#endif
           58  +int sqlite3_remember_init(
           59  +  sqlite3 *db, 
           60  +  char **pzErrMsg, 
           61  +  const sqlite3_api_routines *pApi
           62  +){
           63  +  int rc = SQLITE_OK;
           64  +  SQLITE_EXTENSION_INIT2(pApi);
           65  +  rc = sqlite3_create_function(db, "remember", 2, SQLITE_UTF8, 0,
           66  +                               rememberFunc, 0, 0);
           67  +  return rc;
           68  +}

Added ext/misc/scrub.c.

            1  +/*
            2  +** 2016-05-05
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** This file implements a utility function (and a utility program) that
           14  +** makes a copy of an SQLite database while simultaneously zeroing out all
           15  +** deleted content.
           16  +**
           17  +** Normally (when PRAGMA secure_delete=OFF, which is the default) when SQLite
           18  +** deletes content, it does not overwrite the deleted content but rather marks
           19  +** the region of the file that held that content as being reusable.  This can
           20  +** cause deleted content to recoverable from the database file.  This stale
           21  +** content is removed by the VACUUM command, but VACUUM can be expensive for
           22  +** large databases.  When in PRAGMA secure_delete=ON mode, the deleted content
           23  +** is zeroed, but secure_delete=ON has overhead as well.
           24  +**
           25  +** This utility attempts to make a copy of a complete SQLite database where
           26  +** all of the deleted content is zeroed out in the copy, and it attempts to
           27  +** do so while being faster than running VACUUM.
           28  +**
           29  +** Usage:
           30  +**
           31  +**   int sqlite3_scrub_backup(
           32  +**       const char *zSourceFile,   // Source database filename
           33  +**       const char *zDestFile,     // Destination database filename
           34  +**       char **pzErrMsg            // Write error message here
           35  +**   );
           36  +**
           37  +** Simply call the API above specifying the filename of the source database
           38  +** and the name of the backup copy.  The source database must already exist
           39  +** and can be in active use. (A read lock is held during the backup.)  The
           40  +** destination file should not previously exist.  If the pzErrMsg parameter
           41  +** is non-NULL and if an error occurs, then an error message might be written
           42  +** into memory obtained from sqlite3_malloc() and *pzErrMsg made to point to
           43  +** that error message.  But if the error is an OOM, the error might not be
           44  +** reported.  The routine always returns non-zero if there is an error.
           45  +**
           46  +** If compiled with -DSCRUB_STANDALONE then a main() procedure is added and
           47  +** this file becomes a standalone program that can be run as follows:
           48  +**
           49  +**      ./sqlite3scrub SOURCE DEST
           50  +*/
           51  +#include "sqlite3.h"
           52  +#include <assert.h>
           53  +#include <stdio.h>
           54  +#include <stdlib.h>
           55  +#include <stdarg.h>
           56  +#include <string.h>
           57  +
           58  +typedef struct ScrubState ScrubState;
           59  +typedef unsigned char u8;
           60  +typedef unsigned short u16;
           61  +typedef unsigned int u32;
           62  +
           63  +
           64  +/* State information for a scrub-and-backup operation */
           65  +struct ScrubState {
           66  +  const char *zSrcFile;    /* Name of the source file */
           67  +  const char *zDestFile;   /* Name of the destination file */
           68  +  int rcErr;               /* Error code */
           69  +  char *zErr;              /* Error message text */
           70  +  sqlite3 *dbSrc;          /* Source database connection */
           71  +  sqlite3_file *pSrc;      /* Source file handle */
           72  +  sqlite3 *dbDest;         /* Destination database connection */
           73  +  sqlite3_file *pDest;     /* Destination file handle */
           74  +  u32 szPage;              /* Page size */
           75  +  u32 szUsable;            /* Usable bytes on each page */
           76  +  u32 nPage;               /* Number of pages */
           77  +  u32 iLastPage;           /* Page number of last page written so far*/
           78  +  u8 *page1;               /* Content of page 1 */
           79  +};
           80  +
           81  +/* Store an error message */
           82  +static void scrubBackupErr(ScrubState *p, const char *zFormat, ...){
           83  +  va_list ap;
           84  +  sqlite3_free(p->zErr);
           85  +  va_start(ap, zFormat);
           86  +  p->zErr = sqlite3_vmprintf(zFormat, ap);
           87  +  va_end(ap);
           88  +  if( p->rcErr==0 ) p->rcErr = SQLITE_ERROR;
           89  +}
           90  +
           91  +/* Allocate memory to hold a single page of content */
           92  +static u8 *scrubBackupAllocPage(ScrubState *p){
           93  +  u8 *pPage;
           94  +  if( p->rcErr ) return 0;
           95  +  pPage = sqlite3_malloc( p->szPage );
           96  +  if( pPage==0 ) p->rcErr = SQLITE_NOMEM;
           97  +  return pPage;
           98  +}
           99  +
          100  +/* Read a page from the source database into memory.  Use the memory
          101  +** provided by pBuf if not NULL or allocate a new page if pBuf==NULL.
          102  +*/
          103  +static u8 *scrubBackupRead(ScrubState *p, int pgno, u8 *pBuf){
          104  +  int rc;
          105  +  sqlite3_int64 iOff;
          106  +  u8 *pOut = pBuf;
          107  +  if( p->rcErr ) return 0;
          108  +  if( pOut==0 ){
          109  +    pOut = scrubBackupAllocPage(p);
          110  +    if( pOut==0 ) return 0;
          111  +  }
          112  +  iOff = (pgno-1)*(sqlite3_int64)p->szPage;
          113  +  rc = p->pSrc->pMethods->xRead(p->pSrc, pOut, p->szPage, iOff);
          114  +  if( rc!=SQLITE_OK ){
          115  +    if( pBuf==0 ) sqlite3_free(pOut);
          116  +    pOut = 0;
          117  +    scrubBackupErr(p, "read failed for page %d", pgno);
          118  +    p->rcErr = SQLITE_IOERR;
          119  +  }
          120  +  return pOut;  
          121  +}
          122  +
          123  +/* Write a page to the destination database */
          124  +static void scrubBackupWrite(ScrubState *p, int pgno, const u8 *pData){
          125  +  int rc;
          126  +  sqlite3_int64 iOff;
          127  +  if( p->rcErr ) return;
          128  +  iOff = (pgno-1)*(sqlite3_int64)p->szPage;
          129  +  rc = p->pDest->pMethods->xWrite(p->pDest, pData, p->szPage, iOff);
          130  +  if( rc!=SQLITE_OK ){
          131  +    scrubBackupErr(p, "write failed for page %d", pgno);
          132  +    p->rcErr = SQLITE_IOERR;
          133  +  }
          134  +  if( pgno>p->iLastPage ) p->iLastPage = pgno;
          135  +}
          136  +
          137  +/* Prepare a statement against the "db" database. */
          138  +static sqlite3_stmt *scrubBackupPrepare(
          139  +  ScrubState *p,      /* Backup context */
          140  +  sqlite3 *db,        /* Database to prepare against */
          141  +  const char *zSql    /* SQL statement */
          142  +){
          143  +  sqlite3_stmt *pStmt;
          144  +  if( p->rcErr ) return 0;
          145  +  p->rcErr = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
          146  +  if( p->rcErr ){
          147  +    scrubBackupErr(p, "SQL error \"%s\" on \"%s\"",
          148  +                   sqlite3_errmsg(db), zSql);
          149  +    sqlite3_finalize(pStmt);
          150  +    return 0;
          151  +  }
          152  +  return pStmt;
          153  +}
          154  +
          155  +
          156  +/* Open the source database file */
          157  +static void scrubBackupOpenSrc(ScrubState *p){
          158  +  sqlite3_stmt *pStmt;
          159  +  int rc;
          160  +  /* Open the source database file */
          161  +  p->rcErr = sqlite3_open_v2(p->zSrcFile, &p->dbSrc,
          162  +                 SQLITE_OPEN_READWRITE |
          163  +                 SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0);
          164  +  if( p->rcErr ){
          165  +    scrubBackupErr(p, "cannot open source database: %s",
          166  +                      sqlite3_errmsg(p->dbSrc));
          167  +    return;
          168  +  }
          169  +  p->rcErr = sqlite3_exec(p->dbSrc, "SELECT 1 FROM sqlite_master; BEGIN;",
          170  +                          0, 0, 0);
          171  +  if( p->rcErr ){
          172  +    scrubBackupErr(p,
          173  +       "cannot start a read transaction on the source database: %s",
          174  +       sqlite3_errmsg(p->dbSrc));
          175  +    return;
          176  +  }
          177  +  rc = sqlite3_wal_checkpoint_v2(p->dbSrc, "main", SQLITE_CHECKPOINT_FULL,
          178  +                                 0, 0);
          179  +  if( rc ){
          180  +    scrubBackupErr(p, "cannot checkpoint the source database");
          181  +    return;
          182  +  }
          183  +  pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_size");
          184  +  if( pStmt==0 ) return;
          185  +  rc = sqlite3_step(pStmt);
          186  +  if( rc==SQLITE_ROW ){
          187  +    p->szPage = sqlite3_column_int(pStmt, 0);
          188  +  }else{
          189  +    scrubBackupErr(p, "unable to determine the page size");
          190  +  }
          191  +  sqlite3_finalize(pStmt);
          192  +  if( p->rcErr ) return;
          193  +  pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_count");
          194  +  if( pStmt==0 ) return;
          195  +  rc = sqlite3_step(pStmt);
          196  +  if( rc==SQLITE_ROW ){
          197  +    p->nPage = sqlite3_column_int(pStmt, 0);
          198  +  }else{
          199  +    scrubBackupErr(p, "unable to determine the size of the source database");
          200  +  }
          201  +  sqlite3_finalize(pStmt);
          202  +  sqlite3_file_control(p->dbSrc, "main", SQLITE_FCNTL_FILE_POINTER, &p->pSrc);
          203  +  if( p->pSrc==0 || p->pSrc->pMethods==0 ){
          204  +    scrubBackupErr(p, "cannot get the source file handle");
          205  +    p->rcErr = SQLITE_ERROR;
          206  +  }
          207  +}
          208  +
          209  +/* Create and open the destination file */
          210  +static void scrubBackupOpenDest(ScrubState *p){
          211  +  sqlite3_stmt *pStmt;
          212  +  int rc;
          213  +  char *zSql;
          214  +  if( p->rcErr ) return;
          215  +  p->rcErr = sqlite3_open_v2(p->zDestFile, &p->dbDest,
          216  +                 SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE |
          217  +                 SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0);
          218  +  if( p->rcErr ){
          219  +    scrubBackupErr(p, "cannot open destination database: %s",
          220  +                      sqlite3_errmsg(p->dbDest));
          221  +    return;
          222  +  }
          223  +  zSql = sqlite3_mprintf("PRAGMA page_size(%u);", p->szPage);
          224  +  if( zSql==0 ){
          225  +    p->rcErr = SQLITE_NOMEM;
          226  +    return;
          227  +  }
          228  +  p->rcErr = sqlite3_exec(p->dbDest, zSql, 0, 0, 0);
          229  +  sqlite3_free(zSql);
          230  +  if( p->rcErr ){
          231  +    scrubBackupErr(p,
          232  +       "cannot set the page size on the destination database: %s",
          233  +       sqlite3_errmsg(p->dbDest));
          234  +    return;
          235  +  }
          236  +  sqlite3_exec(p->dbDest, "PRAGMA journal_mode=OFF;", 0, 0, 0);
          237  +  p->rcErr = sqlite3_exec(p->dbDest, "BEGIN EXCLUSIVE;", 0, 0, 0);
          238  +  if( p->rcErr ){
          239  +    scrubBackupErr(p,
          240  +       "cannot start a write transaction on the destination database: %s",
          241  +       sqlite3_errmsg(p->dbDest));
          242  +    return;
          243  +  }
          244  +  pStmt = scrubBackupPrepare(p, p->dbDest, "PRAGMA page_count;");
          245  +  if( pStmt==0 ) return;
          246  +  rc = sqlite3_step(pStmt);
          247  +  if( rc!=SQLITE_ROW ){
          248  +    scrubBackupErr(p, "cannot measure the size of the destination");
          249  +  }else if( sqlite3_column_int(pStmt, 0)>1 ){
          250  +    scrubBackupErr(p, "destination database is not empty - holds %d pages",
          251  +                   sqlite3_column_int(pStmt, 0));
          252  +  }
          253  +  sqlite3_finalize(pStmt);
          254  +  sqlite3_file_control(p->dbDest, "main", SQLITE_FCNTL_FILE_POINTER, &p->pDest);
          255  +  if( p->pDest==0 || p->pDest->pMethods==0 ){
          256  +    scrubBackupErr(p, "cannot get the destination file handle");
          257  +    p->rcErr = SQLITE_ERROR;
          258  +  }
          259  +}
          260  +
          261  +/* Read a 32-bit big-endian integer */
          262  +static u32 scrubBackupInt32(const u8 *a){
          263  +  u32 v = a[3];
          264  +  v += ((u32)a[2])<<8;
          265  +  v += ((u32)a[1])<<16;
          266  +  v += ((u32)a[0])<<24;
          267  +  return v;
          268  +}
          269  +
          270  +/* Read a 16-bit big-endian integer */
          271  +static u32 scrubBackupInt16(const u8 *a){
          272  +  return (a[0]<<8) + a[1];
          273  +}
          274  +
          275  +/*
          276  +** Read a varint.  Put the value in *pVal and return the number of bytes.
          277  +*/
          278  +static int scrubBackupVarint(const u8 *z, sqlite3_int64 *pVal){
          279  +  sqlite3_int64 v = 0;
          280  +  int i;
          281  +  for(i=0; i<8; i++){
          282  +    v = (v<<7) + (z[i]&0x7f);
          283  +    if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; }
          284  +  }
          285  +  v = (v<<8) + (z[i]&0xff);
          286  +  *pVal = v;
          287  +  return 9;
          288  +}
          289  +
          290  +/*
          291  +** Return the number of bytes in a varint.
          292  +*/
          293  +static int scrubBackupVarintSize(const u8 *z){
          294  +  int i;
          295  +  for(i=0; i<8; i++){
          296  +    if( (z[i]&0x80)==0 ){ return i+1; }
          297  +  }
          298  +  return 9;
          299  +}
          300  +
          301  +/*
          302  +** Copy the freelist trunk page given, and all its descendents,
          303  +** zeroing out as much as possible in the process.
          304  +*/
          305  +static void scrubBackupFreelist(ScrubState *p, int pgno, u32 nFree){
          306  +  u8 *a, *aBuf;
          307  +  u32 n, mx;
          308  +
          309  +  if( p->rcErr ) return;
          310  +  aBuf = scrubBackupAllocPage(p);
          311  +  if( aBuf==0 ) return;
          312  + 
          313  +  while( pgno && nFree){
          314  +    a = scrubBackupRead(p, pgno, aBuf);
          315  +    if( a==0 ) break;
          316  +    n = scrubBackupInt32(&a[4]);
          317  +    mx = p->szUsable/4 - 2;
          318  +    if( n<mx ){
          319  +      memset(&a[n*4+8], 0, 4*(mx-n));
          320  +    }
          321  +    scrubBackupWrite(p, pgno, a);
          322  +    pgno = scrubBackupInt32(a);
          323  +#if 0
          324  +    /* There is really no point in copying the freelist leaf pages.
          325  +    ** Simply leave them uninitialized in the destination database.  The
          326  +    ** OS filesystem should zero those pages for us automatically.
          327  +    */
          328  +    for(i=0; i<n && nFree; i++){
          329  +      u32 iLeaf = scrubBackupInt32(&a[i*4+8]);
          330  +      if( aZero==0 ){
          331  +        aZero = scrubBackupAllocPage(p);
          332  +        if( aZero==0 ){ pgno = 0; break; }
          333  +        memset(aZero, 0, p->szPage);
          334  +      }
          335  +      scrubBackupWrite(p, iLeaf, aZero);
          336  +      nFree--;
          337  +    }
          338  +#endif
          339  +  }
          340  +  sqlite3_free(aBuf);
          341  +}
          342  +
          343  +/*
          344  +