/ Check-in [c9d2ec51]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge the enhancements associated with the first 3.22.0 beta.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | apple-osx
Files: files | file ages | folders
SHA3-256: c9d2ec51c873c451b22a476b37b1b09c7f3f8c51e41144d544e7fc677ebf126b
User & Date: drh 2018-01-15 21:59:56
Context
2018-01-22
19:04
Merge all version-3.22.0 changes. check-in: 27e20d69 user: drh tags: apple-osx
2018-01-15
21:59
Merge the enhancements associated with the first 3.22.0 beta. check-in: c9d2ec51 user: drh tags: apple-osx
19:00
Fix a problem in the zipfile module causing it to generate incorrect checksums. Remove the ability to insert compressed data into a zip archive. check-in: b0b7d036 user: dan tags: trunk
2017-12-05
15:00
Merge latest trunk changes, including the compilation fix for builds that define both SQLITE_ENABLE_MULTITHREADED_CHECKS and SQLITE_ENABLE_API_ARMOUR. check-in: 2cb5d2a9 user: dan tags: apple-osx
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Added .fossil-settings/empty-dirs.

            1  +compat

Added .fossil-settings/ignore-glob.

            1  +compat/*

Changes to Makefile.in.

   424    424     $(TOP)/ext/fts3/fts3_test.c  \
   425    425     $(TOP)/ext/session/test_session.c \
   426    426     $(TOP)/ext/rbu/test_rbu.c 
   427    427   
   428    428   # Statically linked extensions
   429    429   #
   430    430   TESTSRC += \
          431  +  $(TOP)/ext/expert/sqlite3expert.c \
          432  +  $(TOP)/ext/expert/test_expert.c \
   431    433     $(TOP)/ext/misc/amatch.c \
   432    434     $(TOP)/ext/misc/carray.c \
   433    435     $(TOP)/ext/misc/closure.c \
   434    436     $(TOP)/ext/misc/csv.c \
   435    437     $(TOP)/ext/misc/eval.c \
   436    438     $(TOP)/ext/misc/fileio.c \
   437    439     $(TOP)/ext/misc/fuzzer.c \
................................................................................
   444    446     $(TOP)/ext/misc/percentile.c \
   445    447     $(TOP)/ext/misc/regexp.c \
   446    448     $(TOP)/ext/misc/remember.c \
   447    449     $(TOP)/ext/misc/series.c \
   448    450     $(TOP)/ext/misc/spellfix.c \
   449    451     $(TOP)/ext/misc/totype.c \
   450    452     $(TOP)/ext/misc/unionvtab.c \
   451         -  $(TOP)/ext/misc/wholenumber.c
          453  +  $(TOP)/ext/misc/wholenumber.c \
          454  +  $(TOP)/ext/misc/zipfile.c
   452    455   
   453    456   # Source code to the library files needed by the test fixture
   454    457   #
   455    458   TESTSRC2 = \
   456    459     $(TOP)/src/attach.c \
   457    460     $(TOP)/src/backup.c \
   458    461     $(TOP)/src/bitvec.c \
................................................................................
   566    569   # executables needed for testing
   567    570   #
   568    571   TESTPROGS = \
   569    572     testfixture$(TEXE) \
   570    573     sqlite3$(TEXE) \
   571    574     sqlite3_analyzer$(TEXE) \
   572    575     sqldiff$(TEXE) \
   573         -  dbhash$(TEXE)
          576  +  dbhash$(TEXE) \
          577  +  sqltclsh$(TEXE)
   574    578   
   575    579   # Databases containing fuzzer test cases
   576    580   #
   577    581   FUZZDATA = \
   578    582     $(TOP)/test/fuzzdata1.db \
   579    583     $(TOP)/test/fuzzdata2.db \
   580    584     $(TOP)/test/fuzzdata3.db \
................................................................................
   584    588   # Standard options to testfixture
   585    589   #
   586    590   TESTOPTS = --verbose=file --output=test-out.txt
   587    591   
   588    592   # Extra compiler options for various shell tools
   589    593   #
   590    594   SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
   591         -# SHELL_OPT += -DSQLITE_ENABLE_FTS5
          595  +#SHELL_OPT += -DSQLITE_ENABLE_FTS5
          596  +SHELL_OPT += -DSQLITE_ENABLE_RTREE
   592    597   SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
   593    598   SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
   594    599   SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
   595    600   SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
   596    601   SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
          602  +SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
          603  +SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
   597    604   FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
   598    605   FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
   599    606   FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
   600    607   FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
   601    608   DBFUZZ_OPT = 
   602    609   
   603    610   # This is the default Makefile target.  The objects listed here
................................................................................
   997   1004   keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
   998   1005   	$(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c
   999   1006   	./mkkeywordhash$(BEXE) >keywordhash.h
  1000   1007   
  1001   1008   # Source files that go into making shell.c
  1002   1009   SHELL_SRC = \
  1003   1010   	$(TOP)/src/shell.c.in \
         1011  +        $(TOP)/ext/misc/appendvfs.c \
  1004   1012   	$(TOP)/ext/misc/shathree.c \
  1005   1013   	$(TOP)/ext/misc/fileio.c \
  1006         -	$(TOP)/ext/misc/completion.c
         1014  +	$(TOP)/ext/misc/completion.c \
         1015  +	$(TOP)/ext/misc/sqlar.c \
         1016  +	$(TOP)/ext/expert/sqlite3expert.c \
         1017  +	$(TOP)/ext/expert/sqlite3expert.h \
         1018  +	$(TOP)/ext/misc/zipfile.c \
         1019  +        $(TOP)/src/test_windirent.c
  1007   1020   
  1008   1021   shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
  1009   1022   	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c
  1010   1023   
  1011   1024   
  1012   1025   
  1013   1026   
................................................................................
  1198   1211   	./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS)
  1199   1212   
  1200   1213   sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in
  1201   1214   	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
  1202   1215   
  1203   1216   sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
  1204   1217   	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
         1218  +
         1219  +sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in	
         1220  +	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
         1221  +
         1222  +sqltclsh$(TEXE): sqltclsh.c
         1223  +	$(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS)
         1224  +
         1225  +sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
         1226  +	$(LTLINK)	$(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(TLIBS)
  1205   1227   
  1206   1228   CHECKER_DEPS =\
  1207   1229     $(TOP)/tool/mkccode.tcl \
  1208   1230     sqlite3.c \
  1209   1231     $(TOP)/src/tclsqlite.c \
  1210   1232     $(TOP)/ext/repair/sqlite3_checker.tcl \
  1211   1233     $(TOP)/ext/repair/checkindex.c \

Changes to Makefile.msc.

    88     88   # be used for debugging with Visual Studio.
    89     89   #
    90     90   !IFNDEF SPLIT_AMALGAMATION
    91     91   SPLIT_AMALGAMATION = 0
    92     92   !ENDIF
    93     93   
    94     94   # <<mark>>
           95  +# Set this non-0 to have this makefile assume the Tcl shell executable
           96  +# (tclsh*.exe) is available in the PATH.  By default, this is disabled
           97  +# for compatibility with older build environments.  This setting only
           98  +# applies if TCLSH_CMD is not set manually.
           99  +#
          100  +!IFNDEF USE_TCLSH_IN_PATH
          101  +USE_TCLSH_IN_PATH = 0
          102  +!ENDIF
          103  +
          104  +# Set this non-0 to use zlib, possibly compiling it from source code.
          105  +#
          106  +!IFNDEF USE_ZLIB
          107  +USE_ZLIB = 0
          108  +!ENDIF
          109  +
          110  +# Set this non-0 to build zlib from source code.  This is enabled by
          111  +# default and in that case it will be assumed that the ZLIBDIR macro
          112  +# points to the top-level source code directory for zlib.
          113  +#
          114  +!IFNDEF BUILD_ZLIB
          115  +BUILD_ZLIB = 1
          116  +!ENDIF
          117  +
    95    118   # Set this non-0 to use the International Components for Unicode (ICU).
    96    119   #
    97    120   !IFNDEF USE_ICU
    98    121   USE_ICU = 0
    99    122   !ENDIF
   100    123   # <</mark>>
   101    124   
................................................................................
   608    631   !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
   609    632   SHELL_CORE_DEP = $(SQLITE3DLL)
   610    633   !ELSE
   611    634   SHELL_CORE_DEP =
   612    635   !ENDIF
   613    636   !ENDIF
   614    637   
          638  +# <<mark>>
          639  +# If zlib support is enabled, add the dependencies for it.
          640  +#
          641  +!IF $(USE_ZLIB)!=0 && $(BUILD_ZLIB)!=0
          642  +SHELL_CORE_DEP = zlib $(SHELL_CORE_DEP)
          643  +TESTFIXTURE_DEP = zlib $(TESTFIXTURE_DEP)
          644  +!ENDIF
          645  +# <</mark>>
          646  +
   615    647   # This is the core library that the shell executable should link with.
   616    648   #
   617    649   !IFNDEF SHELL_CORE_LIB
   618    650   !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
   619    651   SHELL_CORE_LIB = $(SQLITE3LIB)
   620    652   !ELSE
   621    653   SHELL_CORE_LIB =
................................................................................
   798    830   # <<mark>>
   799    831   # The locations of the Tcl header and library files.  Also, the library that
   800    832   # non-stubs enabled programs using Tcl must link against.  These variables
   801    833   # (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment
   802    834   # prior to running nmake in order to match the actual installed location and
   803    835   # version on this machine.
   804    836   #
          837  +!IFNDEF TCLDIR
          838  +TCLDIR = $(TOP)\compat\tcl
          839  +!ENDIF
          840  +
   805    841   !IFNDEF TCLINCDIR
   806         -TCLINCDIR = c:\tcl\include
          842  +TCLINCDIR = $(TCLDIR)\include
   807    843   !ENDIF
   808    844   
   809    845   !IFNDEF TCLLIBDIR
   810         -TCLLIBDIR = c:\tcl\lib
          846  +TCLLIBDIR = $(TCLDIR)\lib
   811    847   !ENDIF
   812    848   
   813    849   !IFNDEF LIBTCL
   814    850   LIBTCL = tcl86.lib
   815    851   !ENDIF
   816    852   
   817    853   !IFNDEF LIBTCLSTUB
   818    854   LIBTCLSTUB = tclstub86.lib
   819    855   !ENDIF
   820    856   
   821    857   !IFNDEF LIBTCLPATH
   822         -LIBTCLPATH = c:\tcl\bin
          858  +LIBTCLPATH = $(TCLDIR)\bin
          859  +!ENDIF
          860  +
          861  +# The locations of the zlib header and library files.  These variables
          862  +# (ZLIBINCDIR, ZLIBLIBDIR, and ZLIBLIB) may be overridden via the environment
          863  +# prior to running nmake in order to match the actual installed (or source
          864  +# code) location on this machine.
          865  +#
          866  +!IFNDEF ZLIBDIR
          867  +ZLIBDIR = $(TOP)\compat\zlib
          868  +!ENDIF
          869  +
          870  +!IFNDEF ZLIBINCDIR
          871  +ZLIBINCDIR = $(ZLIBDIR)
          872  +!ENDIF
          873  +
          874  +!IFNDEF ZLIBLIBDIR
          875  +ZLIBLIBDIR = $(ZLIBDIR)
          876  +!ENDIF
          877  +
          878  +!IFNDEF ZLIBLIB
          879  +!IF $(DYNAMIC_SHELL)!=0
          880  +ZLIBLIB = zdll.lib
          881  +!ELSE
          882  +ZLIBLIB = zlib.lib
          883  +!ENDIF
   823    884   !ENDIF
   824    885   
   825    886   # The locations of the ICU header and library files.  These variables
   826    887   # (ICUINCDIR, ICULIBDIR, and LIBICU) may be overridden via the environment
   827    888   # prior to running nmake in order to match the actual installed location on
   828    889   # this machine.
   829    890   #
          891  +!IFNDEF ICUDIR
          892  +ICUDIR = $(TOP)\compat\icu
          893  +!ENDIF
          894  +
   830    895   !IFNDEF ICUINCDIR
   831         -ICUINCDIR = c:\icu\include
          896  +ICUINCDIR = $(ICUDIR)\include
   832    897   !ENDIF
   833    898   
   834    899   !IFNDEF ICULIBDIR
   835         -ICULIBDIR = c:\icu\lib
          900  +ICULIBDIR = $(ICUDIR)\lib
   836    901   !ENDIF
   837    902   
   838    903   !IFNDEF LIBICU
   839    904   LIBICU = icuuc.lib icuin.lib
   840    905   !ENDIF
   841    906   
   842    907   # This is the command to use for tclsh - normally just "tclsh", but we may
   843    908   # know the specific version we want to use.  This variable (TCLSH_CMD) may be
   844    909   # overridden via the environment prior to running nmake in order to select a
   845    910   # specific Tcl shell to use.
   846    911   #
   847    912   !IFNDEF TCLSH_CMD
          913  +!IF $(USE_TCLSH_IN_PATH)!=0 || !EXIST("$(TCLDIR)\bin\tclsh.exe")
   848    914   TCLSH_CMD = tclsh
          915  +!ELSE
          916  +TCLSH_CMD = $(TCLDIR)\bin\tclsh.exe
          917  +!ENDIF
   849    918   !ENDIF
   850    919   # <</mark>>
   851    920   
   852    921   # Compiler options needed for programs that use the readline() library.
   853    922   #
   854    923   !IFNDEF READLINE_FLAGS
   855    924   READLINE_FLAGS = -DHAVE_READLINE=0
................................................................................
   947   1016   #
   948   1017   !IF $(DEBUG)>1 || $(SYMBOLS)!=0
   949   1018   TCC = $(TCC) -Zi
   950   1019   BCC = $(BCC) -Zi
   951   1020   !ENDIF
   952   1021   
   953   1022   # <<mark>>
         1023  +# If zlib support is enabled, add the compiler options for it.
         1024  +#
         1025  +!IF $(USE_ZLIB)!=0
         1026  +TCC = $(TCC) -DSQLITE_HAVE_ZLIB=1
         1027  +RCC = $(RCC) -DSQLITE_HAVE_ZLIB=1
         1028  +TCC = $(TCC) -I$(ZLIBINCDIR)
         1029  +RCC = $(RCC) -I$(ZLIBINCDIR)
         1030  +!ENDIF
         1031  +
   954   1032   # If ICU support is enabled, add the compiler options for it.
   955   1033   #
   956   1034   !IF $(USE_ICU)!=0
   957   1035   TCC = $(TCC) -DSQLITE_ENABLE_ICU=1
   958   1036   RCC = $(RCC) -DSQLITE_ENABLE_ICU=1
   959   1037   TCC = $(TCC) -I$(TOP)\ext\icu
   960   1038   RCC = $(RCC) -I$(TOP)\ext\icu
................................................................................
  1067   1145   LDFLAGS = $(LDOPTS)
  1068   1146   !ENDIF
  1069   1147   
  1070   1148   # <<mark>>
  1071   1149   # Start with the Tcl related linker options.
  1072   1150   #
  1073   1151   !IF $(NO_TCL)==0
  1074         -LTLIBPATHS = /LIBPATH:$(TCLLIBDIR)
  1075         -LTLIBS = $(LTLIBS) $(LIBTCL)
         1152  +TCLLIBPATHS = $(TCLLIBPATHS) /LIBPATH:$(TCLLIBDIR)
         1153  +TCLLIBS = $(TCLLIBS) $(LIBTCL)
         1154  +!ENDIF
         1155  +
         1156  +# If zlib support is enabled, add the linker options for it.
         1157  +#
         1158  +!IF $(USE_ZLIB)!=0
         1159  +LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ZLIBLIBDIR)
         1160  +LTLIBS = $(LTLIBS) $(ZLIBLIB)
  1076   1161   !ENDIF
  1077   1162   
  1078   1163   # If ICU support is enabled, add the linker options for it.
  1079   1164   #
  1080   1165   !IF $(USE_ICU)!=0
  1081   1166   LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ICULIBDIR)
  1082   1167   LTLIBS = $(LTLIBS) $(LIBICU)
................................................................................
  1396   1481     $(TOP)\ext\fts3\fts3_test.c \
  1397   1482     $(TOP)\ext\rbu\test_rbu.c \
  1398   1483     $(TOP)\ext\session\test_session.c
  1399   1484   
  1400   1485   # Statically linked extensions.
  1401   1486   #
  1402   1487   TESTEXT = \
         1488  +  $(TOP)\ext\expert\sqlite3expert.c \
         1489  +  $(TOP)\ext\expert\test_expert.c \
  1403   1490     $(TOP)\ext\misc\amatch.c \
  1404   1491     $(TOP)\ext\misc\carray.c \
  1405   1492     $(TOP)\ext\misc\closure.c \
  1406   1493     $(TOP)\ext\misc\csv.c \
  1407   1494     $(TOP)\ext\misc\eval.c \
  1408   1495     $(TOP)\ext\misc\fileio.c \
  1409   1496     $(TOP)\ext\misc\fuzzer.c \
................................................................................
  1417   1504     $(TOP)\ext\misc\regexp.c \
  1418   1505     $(TOP)\ext\misc\remember.c \
  1419   1506     $(TOP)\ext\misc\series.c \
  1420   1507     $(TOP)\ext\misc\spellfix.c \
  1421   1508     $(TOP)\ext\misc\totype.c \
  1422   1509     $(TOP)\ext\misc\unionvtab.c \
  1423   1510     $(TOP)\ext\misc\wholenumber.c
         1511  +
         1512  +# If use of zlib is enabled, add the "zipfile.c" source file.
         1513  +#
         1514  +!IF $(USE_ZLIB)!=0
         1515  +TESTEXT = $(TESTEXT) $(TOP)\ext\misc\zipfile.c
         1516  +!ENDIF
  1424   1517   
  1425   1518   # Source code to the library files needed by the test fixture
  1426   1519   # (non-amalgamation)
  1427   1520   #
  1428   1521   TESTSRC2 = \
  1429   1522     $(SRC00) \
  1430   1523     $(SRC01) \
................................................................................
  1489   1582   #
  1490   1583   TESTPROGS = \
  1491   1584     testfixture.exe \
  1492   1585     $(SQLITE3EXE) \
  1493   1586     sqlite3_analyzer.exe \
  1494   1587     sqlite3_checker.exe \
  1495   1588     sqldiff.exe \
  1496         -  dbhash.exe
         1589  +  dbhash.exe \
         1590  +  sqltclsh.exe
  1497   1591   
  1498   1592   # Databases containing fuzzer test cases
  1499   1593   #
  1500   1594   FUZZDATA = \
  1501   1595     $(TOP)\test\fuzzdata1.db \
  1502   1596     $(TOP)\test\fuzzdata2.db \
  1503   1597     $(TOP)\test\fuzzdata3.db \
................................................................................
  1507   1601   
  1508   1602   # Additional compiler options for the shell.  These are only effective
  1509   1603   # when the shell is not being dynamically linked.
  1510   1604   #
  1511   1605   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
  1512   1606   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
  1513   1607   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB
  1514         -# SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS5
  1515         -SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS
  1516   1608   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
         1609  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC -DSQLITE_INTROSPECTION_PRAGMAS
         1610  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_RTREE
  1517   1611   !ENDIF
  1518   1612   
  1519   1613   # <<mark>>
  1520   1614   # Extra compiler options for various test tools.
  1521   1615   #
  1522   1616   MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5
  1523   1617   FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
................................................................................
  1541   1635   ALL_TCL_TARGETS =
  1542   1636   !ENDIF
  1543   1637   # <</mark>>
  1544   1638   
  1545   1639   # This is the default Makefile target.  The objects listed here
  1546   1640   # are what get build when you type just "make" with no arguments.
  1547   1641   #
  1548         -all:	dll libsqlite3.lib shell $(ALL_TCL_TARGETS)
         1642  +core:	dll libsqlite3.lib shell
         1643  +
         1644  +# Targets that require the Tcl library.
         1645  +#
         1646  +tcl:	$(ALL_TCL_TARGETS)
         1647  +
         1648  +# This Makefile target builds all of the standard binaries.
         1649  +#
         1650  +all:	core tcl
  1549   1651   
  1550   1652   # Dynamic link library section.
  1551   1653   #
  1552   1654   dll:	$(SQLITE3DLL)
  1553   1655   
  1554   1656   # Shell executable.
  1555   1657   #
................................................................................
  1936   2038   tclsqlite.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
  1937   2039   	$(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
  1938   2040   
  1939   2041   tclsqlite-shell.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
  1940   2042   	$(LTCOMPILE) $(NO_WARN) -DTCLSH -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
  1941   2043   
  1942   2044   tclsqlite3.exe:	tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
  1943         -	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
         2045  +	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
  1944   2046   
  1945   2047   # Rules to build opcodes.c and opcodes.h
  1946   2048   #
  1947   2049   opcodes.c:	opcodes.h $(TOP)\tool\mkopcodec.tcl
  1948   2050   	$(TCLSH_CMD) $(TOP)\tool\mkopcodec.tcl opcodes.h > opcodes.c
  1949   2051   
  1950   2052   opcodes.h:	parse.h $(TOP)\src\vdbe.c $(TOP)\tool\mkopcodeh.tcl
................................................................................
  1979   2081   
  1980   2082   keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
  1981   2083   	.\mkkeywordhash.exe > keywordhash.h
  1982   2084   
  1983   2085   # Source files that go into making shell.c
  1984   2086   SHELL_SRC = \
  1985   2087   	$(TOP)\src\shell.c.in \
         2088  +        $(TOP)\ext\misc\appendvfs.c \
  1986   2089   	$(TOP)\ext\misc\shathree.c \
  1987   2090   	$(TOP)\ext\misc\fileio.c \
  1988         -	$(TOP)\ext\misc\completion.c
         2091  +	$(TOP)\ext\misc\completion.c \
         2092  +	$(TOP)\ext\expert\sqlite3expert.c \
         2093  +	$(TOP)\ext\expert\sqlite3expert.h \
         2094  +	$(TOP)\src\test_windirent.c
         2095  +
         2096  +# If use of zlib is enabled, add the "zipfile.c" source file.
         2097  +#
         2098  +!IF $(USE_ZLIB)!=0
         2099  +SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\sqlar.c
         2100  +SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\zipfile.c
         2101  +!ENDIF
  1989   2102   
  1990   2103   shell.c:	$(SHELL_SRC) $(TOP)\tool\mkshellc.tcl
  1991   2104   	$(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c
  1992   2105   
         2106  +zlib:
         2107  +	pushd $(ZLIBDIR) && $(MAKE) /f win32\Makefile.msc clean $(ZLIBLIB) && popd
  1993   2108   
  1994   2109   # Rules to build the extension objects.
  1995   2110   #
  1996   2111   icu.lo:	$(TOP)\ext\icu\icu.c $(HDR) $(EXTHDR)
  1997   2112   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\icu\icu.c
  1998   2113   
  1999   2114   fts2.lo:	$(TOP)\ext\fts2\fts2.c $(HDR) $(EXTHDR)
................................................................................
  2074   2189      fts5parse.c fts5parse.h \
  2075   2190      $(TOP)\ext\fts5\fts5_storage.c \
  2076   2191      $(TOP)\ext\fts5\fts5_tokenize.c \
  2077   2192      $(TOP)\ext\fts5\fts5_unicode2.c \
  2078   2193      $(TOP)\ext\fts5\fts5_varint.c \
  2079   2194      $(TOP)\ext\fts5\fts5_vocab.c
  2080   2195   
         2196  +LSM1_SRC = \
         2197  +   $(TOP)\ext\lsm1\lsm.h \
         2198  +   $(TOP)\ext\lsm1\lsmInt.h \
         2199  +   $(TOP)\ext\lsm1\lsm_ckpt.c \
         2200  +   $(TOP)\ext\lsm1\lsm_file.c \
         2201  +   $(TOP)\ext\lsm1\lsm_log.c \
         2202  +   $(TOP)\ext\lsm1\lsm_main.c \
         2203  +   $(TOP)\ext\lsm1\lsm_mem.c \
         2204  +   $(TOP)\ext\lsm1\lsm_mutex.c \
         2205  +   $(TOP)\ext\lsm1\lsm_shared.c \
         2206  +   $(TOP)\ext\lsm1\lsm_sorted.c \
         2207  +   $(TOP)\ext\lsm1\lsm_str.c \
         2208  +   $(TOP)\ext\lsm1\lsm_tree.c \
         2209  +   $(TOP)\ext\lsm1\lsm_unix.c \
         2210  +   $(TOP)\ext\lsm1\lsm_varint.c \
         2211  +   $(TOP)\ext\lsm1\lsm_vtab.c \
         2212  +   $(TOP)\ext\lsm1\lsm_win32.c
         2213  +
  2081   2214   fts5parse.c:	$(TOP)\ext\fts5\fts5parse.y lemon.exe
  2082   2215   	copy $(TOP)\ext\fts5\fts5parse.y .
  2083   2216   	del /Q fts5parse.h 2>NUL
  2084   2217   	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) fts5parse.y
  2085   2218   
  2086   2219   fts5parse.h:	fts5parse.c
  2087   2220   
  2088   2221   fts5.c:	$(FTS5_SRC)
  2089   2222   	$(TCLSH_CMD) $(TOP)\ext\fts5\tool\mkfts5c.tcl
  2090   2223   	copy $(TOP)\ext\fts5\fts5.h .
  2091   2224   
         2225  +lsm1.c:	$(LSM1_SRC)
         2226  +	$(TCLSH_CMD) $(TOP)\ext\lsm1\tool\mklsm1c.tcl
         2227  +	copy $(TOP)\ext\lsm1\lsm.h .
         2228  +
  2092   2229   fts5.lo:	fts5.c $(HDR) $(EXTHDR)
  2093   2230   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c fts5.c
  2094   2231   
  2095   2232   fts5_ext.lo:	fts5.c $(HDR) $(EXTHDR)
  2096   2233   	$(LTCOMPILE) $(NO_WARN) -c fts5.c
  2097   2234   
  2098   2235   fts5.dll:	fts5_ext.lo
................................................................................
  2141   2278   	type "$(TCLINCDIR)\tcl.h" | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact tclDecls.h sqlite_tclDecls.h \
  2142   2279   		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "typedef (.*?)\(Tcl_" "typedef \1 (SQLITE_TCLAPI Tcl_" \
  2143   2280   		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "void (*freeProc)" "void (SQLITE_TCLAPI *freeProc)" \
  2144   2281   		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*findProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *findProc)" \
  2145   2282   		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*createProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *createProc)" >> $(SQLITETCLH)
  2146   2283   !ENDIF
  2147   2284   
  2148         -testfixture.exe:	$(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP)
         2285  +testfixture.exe:	$(TESTFIXTURE_SRC) $(TESTFIXTURE_DEP) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP)
  2149   2286   	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \
  2150   2287   		-DBUILD_sqlite -I$(TCLINCDIR) \
  2151   2288   		$(TESTFIXTURE_SRC) \
  2152         -		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
         2289  +		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
  2153   2290   
  2154   2291   extensiontest:	testfixture.exe testloadext.dll
  2155   2292   	@set PATH=$(LIBTCLPATH);$(PATH)
  2156   2293   	.\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS)
  2157   2294   
  2158   2295   fulltest:	$(TESTPROGS) fuzztest
  2159   2296   	@set PATH=$(LIBTCLPATH);$(PATH)
................................................................................
  2195   2332   	.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)
  2196   2333   
  2197   2334   sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(SQLITE_TCL_DEP)
  2198   2335   	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in > $@
  2199   2336   
  2200   2337   sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
  2201   2338   	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
  2202         -		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
         2339  +		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
         2340  +
         2341  +sqltclsh.c: sqlite3.c $(TOP)\src\tclsqlite.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in
         2342  +	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in >sqltclsh.c
         2343  +
         2344  +sqltclsh.exe: sqltclsh.c  $(SHELL_CORE_DEP) $(LIBRESOBJS)
         2345  +	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqltclsh.c \
         2346  +		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
         2347  +
         2348  +sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c
         2349  +	$(LTLINK) $(NO_WARN)	$(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS)
  2203   2350   
  2204   2351   CHECKER_DEPS =\
  2205   2352     $(TOP)/tool/mkccode.tcl \
  2206   2353     sqlite3.c \
  2207   2354     $(TOP)/src/tclsqlite.c \
  2208   2355     $(TOP)/ext/repair/sqlite3_checker.tcl \
  2209   2356     $(TOP)/ext/repair/checkindex.c \
................................................................................
  2212   2359     $(TOP)/ext/repair/sqlite3_checker.c.in
  2213   2360   
  2214   2361   sqlite3_checker.c:	$(CHECKER_DEPS)
  2215   2362   	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\ext\repair\sqlite3_checker.c.in > $@
  2216   2363   
  2217   2364   sqlite3_checker.exe:	sqlite3_checker.c $(LIBRESOBJS)
  2218   2365   	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \
  2219         -		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
         2366  +		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
  2220   2367   
  2221   2368   dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H)
  2222   2369   	$(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \
  2223   2370   		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS)
  2224   2371   
  2225   2372   testloadext.lo:	$(TOP)\src\test_loadext.c
  2226   2373   	$(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c
................................................................................
  2313   2460   	del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL
  2314   2461   	del /Q sqlite3.c sqlite3-*.c 2>NUL
  2315   2462   	del /Q sqlite3rc.h 2>NUL
  2316   2463   	del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL
  2317   2464   	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
  2318   2465   	del /Q sqlite-*-output.vsix 2>NUL
  2319   2466   	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL
         2467  +	del /Q sqltclsh.exe 2>NUL
  2320   2468   	del /Q fts5.* fts5parse.* 2>NUL
         2469  +	del /Q lsm.h lsm1.c 2>NUL
  2321   2470   # <</mark>>

Changes to README.md.

     1      1   <h1 align="center">SQLite Source Repository</h1>
     2      2   
     3      3   This repository contains the complete source code for the SQLite database
     4         -engine.  Some test scripts are also include.  However, many other test scripts
            4  +engine.  Some test scripts are also included.  However, many other test scripts
     5      5   and most of the documentation are managed separately.
     6      6   
     7      7   If you are reading this on a Git mirror someplace, you are doing it wrong.
     8      8   The [official repository](https://www.sqlite.org/src/) is better.  Go there
     9      9   now.
    10     10   
    11     11   ## Obtaining The Code
................................................................................
   100    100   to the "sqlite3.dll" command line above.  When debugging into the SQLite
   101    101   code, adding the "DEBUG=1" argument to one of the above command lines is
   102    102   recommended.
   103    103   
   104    104   SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation
   105    105   is required by the makefiles (including those for MSVC).  SQLite contains
   106    106   a lot of generated code and Tcl is used to do much of that code generation.
   107         -The makefiles also require AWK.
   108    107   
   109    108   ## Source Code Tour
   110    109   
   111    110   Most of the core source files are in the **src/** subdirectory.  The
   112    111   **src/** folder also contains files used to build the "testfixture" test
   113    112   harness. The names of the source files used by "testfixture" all begin
   114    113   with "test".
   115    114   The **src/** also contains the "shell.c" file
   116    115   which is the main program for the "sqlite3.exe"
   117    116   [command-line shell](https://sqlite.org/cli.html) and
   118    117   the "tclsqlite.c" file which implements the
   119         -[TCL bindings](https://sqlite.org/tclsqlite.html) for SQLite.
          118  +[Tcl bindings](https://sqlite.org/tclsqlite.html) for SQLite.
   120    119   (Historical note:  SQLite began as a Tcl
   121    120   extension and only later escaped to the wild as an independent library.)
   122    121   
   123    122   Test scripts and programs are found in the **test/** subdirectory.
   124    123   Addtional test code is found in other source repositories.
   125    124   See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for
   126    125   additional information.
................................................................................
   159    158   the src/parse.y file.  The conversion of "parse.y" into "parse.c" is done
   160    159   by the [lemon](./doc/lemon.html) LALR(1) parser generator.  The source code
   161    160   for lemon is at tool/lemon.c.  Lemon uses the tool/lempar.c file as a
   162    161   template for generating its parser.
   163    162   
   164    163   Lemon also generates the **parse.h** header file, at the same time it
   165    164   generates parse.c. But the parse.h header file is
   166         -modified further (to add additional symbols) using the ./addopcodes.awk
   167         -AWK script.
          165  +modified further (to add additional symbols) using the ./addopcodes.tcl
          166  +Tcl script.
   168    167   
   169    168   The **opcodes.h** header file contains macros that define the numbers
   170    169   corresponding to opcodes in the "VDBE" virtual machine.  The opcodes.h
   171    170   file is generated by the scanning the src/vdbe.c source file.  The
   172         -AWK script at ./mkopcodeh.awk does this scan and generates opcodes.h.
   173         -A second AWK script, ./mkopcodec.awk, then scans opcodes.h to generate
          171  +Tcl script at ./mkopcodeh.tcl does this scan and generates opcodes.h.
          172  +A second Tcl script, ./mkopcodec.tcl, then scans opcodes.h to generate
   174    173   the **opcodes.c** source file, which contains a reverse mapping from
   175    174   opcode-number to opcode-name that is used for EXPLAIN output.
   176    175   
   177    176   The **keywordhash.h** header file contains the definition of a hash table
   178    177   that maps SQL language keywords (ex: "CREATE", "SELECT", "INDEX", etc.) into
   179    178   the numeric codes used by the parse.c parser.  The keywordhash.h file is
   180    179   generated by a C-language program at tool mkkeywordhash.c.
................................................................................
   203    202   tool/mksqlite3c.tcl script is run to copy them all together in just the
   204    203   right order while resolving internal "#include" references.
   205    204   
   206    205   The amalgamation source file is more than 200K lines long.  Some symbolic
   207    206   debuggers (most notably MSVC) are unable to deal with files longer than 64K
   208    207   lines.  To work around this, a separate Tcl script, tool/split-sqlite3c.tcl,
   209    208   can be run on the amalgamation to break it up into a single small C file
   210         -called **sqlite3-all.c** that does #include on about five other files
   211         -named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-5.c**.  In this way,
          209  +called **sqlite3-all.c** that does #include on about seven other files
          210  +named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-7.c**.  In this way,
   212    211   all of the source code is contained within a single translation unit so
   213    212   that the compiler can do extra cross-procedure optimization, but no
   214    213   individual source file exceeds 32K lines in length.
   215    214   
   216    215   ## How It All Fits Together
   217    216   
   218    217   SQLite is modular in design.
................................................................................
   233    232   Key files:
   234    233   
   235    234     *  **sqlite.h.in** - This file defines the public interface to the SQLite
   236    235        library.  Readers will need to be familiar with this interface before
   237    236        trying to understand how the library works internally.
   238    237   
   239    238     *  **sqliteInt.h** - this header file defines many of the data objects
   240         -     used internally by SQLite.
          239  +     used internally by SQLite.  In addition to "sqliteInt.h", some
          240  +     subsystems have their own header files.
   241    241   
   242    242     *  **parse.y** - This file describes the LALR(1) grammar that SQLite uses
   243    243        to parse SQL statements, and the actions that are taken at each step
   244    244        in the parsing process.
   245    245   
   246    246     *  **vdbe.c** - This file implements the virtual machine that runs
   247    247        prepared statements.  There are various helper files whose names
   248    248        begin with "vdbe".  The VDBE has access to the vdbeInt.h header file
   249    249        which defines internal data objects.  The rest of SQLite interacts
   250    250        with the VDBE through an interface defined by vdbe.h.
   251    251   
   252         -  *  **where.c** - This file analyzes the WHERE clause and generates
          252  +  *  **where.c** - This file (together with its helper files named
          253  +     by "where*.c") analyzes the WHERE clause and generates
   253    254        virtual machine code to run queries efficiently.  This file is
   254    255        sometimes called the "query optimizer".  It has its own private
   255    256        header file, whereInt.h, that defines data objects used internally.
   256    257   
   257    258     *  **btree.c** - This file contains the implementation of the B-Tree
   258         -     storage engine used by SQLite.
          259  +     storage engine used by SQLite.  The interface to the rest of the system
          260  +     is defined by "btree.h".  The "btreeInt.h" header defines objects
          261  +     used internally by btree.c and not published to the rest of the system.
   259    262   
   260    263     *  **pager.c** - This file contains the "pager" implementation, the
   261         -     module that implements transactions.
          264  +     module that implements transactions.  The "pager.h" header file
          265  +     defines the interface between pager.c and the rest of the system.
   262    266   
   263    267     *  **os_unix.c** and **os_win.c** - These two files implement the interface
   264    268        between SQLite and the underlying operating system using the run-time
   265    269        pluggable VFS interface.
   266    270   
   267         -  *  **shell.c** - This file is not part of the core SQLite library.  This
          271  +  *  **shell.c.in** - This file is not part of the core SQLite library.  This
   268    272        is the file that, when linked against sqlite3.a, generates the
   269         -     "sqlite3.exe" command-line shell.
          273  +     "sqlite3.exe" command-line shell.  The "shell.c.in" file is transformed
          274  +     into "shell.c" as part of the build process.
   270    275   
   271    276     *  **tclsqlite.c** - This file implements the Tcl bindings for SQLite.  It
   272    277        is not part of the core SQLite library.  But as most of the tests in this
   273    278        repository are written in Tcl, the Tcl language bindings are important.
          279  +
          280  +  *  **test*.c** - Files in the src/ folder that begin with "test" go into
          281  +     building the "testfixture.exe" program.  The testfixture.exe program is
          282  +     an enhanced Tcl shell.  The testfixture.exe program runs scripts in the
          283  +     test/ folder to validate the core SQLite code.  The testfixture program
          284  +     (and some other test programs too) is build and run when you type
          285  +     "make test".
          286  +
          287  +  *  **ext/misc/json1.c** - This file implements the various JSON functions
          288  +     that are build into SQLite.
   274    289   
   275    290   There are many other source files.  Each has a succinct header comment that
   276    291   describes its purpose and role within the larger system.
   277    292   
   278    293   
   279    294   ## Contacts
   280    295   
   281    296   The main SQLite webpage is [http://www.sqlite.org/](http://www.sqlite.org/)
   282    297   with geographically distributed backups at
   283    298   [http://www2.sqlite.org/](http://www2.sqlite.org) and
   284    299   [http://www3.sqlite.org/](http://www3.sqlite.org).

Changes to autoconf/Makefile.msc.

   556    556   !IFNDEF SHELL_CORE_DEP
   557    557   !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
   558    558   SHELL_CORE_DEP = $(SQLITE3DLL)
   559    559   !ELSE
   560    560   SHELL_CORE_DEP =
   561    561   !ENDIF
   562    562   !ENDIF
          563  +
   563    564   
   564    565   # This is the core library that the shell executable should link with.
   565    566   #
   566    567   !IFNDEF SHELL_CORE_LIB
   567    568   !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
   568    569   SHELL_CORE_LIB = $(SQLITE3LIB)
   569    570   !ELSE
................................................................................
   925    926   
   926    927   # Additional compiler options for the shell.  These are only effective
   927    928   # when the shell is not being dynamically linked.
   928    929   #
   929    930   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
   930    931   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
   931    932   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB
          933  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC -DSQLITE_INTROSPECTION_PRAGMAS
   932    934   !ENDIF
   933    935   
   934    936   
   935    937   # This is the default Makefile target.  The objects listed here
   936    938   # are what get build when you type just "make" with no arguments.
   937    939   #
   938         -all:	dll shell
          940  +core:	dll shell
          941  +
          942  +# Targets that require the Tcl library.
          943  +#
          944  +tcl:	$(ALL_TCL_TARGETS)
          945  +
          946  +# This Makefile target builds all of the standard binaries.
          947  +#
          948  +all:	core tcl
   939    949   
   940    950   # Dynamic link library section.
   941    951   #
   942    952   dll:	$(SQLITE3DLL)
   943    953   
   944    954   # Shell executable.
   945    955   #

Added ext/expert/README.md.

            1  +## SQLite Expert Extension
            2  +
            3  +This folder contains code for a simple system to propose useful indexes
            4  +given a database and a set of SQL queries. It works as follows:
            5  +
            6  +  1. The user database schema is copied to a temporary database.
            7  +
            8  +  1. All SQL queries are prepared against the temporary database.
            9  +     Information regarding the WHERE and ORDER BY clauses, and other query
           10  +     features that affect index selection are recorded.
           11  +
           12  +  1. The information gathered in step 2 is used to create candidate 
           13  +     indexes - indexes that the planner might have made use of in the previous
           14  +     step, had they been available.
           15  +
           16  +  1. A subset of the data in the user database is used to generate statistics
           17  +     for all existing indexes and the candidate indexes generated in step 3
           18  +     above.
           19  +
           20  +  1. The SQL queries are prepared a second time. If the planner uses any
           21  +     of the indexes created in step 3, they are recommended to the user.
           22  +
           23  +# C API
           24  +
           25  +The SQLite expert C API is defined in sqlite3expert.h. Most uses will proceed
           26  +as follows:
           27  +
           28  +  1. An sqlite3expert object is created by calling **sqlite3\_expert\_new()**.
           29  +     A database handle opened by the user is passed as an argument.
           30  +
           31  +  1. The sqlite3expert object is configured with one or more SQL statements
           32  +     by making one or more calls to **sqlite3\_expert\_sql()**. Each call may
           33  +     specify a single SQL statement, or multiple statements separated by
           34  +     semi-colons.
           35  +  
           36  +  1. Optionally, the **sqlite3\_expert\_config()** API may be used to 
           37  +     configure the size of the data subset used to generate index statistics.
           38  +     Using a smaller subset of the data can speed up the analysis.
           39  +
           40  +  1. **sqlite3\_expert\_analyze()** is called to run the analysis.
           41  +
           42  +  1. One or more calls are made to **sqlite3\_expert\_report()** to extract
           43  +     components of the results of the analysis.
           44  +
           45  +  1. **sqlite3\_expert\_destroy()** is called to free all resources.
           46  +
           47  +Refer to comments in sqlite3expert.h for further details.
           48  +
           49  +# sqlite3_expert application
           50  +
           51  +The file "expert.c" contains the code for a command line application that
           52  +uses the API described above. It can be compiled with (for example):
           53  +
           54  +<pre>
           55  +  gcc -O2 sqlite3.c expert.c sqlite3expert.c -o sqlite3_expert
           56  +</pre>
           57  +
           58  +Assuming the database is named "test.db", it can then be run to analyze a
           59  +single query:
           60  +
           61  +<pre>
           62  +  ./sqlite3_expert -sql &lt;sql-query&gt; test.db
           63  +</pre>
           64  +
           65  +Or an entire text file worth of queries with:
           66  +
           67  +<pre>
           68  +  ./sqlite3_expert -file &lt;text-file&gt; test.db
           69  +</pre>
           70  +
           71  +By default, sqlite3\_expert generates index statistics using all the data in
           72  +the user database. For a large database, this may be prohibitively time
           73  +consuming. The "-sample" option may be used to configure sqlite3\_expert to
           74  +generate statistics based on an integer percentage of the user database as
           75  +follows:
           76  +
           77  +<pre>
           78  +  # Generate statistics based on 25% of the user database rows:
           79  +  ./sqlite3_expert -sample 25 -sql &lt;sql-query&gt; test.db
           80  +
           81  +  # Do not generate any statistics at all:
           82  +  ./sqlite3_expert -sample 0 -sql &lt;sql-query&gt; test.db
           83  +</pre>

Added ext/expert/expert.c.

            1  +/*
            2  +** 2017 April 07
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +*/
           13  +
           14  +
           15  +#include <sqlite3.h>
           16  +#include <stdio.h>
           17  +#include <stdlib.h>
           18  +#include <string.h>
           19  +#include "sqlite3expert.h"
           20  +
           21  +
           22  +static void option_requires_argument(const char *zOpt){
           23  +  fprintf(stderr, "Option requires an argument: %s\n", zOpt);
           24  +  exit(-3);
           25  +}
           26  +
           27  +static int option_integer_arg(const char *zVal){
           28  +  return atoi(zVal);
           29  +}
           30  +
           31  +static void usage(char **argv){
           32  +  fprintf(stderr, "\n");
           33  +  fprintf(stderr, "Usage %s ?OPTIONS? DATABASE\n", argv[0]);
           34  +  fprintf(stderr, "\n");
           35  +  fprintf(stderr, "Options are:\n");
           36  +  fprintf(stderr, "  -sql SQL   (analyze SQL statements passed as argument)\n");
           37  +  fprintf(stderr, "  -file FILE (read SQL statements from file FILE)\n");
           38  +  fprintf(stderr, "  -verbose LEVEL (integer verbosity level. default 1)\n");
           39  +  fprintf(stderr, "  -sample PERCENT (percent of db to sample. default 100)\n");
           40  +  exit(-1);
           41  +}
           42  +
           43  +static int readSqlFromFile(sqlite3expert *p, const char *zFile, char **pzErr){
           44  +  FILE *in = fopen(zFile, "rb");
           45  +  long nIn;
           46  +  size_t nRead;
           47  +  char *pBuf;
           48  +  int rc;
           49  +  if( in==0 ){
           50  +    *pzErr = sqlite3_mprintf("failed to open file %s\n", zFile);
           51  +    return SQLITE_ERROR;
           52  +  }
           53  +  fseek(in, 0, SEEK_END);
           54  +  nIn = ftell(in);
           55  +  rewind(in);
           56  +  pBuf = sqlite3_malloc64( nIn+1 );
           57  +  nRead = fread(pBuf, nIn, 1, in);
           58  +  fclose(in);
           59  +  if( nRead!=1 ){
           60  +    sqlite3_free(pBuf);
           61  +    *pzErr = sqlite3_mprintf("failed to read file %s\n", zFile);
           62  +    return SQLITE_ERROR;
           63  +  }
           64  +  pBuf[nIn] = 0;
           65  +  rc = sqlite3_expert_sql(p, pBuf, pzErr);
           66  +  sqlite3_free(pBuf);
           67  +  return rc;
           68  +}
           69  +
           70  +int main(int argc, char **argv){
           71  +  const char *zDb;
           72  +  int rc = 0;
           73  +  char *zErr = 0;
           74  +  int i;
           75  +  int iVerbose = 1;               /* -verbose option */
           76  +
           77  +  sqlite3 *db = 0;
           78  +  sqlite3expert *p = 0;
           79  +
           80  +  if( argc<2 ) usage(argv);
           81  +  zDb = argv[argc-1];
           82  +  if( zDb[0]=='-' ) usage(argv);
           83  +  rc = sqlite3_open(zDb, &db);
           84  +  if( rc!=SQLITE_OK ){
           85  +    fprintf(stderr, "Cannot open db file: %s - %s\n", zDb, sqlite3_errmsg(db));
           86  +    exit(-2);
           87  +  }
           88  +
           89  +  p = sqlite3_expert_new(db, &zErr);
           90  +  if( p==0 ){
           91  +    fprintf(stderr, "Cannot run analysis: %s\n", zErr);
           92  +    rc = 1;
           93  +  }else{
           94  +    for(i=1; i<(argc-1); i++){
           95  +      char *zArg = argv[i];
           96  +      if( zArg[0]=='-' && zArg[1]=='-' && zArg[2]!=0 ) zArg++;
           97  +      int nArg = (int)strlen(zArg);
           98  +      if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-file", nArg) ){
           99  +        if( ++i==(argc-1) ) option_requires_argument("-file");
          100  +        rc = readSqlFromFile(p, argv[i], &zErr);
          101  +      }
          102  +
          103  +      else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){
          104  +        if( ++i==(argc-1) ) option_requires_argument("-sql");
          105  +        rc = sqlite3_expert_sql(p, argv[i], &zErr);
          106  +      }
          107  +
          108  +      else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sample", nArg) ){
          109  +        int iSample;
          110  +        if( ++i==(argc-1) ) option_requires_argument("-sample");
          111  +        iSample = option_integer_arg(argv[i]);
          112  +        sqlite3_expert_config(p, EXPERT_CONFIG_SAMPLE, iSample);
          113  +      }
          114  +
          115  +      else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-verbose", nArg) ){
          116  +        if( ++i==(argc-1) ) option_requires_argument("-verbose");
          117  +        iVerbose = option_integer_arg(argv[i]);
          118  +      }
          119  +
          120  +      else{
          121  +        usage(argv);
          122  +      }
          123  +    }
          124  +  }
          125  +
          126  +  if( rc==SQLITE_OK ){
          127  +    rc = sqlite3_expert_analyze(p, &zErr);
          128  +  }
          129  +
          130  +  if( rc==SQLITE_OK ){
          131  +    int nQuery = sqlite3_expert_count(p);
          132  +    if( iVerbose>0 ){
          133  +      const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES);
          134  +      fprintf(stdout, "-- Candidates -------------------------------\n");
          135  +      fprintf(stdout, "%s\n", zCand);
          136  +    }
          137  +    for(i=0; i<nQuery; i++){
          138  +      const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
          139  +      const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
          140  +      const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
          141  +      if( zIdx==0 ) zIdx = "(no new indexes)\n";
          142  +      if( iVerbose>0 ){
          143  +        fprintf(stdout, "-- Query %d ----------------------------------\n",i+1);
          144  +        fprintf(stdout, "%s\n\n", zSql);
          145  +      }
          146  +      fprintf(stdout, "%s\n%s\n", zIdx, zEQP);
          147  +    }
          148  +  }else{
          149  +    fprintf(stderr, "Error: %s\n", zErr ? zErr : "?");
          150  +  }
          151  +
          152  +  sqlite3_expert_destroy(p);
          153  +  sqlite3_free(zErr);
          154  +  return rc;
          155  +}

Added ext/expert/expert1.test.

            1  +# 2009 Nov 11
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# The focus of this file is testing the CLI shell tool. Specifically,
           13  +# the ".recommend" command.
           14  +#
           15  +#
           16  +
           17  +# Test plan:
           18  +#
           19  +#
           20  +if {![info exists testdir]} {
           21  +  set testdir [file join [file dirname [info script]] .. .. test]
           22  +}
           23  +source $testdir/tester.tcl
           24  +set testprefix expert1
           25  +
           26  +if {[info commands sqlite3_expert_new]==""} {
           27  +  finish_test
           28  +  return
           29  +}
           30  +
           31  +set CLI [test_binary_name sqlite3]
           32  +set CMD [test_binary_name sqlite3_expert]
           33  +
           34  +proc squish {txt} {
           35  +  regsub -all {[[:space:]]+} $txt { }
           36  +}
           37  +
           38  +proc do_setup_rec_test {tn setup sql res} {
           39  +  reset_db
           40  +  db eval $setup
           41  +  uplevel [list do_rec_test $tn $sql $res]
           42  +}
           43  +
           44  +foreach {tn setup} {
           45  +  1 {
           46  +    if {![file executable $CMD]} { continue }
           47  +
           48  +    proc do_rec_test {tn sql res} {
           49  +      set res [squish [string trim $res]]
           50  +      set tst [subst -nocommands { 
           51  +        squish [string trim [exec $::CMD -verbose 0 -sql {$sql;} test.db]]
           52  +      }]
           53  +      uplevel [list do_test $tn $tst $res]
           54  +    }
           55  +  }
           56  +  2 {
           57  +    if {[info commands sqlite3_expert_new]==""} { continue }
           58  +
           59  +    proc do_rec_test {tn sql res} {
           60  +      set expert [sqlite3_expert_new db]
           61  +      $expert sql $sql
           62  +      $expert analyze
           63  +
           64  +      set result [list]
           65  +      for {set i 0} {$i < [$expert count]} {incr i} {
           66  +        set idx [string trim [$expert report $i indexes]]
           67  +        if {$idx==""} {set idx "(no new indexes)"}
           68  +        lappend result $idx
           69  +        lappend result [string trim [$expert report $i plan]]
           70  +      }
           71  +
           72  +      $expert destroy
           73  +
           74  +      set tst [subst -nocommands {set {} [squish [join {$result}]]}]
           75  +      uplevel [list do_test $tn $tst [string trim [squish $res]]]
           76  +    }
           77  +  }
           78  +  3 {
           79  +    if {![file executable $CLI]} { continue }
           80  +
           81  +    proc do_rec_test {tn sql res} {
           82  +      set res [squish [string trim $res]]
           83  +      set tst [subst -nocommands { 
           84  +        squish [string trim [exec $::CLI test.db ".expert" {$sql;}]]
           85  +      }]
           86  +      uplevel [list do_test $tn $tst $res]
           87  +    }
           88  +  }
           89  +} {
           90  +
           91  +  eval $setup
           92  +
           93  +
           94  +do_setup_rec_test $tn.1 { CREATE TABLE t1(a, b, c) } {
           95  +  SELECT * FROM t1
           96  +} {
           97  +  (no new indexes)
           98  +  0|0|0|SCAN TABLE t1
           99  +}
          100  +
          101  +do_setup_rec_test $tn.2 {
          102  +  CREATE TABLE t1(a, b, c);
          103  +} {
          104  +  SELECT * FROM t1 WHERE b>?;
          105  +} {
          106  +  CREATE INDEX t1_idx_00000062 ON t1(b);
          107  +  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
          108  +}
          109  +
          110  +do_setup_rec_test $tn.3 {
          111  +  CREATE TABLE t1(a, b, c);
          112  +} {
          113  +  SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
          114  +} {
          115  +  CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE);
          116  +  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
          117  +}
          118  +
          119  +do_setup_rec_test $tn.4 {
          120  +  CREATE TABLE t1(a, b, c);
          121  +} {
          122  +  SELECT a FROM t1 ORDER BY b;
          123  +} {
          124  +  CREATE INDEX t1_idx_00000062 ON t1(b);
          125  +  0|0|0|SCAN TABLE t1 USING INDEX t1_idx_00000062
          126  +}
          127  +
          128  +do_setup_rec_test $tn.5 {
          129  +  CREATE TABLE t1(a, b, c);
          130  +} {
          131  +  SELECT a FROM t1 WHERE a=? ORDER BY b;
          132  +} {
          133  +  CREATE INDEX t1_idx_000123a7 ON t1(a, b);
          134  +  0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
          135  +}
          136  +
          137  +do_setup_rec_test $tn.6 {
          138  +  CREATE TABLE t1(a, b, c);
          139  +} {
          140  +  SELECT min(a) FROM t1
          141  +} {
          142  +  CREATE INDEX t1_idx_00000061 ON t1(a);
          143  +  0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
          144  +}
          145  +
          146  +do_setup_rec_test $tn.7 {
          147  +  CREATE TABLE t1(a, b, c);
          148  +} {
          149  +  SELECT * FROM t1 ORDER BY a, b, c;
          150  +} {
          151  +  CREATE INDEX t1_idx_033e95fe ON t1(a, b, c);
          152  +  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
          153  +}
          154  +
          155  +#do_setup_rec_test $tn.1.8 {
          156  +#  CREATE TABLE t1(a, b, c);
          157  +#} {
          158  +#  SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
          159  +#} {
          160  +#  CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c);
          161  +#  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
          162  +#}
          163  +
          164  +do_setup_rec_test $tn.8.1 {
          165  +  CREATE TABLE t1(a COLLATE NOCase, b, c);
          166  +} {
          167  +  SELECT * FROM t1 WHERE a=?
          168  +} {
          169  +  CREATE INDEX t1_idx_00000061 ON t1(a);
          170  +  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
          171  +}
          172  +do_setup_rec_test $tn.8.2 {
          173  +  CREATE TABLE t1(a, b COLLATE nocase, c);
          174  +} {
          175  +  SELECT * FROM t1 ORDER BY a ASC, b DESC, c ASC;
          176  +} {
          177  +  CREATE INDEX t1_idx_5cb97285 ON t1(a, b DESC, c);
          178  +  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5cb97285
          179  +}
          180  +
          181  +
          182  +# Tables with names that require quotes.
          183  +#
          184  +do_setup_rec_test $tn.9.1 {
          185  +  CREATE TABLE "t t"(a, b, c);
          186  +} {
          187  +  SELECT * FROM "t t" WHERE a=?
          188  +} {
          189  +  CREATE INDEX 't t_idx_00000061' ON 't t'(a);
          190  +  0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
          191  +}
          192  +
          193  +do_setup_rec_test $tn.9.2 {
          194  +  CREATE TABLE "t t"(a, b, c);
          195  +} {
          196  +  SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
          197  +} {
          198  +  CREATE INDEX 't t_idx_00000062' ON 't t'(b);
          199  +  0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000062 (b>? AND b<?)
          200  +}
          201  +
          202  +# Columns with names that require quotes.
          203  +#
          204  +do_setup_rec_test $tn.10.1 {
          205  +  CREATE TABLE t3(a, "b b", c);
          206  +} {
          207  +  SELECT * FROM t3 WHERE "b b" = ?
          208  +} {
          209  +  CREATE INDEX t3_idx_00050c52 ON t3('b b');
          210  +  0|0|0|SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
          211  +}
          212  +
          213  +do_setup_rec_test $tn.10.2 {
          214  +  CREATE TABLE t3(a, "b b", c);
          215  +} {
          216  +  SELECT * FROM t3 ORDER BY "b b"
          217  +} {
          218  +  CREATE INDEX t3_idx_00050c52 ON t3('b b');
          219  +  0|0|0|SCAN TABLE t3 USING INDEX t3_idx_00050c52
          220  +}
          221  +
          222  +# Transitive constraints
          223  +#
          224  +do_setup_rec_test $tn.11.1 {
          225  +  CREATE TABLE t5(a, b);
          226  +  CREATE TABLE t6(c, d);
          227  +} {
          228  +  SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
          229  +} {
          230  +  CREATE INDEX t5_idx_000123a7 ON t5(a, b);
          231  +  CREATE INDEX t6_idx_00000063 ON t6(c);
          232  +  0|0|1|SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?) 
          233  +  0|1|0|SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
          234  +}
          235  +
          236  +# OR terms.
          237  +#
          238  +do_setup_rec_test $tn.12.1 {
          239  +  CREATE TABLE t7(a, b);
          240  +} {
          241  +  SELECT * FROM t7 WHERE a=? OR b=?
          242  +} {
          243  +  CREATE INDEX t7_idx_00000062 ON t7(b);
          244  +  CREATE INDEX t7_idx_00000061 ON t7(a);
          245  +  0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
          246  +  0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
          247  +}
          248  +
          249  +# rowid terms.
          250  +#
          251  +do_setup_rec_test $tn.13.1 {
          252  +  CREATE TABLE t8(a, b);
          253  +} {
          254  +  SELECT * FROM t8 WHERE rowid=?
          255  +} {
          256  +  (no new indexes)
          257  +  0|0|0|SEARCH TABLE t8 USING INTEGER PRIMARY KEY (rowid=?)
          258  +}
          259  +do_setup_rec_test $tn.13.2 {
          260  +  CREATE TABLE t8(a, b);
          261  +} {
          262  +  SELECT * FROM t8 ORDER BY rowid
          263  +} {
          264  +  (no new indexes)
          265  +  0|0|0|SCAN TABLE t8
          266  +}
          267  +do_setup_rec_test $tn.13.3 {
          268  +  CREATE TABLE t8(a, b);
          269  +} {
          270  +  SELECT * FROM t8 WHERE a=? ORDER BY rowid
          271  +} {
          272  +  CREATE INDEX t8_idx_00000061 ON t8(a); 
          273  +  0|0|0|SEARCH TABLE t8 USING INDEX t8_idx_00000061 (a=?)
          274  +}
          275  +
          276  +# Triggers
          277  +#
          278  +do_setup_rec_test $tn.14 {
          279  +  CREATE TABLE t9(a, b, c);
          280  +  CREATE TABLE t10(a, b, c);
          281  +  CREATE TRIGGER t9t AFTER INSERT ON t9 BEGIN
          282  +    UPDATE t10 SET a=new.a WHERE b = new.b;
          283  +  END;
          284  +} {
          285  +  INSERT INTO t9 VALUES(?, ?, ?);
          286  +} {
          287  +  CREATE INDEX t10_idx_00000062 ON t10(b); 
          288  +  0|0|0|SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
          289  +}
          290  +
          291  +do_setup_rec_test $tn.15 {
          292  +  CREATE TABLE t1(a, b);
          293  +  CREATE TABLE t2(c, d);
          294  +
          295  +  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
          296  +  INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
          297  +
          298  +  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
          299  +  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
          300  +} {
          301  +  SELECT * FROM t2, t1 WHERE b=? AND d=? AND t2.rowid=t1.rowid
          302  +} {
          303  +  CREATE INDEX t2_idx_00000064 ON t2(d);
          304  +  0|0|0|SEARCH TABLE t2 USING INDEX t2_idx_00000064 (d=?) 
          305  +  0|1|1|SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)
          306  +}
          307  +
          308  +do_setup_rec_test $tn.16 {
          309  +  CREATE TABLE t1(a, b);
          310  +} {
          311  +  SELECT * FROM t1 WHERE b IS NOT NULL;
          312  +} {
          313  +  (no new indexes)
          314  +  0|0|0|SCAN TABLE t1
          315  +}
          316  +
          317  +}
          318  +
          319  +proc do_candidates_test {tn sql res} {
          320  +  set res [squish [string trim $res]]
          321  +
          322  +  set expert [sqlite3_expert_new db]
          323  +  $expert sql $sql
          324  +  $expert analyze
          325  +
          326  +  set candidates [squish [string trim [$expert report 0 candidates]]]
          327  +  $expert destroy
          328  +
          329  +  uplevel [list do_test $tn [list set {} $candidates] $res]
          330  +}
          331  +
          332  +
          333  +reset_db
          334  +do_execsql_test 3.0 {
          335  +  CREATE TABLE t1(a, b);
          336  +  CREATE TABLE t2(c, d);
          337  +
          338  +  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
          339  +  INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
          340  +
          341  +  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
          342  +  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
          343  +}
          344  +do_candidates_test 3.1 {
          345  +  SELECT * FROM t1,t2 WHERE (b=? OR a=?) AND (c=? OR d=?)
          346  +} {
          347  +  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
          348  +  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
          349  +  CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 
          350  +  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
          351  +}
          352  +
          353  +do_candidates_test 3.2 {
          354  +  SELECT * FROM t1,t2 WHERE a=? AND b=? AND c=? AND d=?
          355  +} {
          356  +  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 17
          357  +  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
          358  +}
          359  +
          360  +do_execsql_test 3.2 {
          361  +  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
          362  +  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
          363  +  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 16
          364  +
          365  +  CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 
          366  +  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
          367  +  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
          368  +
          369  +  ANALYZE;
          370  +  SELECT * FROM sqlite_stat1 ORDER BY 1, 2;
          371  +} {
          372  +  t1 t1_idx_00000061 {100 50} 
          373  +  t1 t1_idx_00000062 {100 20}
          374  +  t1 t1_idx_000123a7 {100 50 17}
          375  +  t2 t2_idx_00000063 {100 20} 
          376  +  t2 t2_idx_00000064 {100 5} 
          377  +  t2 t2_idx_0001295b {100 20 5}
          378  +}
          379  +
          380  +
          381  +finish_test
          382  +

Added ext/expert/sqlite3expert.c.

            1  +/*
            2  +** 2017 April 09
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +*/
           13  +#include "sqlite3expert.h"
           14  +#include <assert.h>
           15  +#include <string.h>
           16  +#include <stdio.h>
           17  +
           18  +#ifndef SQLITE_OMIT_VIRTUALTABLE 
           19  +
           20  +typedef sqlite3_int64 i64;
           21  +typedef sqlite3_uint64 u64;
           22  +
           23  +typedef struct IdxColumn IdxColumn;
           24  +typedef struct IdxConstraint IdxConstraint;
           25  +typedef struct IdxScan IdxScan;
           26  +typedef struct IdxStatement IdxStatement;
           27  +typedef struct IdxTable IdxTable;
           28  +typedef struct IdxWrite IdxWrite;
           29  +
           30  +#define STRLEN  (int)strlen
           31  +
           32  +/*
           33  +** A temp table name that we assume no user database will actually use.
           34  +** If this assumption proves incorrect triggers on the table with the
           35  +** conflicting name will be ignored.
           36  +*/
           37  +#define UNIQUE_TABLE_NAME "t592690916721053953805701627921227776"
           38  +
           39  +/*
           40  +** A single constraint. Equivalent to either "col = ?" or "col < ?" (or
           41  +** any other type of single-ended range constraint on a column).
           42  +**
           43  +** pLink:
           44  +**   Used to temporarily link IdxConstraint objects into lists while
           45  +**   creating candidate indexes.
           46  +*/
           47  +struct IdxConstraint {
           48  +  char *zColl;                    /* Collation sequence */
           49  +  int bRange;                     /* True for range, false for eq */
           50  +  int iCol;                       /* Constrained table column */
           51  +  int bFlag;                      /* Used by idxFindCompatible() */
           52  +  int bDesc;                      /* True if ORDER BY <expr> DESC */
           53  +  IdxConstraint *pNext;           /* Next constraint in pEq or pRange list */
           54  +  IdxConstraint *pLink;           /* See above */
           55  +};
           56  +
           57  +/*
           58  +** A single scan of a single table.
           59  +*/
           60  +struct IdxScan {
           61  +  IdxTable *pTab;                 /* Associated table object */
           62  +  int iDb;                        /* Database containing table zTable */
           63  +  i64 covering;                   /* Mask of columns required for cov. index */
           64  +  IdxConstraint *pOrder;          /* ORDER BY columns */
           65  +  IdxConstraint *pEq;             /* List of == constraints */
           66  +  IdxConstraint *pRange;          /* List of < constraints */
           67  +  IdxScan *pNextScan;             /* Next IdxScan object for same analysis */
           68  +};
           69  +
           70  +/*
           71  +** Information regarding a single database table. Extracted from 
           72  +** "PRAGMA table_info" by function idxGetTableInfo().
           73  +*/
           74  +struct IdxColumn {
           75  +  char *zName;
           76  +  char *zColl;
           77  +  int iPk;
           78  +};
           79  +struct IdxTable {
           80  +  int nCol;
           81  +  char *zName;                    /* Table name */
           82  +  IdxColumn *aCol;
           83  +  IdxTable *pNext;                /* Next table in linked list of all tables */
           84  +};
           85  +
           86  +/*
           87  +** An object of the following type is created for each unique table/write-op
           88  +** seen. The objects are stored in a singly-linked list beginning at
           89  +** sqlite3expert.pWrite.
           90  +*/
           91  +struct IdxWrite {
           92  +  IdxTable *pTab;
           93  +  int eOp;                        /* SQLITE_UPDATE, DELETE or INSERT */
           94  +  IdxWrite *pNext;
           95  +};
           96  +
           97  +/*
           98  +** Each statement being analyzed is represented by an instance of this
           99  +** structure.
          100  +*/
          101  +struct IdxStatement {
          102  +  int iId;                        /* Statement number */
          103  +  char *zSql;                     /* SQL statement */
          104  +  char *zIdx;                     /* Indexes */
          105  +  char *zEQP;                     /* Plan */
          106  +  IdxStatement *pNext;
          107  +};
          108  +
          109  +
          110  +/*
          111  +** A hash table for storing strings. With space for a payload string
          112  +** with each entry. Methods are:
          113  +**
          114  +**   idxHashInit()
          115  +**   idxHashClear()
          116  +**   idxHashAdd()
          117  +**   idxHashSearch()
          118  +*/
          119  +#define IDX_HASH_SIZE 1023
          120  +typedef struct IdxHashEntry IdxHashEntry;
          121  +typedef struct IdxHash IdxHash;
          122  +struct IdxHashEntry {
          123  +  char *zKey;                     /* nul-terminated key */
          124  +  char *zVal;                     /* nul-terminated value string */
          125  +  char *zVal2;                    /* nul-terminated value string 2 */
          126  +  IdxHashEntry *pHashNext;        /* Next entry in same hash bucket */
          127  +  IdxHashEntry *pNext;            /* Next entry in hash */
          128  +};
          129  +struct IdxHash {
          130  +  IdxHashEntry *pFirst;
          131  +  IdxHashEntry *aHash[IDX_HASH_SIZE];
          132  +};
          133  +
          134  +/*
          135  +** sqlite3expert object.
          136  +*/
          137  +struct sqlite3expert {
          138  +  int iSample;                    /* Percentage of tables to sample for stat1 */
          139  +  sqlite3 *db;                    /* User database */
          140  +  sqlite3 *dbm;                   /* In-memory db for this analysis */
          141  +  sqlite3 *dbv;                   /* Vtab schema for this analysis */
          142  +  IdxTable *pTable;               /* List of all IdxTable objects */
          143  +  IdxScan *pScan;                 /* List of scan objects */
          144  +  IdxWrite *pWrite;               /* List of write objects */
          145  +  IdxStatement *pStatement;       /* List of IdxStatement objects */
          146  +  int bRun;                       /* True once analysis has run */
          147  +  char **pzErrmsg;
          148  +  int rc;                         /* Error code from whereinfo hook */
          149  +  IdxHash hIdx;                   /* Hash containing all candidate indexes */
          150  +  char *zCandidates;              /* For EXPERT_REPORT_CANDIDATES */
          151  +};
          152  +
          153  +
          154  +/*
          155  +** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). 
          156  +** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL.
          157  +*/
          158  +static void *idxMalloc(int *pRc, int nByte){
          159  +  void *pRet;
          160  +  assert( *pRc==SQLITE_OK );
          161  +  assert( nByte>0 );
          162  +  pRet = sqlite3_malloc(nByte);
          163  +  if( pRet ){
          164  +    memset(pRet, 0, nByte);
          165  +  }else{
          166  +    *pRc = SQLITE_NOMEM;
          167  +  }
          168  +  return pRet;
          169  +}
          170  +
          171  +/*
          172  +** Initialize an IdxHash hash table.
          173  +*/
          174  +static void idxHashInit(IdxHash *pHash){
          175  +  memset(pHash, 0, sizeof(IdxHash));
          176  +}
          177  +
          178  +/*
          179  +** Reset an IdxHash hash table.
          180  +*/
          181  +static void idxHashClear(IdxHash *pHash){
          182  +  int i;
          183  +  for(i=0; i<IDX_HASH_SIZE; i++){
          184  +    IdxHashEntry *pEntry;
          185  +    IdxHashEntry *pNext;
          186  +    for(pEntry=pHash->aHash[i]; pEntry; pEntry=pNext){
          187  +      pNext = pEntry->pHashNext;
          188  +      sqlite3_free(pEntry->zVal2);
          189  +      sqlite3_free(pEntry);
          190  +    }
          191  +  }
          192  +  memset(pHash, 0, sizeof(IdxHash));
          193  +}
          194  +
          195  +/*
          196  +** Return the index of the hash bucket that the string specified by the
          197  +** arguments to this function belongs.
          198  +*/
          199  +static int idxHashString(const char *z, int n){
          200  +  unsigned int ret = 0;
          201  +  int i;
          202  +  for(i=0; i<n; i++){
          203  +    ret += (ret<<3) + (unsigned char)(z[i]);
          204  +  }
          205  +  return (int)(ret % IDX_HASH_SIZE);
          206  +}
          207  +
          208  +/*
          209  +** If zKey is already present in the hash table, return non-zero and do
          210  +** nothing. Otherwise, add an entry with key zKey and payload string zVal to
          211  +** the hash table passed as the second argument. 
          212  +*/
          213  +static int idxHashAdd(
          214  +  int *pRc, 
          215  +  IdxHash *pHash, 
          216  +  const char *zKey,
          217  +  const char *zVal
          218  +){
          219  +  int nKey = STRLEN(zKey);
          220  +  int iHash = idxHashString(zKey, nKey);
          221  +  int nVal = (zVal ? STRLEN(zVal) : 0);
          222  +  IdxHashEntry *pEntry;
          223  +  assert( iHash>=0 );
          224  +  for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
          225  +    if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
          226  +      return 1;
          227  +    }
          228  +  }
          229  +  pEntry = idxMalloc(pRc, sizeof(IdxHashEntry) + nKey+1 + nVal+1);
          230  +  if( pEntry ){
          231  +    pEntry->zKey = (char*)&pEntry[1];
          232  +    memcpy(pEntry->zKey, zKey, nKey);
          233  +    if( zVal ){
          234  +      pEntry->zVal = &pEntry->zKey[nKey+1];
          235  +      memcpy(pEntry->zVal, zVal, nVal);
          236  +    }
          237  +    pEntry->pHashNext = pHash->aHash[iHash];
          238  +    pHash->aHash[iHash] = pEntry;
          239  +
          240  +    pEntry->pNext = pHash->pFirst;
          241  +    pHash->pFirst = pEntry;
          242  +  }
          243  +  return 0;
          244  +}
          245  +
          246  +/*
          247  +** If zKey/nKey is present in the hash table, return a pointer to the 
          248  +** hash-entry object.
          249  +*/
          250  +static IdxHashEntry *idxHashFind(IdxHash *pHash, const char *zKey, int nKey){
          251  +  int iHash;
          252  +  IdxHashEntry *pEntry;
          253  +  if( nKey<0 ) nKey = STRLEN(zKey);
          254  +  iHash = idxHashString(zKey, nKey);
          255  +  assert( iHash>=0 );
          256  +  for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
          257  +    if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
          258  +      return pEntry;
          259  +    }
          260  +  }
          261  +  return 0;
          262  +}
          263  +
          264  +/*
          265  +** If the hash table contains an entry with a key equal to the string
          266  +** passed as the final two arguments to this function, return a pointer
          267  +** to the payload string. Otherwise, if zKey/nKey is not present in the
          268  +** hash table, return NULL.
          269  +*/
          270  +static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
          271  +  IdxHashEntry *pEntry = idxHashFind(pHash, zKey, nKey);
          272  +  if( pEntry ) return pEntry->zVal;
          273  +  return 0;
          274  +}
          275  +
          276  +/*
          277  +** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl
          278  +** variable to point to a copy of nul-terminated string zColl.
          279  +*/
          280  +static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
          281  +  IdxConstraint *pNew;
          282  +  int nColl = STRLEN(zColl);
          283  +
          284  +  assert( *pRc==SQLITE_OK );
          285  +  pNew = (IdxConstraint*)idxMalloc(pRc, sizeof(IdxConstraint) * nColl + 1);
          286  +  if( pNew ){
          287  +    pNew->zColl = (char*)&pNew[1];
          288  +    memcpy(pNew->zColl, zColl, nColl+1);
          289  +  }
          290  +  return pNew;
          291  +}
          292  +
          293  +/*
          294  +** An error associated with database handle db has just occurred. Pass
          295  +** the error message to callback function xOut.
          296  +*/
          297  +static void idxDatabaseError(
          298  +  sqlite3 *db,                    /* Database handle */
          299  +  char **pzErrmsg                 /* Write error here */
          300  +){
          301  +  *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
          302  +}
          303  +
          304  +/*
          305  +** Prepare an SQL statement.
          306  +*/
          307  +static int idxPrepareStmt(
          308  +  sqlite3 *db,                    /* Database handle to compile against */
          309  +  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
          310  +  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
          311  +  const char *zSql                /* SQL statement to compile */
          312  +){
          313  +  int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
          314  +  if( rc!=SQLITE_OK ){
          315  +    *ppStmt = 0;
          316  +    idxDatabaseError(db, pzErrmsg);
          317  +  }
          318  +  return rc;
          319  +}
          320  +
          321  +/*
          322  +** Prepare an SQL statement using the results of a printf() formatting.
          323  +*/
          324  +static int idxPrintfPrepareStmt(
          325  +  sqlite3 *db,                    /* Database handle to compile against */
          326  +  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
          327  +  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
          328  +  const char *zFmt,               /* printf() format of SQL statement */
          329  +  ...                             /* Trailing printf() arguments */
          330  +){
          331  +  va_list ap;
          332  +  int rc;
          333  +  char *zSql;
          334  +  va_start(ap, zFmt);
          335  +  zSql = sqlite3_vmprintf(zFmt, ap);
          336  +  if( zSql==0 ){
          337  +    rc = SQLITE_NOMEM;
          338  +  }else{
          339  +    rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql);
          340  +    sqlite3_free(zSql);
          341  +  }
          342  +  va_end(ap);
          343  +  return rc;
          344  +}
          345  +
          346  +
          347  +/*************************************************************************
          348  +** Beginning of virtual table implementation.
          349  +*/
          350  +typedef struct ExpertVtab ExpertVtab;
          351  +struct ExpertVtab {
          352  +  sqlite3_vtab base;
          353  +  IdxTable *pTab;
          354  +  sqlite3expert *pExpert;
          355  +};
          356  +
          357  +typedef struct ExpertCsr ExpertCsr;
          358  +struct ExpertCsr {
          359  +  sqlite3_vtab_cursor base;
          360  +  sqlite3_stmt *pData;
          361  +};
          362  +
          363  +static char *expertDequote(const char *zIn){
          364  +  int n = STRLEN(zIn);
          365  +  char *zRet = sqlite3_malloc(n);
          366  +
          367  +  assert( zIn[0]=='\'' );
          368  +  assert( zIn[n-1]=='\'' );
          369  +
          370  +  if( zRet ){
          371  +    int iOut = 0;
          372  +    int iIn = 0;
          373  +    for(iIn=1; iIn<(n-1); iIn++){
          374  +      if( zIn[iIn]=='\'' ){
          375  +        assert( zIn[iIn+1]=='\'' );
          376  +        iIn++;
          377  +      }
          378  +      zRet[iOut++] = zIn[iIn];
          379  +    }
          380  +    zRet[iOut] = '\0';
          381  +  }
          382  +
          383  +  return zRet;
          384  +}
          385  +
          386  +/* 
          387  +** This function is the implementation of both the xConnect and xCreate
          388  +** methods of the r-tree virtual table.
          389  +**
          390  +**   argv[0]   -> module name
          391  +**   argv[1]   -> database name
          392  +**   argv[2]   -> table name
          393  +**   argv[...] -> column names...
          394  +*/
          395  +static int expertConnect(
          396  +  sqlite3 *db,
          397  +  void *pAux,
          398  +  int argc, const char *const*argv,
          399  +  sqlite3_vtab **ppVtab,
          400  +  char **pzErr
          401  +){
          402  +  sqlite3expert *pExpert = (sqlite3expert*)pAux;
          403  +  ExpertVtab *p = 0;
          404  +  int rc;
          405  +
          406  +  if( argc!=4 ){
          407  +    *pzErr = sqlite3_mprintf("internal error!");
          408  +    rc = SQLITE_ERROR;
          409  +  }else{
          410  +    char *zCreateTable = expertDequote(argv[3]);
          411  +    if( zCreateTable ){
          412  +      rc = sqlite3_declare_vtab(db, zCreateTable);
          413  +      if( rc==SQLITE_OK ){
          414  +        p = idxMalloc(&rc, sizeof(ExpertVtab));
          415  +      }
          416  +      if( rc==SQLITE_OK ){
          417  +        p->pExpert = pExpert;
          418  +        p->pTab = pExpert->pTable;
          419  +        assert( sqlite3_stricmp(p->pTab->zName, argv[2])==0 );
          420  +      }
          421  +      sqlite3_free(zCreateTable);
          422  +    }else{
          423  +      rc = SQLITE_NOMEM;
          424  +    }
          425  +  }
          426  +
          427  +  *ppVtab = (sqlite3_vtab*)p;
          428  +  return rc;
          429  +}
          430  +
          431  +static int expertDisconnect(sqlite3_vtab *pVtab){
          432  +  ExpertVtab *p = (ExpertVtab*)pVtab;
          433  +  sqlite3_free(p);
          434  +  return SQLITE_OK;
          435  +}
          436  +
          437  +static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
          438  +  ExpertVtab *p = (ExpertVtab*)pVtab;
          439  +  int rc = SQLITE_OK;
          440  +  int n = 0;
          441  +  IdxScan *pScan;
          442  +  const int opmask = 
          443  +    SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_GT |
          444  +    SQLITE_INDEX_CONSTRAINT_LT | SQLITE_INDEX_CONSTRAINT_GE |
          445  +    SQLITE_INDEX_CONSTRAINT_LE;
          446  +
          447  +  pScan = idxMalloc(&rc, sizeof(IdxScan));
          448  +  if( pScan ){
          449  +    int i;
          450  +
          451  +    /* Link the new scan object into the list */
          452  +    pScan->pTab = p->pTab;
          453  +    pScan->pNextScan = p->pExpert->pScan;
          454  +    p->pExpert->pScan = pScan;
          455  +
          456  +    /* Add the constraints to the IdxScan object */
          457  +    for(i=0; i<pIdxInfo->nConstraint; i++){
          458  +      struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
          459  +      if( pCons->usable 
          460  +       && pCons->iColumn>=0 
          461  +       && p->pTab->aCol[pCons->iColumn].iPk==0
          462  +       && (pCons->op & opmask) 
          463  +      ){
          464  +        IdxConstraint *pNew;
          465  +        const char *zColl = sqlite3_vtab_collation(pIdxInfo, i);
          466  +        pNew = idxNewConstraint(&rc, zColl);
          467  +        if( pNew ){
          468  +          pNew->iCol = pCons->iColumn;
          469  +          if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){
          470  +            pNew->pNext = pScan->pEq;
          471  +            pScan->pEq = pNew;
          472  +          }else{
          473  +            pNew->bRange = 1;
          474  +            pNew->pNext = pScan->pRange;
          475  +            pScan->pRange = pNew;
          476  +          }
          477  +        }
          478  +        n++;
          479  +        pIdxInfo->aConstraintUsage[i].argvIndex = n;
          480  +      }
          481  +    }
          482  +
          483  +    /* Add the ORDER BY to the IdxScan object */
          484  +    for(i=pIdxInfo->nOrderBy-1; i>=0; i--){
          485  +      int iCol = pIdxInfo->aOrderBy[i].iColumn;
          486  +      if( iCol>=0 ){
          487  +        IdxConstraint *pNew = idxNewConstraint(&rc, p->pTab->aCol[iCol].zColl);
          488  +        if( pNew ){
          489  +          pNew->iCol = iCol;
          490  +          pNew->bDesc = pIdxInfo->aOrderBy[i].desc;
          491  +          pNew->pNext = pScan->pOrder;
          492  +          pNew->pLink = pScan->pOrder;
          493  +          pScan->pOrder = pNew;
          494  +          n++;
          495  +        }
          496  +      }
          497  +    }
          498  +  }
          499  +
          500  +  pIdxInfo->estimatedCost = 1000000.0 / (n+1);
          501  +  return rc;
          502  +}
          503  +
          504  +static int expertUpdate(
          505  +  sqlite3_vtab *pVtab, 
          506  +  int nData, 
          507  +  sqlite3_value **azData, 
          508  +  sqlite_int64 *pRowid
          509  +){
          510  +  return SQLITE_OK;
          511  +}
          512  +
          513  +/* 
          514  +** Virtual table module xOpen method.
          515  +*/
          516  +static int expertOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
          517  +  int rc = SQLITE_OK;
          518  +  ExpertCsr *pCsr;
          519  +  pCsr = idxMalloc(&rc, sizeof(ExpertCsr));
          520  +  *ppCursor = (sqlite3_vtab_cursor*)pCsr;
          521  +  return rc;
          522  +}
          523  +
          524  +/* 
          525  +** Virtual table module xClose method.
          526  +*/
          527  +static int expertClose(sqlite3_vtab_cursor *cur){
          528  +  ExpertCsr *pCsr = (ExpertCsr*)cur;
          529  +  sqlite3_finalize(pCsr->pData);
          530  +  sqlite3_free(pCsr);
          531  +  return SQLITE_OK;
          532  +}
          533  +
          534  +/*
          535  +** Virtual table module xEof method.
          536  +**
          537  +** Return non-zero if the cursor does not currently point to a valid 
          538  +** record (i.e if the scan has finished), or zero otherwise.
          539  +*/
          540  +static int expertEof(sqlite3_vtab_cursor *cur){
          541  +  ExpertCsr *pCsr = (ExpertCsr*)cur;
          542  +  return pCsr->pData==0;
          543  +}
          544  +
          545  +/* 
          546  +** Virtual table module xNext method.
          547  +*/
          548  +static int expertNext(sqlite3_vtab_cursor *cur){
          549  +  ExpertCsr *pCsr = (ExpertCsr*)cur;
          550  +  int rc = SQLITE_OK;
          551  +
          552  +  assert( pCsr->pData );
          553  +  rc = sqlite3_step(pCsr->pData);
          554  +  if( rc!=SQLITE_ROW ){
          555  +    rc = sqlite3_finalize(pCsr->pData);
          556  +    pCsr->pData = 0;
          557  +  }else{
          558  +    rc = SQLITE_OK;
          559  +  }
          560  +
          561  +  return rc;
          562  +}
          563  +
          564  +/* 
          565  +** Virtual table module xRowid method.
          566  +*/
          567  +static int expertRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
          568  +  *pRowid = 0;
          569  +  return SQLITE_OK;
          570  +}
          571  +
          572  +/* 
          573  +** Virtual table module xColumn method.
          574  +*/
          575  +static int expertColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
          576  +  ExpertCsr *pCsr = (ExpertCsr*)cur;
          577  +  sqlite3_value *pVal;
          578  +  pVal = sqlite3_column_value(pCsr->pData, i);
          579  +  if( pVal ){
          580  +    sqlite3_result_value(ctx, pVal);
          581  +  }
          582  +  return SQLITE_OK;
          583  +}
          584  +
          585  +/* 
          586  +** Virtual table module xFilter method.
          587  +*/
          588  +static int expertFilter(
          589  +  sqlite3_vtab_cursor *cur, 
          590  +  int idxNum, const char *idxStr,
          591  +  int argc, sqlite3_value **argv
          592  +){
          593  +  ExpertCsr *pCsr = (ExpertCsr*)cur;
          594  +  ExpertVtab *pVtab = (ExpertVtab*)(cur->pVtab);
          595  +  sqlite3expert *pExpert = pVtab->pExpert;
          596  +  int rc;
          597  +
          598  +  rc = sqlite3_finalize(pCsr->pData);
          599  +  pCsr->pData = 0;
          600  +  if( rc==SQLITE_OK ){
          601  +    rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg,
          602  +        "SELECT * FROM main.%Q WHERE sample()", pVtab->pTab->zName
          603  +    );
          604  +  }
          605  +
          606  +  if( rc==SQLITE_OK ){
          607  +    rc = expertNext(cur);
          608  +  }
          609  +  return rc;
          610  +}
          611  +
          612  +static int idxRegisterVtab(sqlite3expert *p){
          613  +  static sqlite3_module expertModule = {
          614  +    2,                            /* iVersion */
          615  +    expertConnect,                /* xCreate - create a table */
          616  +    expertConnect,                /* xConnect - connect to an existing table */
          617  +    expertBestIndex,              /* xBestIndex - Determine search strategy */
          618  +    expertDisconnect,             /* xDisconnect - Disconnect from a table */
          619  +    expertDisconnect,             /* xDestroy - Drop a table */
          620  +    expertOpen,                   /* xOpen - open a cursor */
          621  +    expertClose,                  /* xClose - close a cursor */
          622  +    expertFilter,                 /* xFilter - configure scan constraints */
          623  +    expertNext,                   /* xNext - advance a cursor */
          624  +    expertEof,                    /* xEof */
          625  +    expertColumn,                 /* xColumn - read data */
          626  +    expertRowid,                  /* xRowid - read data */
          627  +    expertUpdate,                 /* xUpdate - write data */
          628  +    0,                            /* xBegin - begin transaction */
          629  +    0,                            /* xSync - sync transaction */
          630  +    0,                            /* xCommit - commit transaction */
          631  +    0,                            /* xRollback - rollback transaction */
          632  +    0,                            /* xFindFunction - function overloading */
          633  +    0,                            /* xRename - rename the table */
          634  +    0,                            /* xSavepoint */
          635  +    0,                            /* xRelease */
          636  +    0,                            /* xRollbackTo */
          637  +  };
          638  +
          639  +  return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p);
          640  +}
          641  +/*
          642  +** End of virtual table implementation.
          643  +*************************************************************************/
          644  +/*
          645  +** Finalize SQL statement pStmt. If (*pRc) is SQLITE_OK when this function
          646  +** is called, set it to the return value of sqlite3_finalize() before
          647  +** returning. Otherwise, discard the sqlite3_finalize() return value.
          648  +*/
          649  +static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){
          650  +  int rc = sqlite3_finalize(pStmt);
          651  +  if( *pRc==SQLITE_OK ) *pRc = rc;
          652  +}
          653  +
          654  +/*
          655  +** Attempt to allocate an IdxTable structure corresponding to table zTab
          656  +** in the main database of connection db. If successful, set (*ppOut) to
          657  +** point to the new object and return SQLITE_OK. Otherwise, return an
          658  +** SQLite error code and set (*ppOut) to NULL. In this case *pzErrmsg may be
          659  +** set to point to an error string.
          660  +**
          661  +** It is the responsibility of the caller to eventually free either the
          662  +** IdxTable object or error message using sqlite3_free().
          663  +*/
          664  +static int idxGetTableInfo(
          665  +  sqlite3 *db,                    /* Database connection to read details from */
          666  +  const char *zTab,               /* Table name */
          667  +  IdxTable **ppOut,               /* OUT: New object (if successful) */
          668  +  char **pzErrmsg                 /* OUT: Error message (if not) */
          669  +){
          670  +  sqlite3_stmt *p1 = 0;
          671  +  int nCol = 0;
          672  +  int nTab = STRLEN(zTab);
          673  +  int nByte = sizeof(IdxTable) + nTab + 1;
          674  +  IdxTable *pNew = 0;
          675  +  int rc, rc2;
          676  +  char *pCsr = 0;
          677  +
          678  +  rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTab);
          679  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
          680  +    const char *zCol = (const char*)sqlite3_column_text(p1, 1);
          681  +    nByte += 1 + STRLEN(zCol);
          682  +    rc = sqlite3_table_column_metadata(
          683  +        db, "main", zTab, zCol, 0, &zCol, 0, 0, 0
          684  +    );
          685  +    nByte += 1 + STRLEN(zCol);
          686  +    nCol++;
          687  +  }
          688  +  rc2 = sqlite3_reset(p1);
          689  +  if( rc==SQLITE_OK ) rc = rc2;
          690  +
          691  +  nByte += sizeof(IdxColumn) * nCol;
          692  +  if( rc==SQLITE_OK ){
          693  +    pNew = idxMalloc(&rc, nByte);
          694  +  }
          695  +  if( rc==SQLITE_OK ){
          696  +    pNew->aCol = (IdxColumn*)&pNew[1];
          697  +    pNew->nCol = nCol;
          698  +    pCsr = (char*)&pNew->aCol[nCol];
          699  +  }
          700  +
          701  +  nCol = 0;
          702  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
          703  +    const char *zCol = (const char*)sqlite3_column_text(p1, 1);
          704  +    int nCopy = STRLEN(zCol) + 1;
          705  +    pNew->aCol[nCol].zName = pCsr;
          706  +    pNew->aCol[nCol].iPk = sqlite3_column_int(p1, 5);
          707  +    memcpy(pCsr, zCol, nCopy);
          708  +    pCsr += nCopy;
          709  +
          710  +    rc = sqlite3_table_column_metadata(
          711  +        db, "main", zTab, zCol, 0, &zCol, 0, 0, 0
          712  +    );
          713  +    if( rc==SQLITE_OK ){
          714  +      nCopy = STRLEN(zCol) + 1;
          715  +      pNew->aCol[nCol].zColl = pCsr;
          716  +      memcpy(pCsr, zCol, nCopy);
          717  +      pCsr += nCopy;
          718  +    }
          719  +
          720  +    nCol++;
          721  +  }
          722  +  idxFinalize(&rc, p1);
          723  +
          724  +  if( rc!=SQLITE_OK ){
          725  +    sqlite3_free(pNew);
          726  +    pNew = 0;
          727  +  }else{
          728  +    pNew->zName = pCsr;
          729  +    memcpy(pNew->zName, zTab, nTab+1);
          730  +  }
          731  +
          732  +  *ppOut = pNew;
          733  +  return rc;
          734  +}
          735  +
          736  +/*
          737  +** This function is a no-op if *pRc is set to anything other than 
          738  +** SQLITE_OK when it is called.
          739  +**
          740  +** If *pRc is initially set to SQLITE_OK, then the text specified by
          741  +** the printf() style arguments is appended to zIn and the result returned
          742  +** in a buffer allocated by sqlite3_malloc(). sqlite3_free() is called on
          743  +** zIn before returning.
          744  +*/
          745  +static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
          746  +  va_list ap;
          747  +  char *zAppend = 0;
          748  +  char *zRet = 0;
          749  +  int nIn = zIn ? STRLEN(zIn) : 0;
          750  +  int nAppend = 0;
          751  +  va_start(ap, zFmt);
          752  +  if( *pRc==SQLITE_OK ){
          753  +    zAppend = sqlite3_vmprintf(zFmt, ap);
          754  +    if( zAppend ){
          755  +      nAppend = STRLEN(zAppend);
          756  +      zRet = (char*)sqlite3_malloc(nIn + nAppend + 1);
          757  +    }
          758  +    if( zAppend && zRet ){
          759  +      if( nIn ) memcpy(zRet, zIn, nIn);
          760  +      memcpy(&zRet[nIn], zAppend, nAppend+1);
          761  +    }else{
          762  +      sqlite3_free(zRet);
          763  +      zRet = 0;
          764  +      *pRc = SQLITE_NOMEM;
          765  +    }
          766  +    sqlite3_free(zAppend);
          767  +    sqlite3_free(zIn);
          768  +  }
          769  +  va_end(ap);
          770  +  return zRet;
          771  +}
          772  +
          773  +/*
          774  +** Return true if zId must be quoted in order to use it as an SQL
          775  +** identifier, or false otherwise.
          776  +*/
          777  +static int idxIdentifierRequiresQuotes(const char *zId){
          778  +  int i;
          779  +  for(i=0; zId[i]; i++){
          780  +    if( !(zId[i]=='_')
          781  +     && !(zId[i]>='0' && zId[i]<='9')
          782  +     && !(zId[i]>='a' && zId[i]<='z')
          783  +     && !(zId[i]>='A' && zId[i]<='Z')
          784  +    ){
          785  +      return 1;
          786  +    }
          787  +  }
          788  +  return 0;
          789  +}
          790  +
          791  +/*
          792  +** This function appends an index column definition suitable for constraint
          793  +** pCons to the string passed as zIn and returns the result.
          794  +*/
          795  +static char *idxAppendColDefn(
          796  +  int *pRc,                       /* IN/OUT: Error code */
          797  +  char *zIn,                      /* Column defn accumulated so far */
          798  +  IdxTable *pTab,                 /* Table index will be created on */
          799  +  IdxConstraint *pCons
          800  +){
          801  +  char *zRet = zIn;
          802  +  IdxColumn *p = &pTab->aCol[pCons->iCol];
          803  +  if( zRet ) zRet = idxAppendText(pRc, zRet, ", ");
          804  +
          805  +  if( idxIdentifierRequiresQuotes(p->zName) ){
          806  +    zRet = idxAppendText(pRc, zRet, "%Q", p->zName);
          807  +  }else{
          808  +    zRet = idxAppendText(pRc, zRet, "%s", p->zName);
          809  +  }
          810  +
          811  +  if( sqlite3_stricmp(p->zColl, pCons->zColl) ){
          812  +    if( idxIdentifierRequiresQuotes(pCons->zColl) ){
          813  +      zRet = idxAppendText(pRc, zRet, " COLLATE %Q", pCons->zColl);
          814  +    }else{
          815  +      zRet = idxAppendText(pRc, zRet, " COLLATE %s", pCons->zColl);
          816  +    }
          817  +  }
          818  +
          819  +  if( pCons->bDesc ){
          820  +    zRet = idxAppendText(pRc, zRet, " DESC");
          821  +  }
          822  +  return zRet;
          823  +}
          824  +
          825  +/*
          826  +** Search database dbm for an index compatible with the one idxCreateFromCons()
          827  +** would create from arguments pScan, pEq and pTail. If no error occurs and 
          828  +** such an index is found, return non-zero. Or, if no such index is found,
          829  +** return zero.
          830  +**
          831  +** If an error occurs, set *pRc to an SQLite error code and return zero.
          832  +*/
          833  +static int idxFindCompatible(
          834  +  int *pRc,                       /* OUT: Error code */
          835  +  sqlite3* dbm,                   /* Database to search */
          836  +  IdxScan *pScan,                 /* Scan for table to search for index on */
          837  +  IdxConstraint *pEq,             /* List of == constraints */
          838  +  IdxConstraint *pTail            /* List of range constraints */
          839  +){
          840  +  const char *zTbl = pScan->pTab->zName;
          841  +  sqlite3_stmt *pIdxList = 0;
          842  +  IdxConstraint *pIter;
          843  +  int nEq = 0;                    /* Number of elements in pEq */
          844  +  int rc;
          845  +
          846  +  /* Count the elements in list pEq */
          847  +  for(pIter=pEq; pIter; pIter=pIter->pLink) nEq++;
          848  +
          849  +  rc = idxPrintfPrepareStmt(dbm, &pIdxList, 0, "PRAGMA index_list=%Q", zTbl);
          850  +  while( rc==SQLITE_OK && sqlite3_step(pIdxList)==SQLITE_ROW ){
          851  +    int bMatch = 1;
          852  +    IdxConstraint *pT = pTail;
          853  +    sqlite3_stmt *pInfo = 0;
          854  +    const char *zIdx = (const char*)sqlite3_column_text(pIdxList, 1);
          855  +
          856  +    /* Zero the IdxConstraint.bFlag values in the pEq list */
          857  +    for(pIter=pEq; pIter; pIter=pIter->pLink) pIter->bFlag = 0;
          858  +
          859  +    rc = idxPrintfPrepareStmt(dbm, &pInfo, 0, "PRAGMA index_xInfo=%Q", zIdx);
          860  +    while( rc==SQLITE_OK && sqlite3_step(pInfo)==SQLITE_ROW ){
          861  +      int iIdx = sqlite3_column_int(pInfo, 0);
          862  +      int iCol = sqlite3_column_int(pInfo, 1);
          863  +      const char *zColl = (const char*)sqlite3_column_text(pInfo, 4);
          864  +
          865  +      if( iIdx<nEq ){
          866  +        for(pIter=pEq; pIter; pIter=pIter->pLink){
          867  +          if( pIter->bFlag ) continue;
          868  +          if( pIter->iCol!=iCol ) continue;
          869  +          if( sqlite3_stricmp(pIter->zColl, zColl) ) continue;
          870  +          pIter->bFlag = 1;
          871  +          break;
          872  +        }
          873  +        if( pIter==0 ){
          874  +          bMatch = 0;
          875  +          break;
          876  +        }
          877  +      }else{
          878  +        if( pT ){
          879  +          if( pT->iCol!=iCol || sqlite3_stricmp(pT->zColl, zColl) ){
          880  +            bMatch = 0;
          881  +            break;
          882  +          }
          883  +          pT = pT->pLink;
          884  +        }
          885  +      }
          886  +    }
          887  +    idxFinalize(&rc, pInfo);
          888  +
          889  +    if( rc==SQLITE_OK && bMatch ){
          890  +      sqlite3_finalize(pIdxList);
          891  +      return 1;
          892  +    }
          893  +  }
          894  +  idxFinalize(&rc, pIdxList);
          895  +
          896  +  *pRc = rc;
          897  +  return 0;
          898  +}
          899  +
          900  +static int idxCreateFromCons(
          901  +  sqlite3expert *p,
          902  +  IdxScan *pScan,
          903  +  IdxConstraint *pEq, 
          904  +  IdxConstraint *pTail
          905  +){
          906  +  sqlite3 *dbm = p->dbm;
          907  +  int rc = SQLITE_OK;
          908  +  if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){
          909  +    IdxTable *pTab = pScan->pTab;
          910  +    char *zCols = 0;
          911  +    char *zIdx = 0;
          912  +    IdxConstraint *pCons;
          913  +    unsigned int h = 0;
          914  +    const char *zFmt;
          915  +
          916  +    for(pCons=pEq; pCons; pCons=pCons->pLink){
          917  +      zCols = idxAppendColDefn(&rc, zCols, pTab, pCons);
          918  +    }
          919  +    for(pCons=pTail; pCons; pCons=pCons->pLink){
          920  +      zCols = idxAppendColDefn(&rc, zCols, pTab, pCons);
          921  +    }
          922  +
          923  +    if( rc==SQLITE_OK ){
          924  +      /* Hash the list of columns to come up with a name for the index */
          925  +      const char *zTable = pScan->pTab->zName;
          926  +      char *zName;                /* Index name */
          927  +      int i;
          928  +      for(i=0; zCols[i]; i++){
          929  +        h += ((h<<3) + zCols[i]);
          930  +      }
          931  +      zName = sqlite3_mprintf("%s_idx_%08x", zTable, h);
          932  +      if( zName==0 ){ 
          933  +        rc = SQLITE_NOMEM;
          934  +      }else{
          935  +        if( idxIdentifierRequiresQuotes(zTable) ){
          936  +          zFmt = "CREATE INDEX '%q' ON %Q(%s)";
          937  +        }else{
          938  +          zFmt = "CREATE INDEX %s ON %s(%s)";
          939  +        }
          940  +        zIdx = sqlite3_mprintf(zFmt, zName, zTable, zCols);
          941  +        if( !zIdx ){
          942  +          rc = SQLITE_NOMEM;
          943  +        }else{
          944  +          rc = sqlite3_exec(dbm, zIdx, 0, 0, p->pzErrmsg);
          945  +          idxHashAdd(&rc, &p->hIdx, zName, zIdx);
          946  +        }
          947  +        sqlite3_free(zName);
          948  +        sqlite3_free(zIdx);
          949  +      }
          950  +    }
          951  +
          952  +    sqlite3_free(zCols);
          953  +  }
          954  +  return rc;
          955  +}
          956  +
          957  +/*
          958  +** Return true if list pList (linked by IdxConstraint.pLink) contains
          959  +** a constraint compatible with *p. Otherwise return false.
          960  +*/
          961  +static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){
          962  +  IdxConstraint *pCmp;
          963  +  for(pCmp=pList; pCmp; pCmp=pCmp->pLink){
          964  +    if( p->iCol==pCmp->iCol ) return 1;
          965  +  }
          966  +  return 0;
          967  +}
          968  +
          969  +static int idxCreateFromWhere(
          970  +  sqlite3expert *p, 
          971  +  IdxScan *pScan,                 /* Create indexes for this scan */
          972  +  IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
          973  +){
          974  +  IdxConstraint *p1 = 0;
          975  +  IdxConstraint *pCon;
          976  +  int rc;
          977  +
          978  +  /* Gather up all the == constraints. */
          979  +  for(pCon=pScan->pEq; pCon; pCon=pCon->pNext){
          980  +    if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){
          981  +      pCon->pLink = p1;
          982  +      p1 = pCon;
          983  +    }
          984  +  }
          985  +
          986  +  /* Create an index using the == constraints collected above. And the
          987  +  ** range constraint/ORDER BY terms passed in by the caller, if any. */
          988  +  rc = idxCreateFromCons(p, pScan, p1, pTail);
          989  +
          990  +  /* If no range/ORDER BY passed by the caller, create a version of the
          991  +  ** index for each range constraint.  */
          992  +  if( pTail==0 ){
          993  +    for(pCon=pScan->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){
          994  +      assert( pCon->pLink==0 );
          995  +      if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){
          996  +        rc = idxCreateFromCons(p, pScan, p1, pCon);
          997  +      }
          998  +    }
          999  +  }
         1000  +
         1001  +  return rc;
         1002  +}
         1003  +
         1004  +/*
         1005  +** Create candidate indexes in database [dbm] based on the data in 
         1006  +** linked-list pScan.
         1007  +*/
         1008  +static int idxCreateCandidates(sqlite3expert *p, char **pzErr){
         1009  +  int rc = SQLITE_OK;
         1010  +  IdxScan *pIter;
         1011  +
         1012  +  for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
         1013  +    rc = idxCreateFromWhere(p, pIter, 0);
         1014  +    if( rc==SQLITE_OK && pIter->pOrder ){
         1015  +      rc = idxCreateFromWhere(p, pIter, pIter->pOrder);
         1016  +    }
         1017  +  }
         1018  +
         1019  +  return rc;
         1020  +}
         1021  +
         1022  +/*
         1023  +** Free all elements of the linked list starting at pConstraint.
         1024  +*/
         1025  +static void idxConstraintFree(IdxConstraint *pConstraint){
         1026  +  IdxConstraint *pNext;
         1027  +  IdxConstraint *p;
         1028  +
         1029  +  for(p=pConstraint; p; p=pNext){
         1030  +    pNext = p->pNext;
         1031  +    sqlite3_free(p);
         1032  +  }
         1033  +}
         1034  +
         1035  +/*
         1036  +** Free all elements of the linked list starting from pScan up until pLast
         1037  +** (pLast is not freed).
         1038  +*/
         1039  +static void idxScanFree(IdxScan *pScan, IdxScan *pLast){
         1040  +  IdxScan *p;
         1041  +  IdxScan *pNext;
         1042  +  for(p=pScan; p!=pLast; p=pNext){
         1043  +    pNext = p->pNextScan;
         1044  +    idxConstraintFree(p->pOrder);
         1045  +    idxConstraintFree(p->pEq);
         1046  +    idxConstraintFree(p->pRange);
         1047  +    sqlite3_free(p);
         1048  +  }
         1049  +}
         1050  +
         1051  +/*
         1052  +** Free all elements of the linked list starting from pStatement up 
         1053  +** until pLast (pLast is not freed).
         1054  +*/
         1055  +static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){
         1056  +  IdxStatement *p;
         1057  +  IdxStatement *pNext;
         1058  +  for(p=pStatement; p!=pLast; p=pNext){
         1059  +    pNext = p->pNext;
         1060  +    sqlite3_free(p->zEQP);
         1061  +    sqlite3_free(p->zIdx);
         1062  +    sqlite3_free(p);
         1063  +  }
         1064  +}
         1065  +
         1066  +/*
         1067  +** Free the linked list of IdxTable objects starting at pTab.
         1068  +*/
         1069  +static void idxTableFree(IdxTable *pTab){
         1070  +  IdxTable *pIter;
         1071  +  IdxTable *pNext;
         1072  +  for(pIter=pTab; pIter; pIter=pNext){
         1073  +    pNext = pIter->pNext;
         1074  +    sqlite3_free(pIter);
         1075  +  }
         1076  +}
         1077  +
         1078  +/*
         1079  +** Free the linked list of IdxWrite objects starting at pTab.
         1080  +*/
         1081  +static void idxWriteFree(IdxWrite *pTab){
         1082  +  IdxWrite *pIter;
         1083  +  IdxWrite *pNext;
         1084  +  for(pIter=pTab; pIter; pIter=pNext){
         1085  +    pNext = pIter->pNext;
         1086  +    sqlite3_free(pIter);
         1087  +  }
         1088  +}
         1089  +
         1090  +
         1091  +
         1092  +/*
         1093  +** This function is called after candidate indexes have been created. It
         1094  +** runs all the queries to see which indexes they prefer, and populates
         1095  +** IdxStatement.zIdx and IdxStatement.zEQP with the results.
         1096  +*/
         1097  +int idxFindIndexes(
         1098  +  sqlite3expert *p,
         1099  +  char **pzErr                         /* OUT: Error message (sqlite3_malloc) */
         1100  +){
         1101  +  IdxStatement *pStmt;
         1102  +  sqlite3 *dbm = p->dbm;
         1103  +  int rc = SQLITE_OK;
         1104  +
         1105  +  IdxHash hIdx;
         1106  +  idxHashInit(&hIdx);
         1107  +
         1108  +  for(pStmt=p->pStatement; rc==SQLITE_OK && pStmt; pStmt=pStmt->pNext){
         1109  +    IdxHashEntry *pEntry;
         1110  +    sqlite3_stmt *pExplain = 0;
         1111  +    idxHashClear(&hIdx);
         1112  +    rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,
         1113  +        "EXPLAIN QUERY PLAN %s", pStmt->zSql
         1114  +    );
         1115  +    while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
         1116  +      int iSelectid = sqlite3_column_int(pExplain, 0);
         1117  +      int iOrder = sqlite3_column_int(pExplain, 1);
         1118  +      int iFrom = sqlite3_column_int(pExplain, 2);
         1119  +      const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
         1120  +      int nDetail = STRLEN(zDetail);
         1121  +      int i;
         1122  +
         1123  +      for(i=0; i<nDetail; i++){
         1124  +        const char *zIdx = 0;
         1125  +        if( memcmp(&zDetail[i], " USING INDEX ", 13)==0 ){
         1126  +          zIdx = &zDetail[i+13];
         1127  +        }else if( memcmp(&zDetail[i], " USING COVERING INDEX ", 22)==0 ){
         1128  +          zIdx = &zDetail[i+22];
         1129  +        }
         1130  +        if( zIdx ){
         1131  +          const char *zSql;
         1132  +          int nIdx = 0;
         1133  +          while( zIdx[nIdx]!='\0' && (zIdx[nIdx]!=' ' || zIdx[nIdx+1]!='(') ){
         1134  +            nIdx++;
         1135  +          }
         1136  +          zSql = idxHashSearch(&p->hIdx, zIdx, nIdx);
         1137  +          if( zSql ){
         1138  +            idxHashAdd(&rc, &hIdx, zSql, 0);
         1139  +            if( rc ) goto find_indexes_out;
         1140  +          }
         1141  +          break;
         1142  +        }
         1143  +      }
         1144  +
         1145  +      pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%d|%d|%d|%s\n", 
         1146  +          iSelectid, iOrder, iFrom, zDetail
         1147  +      );
         1148  +    }
         1149  +
         1150  +    for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
         1151  +      pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s;\n", pEntry->zKey);
         1152  +    }
         1153  +
         1154  +    idxFinalize(&rc, pExplain);
         1155  +  }
         1156  +
         1157  + find_indexes_out:
         1158  +  idxHashClear(&hIdx);
         1159  +  return rc;
         1160  +}
         1161  +
         1162  +static int idxAuthCallback(
         1163  +  void *pCtx,
         1164  +  int eOp,
         1165  +  const char *z3,
         1166  +  const char *z4,
         1167  +  const char *zDb,
         1168  +  const char *zTrigger
         1169  +){
         1170  +  int rc = SQLITE_OK;
         1171  +  if( eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE || eOp==SQLITE_DELETE ){
         1172  +    if( sqlite3_stricmp(zDb, "main")==0 ){
         1173  +      sqlite3expert *p = (sqlite3expert*)pCtx;
         1174  +      IdxTable *pTab;
         1175  +      for(pTab=p->pTable; pTab; pTab=pTab->pNext){
         1176  +        if( 0==sqlite3_stricmp(z3, pTab->zName) ) break;
         1177  +      }
         1178  +      if( pTab ){
         1179  +        IdxWrite *pWrite;
         1180  +        for(pWrite=p->pWrite; pWrite; pWrite=pWrite->pNext){
         1181  +          if( pWrite->pTab==pTab && pWrite->eOp==eOp ) break;
         1182  +        }
         1183  +        if( pWrite==0 ){
         1184  +          pWrite = idxMalloc(&rc, sizeof(IdxWrite));
         1185  +          if( rc==SQLITE_OK ){
         1186  +            pWrite->pTab = pTab;
         1187  +            pWrite->eOp = eOp;
         1188  +            pWrite->pNext = p->pWrite;
         1189  +            p->pWrite = pWrite;
         1190  +          }
         1191  +        }
         1192  +      }
         1193  +    }
         1194  +  }
         1195  +  return rc;
         1196  +}
         1197  +
         1198  +static int idxProcessOneTrigger(
         1199  +  sqlite3expert *p, 
         1200  +  IdxWrite *pWrite, 
         1201  +  char **pzErr
         1202  +){
         1203  +  static const char *zInt = UNIQUE_TABLE_NAME;
         1204  +  static const char *zDrop = "DROP TABLE " UNIQUE_TABLE_NAME;
         1205  +  IdxTable *pTab = pWrite->pTab;
         1206  +  const char *zTab = pTab->zName;
         1207  +  const char *zSql = 
         1208  +    "SELECT 'CREATE TEMP' || substr(sql, 7) FROM sqlite_master "
         1209  +    "WHERE tbl_name = %Q AND type IN ('table', 'trigger') "
         1210  +    "ORDER BY type;";
         1211  +  sqlite3_stmt *pSelect = 0;
         1212  +  int rc = SQLITE_OK;
         1213  +  char *zWrite = 0;
         1214  +
         1215  +  /* Create the table and its triggers in the temp schema */
         1216  +  rc = idxPrintfPrepareStmt(p->db, &pSelect, pzErr, zSql, zTab, zTab);
         1217  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSelect) ){
         1218  +    const char *zCreate = (const char*)sqlite3_column_text(pSelect, 0);
         1219  +    rc = sqlite3_exec(p->dbv, zCreate, 0, 0, pzErr);
         1220  +  }
         1221  +  idxFinalize(&rc, pSelect);
         1222  +
         1223  +  /* Rename the table in the temp schema to zInt */
         1224  +  if( rc==SQLITE_OK ){
         1225  +    char *z = sqlite3_mprintf("ALTER TABLE temp.%Q RENAME TO %Q", zTab, zInt);
         1226  +    if( z==0 ){
         1227  +      rc = SQLITE_NOMEM;
         1228  +    }else{
         1229  +      rc = sqlite3_exec(p->dbv, z, 0, 0, pzErr);
         1230  +      sqlite3_free(z);
         1231  +    }
         1232  +  }
         1233  +
         1234  +  switch( pWrite->eOp ){
         1235  +    case SQLITE_INSERT: {
         1236  +      int i;
         1237  +      zWrite = idxAppendText(&rc, zWrite, "INSERT INTO %Q VALUES(", zInt);
         1238  +      for(i=0; i<pTab->nCol; i++){
         1239  +        zWrite = idxAppendText(&rc, zWrite, "%s?", i==0 ? "" : ", ");
         1240  +      }
         1241  +      zWrite = idxAppendText(&rc, zWrite, ")");
         1242  +      break;
         1243  +    }
         1244  +    case SQLITE_UPDATE: {
         1245  +      int i;
         1246  +      zWrite = idxAppendText(&rc, zWrite, "UPDATE %Q SET ", zInt);
         1247  +      for(i=0; i<pTab->nCol; i++){
         1248  +        zWrite = idxAppendText(&rc, zWrite, "%s%Q=?", i==0 ? "" : ", ", 
         1249  +            pTab->aCol[i].zName
         1250  +        );
         1251  +      }
         1252  +      break;
         1253  +    }
         1254  +    default: {
         1255  +      assert( pWrite->eOp==SQLITE_DELETE );
         1256  +      if( rc==SQLITE_OK ){
         1257  +        zWrite = sqlite3_mprintf("DELETE FROM %Q", zInt);
         1258  +        if( zWrite==0 ) rc = SQLITE_NOMEM;
         1259  +      }
         1260  +    }
         1261  +  }
         1262  +
         1263  +  if( rc==SQLITE_OK ){
         1264  +    sqlite3_stmt *pX = 0;
         1265  +    rc = sqlite3_prepare_v2(p->dbv, zWrite, -1, &pX, 0);
         1266  +    idxFinalize(&rc, pX);
         1267  +    if( rc!=SQLITE_OK ){
         1268  +      idxDatabaseError(p->dbv, pzErr);
         1269  +    }
         1270  +  }
         1271  +  sqlite3_free(zWrite);
         1272  +
         1273  +  if( rc==SQLITE_OK ){
         1274  +    rc = sqlite3_exec(p->dbv, zDrop, 0, 0, pzErr);
         1275  +  }
         1276  +
         1277  +  return rc;
         1278  +}
         1279  +
         1280  +static int idxProcessTriggers(sqlite3expert *p, char **pzErr){
         1281  +  int rc = SQLITE_OK;
         1282  +  IdxWrite *pEnd = 0;
         1283  +  IdxWrite *pFirst = p->pWrite;
         1284  +
         1285  +  while( rc==SQLITE_OK && pFirst!=pEnd ){
         1286  +    IdxWrite *pIter;
         1287  +    for(pIter=pFirst; rc==SQLITE_OK && pIter!=pEnd; pIter=pIter->pNext){
         1288  +      rc = idxProcessOneTrigger(p, pIter, pzErr);
         1289  +    }
         1290  +    pEnd = pFirst;
         1291  +    pFirst = p->pWrite;
         1292  +  }
         1293  +
         1294  +  return rc;
         1295  +}
         1296  +
         1297  +
         1298  +static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
         1299  +  int rc = idxRegisterVtab(p);
         1300  +  sqlite3_stmt *pSchema = 0;
         1301  +
         1302  +  /* For each table in the main db schema:
         1303  +  **
         1304  +  **   1) Add an entry to the p->pTable list, and
         1305  +  **   2) Create the equivalent virtual table in dbv.
         1306  +  */
         1307  +  rc = idxPrepareStmt(p->db, &pSchema, pzErrmsg,
         1308  +      "SELECT type, name, sql, 1 FROM sqlite_master "
         1309  +      "WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%%' "
         1310  +      " UNION ALL "
         1311  +      "SELECT type, name, sql, 2 FROM sqlite_master "
         1312  +      "WHERE type = 'trigger'"
         1313  +      "  AND tbl_name IN(SELECT name FROM sqlite_master WHERE type = 'view') "
         1314  +      "ORDER BY 4, 1"
         1315  +  );
         1316  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSchema) ){
         1317  +    const char *zType = (const char*)sqlite3_column_text(pSchema, 0);
         1318  +    const char *zName = (const char*)sqlite3_column_text(pSchema, 1);
         1319  +    const char *zSql = (const char*)sqlite3_column_text(pSchema, 2);
         1320  +
         1321  +    if( zType[0]=='v' || zType[1]=='r' ){
         1322  +      rc = sqlite3_exec(p->dbv, zSql, 0, 0, pzErrmsg);
         1323  +    }else{
         1324  +      IdxTable *pTab;
         1325  +      rc = idxGetTableInfo(p->db, zName, &pTab, pzErrmsg);
         1326  +      if( rc==SQLITE_OK ){
         1327  +        int i;
         1328  +        char *zInner = 0;
         1329  +        char *zOuter = 0;
         1330  +        pTab->pNext = p->pTable;
         1331  +        p->pTable = pTab;
         1332  +
         1333  +        /* The statement the vtab will pass to sqlite3_declare_vtab() */
         1334  +        zInner = idxAppendText(&rc, 0, "CREATE TABLE x(");
         1335  +        for(i=0; i<pTab->nCol; i++){
         1336  +          zInner = idxAppendText(&rc, zInner, "%s%Q COLLATE %s", 
         1337  +              (i==0 ? "" : ", "), pTab->aCol[i].zName, pTab->aCol[i].zColl
         1338  +          );
         1339  +        }
         1340  +        zInner = idxAppendText(&rc, zInner, ")");
         1341  +
         1342  +        /* The CVT statement to create the vtab */
         1343  +        zOuter = idxAppendText(&rc, 0, 
         1344  +            "CREATE VIRTUAL TABLE %Q USING expert(%Q)", zName, zInner
         1345  +        );
         1346  +        if( rc==SQLITE_OK ){
         1347  +          rc = sqlite3_exec(p->dbv, zOuter, 0, 0, pzErrmsg);
         1348  +        }
         1349  +        sqlite3_free(zInner);
         1350  +        sqlite3_free(zOuter);
         1351  +      }
         1352  +    }
         1353  +  }
         1354  +  idxFinalize(&rc, pSchema);
         1355  +  return rc;
         1356  +}
         1357  +
         1358  +struct IdxSampleCtx {
         1359  +  int iTarget;
         1360  +  double target;                  /* Target nRet/nRow value */
         1361  +  double nRow;                    /* Number of rows seen */
         1362  +  double nRet;                    /* Number of rows returned */
         1363  +};
         1364  +
         1365  +static void idxSampleFunc(
         1366  +  sqlite3_context *pCtx,
         1367  +  int argc,
         1368  +  sqlite3_value **argv
         1369  +){
         1370  +  struct IdxSampleCtx *p = (struct IdxSampleCtx*)sqlite3_user_data(pCtx);
         1371  +  int bRet;
         1372  +
         1373  +  assert( argc==0 );
         1374  +  if( p->nRow==0.0 ){
         1375  +    bRet = 1;
         1376  +  }else{
         1377  +    bRet = (p->nRet / p->nRow) <= p->target;
         1378  +    if( bRet==0 ){
         1379  +      unsigned short rnd;
         1380  +      sqlite3_randomness(2, (void*)&rnd);
         1381  +      bRet = ((int)rnd % 100) <= p->iTarget;
         1382  +    }
         1383  +  }
         1384  +
         1385  +  sqlite3_result_int(pCtx, bRet);
         1386  +  p->nRow += 1.0;
         1387  +  p->nRet += (double)bRet;
         1388  +}
         1389  +
         1390  +struct IdxRemCtx {
         1391  +  int nSlot;
         1392  +  struct IdxRemSlot {
         1393  +    int eType;                    /* SQLITE_NULL, INTEGER, REAL, TEXT, BLOB */
         1394  +    i64 iVal;                     /* SQLITE_INTEGER value */
         1395  +    double rVal;                  /* SQLITE_FLOAT value */
         1396  +    int nByte;                    /* Bytes of space allocated at z */
         1397  +    int n;                        /* Size of buffer z */
         1398  +    char *z;                      /* SQLITE_TEXT/BLOB value */
         1399  +  } aSlot[1];
         1400  +};
         1401  +
         1402  +/*
         1403  +** Implementation of scalar function rem().
         1404  +*/
         1405  +static void idxRemFunc(
         1406  +  sqlite3_context *pCtx,
         1407  +  int argc,
         1408  +  sqlite3_value **argv
         1409  +){
         1410  +  struct IdxRemCtx *p = (struct IdxRemCtx*)sqlite3_user_data(pCtx);
         1411  +  struct IdxRemSlot *pSlot;
         1412  +  int iSlot;
         1413  +  assert( argc==2 );
         1414  +
         1415  +  iSlot = sqlite3_value_int(argv[0]);
         1416  +  assert( iSlot<=p->nSlot );
         1417  +  pSlot = &p->aSlot[iSlot];
         1418  +
         1419  +  switch( pSlot->eType ){
         1420  +    case SQLITE_NULL:
         1421  +      /* no-op */
         1422  +      break;
         1423  +
         1424  +    case SQLITE_INTEGER:
         1425  +      sqlite3_result_int64(pCtx, pSlot->iVal);
         1426  +      break;
         1427  +
         1428  +    case SQLITE_FLOAT:
         1429  +      sqlite3_result_double(pCtx, pSlot->rVal);
         1430  +      break;
         1431  +
         1432  +    case SQLITE_BLOB:
         1433  +      sqlite3_result_blob(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT);
         1434  +      break;
         1435  +
         1436  +    case SQLITE_TEXT:
         1437  +      sqlite3_result_text(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT);
         1438  +      break;
         1439  +  }
         1440  +
         1441  +  pSlot->eType = sqlite3_value_type(argv[1]);
         1442  +  switch( pSlot->eType ){
         1443  +    case SQLITE_NULL:
         1444  +      /* no-op */
         1445  +      break;
         1446  +
         1447  +    case SQLITE_INTEGER:
         1448  +      pSlot->iVal = sqlite3_value_int64(argv[1]);
         1449  +      break;
         1450  +
         1451  +    case SQLITE_FLOAT:
         1452  +      pSlot->rVal = sqlite3_value_double(argv[1]);
         1453  +      break;
         1454  +
         1455  +    case SQLITE_BLOB:
         1456  +    case SQLITE_TEXT: {
         1457  +      int nByte = sqlite3_value_bytes(argv[1]);
         1458  +      if( nByte>pSlot->nByte ){
         1459  +        char *zNew = (char*)sqlite3_realloc(pSlot->z, nByte*2);
         1460  +        if( zNew==0 ){
         1461  +          sqlite3_result_error_nomem(pCtx);
         1462  +          return;
         1463  +        }
         1464  +        pSlot->nByte = nByte*2;
         1465  +        pSlot->z = zNew;
         1466  +      }
         1467  +      pSlot->n = nByte;
         1468  +      if( pSlot->eType==SQLITE_BLOB ){
         1469  +        memcpy(pSlot->z, sqlite3_value_blob(argv[1]), nByte);
         1470  +      }else{
         1471  +        memcpy(pSlot->z, sqlite3_value_text(argv[1]), nByte);
         1472  +      }
         1473  +      break;
         1474  +    }
         1475  +  }
         1476  +}
         1477  +
         1478  +static int idxLargestIndex(sqlite3 *db, int *pnMax, char **pzErr){
         1479  +  int rc = SQLITE_OK;
         1480  +  const char *zMax = 
         1481  +    "SELECT max(i.seqno) FROM "
         1482  +    "  sqlite_master AS s, "
         1483  +    "  pragma_index_list(s.name) AS l, "
         1484  +    "  pragma_index_info(l.name) AS i "
         1485  +    "WHERE s.type = 'table'";
         1486  +  sqlite3_stmt *pMax = 0;
         1487  +
         1488  +  *pnMax = 0;
         1489  +  rc = idxPrepareStmt(db, &pMax, pzErr, zMax);
         1490  +  if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){
         1491  +    *pnMax = sqlite3_column_int(pMax, 0) + 1;
         1492  +  }
         1493  +  idxFinalize(&rc, pMax);
         1494  +
         1495  +  return rc;
         1496  +}
         1497  +
         1498  +static int idxPopulateOneStat1(
         1499  +  sqlite3expert *p,
         1500  +  sqlite3_stmt *pIndexXInfo,
         1501  +  sqlite3_stmt *pWriteStat,
         1502  +  const char *zTab,
         1503  +  const char *zIdx,
         1504  +  char **pzErr
         1505  +){
         1506  +  char *zCols = 0;
         1507  +  char *zOrder = 0;
         1508  +  char *zQuery = 0;
         1509  +  int nCol = 0;
         1510  +  int i;
         1511  +  sqlite3_stmt *pQuery = 0;
         1512  +  int *aStat = 0;
         1513  +  int rc = SQLITE_OK;
         1514  +
         1515  +  assert( p->iSample>0 );
         1516  +
         1517  +  /* Formulate the query text */
         1518  +  sqlite3_bind_text(pIndexXInfo, 1, zIdx, -1, SQLITE_STATIC);
         1519  +  while( SQLITE_OK==rc && SQLITE_ROW==sqlite3_step(pIndexXInfo) ){
         1520  +    const char *zComma = zCols==0 ? "" : ", ";
         1521  +    const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0);
         1522  +    const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1);
         1523  +    zCols = idxAppendText(&rc, zCols, 
         1524  +        "%sx.%Q IS rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl
         1525  +    );
         1526  +    zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol);
         1527  +  }
         1528  +  sqlite3_reset(pIndexXInfo);
         1529  +  if( rc==SQLITE_OK ){
         1530  +    if( p->iSample==100 ){
         1531  +      zQuery = sqlite3_mprintf(
         1532  +          "SELECT %s FROM %Q x ORDER BY %s", zCols, zTab, zOrder
         1533  +      );
         1534  +    }else{
         1535  +      zQuery = sqlite3_mprintf(
         1536  +          "SELECT %s FROM temp."UNIQUE_TABLE_NAME" x ORDER BY %s", zCols, zOrder
         1537  +      );
         1538  +    }
         1539  +  }
         1540  +  sqlite3_free(zCols);
         1541  +  sqlite3_free(zOrder);
         1542  +
         1543  +  /* Formulate the query text */
         1544  +  if( rc==SQLITE_OK ){
         1545  +    sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
         1546  +    rc = idxPrepareStmt(dbrem, &pQuery, pzErr, zQuery);
         1547  +  }
         1548  +  sqlite3_free(zQuery);
         1549  +
         1550  +  if( rc==SQLITE_OK ){
         1551  +    aStat = (int*)idxMalloc(&rc, sizeof(int)*(nCol+1));
         1552  +  }
         1553  +  if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){
         1554  +    IdxHashEntry *pEntry;
         1555  +    char *zStat = 0;
         1556  +    for(i=0; i<=nCol; i++) aStat[i] = 1;
         1557  +    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){
         1558  +      aStat[0]++;
         1559  +      for(i=0; i<nCol; i++){
         1560  +        if( sqlite3_column_int(pQuery, i)==0 ) break;
         1561  +      }
         1562  +      for(/*no-op*/; i<nCol; i++){
         1563  +        aStat[i+1]++;
         1564  +      }
         1565  +    }
         1566  +
         1567  +    if( rc==SQLITE_OK ){
         1568  +      int s0 = aStat[0];
         1569  +      zStat = sqlite3_mprintf("%d", s0);
         1570  +      if( zStat==0 ) rc = SQLITE_NOMEM;
         1571  +      for(i=1; rc==SQLITE_OK && i<=nCol; i++){
         1572  +        zStat = idxAppendText(&rc, zStat, " %d", (s0+aStat[i]/2) / aStat[i]);
         1573  +      }
         1574  +    }
         1575  +
         1576  +    if( rc==SQLITE_OK ){
         1577  +      sqlite3_bind_text(pWriteStat, 1, zTab, -1, SQLITE_STATIC);
         1578  +      sqlite3_bind_text(pWriteStat, 2, zIdx, -1, SQLITE_STATIC);
         1579  +      sqlite3_bind_text(pWriteStat, 3, zStat, -1, SQLITE_STATIC);
         1580  +      sqlite3_step(pWriteStat);
         1581  +      rc = sqlite3_reset(pWriteStat);
         1582  +    }
         1583  +
         1584  +    pEntry = idxHashFind(&p->hIdx, zIdx, STRLEN(zIdx));
         1585  +    if( pEntry ){
         1586  +      assert( pEntry->zVal2==0 );
         1587  +      pEntry->zVal2 = zStat;
         1588  +    }else{
         1589  +      sqlite3_free(zStat);
         1590  +    }
         1591  +  }
         1592  +  sqlite3_free(aStat);
         1593  +  idxFinalize(&rc, pQuery);
         1594  +
         1595  +  return rc;
         1596  +}
         1597  +
         1598  +static int idxBuildSampleTable(sqlite3expert *p, const char *zTab){
         1599  +  int rc;
         1600  +  char *zSql;
         1601  +
         1602  +  rc = sqlite3_exec(p->dbv,"DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
         1603  +  if( rc!=SQLITE_OK ) return rc;
         1604  +
         1605  +  zSql = sqlite3_mprintf(
         1606  +      "CREATE TABLE temp." UNIQUE_TABLE_NAME " AS SELECT * FROM %Q", zTab
         1607  +  );
         1608  +  if( zSql==0 ) return SQLITE_NOMEM;
         1609  +  rc = sqlite3_exec(p->dbv, zSql, 0, 0, 0);
         1610  +  sqlite3_free(zSql);
         1611  +
         1612  +  return rc;
         1613  +}
         1614  +
         1615  +/*
         1616  +** This function is called as part of sqlite3_expert_analyze(). Candidate
         1617  +** indexes have already been created in database sqlite3expert.dbm, this
         1618  +** function populates sqlite_stat1 table in the same database.
         1619  +**
         1620  +** The stat1 data is generated by querying the 
         1621  +*/
         1622  +static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
         1623  +  int rc = SQLITE_OK;
         1624  +  int nMax =0;
         1625  +  struct IdxRemCtx *pCtx = 0;
         1626  +  struct IdxSampleCtx samplectx; 
         1627  +  int i;
         1628  +  i64 iPrev = -100000;
         1629  +  sqlite3_stmt *pAllIndex = 0;
         1630  +  sqlite3_stmt *pIndexXInfo = 0;
         1631  +  sqlite3_stmt *pWrite = 0;
         1632  +
         1633  +  const char *zAllIndex =
         1634  +    "SELECT s.rowid, s.name, l.name FROM "
         1635  +    "  sqlite_master AS s, "
         1636  +    "  pragma_index_list(s.name) AS l "
         1637  +    "WHERE s.type = 'table'";
         1638  +  const char *zIndexXInfo = 
         1639  +    "SELECT name, coll FROM pragma_index_xinfo(?) WHERE key";
         1640  +  const char *zWrite = "INSERT INTO sqlite_stat1 VALUES(?, ?, ?)";
         1641  +
         1642  +  /* If iSample==0, no sqlite_stat1 data is required. */
         1643  +  if( p->iSample==0 ) return SQLITE_OK;
         1644  +
         1645  +  rc = idxLargestIndex(p->dbm, &nMax, pzErr);
         1646  +  if( nMax<=0 || rc!=SQLITE_OK ) return rc;
         1647  +
         1648  +  rc = sqlite3_exec(p->dbm, "ANALYZE; PRAGMA writable_schema=1", 0, 0, 0);
         1649  +
         1650  +  if( rc==SQLITE_OK ){
         1651  +    int nByte = sizeof(struct IdxRemCtx) + (sizeof(struct IdxRemSlot) * nMax);
         1652  +    pCtx = (struct IdxRemCtx*)idxMalloc(&rc, nByte);
         1653  +  }
         1654  +
         1655  +  if( rc==SQLITE_OK ){
         1656  +    sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
         1657  +    rc = sqlite3_create_function(
         1658  +        dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
         1659  +    );
         1660  +  }
         1661  +  if( rc==SQLITE_OK ){
         1662  +    rc = sqlite3_create_function(
         1663  +        p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0
         1664  +    );
         1665  +  }
         1666  +
         1667  +  if( rc==SQLITE_OK ){
         1668  +    pCtx->nSlot = nMax+1;
         1669  +    rc = idxPrepareStmt(p->dbm, &pAllIndex, pzErr, zAllIndex);
         1670  +  }
         1671  +  if( rc==SQLITE_OK ){
         1672  +    rc = idxPrepareStmt(p->dbm, &pIndexXInfo, pzErr, zIndexXInfo);
         1673  +  }
         1674  +  if( rc==SQLITE_OK ){
         1675  +    rc = idxPrepareStmt(p->dbm, &pWrite, pzErr, zWrite);
         1676  +  }
         1677  +
         1678  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pAllIndex) ){
         1679  +    i64 iRowid = sqlite3_column_int64(pAllIndex, 0);
         1680  +    const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 1);
         1681  +    const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 2);
         1682  +    if( p->iSample<100 && iPrev!=iRowid ){
         1683  +      samplectx.target = (double)p->iSample / 100.0;
         1684  +      samplectx.iTarget = p->iSample;
         1685  +      samplectx.nRow = 0.0;
         1686  +      samplectx.nRet = 0.0;
         1687  +      rc = idxBuildSampleTable(p, zTab);
         1688  +      if( rc!=SQLITE_OK ) break;
         1689  +    }
         1690  +    rc = idxPopulateOneStat1(p, pIndexXInfo, pWrite, zTab, zIdx, pzErr);
         1691  +    iPrev = iRowid;
         1692  +  }
         1693  +  if( rc==SQLITE_OK && p->iSample<100 ){
         1694  +    rc = sqlite3_exec(p->dbv, 
         1695  +        "DROP TABLE IF EXISTS temp." UNIQUE_TABLE_NAME, 0,0,0
         1696  +    );
         1697  +  }
         1698  +
         1699  +  idxFinalize(&rc, pAllIndex);
         1700  +  idxFinalize(&rc, pIndexXInfo);
         1701  +  idxFinalize(&rc, pWrite);
         1702  +
         1703  +  for(i=0; i<pCtx->nSlot; i++){
         1704  +    sqlite3_free(pCtx->aSlot[i].z);
         1705  +  }
         1706  +  sqlite3_free(pCtx);
         1707  +
         1708  +  if( rc==SQLITE_OK ){
         1709  +    rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_master", 0, 0, 0);
         1710  +  }
         1711  +
         1712  +  sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
         1713  +  return rc;
         1714  +}
         1715  +
         1716  +/*
         1717  +** Allocate a new sqlite3expert object.
         1718  +*/
         1719  +sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
         1720  +  int rc = SQLITE_OK;
         1721  +  sqlite3expert *pNew;
         1722  +
         1723  +  pNew = (sqlite3expert*)idxMalloc(&rc, sizeof(sqlite3expert));
         1724  +
         1725  +  /* Open two in-memory databases to work with. The "vtab database" (dbv)
         1726  +  ** will contain a virtual table corresponding to each real table in
         1727  +  ** the user database schema, and a copy of each view. It is used to
         1728  +  ** collect information regarding the WHERE, ORDER BY and other clauses
         1729  +  ** of the user's query.
         1730  +  */
         1731  +  if( rc==SQLITE_OK ){
         1732  +    pNew->db = db;
         1733  +    pNew->iSample = 100;
         1734  +    rc = sqlite3_open(":memory:", &pNew->dbv);
         1735  +  }
         1736  +  if( rc==SQLITE_OK ){
         1737  +    rc = sqlite3_open(":memory:", &pNew->dbm);
         1738  +    if( rc==SQLITE_OK ){
         1739  +      sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_TRIGGER_EQP, 1, (int*)0);
         1740  +    }
         1741  +  }
         1742  +  
         1743  +
         1744  +  /* Copy the entire schema of database [db] into [dbm]. */
         1745  +  if( rc==SQLITE_OK ){
         1746  +    sqlite3_stmt *pSql;
         1747  +    rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, 
         1748  +        "SELECT sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%%'"
         1749  +        " AND sql NOT LIKE 'CREATE VIRTUAL %%'"
         1750  +    );
         1751  +    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
         1752  +      const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
         1753  +      rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg);
         1754  +    }
         1755  +    idxFinalize(&rc, pSql);
         1756  +  }
         1757  +
         1758  +  /* Create the vtab schema */
         1759  +  if( rc==SQLITE_OK ){
         1760  +    rc = idxCreateVtabSchema(pNew, pzErrmsg);
         1761  +  }
         1762  +
         1763  +  /* Register the auth callback with dbv */
         1764  +  if( rc==SQLITE_OK ){
         1765  +    sqlite3_set_authorizer(pNew->dbv, idxAuthCallback, (void*)pNew);
         1766  +  }
         1767  +
         1768  +  /* If an error has occurred, free the new object and reutrn NULL. Otherwise,
         1769  +  ** return the new sqlite3expert handle.  */
         1770  +  if( rc!=SQLITE_OK ){
         1771  +    sqlite3_expert_destroy(pNew);
         1772  +    pNew = 0;
         1773  +  }
         1774  +  return pNew;
         1775  +}
         1776  +
         1777  +/*
         1778  +** Configure an sqlite3expert object.
         1779  +*/
         1780  +int sqlite3_expert_config(sqlite3expert *p, int op, ...){
         1781  +  int rc = SQLITE_OK;
         1782  +  va_list ap;
         1783  +  va_start(ap, op);
         1784  +  switch( op ){
         1785  +    case EXPERT_CONFIG_SAMPLE: {
         1786  +      int iVal = va_arg(ap, int);
         1787  +      if( iVal<0 ) iVal = 0;
         1788  +      if( iVal>100 ) iVal = 100;
         1789  +      p->iSample = iVal;
         1790  +      break;
         1791  +    }
         1792  +    default:
         1793  +      rc = SQLITE_NOTFOUND;
         1794  +      break;
         1795  +  }
         1796  +
         1797  +  va_end(ap);
         1798  +  return rc;
         1799  +}
         1800  +
         1801  +/*
         1802  +** Add an SQL statement to the analysis.
         1803  +*/
         1804  +int sqlite3_expert_sql(
         1805  +  sqlite3expert *p,               /* From sqlite3_expert_new() */
         1806  +  const char *zSql,               /* SQL statement to add */
         1807  +  char **pzErr                    /* OUT: Error message (if any) */
         1808  +){
         1809  +  IdxScan *pScanOrig = p->pScan;
         1810  +  IdxStatement *pStmtOrig = p->pStatement;
         1811  +  int rc = SQLITE_OK;
         1812  +  const char *zStmt = zSql;
         1813  +
         1814  +  if( p->bRun ) return SQLITE_MISUSE;
         1815  +
         1816  +  while( rc==SQLITE_OK && zStmt && zStmt[0] ){
         1817  +    sqlite3_stmt *pStmt = 0;
         1818  +    rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt);
         1819  +    if( rc==SQLITE_OK ){
         1820  +      if( pStmt ){
         1821  +        IdxStatement *pNew;
         1822  +        const char *z = sqlite3_sql(pStmt);
         1823  +        int n = STRLEN(z);
         1824  +        pNew = (IdxStatement*)idxMalloc(&rc, sizeof(IdxStatement) + n+1);
         1825  +        if( rc==SQLITE_OK ){
         1826  +          pNew->zSql = (char*)&pNew[1];
         1827  +          memcpy(pNew->zSql, z, n+1);
         1828  +          pNew->pNext = p->pStatement;
         1829  +          if( p->pStatement ) pNew->iId = p->pStatement->iId+1;
         1830  +          p->pStatement = pNew;
         1831  +        }
         1832  +        sqlite3_finalize(pStmt);
         1833  +      }
         1834  +    }else{
         1835  +      idxDatabaseError(p->dbv, pzErr);
         1836  +    }
         1837  +  }
         1838  +
         1839  +  if( rc!=SQLITE_OK ){
         1840  +    idxScanFree(p->pScan, pScanOrig);
         1841  +    idxStatementFree(p->pStatement, pStmtOrig);
         1842  +    p->pScan = pScanOrig;
         1843  +    p->pStatement = pStmtOrig;
         1844  +  }
         1845  +
         1846  +  return rc;
         1847  +}
         1848  +
         1849  +int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
         1850  +  int rc;
         1851  +  IdxHashEntry *pEntry;
         1852  +
         1853  +  /* Do trigger processing to collect any extra IdxScan structures */
         1854  +  rc = idxProcessTriggers(p, pzErr);
         1855  +
         1856  +  /* Create candidate indexes within the in-memory database file */
         1857  +  if( rc==SQLITE_OK ){
         1858  +    rc = idxCreateCandidates(p, pzErr);
         1859  +  }
         1860  +
         1861  +  /* Generate the stat1 data */
         1862  +  if( rc==SQLITE_OK ){
         1863  +    rc = idxPopulateStat1(p, pzErr);
         1864  +  }
         1865  +
         1866  +  /* Formulate the EXPERT_REPORT_CANDIDATES text */
         1867  +  for(pEntry=p->hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
         1868  +    p->zCandidates = idxAppendText(&rc, p->zCandidates, 
         1869  +        "%s;%s%s\n", pEntry->zVal, 
         1870  +        pEntry->zVal2 ? " -- stat1: " : "", pEntry->zVal2
         1871  +    );
         1872  +  }
         1873  +
         1874  +  /* Figure out which of the candidate indexes are preferred by the query
         1875  +  ** planner and report the results to the user.  */
         1876  +  if( rc==SQLITE_OK ){
         1877  +    rc = idxFindIndexes(p, pzErr);
         1878  +  }
         1879  +
         1880  +  if( rc==SQLITE_OK ){
         1881  +    p->bRun = 1;
         1882  +  }
         1883  +  return rc;
         1884  +}
         1885  +
         1886  +/*
         1887  +** Return the total number of statements that have been added to this
         1888  +** sqlite3expert using sqlite3_expert_sql().
         1889  +*/
         1890  +int sqlite3_expert_count(sqlite3expert *p){
         1891  +  int nRet = 0;
         1892  +  if( p->pStatement ) nRet = p->pStatement->iId+1;
         1893  +  return nRet;
         1894  +}
         1895  +
         1896  +/*
         1897  +** Return a component of the report.
         1898  +*/
         1899  +const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){
         1900  +  const char *zRet = 0;
         1901  +  IdxStatement *pStmt;
         1902  +
         1903  +  if( p->bRun==0 ) return 0;
         1904  +  for(pStmt=p->pStatement; pStmt && pStmt->iId!=iStmt; pStmt=pStmt->pNext);
         1905  +  switch( eReport ){
         1906  +    case EXPERT_REPORT_SQL:
         1907  +      if( pStmt ) zRet = pStmt->zSql;
         1908  +      break;
         1909  +    case EXPERT_REPORT_INDEXES:
         1910  +      if( pStmt ) zRet = pStmt->zIdx;
         1911  +      break;
         1912  +    case EXPERT_REPORT_PLAN:
         1913  +      if( pStmt ) zRet = pStmt->zEQP;
         1914  +      break;
         1915  +    case EXPERT_REPORT_CANDIDATES:
         1916  +      zRet = p->zCandidates;
         1917  +      break;
         1918  +  }
         1919  +  return zRet;
         1920  +}
         1921  +
         1922  +/*
         1923  +** Free an sqlite3expert object.
         1924  +*/
         1925  +void sqlite3_expert_destroy(sqlite3expert *p){
         1926  +  if( p ){
         1927  +    sqlite3_close(p->dbm);
         1928  +    sqlite3_close(p->dbv);
         1929  +    idxScanFree(p->pScan, 0);
         1930  +    idxStatementFree(p->pStatement, 0);
         1931  +    idxTableFree(p->pTable);
         1932  +    idxWriteFree(p->pWrite);
         1933  +    idxHashClear(&p->hIdx);
         1934  +    sqlite3_free(p->zCandidates);
         1935  +    sqlite3_free(p);
         1936  +  }
         1937  +}
         1938  +
         1939  +#endif /* ifndef SQLITE_OMIT_VIRTUAL_TABLE */

Added ext/expert/sqlite3expert.h.

            1  +/*
            2  +** 2017 April 07
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +*/
           13  +
           14  +
           15  +#include "sqlite3.h"
           16  +
           17  +typedef struct sqlite3expert sqlite3expert;
           18  +
           19  +/*
           20  +** Create a new sqlite3expert object.
           21  +**
           22  +** If successful, a pointer to the new object is returned and (*pzErr) set
           23  +** to NULL. Or, if an error occurs, NULL is returned and (*pzErr) set to
           24  +** an English-language error message. In this case it is the responsibility
           25  +** of the caller to eventually free the error message buffer using
           26  +** sqlite3_free().
           27  +*/
           28  +sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr);
           29  +
           30  +/*
           31  +** Configure an sqlite3expert object.
           32  +**
           33  +** EXPERT_CONFIG_SAMPLE:
           34  +**   By default, sqlite3_expert_analyze() generates sqlite_stat1 data for
           35  +**   each candidate index. This involves scanning and sorting the entire
           36  +**   contents of each user database table once for each candidate index
           37  +**   associated with the table. For large databases, this can be 
           38  +**   prohibitively slow. This option allows the sqlite3expert object to
           39  +**   be configured so that sqlite_stat1 data is instead generated based on a
           40  +**   subset of each table, or so that no sqlite_stat1 data is used at all.
           41  +**
           42  +**   A single integer argument is passed to this option. If the value is less
           43  +**   than or equal to zero, then no sqlite_stat1 data is generated or used by
           44  +**   the analysis - indexes are recommended based on the database schema only.
           45  +**   Or, if the value is 100 or greater, complete sqlite_stat1 data is
           46  +**   generated for each candidate index (this is the default). Finally, if the
           47  +**   value falls between 0 and 100, then it represents the percentage of user
           48  +**   table rows that should be considered when generating sqlite_stat1 data.
           49  +**
           50  +**   Examples:
           51  +**
           52  +**     // Do not generate any sqlite_stat1 data
           53  +**     sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 0);
           54  +**
           55  +**     // Generate sqlite_stat1 data based on 10% of the rows in each table.
           56  +**     sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 10);
           57  +*/
           58  +int sqlite3_expert_config(sqlite3expert *p, int op, ...);
           59  +
           60  +#define EXPERT_CONFIG_SAMPLE 1    /* int */
           61  +
           62  +/*
           63  +** Specify zero or more SQL statements to be included in the analysis.
           64  +**
           65  +** Buffer zSql must contain zero or more complete SQL statements. This
           66  +** function parses all statements contained in the buffer and adds them
           67  +** to the internal list of statements to analyze. If successful, SQLITE_OK
           68  +** is returned and (*pzErr) set to NULL. Or, if an error occurs - for example
           69  +** due to a error in the SQL - an SQLite error code is returned and (*pzErr)
           70  +** may be set to point to an English language error message. In this case
           71  +** the caller is responsible for eventually freeing the error message buffer
           72  +** using sqlite3_free().
           73  +**
           74  +** If an error does occur while processing one of the statements in the
           75  +** buffer passed as the second argument, none of the statements in the
           76  +** buffer are added to the analysis.
           77  +**
           78  +** This function must be called before sqlite3_expert_analyze(). If a call
           79  +** to this function is made on an sqlite3expert object that has already
           80  +** been passed to sqlite3_expert_analyze() SQLITE_MISUSE is returned
           81  +** immediately and no statements are added to the analysis.
           82  +*/
           83  +int sqlite3_expert_sql(
           84  +  sqlite3expert *p,               /* From a successful sqlite3_expert_new() */
           85  +  const char *zSql,               /* SQL statement(s) to add */
           86  +  char **pzErr                    /* OUT: Error message (if any) */
           87  +);
           88  +
           89  +
           90  +/*
           91  +** This function is called after the sqlite3expert object has been configured
           92  +** with all SQL statements using sqlite3_expert_sql() to actually perform
           93  +** the analysis. Once this function has been called, it is not possible to
           94  +** add further SQL statements to the analysis.
           95  +**
           96  +** If successful, SQLITE_OK is returned and (*pzErr) is set to NULL. Or, if
           97  +** an error occurs, an SQLite error code is returned and (*pzErr) set to 
           98  +** point to a buffer containing an English language error message. In this
           99  +** case it is the responsibility of the caller to eventually free the buffer
          100  +** using sqlite3_free().
          101  +**
          102  +** If an error does occur within this function, the sqlite3expert object
          103  +** is no longer useful for any purpose. At that point it is no longer
          104  +** possible to add further SQL statements to the object or to re-attempt
          105  +** the analysis. The sqlite3expert object must still be freed using a call
          106  +** sqlite3_expert_destroy().
          107  +*/
          108  +int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr);
          109  +
          110  +/*
          111  +** Return the total number of statements loaded using sqlite3_expert_sql().
          112  +** The total number of SQL statements may be different from the total number
          113  +** to calls to sqlite3_expert_sql().
          114  +*/
          115  +int sqlite3_expert_count(sqlite3expert*);
          116  +
          117  +/*
          118  +** Return a component of the report.
          119  +**
          120  +** This function is called after sqlite3_expert_analyze() to extract the
          121  +** results of the analysis. Each call to this function returns either a
          122  +** NULL pointer or a pointer to a buffer containing a nul-terminated string.
          123  +** The value passed as the third argument must be one of the EXPERT_REPORT_*
          124  +** #define constants defined below.
          125  +**
          126  +** For some EXPERT_REPORT_* parameters, the buffer returned contains 
          127  +** information relating to a specific SQL statement. In these cases that
          128  +** SQL statement is identified by the value passed as the second argument.
          129  +** SQL statements are numbered from 0 in the order in which they are parsed.
          130  +** If an out-of-range value (less than zero or equal to or greater than the
          131  +** value returned by sqlite3_expert_count()) is passed as the second argument
          132  +** along with such an EXPERT_REPORT_* parameter, NULL is always returned.
          133  +**
          134  +** EXPERT_REPORT_SQL:
          135  +**   Return the text of SQL statement iStmt.
          136  +**
          137  +** EXPERT_REPORT_INDEXES:
          138  +**   Return a buffer containing the CREATE INDEX statements for all recommended
          139  +**   indexes for statement iStmt. If there are no new recommeded indexes, NULL 
          140  +**   is returned.
          141  +**
          142  +** EXPERT_REPORT_PLAN:
          143  +**   Return a buffer containing the EXPLAIN QUERY PLAN output for SQL query
          144  +**   iStmt after the proposed indexes have been added to the database schema.
          145  +**
          146  +** EXPERT_REPORT_CANDIDATES:
          147  +**   Return a pointer to a buffer containing the CREATE INDEX statements 
          148  +**   for all indexes that were tested (for all SQL statements). The iStmt
          149  +**   parameter is ignored for EXPERT_REPORT_CANDIDATES calls.
          150  +*/
          151  +const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport);
          152  +
          153  +/*
          154  +** Values for the third argument passed to sqlite3_expert_report().
          155  +*/
          156  +#define EXPERT_REPORT_SQL        1
          157  +#define EXPERT_REPORT_INDEXES    2
          158  +#define EXPERT_REPORT_PLAN       3
          159  +#define EXPERT_REPORT_CANDIDATES 4
          160  +
          161  +/*
          162  +** Free an (sqlite3expert*) handle and all associated resources. There 
          163  +** should be one call to this function for each successful call to 
          164  +** sqlite3-expert_new().
          165  +*/
          166  +void sqlite3_expert_destroy(sqlite3expert*);
          167  +
          168  +

Added ext/expert/test_expert.c.

            1  +/*
            2  +** 2017 April 07
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +*/
           13  +
           14  +#if defined(SQLITE_TEST)
           15  +
           16  +#include "sqlite3expert.h"
           17  +#include <assert.h>
           18  +#include <string.h>
           19  +
           20  +#if defined(INCLUDE_SQLITE_TCL_H)
           21  +#  include "sqlite_tcl.h"
           22  +#else
           23  +#  include "tcl.h"
           24  +#  ifndef SQLITE_TCLAPI
           25  +#    define SQLITE_TCLAPI
           26  +#  endif
           27  +#endif
           28  +
           29  +#ifndef SQLITE_OMIT_VIRTUALTABLE
           30  +
           31  +/*
           32  +** Extract an sqlite3* db handle from the object passed as the second
           33  +** argument. If successful, set *pDb to point to the db handle and return
           34  +** TCL_OK. Otherwise, return TCL_ERROR.
           35  +*/
           36  +static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){
           37  +  Tcl_CmdInfo info;
           38  +  if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){
           39  +    Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0);
           40  +    return TCL_ERROR;
           41  +  }
           42  +
           43  +  *pDb = *(sqlite3 **)info.objClientData;
           44  +  return TCL_OK;
           45  +}
           46  +
           47  +
           48  +/*
           49  +** Tclcmd:  $expert sql SQL
           50  +**          $expert analyze
           51  +**          $expert count
           52  +**          $expert report STMT EREPORT
           53  +**          $expert destroy
           54  +*/
           55  +static int SQLITE_TCLAPI testExpertCmd(
           56  +  void *clientData,
           57  +  Tcl_Interp *interp,
           58  +  int objc,
           59  +  Tcl_Obj *CONST objv[]
           60  +){
           61  +  sqlite3expert *pExpert = (sqlite3expert*)clientData;
           62  +  struct Subcmd {
           63  +    const char *zSub;
           64  +    int nArg;
           65  +    const char *zMsg;
           66  +  } aSub[] = {
           67  +    { "sql",       1, "TABLE",        }, /* 0 */
           68  +    { "analyze",   0, "",             }, /* 1 */
           69  +    { "count",     0, "",             }, /* 2 */
           70  +    { "report",    2, "STMT EREPORT", }, /* 3 */
           71  +    { "destroy",   0, "",             }, /* 4 */
           72  +    { 0 }
           73  +  };
           74  +  int iSub;
           75  +  int rc = TCL_OK;
           76  +  char *zErr = 0;
           77  +
           78  +  if( objc<2 ){
           79  +    Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ...");
           80  +    return TCL_ERROR;
           81  +  }
           82  +  rc = Tcl_GetIndexFromObjStruct(interp, 
           83  +      objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub
           84  +  );
           85  +  if( rc!=TCL_OK ) return rc;
           86  +  if( objc!=2+aSub[iSub].nArg ){
           87  +    Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg);
           88  +    return TCL_ERROR;
           89  +  }
           90  +
           91  +  switch( iSub ){
           92  +    case 0: {      /* sql */
           93  +      char *zArg = Tcl_GetString(objv[2]);
           94  +      rc = sqlite3_expert_sql(pExpert, zArg, &zErr);
           95  +      break;
           96  +    }
           97  +
           98  +    case 1: {      /* analyze */
           99  +      rc = sqlite3_expert_analyze(pExpert, &zErr);
          100  +      break;
          101  +    }
          102  +
          103  +    case 2: {      /* count */
          104  +      int n = sqlite3_expert_count(pExpert);
          105  +      Tcl_SetObjResult(interp, Tcl_NewIntObj(n));
          106  +      break;
          107  +    }
          108  +
          109  +    case 3: {      /* report */
          110  +      const char *aEnum[] = {
          111  +        "sql", "indexes", "plan", "candidates", 0
          112  +      };
          113  +      int iEnum;
          114  +      int iStmt;
          115  +      const char *zReport;
          116  +
          117  +      if( Tcl_GetIntFromObj(interp, objv[2], &iStmt) 
          118  +       || Tcl_GetIndexFromObj(interp, objv[3], aEnum, "report", 0, &iEnum)
          119  +      ){
          120  +        return TCL_ERROR;
          121  +      }
          122  +
          123  +      assert( EXPERT_REPORT_SQL==1 );
          124  +      assert( EXPERT_REPORT_INDEXES==2 );
          125  +      assert( EXPERT_REPORT_PLAN==3 );
          126  +      assert( EXPERT_REPORT_CANDIDATES==4 );
          127  +      zReport = sqlite3_expert_report(pExpert, iStmt, 1+iEnum);
          128  +      Tcl_SetObjResult(interp, Tcl_NewStringObj(zReport, -1));
          129  +      break;
          130  +    }
          131  +
          132  +    default:       /* destroy */
          133  +      assert( iSub==4 );     
          134  +      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
          135  +      break;
          136  +  }
          137  +
          138  +  if( rc!=TCL_OK ){
          139  +    if( zErr ){
          140  +      Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1));
          141  +    }else{
          142  +      extern const char *sqlite3ErrName(int);
          143  +      Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
          144  +    }
          145  +  }
          146  +  sqlite3_free(zErr);
          147  +  return rc;
          148  +}
          149  +
          150  +static void SQLITE_TCLAPI testExpertDel(void *clientData){
          151  +  sqlite3expert *pExpert = (sqlite3expert*)clientData;
          152  +  sqlite3_expert_destroy(pExpert);
          153  +}
          154  +
          155  +/*
          156  +** sqlite3_expert_new DB
          157  +*/
          158  +static int SQLITE_TCLAPI test_sqlite3_expert_new(
          159  +  void * clientData,
          160  +  Tcl_Interp *interp,
          161  +  int objc,
          162  +  Tcl_Obj *CONST objv[]
          163  +){
          164  +  static int iCmd = 0;
          165  +  sqlite3 *db;
          166  +  char *zCmd = 0;
          167  +  char *zErr = 0;
          168  +  sqlite3expert *pExpert;
          169  +  int rc = TCL_OK;
          170  +
          171  +  if( objc!=2 ){
          172  +    Tcl_WrongNumArgs(interp, 1, objv, "DB");
          173  +    return TCL_ERROR;
          174  +  }
          175  +  if( dbHandleFromObj(interp, objv[1], &db) ){
          176  +    return TCL_ERROR;
          177  +  }
          178  +
          179  +  zCmd = sqlite3_mprintf("sqlite3expert%d", ++iCmd);
          180  +  if( zCmd==0 ){
          181  +    Tcl_AppendResult(interp, "out of memory", (char*)0);
          182  +    return TCL_ERROR;
          183  +  }
          184  +
          185  +  pExpert = sqlite3_expert_new(db, &zErr);
          186  +  if( pExpert==0 ){
          187  +    Tcl_AppendResult(interp, zErr, (char*)0);
          188  +    rc = TCL_ERROR;
          189  +  }else{
          190  +    void *p = (void*)pExpert;
          191  +    Tcl_CreateObjCommand(interp, zCmd, testExpertCmd, p, testExpertDel);
          192  +    Tcl_SetObjResult(interp, Tcl_NewStringObj(zCmd, -1));
          193  +  }
          194  +
          195  +  sqlite3_free(zCmd);
          196  +  sqlite3_free(zErr);
          197  +  return rc;
          198  +}
          199  +
          200  +#endif  /* ifndef SQLITE_OMIT_VIRTUALTABLE */
          201  +
          202  +int TestExpert_Init(Tcl_Interp *interp){
          203  +#ifndef SQLITE_OMIT_VIRTUALTABLE
          204  +  struct Cmd {
          205  +    const char *zCmd;
          206  +    Tcl_ObjCmdProc *xProc;
          207  +  } aCmd[] = {
          208  +    { "sqlite3_expert_new", test_sqlite3_expert_new },
          209  +  };
          210  +  int i;
          211  +
          212  +  for(i=0; i<sizeof(aCmd)/sizeof(struct Cmd); i++){
          213  +    struct Cmd *p = &aCmd[i];
          214  +    Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, 0, 0);
          215  +  }
          216  +#endif
          217  +  return TCL_OK;
          218  +}
          219  +
          220  +#endif

Changes to ext/fts5/fts5_aux.c.

   353    353       if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
   354    354       if( iAdj<0 ) iAdj = 0;
   355    355       *piPos = iAdj;
   356    356     }
   357    357   
   358    358     return rc;
   359    359   }
          360  +
          361  +/*
          362  +** Return the value in pVal interpreted as utf-8 text. Except, if pVal 
          363  +** contains a NULL value, return a pointer to a static string zero
          364  +** bytes in length instead of a NULL pointer.
          365  +*/
          366  +static const char *fts5ValueToText(sqlite3_value *pVal){
          367  +  const char *zRet = (const char*)sqlite3_value_text(pVal);
          368  +  return zRet ? zRet : "";
          369  +}
   360    370   
   361    371   /*
   362    372   ** Implementation of snippet() function.
   363    373   */
   364    374   static void fts5SnippetFunction(
   365    375     const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
   366    376     Fts5Context *pFts,              /* First arg to pass to pApi functions */
................................................................................
   389    399       sqlite3_result_error(pCtx, zErr, -1);
   390    400       return;
   391    401     }
   392    402   
   393    403     nCol = pApi->xColumnCount(pFts);
   394    404     memset(&ctx, 0, sizeof(HighlightContext));
   395    405     iCol = sqlite3_value_int(apVal[0]);
   396         -  ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
   397         -  ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
   398         -  zEllips = (const char*)sqlite3_value_text(apVal[3]);
          406  +  ctx.zOpen = fts5ValueToText(apVal[1]);
          407  +  ctx.zClose = fts5ValueToText(apVal[2]);
          408  +  zEllips = fts5ValueToText(apVal[3]);
   399    409     nToken = sqlite3_value_int(apVal[4]);
   400    410   
   401    411     iBestCol = (iCol>=0 ? iCol : 0);
   402    412     nPhrase = pApi->xPhraseCount(pFts);
   403    413     aSeen = sqlite3_malloc(nPhrase);
   404    414     if( aSeen==0 ){
   405    415       rc = SQLITE_NOMEM;

Changes to ext/fts5/fts5_index.c.

  4905   4905     if( p2->n ){
  4906   4906       i64 iLastRowid = 0;
  4907   4907       Fts5DoclistIter i1;
  4908   4908       Fts5DoclistIter i2;
  4909   4909       Fts5Buffer out = {0, 0, 0};
  4910   4910       Fts5Buffer tmp = {0, 0, 0};
  4911   4911   
  4912         -    if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return;
         4912  +    /* The maximum size of the output is equal to the sum of the two 
         4913  +    ** input sizes + 1 varint (9 bytes). The extra varint is because if the
         4914  +    ** first rowid in one input is a large negative number, and the first in
         4915  +    ** the other a non-negative number, the delta for the non-negative
         4916  +    ** number will be larger on disk than the literal integer value
         4917  +    ** was.  */
         4918  +    if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n + 9) ) return;
  4913   4919       fts5DoclistIterInit(p1, &i1);
  4914   4920       fts5DoclistIterInit(p2, &i2);
  4915   4921   
  4916   4922       while( 1 ){
  4917   4923         if( i1.iRowid<i2.iRowid ){
  4918   4924           /* Copy entry from i1 */
  4919   4925           fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
................................................................................
  4999   5005         fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
  5000   5006         fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.aEof - i1.aPoslist);
  5001   5007       }
  5002   5008       else if( i2.aPoslist ){
  5003   5009         fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
  5004   5010         fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist);
  5005   5011       }
         5012  +    assert( out.n<=(p1->n+p2->n+9) );
  5006   5013   
  5007   5014       fts5BufferSet(&p->rc, p1, out.n, out.p);
  5008   5015       fts5BufferFree(&tmp);
  5009   5016       fts5BufferFree(&out);
  5010   5017     }
  5011   5018   }
  5012   5019   

Changes to ext/fts5/fts5_tcl.c.

   429    429         rc = p->pApi->xSetAuxdata(p->pFts, (void*)((char*)0 + iVal), 0);
   430    430         break;
   431    431       }
   432    432       CASE(15, "xGetAuxdataInt") {
   433    433         int iVal;
   434    434         int bClear;
   435    435         if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR;
   436         -      iVal = ((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
          436  +      iVal = (int)((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
   437    437         Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
   438    438         break;
   439    439       }
   440    440   
   441    441       CASE(16, "xPhraseForeach") {
   442    442         int iPhrase;
   443    443         int iCol;

Changes to ext/fts5/test/fts5af.test.

   170    170       'x a a a a a a a a a a',
   171    171       'a a a a a a a a a a a a a a a a a a a x'
   172    172     );
   173    173   }
   174    174   do_execsql_test 5.1 {
   175    175     SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
   176    176   } {{[x] a a a a a...}}
          177  +
          178  +do_execsql_test 5.2 {
          179  +  SELECT snippet(p1, 0, '[', ']', NULL, 6) FROM p1('x');
          180  +} {{[x] a a a a a}}
          181  +do_execsql_test 5.3 {
          182  +  SELECT snippet(p1, 0, NULL, ']', '...', 6) FROM p1('x');
          183  +} {{x] a a a a a...}}
          184  +do_execsql_test 5.4 {
          185  +  SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x');
          186  +} {{[x a a a a a...}}
   177    187   
   178    188   } ;# foreach_detail_mode 
   179    189   
   180    190   finish_test

Changes to ext/fts5/test/fts5query.test.

    60     60       foreach x [list bbb ddd fff hhh jjj lll nnn ppp rrr ttt] {
    61     61         set doc [string repeat "$x " 30]
    62     62         execsql { INSERT INTO t1 VALUES($doc) }
    63     63       }
    64     64       execsql COMMIT
    65     65     } {}
    66     66   
    67         -  do_execsql_test 1.$tn.2 {
           67  +  do_execsql_test 2.$tn.2 {
    68     68       INSERT INTO t1(t1) VALUES('integrity-check');
    69     69     }
    70     70   
    71     71     set ret 1
    72     72     foreach x [list a c e g i k m o q s u] {
    73     73       do_execsql_test 2.$tn.3.$ret {
    74     74         SELECT rowid FROM t1 WHERE t1 MATCH $x || '*';
    75     75       } {}
    76     76       incr ret
    77     77     }
    78     78   }
    79     79   
           80  +reset_db
           81  +do_execsql_test 3.0 {
           82  +  CREATE VIRTUAL TABLE x1 USING fts5(a);
           83  +  INSERT INTO x1(rowid, a) VALUES(-1000000000000, 'toyota');
           84  +  INSERT INTO x1(rowid, a) VALUES(1, 'tarago');
           85  +}
           86  +do_execsql_test 3.1 {
           87  +  SELECT rowid FROM x1('t*');
           88  +} {-1000000000000 1}
           89  +
    80     90   
    81     91   finish_test

Changes to ext/icu/icu.c.

    24     24   **
    25     25   **   * Integration of ICU and SQLite collation sequences.
    26     26   **
    27     27   **   * An implementation of the LIKE operator that uses ICU to 
    28     28   **     provide case-independent matching.
    29     29   */
    30     30   
    31         -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
           31  +#if !defined(SQLITE_CORE)                  \
           32  + || defined(SQLITE_ENABLE_ICU)             \
           33  + || defined(SQLITE_ENABLE_ICU_COLLATIONS)
    32     34   
    33     35   /* Include ICU headers */
    34     36   #include <unicode/utypes.h>
    35     37   #include <unicode/uregex.h>
    36     38   #include <unicode/ustring.h>
    37     39   #include <unicode/ucol.h>
    38     40   
................................................................................
    41     43   #ifndef SQLITE_CORE
    42     44     #include "sqlite3ext.h"
    43     45     SQLITE_EXTENSION_INIT1
    44     46   #else
    45     47     #include "sqlite3.h"
    46     48   #endif
    47     49   
           50  +/*
           51  +** This function is called when an ICU function called from within
           52  +** the implementation of an SQL scalar function returns an error.
           53  +**
           54  +** The scalar function context passed as the first argument is 
           55  +** loaded with an error message based on the following two args.
           56  +*/
           57  +static void icuFunctionError(
           58  +  sqlite3_context *pCtx,       /* SQLite scalar function context */
           59  +  const char *zName,           /* Name of ICU function that failed */
           60  +  UErrorCode e                 /* Error code returned by ICU function */
           61  +){
           62  +  char zBuf[128];
           63  +  sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
           64  +  zBuf[127] = '\0';
           65  +  sqlite3_result_error(pCtx, zBuf, -1);
           66  +}
           67  +
           68  +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
           69  +
    48     70   /*
    49     71   ** Maximum length (in bytes) of the pattern in a LIKE or GLOB
    50     72   ** operator.
    51     73   */
    52     74   #ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH
    53     75   # define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000
    54     76   #endif
................................................................................
   220    242     }
   221    243   
   222    244     if( zA && zB ){
   223    245       sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc));
   224    246     }
   225    247   }
   226    248   
   227         -/*
   228         -** This function is called when an ICU function called from within
   229         -** the implementation of an SQL scalar function returns an error.
   230         -**
   231         -** The scalar function context passed as the first argument is 
   232         -** loaded with an error message based on the following two args.
   233         -*/
   234         -static void icuFunctionError(
   235         -  sqlite3_context *pCtx,       /* SQLite scalar function context */
   236         -  const char *zName,           /* Name of ICU function that failed */
   237         -  UErrorCode e                 /* Error code returned by ICU function */
   238         -){
   239         -  char zBuf[128];
   240         -  sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
   241         -  zBuf[127] = '\0';
   242         -  sqlite3_result_error(pCtx, zBuf, -1);
   243         -}
   244         -
   245    249   /*
   246    250   ** Function to delete compiled regexp objects. Registered as
   247    251   ** a destructor function with sqlite3_set_auxdata().
   248    252   */
   249    253   static void icuRegexpDelete(void *p){
   250    254     URegularExpression *pExpr = (URegularExpression *)p;
   251    255     uregex_close(pExpr);
................................................................................
   403    407         icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status);
   404    408       }
   405    409       return;
   406    410     }
   407    411     assert( 0 );     /* Unreachable */
   408    412   }
   409    413   
          414  +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */
          415  +
   410    416   /*
   411    417   ** Collation sequence destructor function. The pCtx argument points to
   412    418   ** a UCollator structure previously allocated using ucol_open().
   413    419   */
   414    420   static void icuCollationDel(void *pCtx){
   415    421     UCollator *p = (UCollator *)pCtx;
   416    422     ucol_close(p);
................................................................................
   497    503       const char *zName;                        /* Function name */
   498    504       unsigned char nArg;                       /* Number of arguments */
   499    505       unsigned short enc;                       /* Optimal text encoding */
   500    506       unsigned char iContext;                   /* sqlite3_user_data() context */
   501    507       void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
   502    508     } scalars[] = {
   503    509       {"icu_load_collation",  2, SQLITE_UTF8,                1, icuLoadCollation},
          510  +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
   504    511       {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC,         0, icuRegexpFunc},
   505    512       {"lower",  1, SQLITE_UTF16|SQLITE_DETERMINISTIC,       0, icuCaseFunc16},
   506    513       {"lower",  2, SQLITE_UTF16|SQLITE_DETERMINISTIC,       0, icuCaseFunc16},
   507    514       {"upper",  1, SQLITE_UTF16|SQLITE_DETERMINISTIC,       1, icuCaseFunc16},
   508    515       {"upper",  2, SQLITE_UTF16|SQLITE_DETERMINISTIC,       1, icuCaseFunc16},
   509    516       {"lower",  1, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuCaseFunc16},
   510    517       {"lower",  2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuCaseFunc16},
   511    518       {"upper",  1, SQLITE_UTF8|SQLITE_DETERMINISTIC,        1, icuCaseFunc16},
   512    519       {"upper",  2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        1, icuCaseFunc16},
   513    520       {"like",   2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuLikeFunc},
   514    521       {"like",   3, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuLikeFunc},
          522  +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */
   515    523     };
   516    524     int rc = SQLITE_OK;
   517    525     int i;
   518         -
   519    526     
   520    527     for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){
   521    528       const struct IcuScalar *p = &scalars[i];
   522    529       rc = sqlite3_create_function(
   523    530           db, p->zName, p->nArg, p->enc, 
   524    531           p->iContext ? (void*)db : (void*)0,
   525    532           p->xFunc, 0, 0

Changes to ext/lsm1/lsm-test/lsmtest.h.

   117    117   ** Functions in wrapper3.c. This file contains the tdb wrapper for lsm.
   118    118   ** The wrapper for lsm is a bit more involved than the others, as it 
   119    119   ** includes code for a couple of different lsm configurations, and for
   120    120   ** various types of fault injection and robustness testing.
   121    121   */
   122    122   int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
   123    123   int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb);
          124  +int test_lsm_lomem2_open(const char*, const char*, int bClear, TestDb **ppDb);
   124    125   int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb);
   125    126   int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb);
   126    127   int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb);
   127    128   int test_lsm_mt3(const char*, const char *zFile, int bClear, TestDb **ppDb);
   128    129   
   129    130   int tdb_lsm_configure(lsm_db *, const char *);
   130    131   

Changes to ext/lsm1/lsm-test/lsmtest_tdb.c.

   717    717     const char *zName;
   718    718     const char *zDefaultDb;
   719    719     int (*xOpen)(const char *, const char *zFilename, int bClear, TestDb **ppDb);
   720    720   } aLib[] = {
   721    721     { "sqlite3",      "testdb.sqlite",    sql_open },
   722    722     { "lsm_small",    "testdb.lsm_small", test_lsm_small_open },
   723    723     { "lsm_lomem",    "testdb.lsm_lomem", test_lsm_lomem_open },
          724  +  { "lsm_lomem2",   "testdb.lsm_lomem2", test_lsm_lomem2_open },
   724    725   #ifdef HAVE_ZLIB
   725    726     { "lsm_zip",      "testdb.lsm_zip",   test_lsm_zip_open },
   726    727   #endif
   727    728     { "lsm",          "testdb.lsm",       test_lsm_open },
   728    729   #ifdef LSM_MUTEX_PTHREADS
   729    730     { "lsm_mt2",      "testdb.lsm_mt2",   test_lsm_mt2 },
   730    731     { "lsm_mt3",      "testdb.lsm_mt3",   test_lsm_mt3 },

Changes to ext/lsm1/lsm-test/lsmtest_tdb3.c.

  1028   1028     const char *zCfg = 
  1029   1029       "page_size=256 block_size=64 autoflush=16 "
  1030   1030       "autocheckpoint=32"
  1031   1031       "mmap=0 "
  1032   1032     ;
  1033   1033     return testLsmOpen(zCfg, zFilename, bClear, ppDb);
  1034   1034   }
         1035  +
         1036  +int test_lsm_lomem2_open(
         1037  +  const char *zSpec, 
         1038  +  const char *zFilename, 
         1039  +  int bClear, 
         1040  +  TestDb **ppDb
         1041  +){
         1042  +    /* "max_freelist=4 autocheckpoint=32" */
         1043  +  const char *zCfg = 
         1044  +    "page_size=512 block_size=64 autoflush=0 mmap=0 "
         1045  +  ;
         1046  +  return testLsmOpen(zCfg, zFilename, bClear, ppDb);
         1047  +}
  1035   1048   
  1036   1049   int test_lsm_zip_open(
  1037   1050     const char *zSpec, 
  1038   1051     const char *zFilename, 
  1039   1052     int bClear, 
  1040   1053     TestDb **ppDb
  1041   1054   ){

Changes to ext/lsm1/lsmInt.h.

   106    106   typedef unsigned short int u16;
   107    107   typedef unsigned int u32;
   108    108   typedef lsm_i64 i64;
   109    109   typedef unsigned long long int u64;
   110    110   #endif
   111    111   
   112    112   /* A page number is a 64-bit integer. */
   113         -typedef i64 Pgno;
          113  +typedef i64 LsmPgno;
   114    114   
   115    115   #ifdef LSM_DEBUG
   116    116   int lsmErrorBkpt(int);
   117    117   #else
   118    118   # define lsmErrorBkpt(x) (x)
   119    119   #endif
   120    120   
................................................................................
   398    398     void **apShm;                   /* Shared memory chunks */
   399    399     ShmHeader *pShmhdr;             /* Live shared-memory header */
   400    400     TreeHeader treehdr;             /* Local copy of tree-header */
   401    401     u32 aSnapshot[LSM_META_PAGE_SIZE / sizeof(u32)];
   402    402   };
   403    403   
   404    404   struct Segment {
   405         -  Pgno iFirst;                     /* First page of this run */
   406         -  Pgno iLastPg;                    /* Last page of this run */
   407         -  Pgno iRoot;                      /* Root page number (if any) */
          405  +  LsmPgno iFirst;                  /* First page of this run */
          406  +  LsmPgno iLastPg;                 /* Last page of this run */
          407  +  LsmPgno iRoot;                   /* Root page number (if any) */
   408    408     int nSize;                       /* Size of this run in pages */
   409    409   
   410    410     Redirect *pRedirect;             /* Block redirects (or NULL) */
   411    411   };
   412    412   
   413    413   /*
   414    414   ** iSplitTopic/pSplitKey/nSplitKey:
................................................................................
   452    452   ** access to the associated Level struct.
   453    453   **
   454    454   ** iOutputOff:
   455    455   **   The byte offset to write to next within the last page of the 
   456    456   **   output segment.
   457    457   */
   458    458   struct MergeInput {
   459         -  Pgno iPg;                       /* Page on which next input is stored */
          459  +  LsmPgno iPg;                    /* Page on which next input is stored */
   460    460     int iCell;                      /* Cell containing next input to merge */
   461    461   };
   462    462   struct Merge {
   463    463     int nInput;                     /* Number of input runs being merged */
   464    464     MergeInput *aInput;             /* Array nInput entries in size */
   465    465     MergeInput splitkey;            /* Location in file of current splitkey */
   466    466     int nSkip;                      /* Number of separators entries to skip */
   467    467     int iOutputOff;                 /* Write offset on output page */
   468         -  Pgno iCurrentPtr;               /* Current pointer value */
          468  +  LsmPgno iCurrentPtr;            /* Current pointer value */
   469    469   };
   470    470   
   471    471   /* 
   472    472   ** The first argument to this macro is a pointer to a Segment structure.
   473    473   ** Returns true if the structure instance indicates that the separators
   474    474   ** array is valid.
   475    475   */
................................................................................
   575    575     u32 iCmpId;                     /* Id of compression scheme */
   576    576     Level *pLevel;                  /* Pointer to level 0 of snapshot (or NULL) */
   577    577     i64 iId;                        /* Snapshot id */
   578    578     i64 iLogOff;                    /* Log file offset */
   579    579     Redirect redirect;              /* Block redirection array */
   580    580   
   581    581     /* Used by worker snapshots only */
   582         -  int nBlock;                     /* Number of blocks in database file */
   583         -  Pgno aiAppend[LSM_APPLIST_SZ];  /* Append point list */
   584         -  Freelist freelist;              /* Free block list */
   585         -  u32 nWrite;                     /* Total number of pages written to disk */
          582  +  int nBlock;                        /* Number of blocks in database file */
          583  +  LsmPgno aiAppend[LSM_APPLIST_SZ];  /* Append point list */
          584  +  Freelist freelist;                 /* Free block list */
          585  +  u32 nWrite;                        /* Total number of pages written to disk */
   586    586   };
   587    587   #define LSM_INITIAL_SNAPSHOT_ID 11
   588    588   
   589    589   /*
   590    590   ** Functions from file "lsm_ckpt.c".
   591    591   */
   592    592   int lsmCheckpointWrite(lsm_db *, u32 *);
................................................................................
   706    706   
   707    707   int lsmFsPageSize(FileSystem *);
   708    708   void lsmFsSetPageSize(FileSystem *, int);
   709    709   
   710    710   int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId);
   711    711   
   712    712   /* Creating, populating, gobbling and deleting sorted runs. */
   713         -void lsmFsGobble(lsm_db *, Segment *, Pgno *, int);
          713  +void lsmFsGobble(lsm_db *, Segment *, LsmPgno *, int);
   714    714   int lsmFsSortedDelete(FileSystem *, Snapshot *, int, Segment *);
   715    715   int lsmFsSortedFinish(FileSystem *, Segment *);
   716    716   int lsmFsSortedAppend(FileSystem *, Snapshot *, Level *, int, Page **);
   717    717   int lsmFsSortedPadding(FileSystem *, Snapshot *, Segment *);
   718    718   
   719    719   /* Functions to retrieve the lsm_env pointer from a FileSystem or Page object */
   720    720   lsm_env *lsmFsEnv(FileSystem *);
................................................................................
   723    723   
   724    724   int lsmFsSectorSize(FileSystem *);
   725    725   
   726    726   void lsmSortedSplitkey(lsm_db *, Level *, int *);
   727    727   
   728    728   /* Reading sorted run content. */
   729    729   int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg);
   730         -int lsmFsDbPageGet(FileSystem *, Segment *, Pgno, Page **);
          730  +int lsmFsDbPageGet(FileSystem *, Segment *, LsmPgno, Page **);
   731    731   int lsmFsDbPageNext(Segment *, Page *, int eDir, Page **);
   732    732   
   733    733   u8 *lsmFsPageData(Page *, int *);
   734    734   int lsmFsPageRelease(Page *);
   735    735   int lsmFsPagePersist(Page *);
   736    736   void lsmFsPageRef(Page *);
   737         -Pgno lsmFsPageNumber(Page *);
          737  +LsmPgno lsmFsPageNumber(Page *);
   738    738   
   739    739   int lsmFsNRead(FileSystem *);
   740    740   int lsmFsNWrite(FileSystem *);
   741    741   
   742    742   int lsmFsMetaPageGet(FileSystem *, int, int, MetaPage **);
   743    743   int lsmFsMetaPageRelease(MetaPage *);
   744    744   u8 *lsmFsMetaPageData(MetaPage *, int *);
   745    745   
   746    746   #ifdef LSM_DEBUG
   747    747   int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg);
   748    748   int lsmFsIntegrityCheck(lsm_db *);
   749    749   #endif
   750    750   
   751         -Pgno lsmFsRedirectPage(FileSystem *, Redirect *, Pgno);
          751  +LsmPgno lsmFsRedirectPage(FileSystem *, Redirect *, LsmPgno);
   752    752   
   753    753   int lsmFsPageWritable(Page *);
   754    754   
   755    755   /* Functions to read, write and sync the log file. */
   756    756   int lsmFsWriteLog(FileSystem *pFS, i64 iOff, LsmString *pStr);
   757    757   int lsmFsSyncLog(FileSystem *pFS);
   758    758   int lsmFsReadLog(FileSystem *pFS, i64 iOff, int nRead, LsmString *pStr);
................................................................................
   764    764   
   765    765   /* And to sync the db file */
   766    766   int lsmFsSyncDb(FileSystem *, int);
   767    767   
   768    768   void lsmFsFlushWaiting(FileSystem *, int *);
   769    769   
   770    770   /* Used by lsm_info(ARRAY_STRUCTURE) and lsm_config(MMAP) */
   771         -int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, Pgno iFirst, char **pzOut);
   772         -int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut);
          771  +int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, LsmPgno iFirst, char **pz);
          772  +int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut);
   773    773   int lsmConfigMmap(lsm_db *pDb, int *piParam);
   774    774   
   775    775   int lsmEnvOpen(lsm_env *, const char *, int, lsm_file **);
   776    776   int lsmEnvClose(lsm_env *pEnv, lsm_file *pFile);
   777    777   int lsmEnvLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int eLock);
   778    778   int lsmEnvTestLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int nLock, int);
   779    779   
................................................................................
   781    781   void lsmEnvShmBarrier(lsm_env *);
   782    782   void lsmEnvShmUnmap(lsm_env *, lsm_file *, int);
   783    783   
   784    784   void lsmEnvSleep(lsm_env *, int);
   785    785   
   786    786   int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal);
   787    787   
   788         -int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, Pgno, int *);
          788  +int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, LsmPgno, int *);
   789    789   
   790    790   void lsmFsPurgeCache(FileSystem *);
   791    791   
   792    792   /*
   793    793   ** End of functions from "lsm_file.c".
   794    794   **************************************************************************/
   795    795   
   796    796   /* 
   797    797   ** Functions from file "lsm_sorted.c".
   798    798   */
   799         -int lsmInfoPageDump(lsm_db *, Pgno, int, char **);
          799  +int lsmInfoPageDump(lsm_db *, LsmPgno, int, char **);
   800    800   void lsmSortedCleanup(lsm_db *);
   801    801   int lsmSortedAutoWork(lsm_db *, int nUnit);
   802    802   
   803    803   int lsmSortedWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *);
   804    804   
   805    805   int lsmSaveWorker(lsm_db *, int);
   806    806   

Changes to ext/lsm1/lsm_ckpt.c.

   385    385   static void ckptExportAppendlist(
   386    386     lsm_db *db,                     /* Database connection */
   387    387     CkptBuffer *p,                  /* Checkpoint buffer to write to */
   388    388     int *piOut,                     /* IN/OUT: Offset within checkpoint buffer */
   389    389     int *pRc                        /* IN/OUT: Error code */
   390    390   ){
   391    391     int i;
   392         -  Pgno *aiAppend = db->pWorker->aiAppend;
          392  +  LsmPgno *aiAppend = db->pWorker->aiAppend;
   393    393   
   394    394     for(i=0; i<LSM_APPLIST_SZ; i++){
   395    395       ckptAppend64(p, piOut, aiAppend[i], pRc);
   396    396     }
   397    397   };
   398    398   
   399    399   static int ckptExportSnapshot( 

Changes to ext/lsm1/lsm_file.c.

   265    265   **   The lsmFsSortedAppend() function sets the pSeg pointer to point to the
   266    266   **   segment that the new page will be a part of. It is unset by
   267    267   **   lsmFsPagePersist() after the page is written to disk.
   268    268   */
   269    269   struct Page {
   270    270     u8 *aData;                      /* Buffer containing page data */
   271    271     int nData;                      /* Bytes of usable data at aData[] */
   272         -  Pgno iPg;                       /* Page number */
          272  +  LsmPgno iPg;                    /* Page number */
   273    273     int nRef;                       /* Number of outstanding references */
   274    274     int flags;                      /* Combination of PAGE_XXX flags */
   275    275     Page *pHashNext;                /* Next page in hash table slot */
   276    276     Page *pLruNext;                 /* Next page in LRU list */
   277    277     Page *pLruPrev;                 /* Previous page in LRU list */
   278    278     FileSystem *pFS;                /* File system that owns this page */
   279    279   
................................................................................
   328    328   #else
   329    329   # define IOERR_WRAPPER(rc) (rc)
   330    330   #endif
   331    331   
   332    332   #ifdef NDEBUG
   333    333   # define assert_lists_are_ok(x)
   334    334   #else
   335         -static Page *fsPageFindInHash(FileSystem *pFS, Pgno iPg, int *piHash);
          335  +static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash);
   336    336   
   337    337   static void assert_lists_are_ok(FileSystem *pFS){
   338    338   #if 0
   339    339     Page *p;
   340    340   
   341    341     assert( pFS->nMapLimit>=0 );
   342    342   
................................................................................
   528    528     return LSM_OK;
   529    529   }
   530    530   
   531    531   /*
   532    532   ** Return true if page iReal of the database should be accessed using mmap.
   533    533   ** False otherwise.
   534    534   */
   535         -static int fsMmapPage(FileSystem *pFS, Pgno iReal){
          535  +static int fsMmapPage(FileSystem *pFS, LsmPgno iReal){
   536    536     return ((i64)iReal*pFS->nPagesize <= pFS->nMapLimit);
   537    537   }
   538    538   
   539    539   /*
   540    540   ** Given that there are currently nHash slots in the hash table, return 
   541    541   ** the hash key for file iFile, page iPg.
   542    542   */
   543         -static int fsHashKey(int nHash, Pgno iPg){
          543  +static int fsHashKey(int nHash, LsmPgno iPg){
   544    544     return (iPg % nHash);
   545    545   }
   546    546   
   547    547   /*
   548    548   ** This is a helper function for lsmFsOpen(). It opens a single file on
   549    549   ** disk (either the database or log file).
   550    550   */
................................................................................
   876    876   ** Return the page number of the first page on block iBlock. Blocks are
   877    877   ** numbered starting from 1.
   878    878   **
   879    879   ** For a compressed database, page numbers are byte offsets. The first
   880    880   ** page on each block is the byte offset immediately following the 4-byte
   881    881   ** "previous block" pointer at the start of each block.
   882    882   */
   883         -static Pgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
   884         -  Pgno iPg;
          883  +static LsmPgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
          884  +  LsmPgno iPg;
   885    885     if( pFS->pCompress ){
   886    886       if( iBlock==1 ){
   887    887         iPg = pFS->nMetasize * 2 + 4;
   888    888       }else{
   889         -      iPg = pFS->nBlocksize * (Pgno)(iBlock-1) + 4;
          889  +      iPg = pFS->nBlocksize * (LsmPgno)(iBlock-1) + 4;
   890    890       }
   891    891     }else{
   892    892       const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   893    893       if( iBlock==1 ){
   894    894         iPg = 1 + ((pFS->nMetasize*2 + pFS->nPagesize - 1) / pFS->nPagesize);
   895    895       }else{
   896    896         iPg = 1 + (iBlock-1) * nPagePerBlock;
................................................................................
   903    903   ** Return the page number of the last page on block iBlock. Blocks are
   904    904   ** numbered starting from 1.
   905    905   **
   906    906   ** For a compressed database, page numbers are byte offsets. The first
   907    907   ** page on each block is the byte offset of the byte immediately before 
   908    908   ** the 4-byte "next block" pointer at the end of each block.
   909    909   */
   910         -static Pgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
          910  +static LsmPgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
   911    911     if( pFS->pCompress ){
   912         -    return pFS->nBlocksize * (Pgno)iBlock - 1 - 4;
          912  +    return pFS->nBlocksize * (LsmPgno)iBlock - 1 - 4;
   913    913     }else{
   914    914       const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   915    915       return iBlock * nPagePerBlock;
   916    916     }
   917    917   }
   918    918   
   919    919   /*
   920    920   ** Return the block number of the block that page iPg is located on. 
   921    921   ** Blocks are numbered starting from 1.
   922    922   */
   923         -static int fsPageToBlock(FileSystem *pFS, Pgno iPg){
          923  +static int fsPageToBlock(FileSystem *pFS, LsmPgno iPg){
   924    924     if( pFS->pCompress ){
   925    925       return (int)((iPg / pFS->nBlocksize) + 1);
   926    926     }else{
   927    927       return (int)(1 + ((iPg-1) / (pFS->nBlocksize / pFS->nPagesize)));
   928    928     }
   929    929   }
   930    930   
   931    931   /*
   932    932   ** Return true if page iPg is the last page on its block.
   933    933   **
   934    934   ** This function is only called in non-compressed database mode.
   935    935   */
   936         -static int fsIsLast(FileSystem *pFS, Pgno iPg){
          936  +static int fsIsLast(FileSystem *pFS, LsmPgno iPg){
   937    937     const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   938    938     assert( !pFS->pCompress );
   939    939     return ( iPg && (iPg % nPagePerBlock)==0 );
   940    940   }
   941    941   
   942    942   /*
   943    943   ** Return true if page iPg is the first page on its block.
   944    944   **
   945    945   ** This function is only called in non-compressed database mode.
   946    946   */
   947         -static int fsIsFirst(FileSystem *pFS, Pgno iPg){
          947  +static int fsIsFirst(FileSystem *pFS, LsmPgno iPg){
   948    948     const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   949    949     assert( !pFS->pCompress );
   950    950     return ( (iPg % nPagePerBlock)==1
   951    951           || (iPg<nPagePerBlock && iPg==fsFirstPageOnBlock(pFS, 1))
   952    952     );
   953    953   }
   954    954   
................................................................................
   963    963     }
   964    964     return pPage->aData;
   965    965   }
   966    966   
   967    967   /*
   968    968   ** Return the page number of a page.
   969    969   */
   970         -Pgno lsmFsPageNumber(Page *pPage){
          970  +LsmPgno lsmFsPageNumber(Page *pPage){
   971    971     /* assert( (pPage->flags & PAGE_DIRTY)==0 ); */
   972    972     return pPage ? pPage->iPg : 0;
   973    973   }
   974    974   
   975    975   /*
   976    976   ** Page pPg is currently part of the LRU list belonging to pFS. Remove
   977    977   ** it from the list. pPg->pLruNext and pPg->pLruPrev are cleared by this
................................................................................
  1054   1054   /*
  1055   1055   ** Search the hash-table for page iPg. If an entry is round, return a pointer
  1056   1056   ** to it. Otherwise, return NULL.
  1057   1057   **
  1058   1058   ** Either way, if argument piHash is not NULL set *piHash to the hash slot
  1059   1059   ** number that page iPg would be stored in before returning.
  1060   1060   */
  1061         -static Page *fsPageFindInHash(FileSystem *pFS, Pgno iPg, int *piHash){
         1061  +static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash){
  1062   1062     Page *p;                        /* Return value */
  1063   1063     int iHash = fsHashKey(pFS->nHash, iPg);
  1064   1064   
  1065   1065     if( piHash ) *piHash = iHash;
  1066   1066     for(p=pFS->apHash[iHash]; p; p=p->pHashNext){
  1067   1067       if( p->iPg==iPg) break;
  1068   1068     }
................................................................................
  1185   1185   }
  1186   1186   
  1187   1187   /*
  1188   1188   ** If page iPg has been redirected according to the redirections in the
  1189   1189   ** object passed as the second argument, return the destination page to
  1190   1190   ** which it is redirected. Otherwise, return a copy of iPg.
  1191   1191   */
  1192         -Pgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, Pgno iPg){
  1193         -  Pgno iReal = iPg;
         1192  +LsmPgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, LsmPgno iPg){
         1193  +  LsmPgno iReal = iPg;
  1194   1194   
  1195   1195     if( pRedir ){
  1196   1196       const int nPagePerBlock = (
  1197   1197           pFS->pCompress ? pFS->nBlocksize : (pFS->nBlocksize / pFS->nPagesize)
  1198   1198       );
  1199   1199       int iBlk = fsPageToBlock(pFS, iPg);
  1200   1200       int i;
  1201   1201       for(i=0; i<pRedir->n; i++){
  1202   1202         int iFrom = pRedir->a[i].iFrom;
  1203   1203         if( iFrom>iBlk ) break;
  1204   1204         if( iFrom==iBlk ){
  1205   1205           int iTo = pRedir->a[i].iTo;
  1206         -        iReal = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock;
         1206  +        iReal = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock;
  1207   1207           if( iTo==1 ){
  1208   1208             iReal += (fsFirstPageOnBlock(pFS, 1)-1);
  1209   1209           }
  1210   1210           break;
  1211   1211         }
  1212   1212       }
  1213   1213     }
  1214   1214   
  1215   1215     assert( iReal!=0 );
  1216   1216     return iReal;
  1217   1217   }
  1218   1218   
  1219   1219   /* Required by the circular fsBlockNext<->fsPageGet dependency. */
  1220         -static int fsPageGet(FileSystem *, Segment *, Pgno, int, Page **, int *);
         1220  +static int fsPageGet(FileSystem *, Segment *, LsmPgno, int, Page **, int *);
  1221   1221   
  1222   1222   /*
  1223   1223   ** Parameter iBlock is a database file block. This function reads the value 
  1224   1224   ** stored in the blocks "next block" pointer and stores it in *piNext.
  1225   1225   ** LSM_OK is returned if everything is successful, or an LSM error code
  1226   1226   ** otherwise.
  1227   1227   */
................................................................................
  1265   1265     }
  1266   1266     return rc;
  1267   1267   }
  1268   1268   
  1269   1269   /*
  1270   1270   ** Return the page number of the last page on the same block as page iPg.
  1271   1271   */
  1272         -Pgno fsLastPageOnPagesBlock(FileSystem *pFS, Pgno iPg){
         1272  +LsmPgno fsLastPageOnPagesBlock(FileSystem *pFS, LsmPgno iPg){
  1273   1273     return fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iPg));
  1274   1274   }
  1275   1275   
  1276   1276   /*
  1277   1277   ** Read nData bytes of data from offset iOff of the database file into
  1278   1278   ** buffer aData. If this means reading past the end of a block, follow
  1279   1279   ** the block pointer to the next block and continue reading.
................................................................................
  1533   1533   ** to the total number of free bytes before returning.
  1534   1534   **
  1535   1535   ** If no error occurs, LSM_OK is returned. Otherwise, an lsm error code.
  1536   1536   */
  1537   1537   static int fsPageGet(
  1538   1538     FileSystem *pFS,                /* File-system handle */
  1539   1539     Segment *pSeg,                  /* Block redirection to use (or NULL) */
  1540         -  Pgno iPg,                       /* Page id */
         1540  +  LsmPgno iPg,                    /* Page id */
  1541   1541     int noContent,                  /* True to not load content from disk */
  1542   1542     Page **ppPg,                    /* OUT: New page handle */
  1543   1543     int *pnSpace                    /* OUT: Bytes of free space */
  1544   1544   ){
  1545   1545     Page *p;
  1546   1546     int iHash;
  1547   1547     int rc = LSM_OK;
  1548   1548   
  1549   1549     /* In most cases iReal is the same as iPg. Except, if pSeg->pRedirect is 
  1550   1550     ** not NULL, and the block containing iPg has been redirected, then iReal
  1551   1551     ** is the page number after redirection.  */
  1552         -  Pgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg);
         1552  +  LsmPgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg);
  1553   1553   
  1554   1554     assert_lists_are_ok(pFS);
  1555   1555     assert( iPg>=fsFirstPageOnBlock(pFS, 1) );
  1556   1556     assert( iReal>=fsFirstPageOnBlock(pFS, 1) );
  1557   1557     *ppPg = 0;
  1558   1558   
  1559   1559     /* Search the hash-table for the page */
................................................................................
  1685   1685   /*
  1686   1686   ** Return true if the first or last page of segment pRun falls between iFirst
  1687   1687   ** and iLast, inclusive, and pRun is not equal to pIgnore.
  1688   1688   */
  1689   1689   static int fsRunEndsBetween(
  1690   1690     Segment *pRun, 
  1691   1691     Segment *pIgnore, 
  1692         -  Pgno iFirst, 
  1693         -  Pgno iLast
         1692  +  LsmPgno iFirst, 
         1693  +  LsmPgno iLast
  1694   1694   ){
  1695   1695     return (pRun!=pIgnore && (
  1696   1696           (pRun->iFirst>=iFirst && pRun->iFirst<=iLast)
  1697   1697        || (pRun->iLastPg>=iFirst && pRun->iLastPg<=iLast)
  1698   1698     ));
  1699   1699   }
  1700   1700   
................................................................................
  1701   1701   /*
  1702   1702   ** Return true if level pLevel contains a segment other than pIgnore for
  1703   1703   ** which the first or last page is between iFirst and iLast, inclusive.
  1704   1704   */
  1705   1705   static int fsLevelEndsBetween(
  1706   1706     Level *pLevel, 
  1707   1707     Segment *pIgnore, 
  1708         -  Pgno iFirst, 
  1709         -  Pgno iLast
         1708  +  LsmPgno iFirst, 
         1709  +  LsmPgno iLast
  1710   1710   ){
  1711   1711     int i;
  1712   1712   
  1713   1713     if( fsRunEndsBetween(&pLevel->lhs, pIgnore, iFirst, iLast) ){
  1714   1714       return 1;
  1715   1715     }
  1716   1716     for(i=0; i<pLevel->nRight; i++){
................................................................................
  1729   1729   static int fsFreeBlock(
  1730   1730     FileSystem *pFS,                /* File system object */
  1731   1731     Snapshot *pSnapshot,            /* Worker snapshot */
  1732   1732     Segment *pIgnore,               /* Ignore this run when searching */
  1733   1733     int iBlk                        /* Block number of block to free */
  1734   1734   ){
  1735   1735     int rc = LSM_OK;                /* Return code */
  1736         -  Pgno iFirst;                    /* First page on block iBlk */
  1737         -  Pgno iLast;                     /* Last page on block iBlk */
         1736  +  LsmPgno iFirst;                 /* First page on block iBlk */
         1737  +  LsmPgno iLast;                  /* Last page on block iBlk */
  1738   1738     Level *pLevel;                  /* Used to iterate through levels */
  1739   1739   
  1740   1740     int iIn;                        /* Used to iterate through append points */
  1741   1741     int iOut = 0;                   /* Used to output append points */
  1742         -  Pgno *aApp = pSnapshot->aiAppend;
         1742  +  LsmPgno *aApp = pSnapshot->aiAppend;
  1743   1743   
  1744   1744     iFirst = fsFirstPageOnBlock(pFS, iBlk);
  1745   1745     iLast = fsLastPageOnBlock(pFS, iBlk);
  1746   1746   
  1747   1747     /* Check if any other run in the snapshot has a start or end page 
  1748   1748     ** within this block. If there is such a run, return early. */
  1749   1749     for(pLevel=lsmDbSnapshotLevel(pSnapshot); pLevel; pLevel=pLevel->pNext){
................................................................................
  1807   1807   }
  1808   1808   
  1809   1809   /*
  1810   1810   ** aPgno is an array containing nPgno page numbers. Return the smallest page
  1811   1811   ** number from the array that falls on block iBlk. Or, if none of the pages
  1812   1812   ** in aPgno[] fall on block iBlk, return 0.
  1813   1813   */
  1814         -static Pgno firstOnBlock(FileSystem *pFS, int iBlk, Pgno *aPgno, int nPgno){
  1815         -  Pgno iRet = 0;
         1814  +static LsmPgno firstOnBlock(
         1815  +  FileSystem *pFS, 
         1816  +  int iBlk, 
         1817  +  LsmPgno *aPgno, 
         1818  +  int nPgno
         1819  +){
         1820  +  LsmPgno iRet = 0;
  1816   1821     int i;
  1817   1822     for(i=0; i<nPgno; i++){
  1818         -    Pgno iPg = aPgno[i];
         1823  +    LsmPgno iPg = aPgno[i];
  1819   1824       if( fsPageToBlock(pFS, iPg)==iBlk && (iRet==0 || iPg<iRet) ){
  1820   1825         iRet = iPg;
  1821   1826       }
  1822   1827     }
  1823   1828     return iRet;
  1824   1829   }
  1825   1830   
  1826   1831   #ifndef NDEBUG
  1827   1832   /*
  1828   1833   ** Return true if page iPg, which is a part of segment p, lies on
  1829   1834   ** a redirected block. 
  1830   1835   */
  1831         -static int fsPageRedirects(FileSystem *pFS, Segment *p, Pgno iPg){
         1836  +static int fsPageRedirects(FileSystem *pFS, Segment *p, LsmPgno iPg){
  1832   1837     return (iPg!=0 && iPg!=lsmFsRedirectPage(pFS, p->pRedirect, iPg));
  1833   1838   }
  1834   1839   
  1835   1840   /*
  1836   1841   ** Return true if the second argument is not NULL and any of the first
  1837   1842   ** last or root pages lie on a redirected block. 
  1838   1843   */
................................................................................
  1850   1855   ** the segment pRun. This function gobbles from the start of the run to the
  1851   1856   ** first page that appears in aPgno[] (i.e. so that the aPgno[] entry is
  1852   1857   ** the new first page of the run).
  1853   1858   */
  1854   1859   void lsmFsGobble(
  1855   1860     lsm_db *pDb,
  1856   1861     Segment *pRun, 
  1857         -  Pgno *aPgno,
         1862  +  LsmPgno *aPgno,
  1858   1863     int nPgno
  1859   1864   ){
  1860   1865     int rc = LSM_OK;
  1861   1866     FileSystem *pFS = pDb->pFS;
  1862   1867     Snapshot *pSnapshot = pDb->pWorker;
  1863   1868     int iBlk;
  1864   1869   
................................................................................
  1867   1872     assert( nPgno>0 && 0==fsPageRedirects(pFS, pRun, aPgno[0]) );
  1868   1873   
  1869   1874     iBlk = fsPageToBlock(pFS, pRun->iFirst);
  1870   1875     pRun->nSize += (int)(pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk));
  1871   1876   
  1872   1877     while( rc==LSM_OK ){
  1873   1878       int iNext = 0;
  1874         -    Pgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno);
         1879  +    LsmPgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno);
  1875   1880       if( iFirst ){
  1876   1881         pRun->iFirst = iFirst;
  1877   1882         break;
  1878   1883       }
  1879   1884       rc = fsBlockNext(pFS, pRun, iBlk, &iNext);
  1880   1885       if( rc==LSM_OK ) rc = fsFreeBlock(pFS, pSnapshot, pRun, iBlk);
  1881   1886       pRun->nSize -= (int)(
................................................................................
  1901   1906   **   *piNext = iPg + nByte;
  1902   1907   **
  1903   1908   ** But take block overflow and redirection into account.
  1904   1909   */
  1905   1910   static int fsNextPageOffset(
  1906   1911     FileSystem *pFS,                /* File system object */
  1907   1912     Segment *pSeg,                  /* Segment to move within */
  1908         -  Pgno iPg,                       /* Offset of current page */
         1913  +  LsmPgno iPg,                    /* Offset of current page */
  1909   1914     int nByte,                      /* Size of current page including headers */
  1910         -  Pgno *piNext                    /* OUT: Offset of next page. Or zero (EOF) */
         1915  +  LsmPgno *piNext                 /* OUT: Offset of next page. Or zero (EOF) */
  1911   1916   ){
  1912         -  Pgno iNext;
         1917  +  LsmPgno iNext;
  1913   1918     int rc;
  1914   1919   
  1915   1920     assert( pFS->pCompress );
  1916   1921   
  1917   1922     rc = fsAddOffset(pFS, pSeg, iPg, nByte-1, &iNext);
  1918   1923     if( pSeg && iNext==pSeg->iLastPg ){
  1919   1924       iNext = 0;
................................................................................
  1935   1940   ** LSM_OK is returned if no error occurs. Otherwise, an lsm error code.
  1936   1941   ** If any value other than LSM_OK is returned, then the final value of
  1937   1942   ** *piPrev is undefined.
  1938   1943   */
  1939   1944   static int fsGetPageBefore(
  1940   1945     FileSystem *pFS, 
  1941   1946     Segment *pSeg, 
  1942         -  Pgno iPg, 
  1943         -  Pgno *piPrev
         1947  +  LsmPgno iPg, 
         1948  +  LsmPgno *piPrev
  1944   1949   ){
  1945   1950     u8 aSz[3];
  1946   1951     int rc;
  1947   1952     i64 iRead;
  1948   1953   
  1949   1954     assert( pFS->pCompress );
  1950   1955   
................................................................................
  1986   1991   **
  1987   1992   ** Page references returned by this function should be released by the 
  1988   1993   ** caller using lsmFsPageRelease().
  1989   1994   */
  1990   1995   int lsmFsDbPageNext(Segment *pRun, Page *pPg, int eDir, Page **ppNext){
  1991   1996     int rc = LSM_OK;
  1992   1997     FileSystem *pFS = pPg->pFS;
  1993         -  Pgno iPg = pPg->iPg;
         1998  +  LsmPgno iPg = pPg->iPg;
  1994   1999   
  1995   2000     assert( 0==fsSegmentRedirects(pFS, pRun) );
  1996   2001     if( pFS->pCompress ){
  1997   2002       int nSpace = pPg->nCompress + 2*3;
  1998   2003   
  1999   2004       do {
  2000   2005         if( eDir>0 ){
................................................................................
  2058   2063   ** already allocated block. If it is possible, the page number of the first
  2059   2064   ** page to use for the new segment is returned. Otherwise zero.
  2060   2065   **
  2061   2066   ** If argument pLvl is not NULL, then this function will not attempt to
  2062   2067   ** start the new segment immediately following any segment that is part
  2063   2068   ** of the right-hand-side of pLvl.
  2064   2069   */
  2065         -static Pgno findAppendPoint(FileSystem *pFS, Level *pLvl){
         2070  +static LsmPgno findAppendPoint(FileSystem *pFS, Level *pLvl){
  2066   2071     int i;
  2067         -  Pgno *aiAppend = pFS->pDb->pWorker->aiAppend;
  2068         -  Pgno iRet = 0;
         2072  +  LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend;
         2073  +  LsmPgno iRet = 0;
  2069   2074   
  2070   2075     for(i=LSM_APPLIST_SZ-1; iRet==0 && i>=0; i--){
  2071   2076       if( (iRet = aiAppend[i]) ){
  2072   2077         if( pLvl ){
  2073   2078           int iBlk = fsPageToBlock(pFS, iRet);
  2074   2079           int j;
  2075   2080           for(j=0; iRet && j<pLvl->nRight; j++){
................................................................................
  2094   2099     Snapshot *pSnapshot,
  2095   2100     Level *pLvl,
  2096   2101     int bDefer,
  2097   2102     Page **ppOut
  2098   2103   ){
  2099   2104     int rc = LSM_OK;
  2100   2105     Page *pPg = 0;
  2101         -  Pgno iApp = 0;
  2102         -  Pgno iNext = 0;
         2106  +  LsmPgno iApp = 0;
         2107  +  LsmPgno iNext = 0;
  2103   2108     Segment *p = &pLvl->lhs;
  2104         -  Pgno iPrev = p->iLastPg;
         2109  +  LsmPgno iPrev = p->iLastPg;
  2105   2110   
  2106   2111     *ppOut = 0;
  2107   2112     assert( p->pRedirect==0 );
  2108   2113   
  2109   2114     if( pFS->pCompress || bDefer ){
  2110   2115       /* In compressed database mode the page is not assigned a page number
  2111   2116       ** or location in the database file at this point. This will be done
................................................................................
  2191   2196       ** Shift this extra block back to the free-block list. 
  2192   2197       **
  2193   2198       ** Otherwise, add the first free page in the last block used by the run
  2194   2199       ** to the lAppend list.
  2195   2200       */
  2196   2201       if( fsLastPageOnPagesBlock(pFS, p->iLastPg)!=p->iLastPg ){
  2197   2202         int i;
  2198         -      Pgno *aiAppend = pFS->pDb->pWorker->aiAppend;
         2203  +      LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend;
  2199   2204         for(i=0; i<LSM_APPLIST_SZ; i++){
  2200   2205           if( aiAppend[i]==0 ){
  2201   2206             aiAppend[i] = p->iLastPg+1;
  2202   2207             break;
  2203   2208           }
  2204   2209         }
  2205   2210       }else if( pFS->pCompress==0 ){
................................................................................
  2222   2227   }
  2223   2228   
  2224   2229   /*
  2225   2230   ** Obtain a reference to page number iPg.
  2226   2231   **
  2227   2232   ** Return LSM_OK if successful, or an lsm error code if an error occurs.
  2228   2233   */
  2229         -int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, Pgno iPg, Page **ppPg){
         2234  +int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, LsmPgno iPg, Page **ppPg){
  2230   2235     return fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0);
  2231   2236   }
  2232   2237   
  2233   2238   /*
  2234   2239   ** Obtain a reference to the last page in the segment passed as the 
  2235   2240   ** second argument.
  2236   2241   **
  2237   2242   ** Return LSM_OK if successful, or an lsm error code if an error occurs.
  2238   2243   */
  2239   2244   int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg){
  2240   2245     int rc;
  2241         -  Pgno iPg = pSeg->iLastPg;
         2246  +  LsmPgno iPg = pSeg->iLastPg;
  2242   2247     if( pFS->pCompress ){
  2243   2248       int nSpace;
  2244   2249       iPg++;
  2245   2250       do {
  2246   2251         nSpace = 0;
  2247   2252         rc = fsGetPageBefore(pFS, pSeg, iPg, &iPg);
  2248   2253         if( rc==LSM_OK ){
................................................................................
  2362   2367   ** number (*piPg) lies on block iFrom, then calculate the equivalent
  2363   2368   ** page on block iTo and set *piPg to this value before returning.
  2364   2369   */
  2365   2370   static void fsMovePage(
  2366   2371     FileSystem *pFS,                /* File system object */
  2367   2372     int iTo,                        /* Destination block */
  2368   2373     int iFrom,                      /* Source block */
  2369         -  Pgno *piPg                      /* IN/OUT: Page number */
         2374  +  LsmPgno *piPg                   /* IN/OUT: Page number */
  2370   2375   ){
  2371         -  Pgno iPg = *piPg;
         2376  +  LsmPgno iPg = *piPg;
  2372   2377     if( iFrom==fsPageToBlock(pFS, iPg) ){
  2373   2378       const int nPagePerBlock = (
  2374   2379           pFS->pCompress ? pFS ->nBlocksize : (pFS->nBlocksize / pFS->nPagesize)
  2375   2380       );
  2376         -    *piPg = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock;
         2381  +    *piPg = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock;
  2377   2382     }
  2378   2383   }
  2379   2384   
  2380   2385   /*
  2381   2386   ** Copy the contents of block iFrom to block iTo. 
  2382   2387   **
  2383   2388   ** It is safe to assume that there are no outstanding references to pages 
................................................................................
  2453   2458   /*
  2454   2459   ** Append raw data to a segment. Return the database file offset that the
  2455   2460   ** data is written to (this may be used as the page number if the data
  2456   2461   ** being appended is a new page record).
  2457   2462   **
  2458   2463   ** This function is only used in compressed database mode.
  2459   2464   */
  2460         -static Pgno fsAppendData(
         2465  +static LsmPgno fsAppendData(
  2461   2466     FileSystem *pFS,                /* File-system handle */
  2462   2467     Segment *pSeg,                  /* Segment to append to */
  2463   2468     const u8 *aData,                /* Buffer containing data to write */
  2464   2469     int nData,                      /* Size of buffer aData[] in bytes */
  2465   2470     int *pRc                        /* IN/OUT: Error code */
  2466   2471   ){
  2467         -  Pgno iRet = 0;
         2472  +  LsmPgno iRet = 0;
  2468   2473     int rc = *pRc;
  2469   2474     assert( pFS->pCompress );
  2470   2475     if( rc==LSM_OK ){
  2471   2476       int nRem = 0;
  2472   2477       int nWrite = 0;
  2473         -    Pgno iLastOnBlock;
  2474         -    Pgno iApp = pSeg->iLastPg+1;
         2478  +    LsmPgno iLastOnBlock;
         2479  +    LsmPgno iApp = pSeg->iLastPg+1;
  2475   2480   
  2476   2481       /* If this is the first data written into the segment, find an append-point
  2477   2482       ** or allocate a new block.  */
  2478   2483       if( iApp==1 ){
  2479   2484         pSeg->iFirst = iApp = findAppendPoint(pFS, 0);
  2480   2485         if( iApp==0 ){
  2481   2486           int iBlk;
................................................................................
  2515   2520             assert( iApp==(fsPageToBlock(pFS, iApp)*pFS->nBlocksize)-4 );
  2516   2521             lsmPutU32(aPtr, iBlk);
  2517   2522             rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, aPtr, sizeof(aPtr));
  2518   2523           }
  2519   2524   
  2520   2525           /* Set the "prev" pointer on the new block */
  2521   2526           if( rc==LSM_OK ){
  2522         -          Pgno iWrite;
         2527  +          LsmPgno iWrite;
  2523   2528             lsmPutU32(aPtr, fsPageToBlock(pFS, iApp));
  2524   2529             iWrite = fsFirstPageOnBlock(pFS, iBlk);
  2525   2530             rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iWrite-4, aPtr, sizeof(aPtr));
  2526   2531             if( nRem>0 ) iApp = iWrite;
  2527   2532           }
  2528   2533         }else{
  2529   2534           /* The next block is already allocated. */
................................................................................
  2584   2589   ** LSM_OK is returned if successful, or an lsm error code otherwise. If
  2585   2590   ** any value other than LSM_OK is returned, then the final value of all
  2586   2591   ** output variables is undefined.
  2587   2592   */
  2588   2593   static int fsAppendPage(
  2589   2594     FileSystem *pFS, 
  2590   2595     Segment *pSeg,
  2591         -  Pgno *piNew,
         2596  +  LsmPgno *piNew,
  2592   2597     int *piPrev,
  2593   2598     int *piNext
  2594   2599   ){
  2595         -  Pgno iPrev = pSeg->iLastPg;
         2600  +  LsmPgno iPrev = pSeg->iLastPg;
  2596   2601     int rc;
  2597   2602     assert( iPrev!=0 );
  2598   2603   
  2599   2604     *piPrev = 0;
  2600   2605     *piNext = 0;
  2601   2606   
  2602   2607     if( fsIsLast(pFS, iPrev) ){
................................................................................
  2646   2651     }
  2647   2652     *pRc = rc;
  2648   2653   }
  2649   2654   
  2650   2655   /*
  2651   2656   ** If there exists a hash-table entry associated with page iPg, remove it.
  2652   2657   */
  2653         -static void fsRemoveHashEntry(FileSystem *pFS, Pgno iPg){
         2658  +static void fsRemoveHashEntry(FileSystem *pFS, LsmPgno iPg){
  2654   2659     Page *p;
  2655   2660     int iHash = fsHashKey(pFS->nHash, iPg);
  2656   2661   
  2657   2662     for(p=pFS->apHash[iHash]; p && p->iPg!=iPg; p=p->pHashNext);
  2658   2663   
  2659   2664     if( p ){
  2660   2665       assert( p->nRef==0 || (p->flags & PAGE_FREE)==0 );
................................................................................
  2800   2805   int lsmFsSortedPadding(
  2801   2806     FileSystem *pFS, 
  2802   2807     Snapshot *pSnapshot,
  2803   2808     Segment *pSeg
  2804   2809   ){
  2805   2810     int rc = LSM_OK;
  2806   2811     if( pFS->pCompress && pSeg->iFirst ){
  2807         -    Pgno iLast2;
  2808         -    Pgno iLast = pSeg->iLastPg;     /* Current last page of segment */
         2812  +    LsmPgno iLast2;
         2813  +    LsmPgno iLast = pSeg->iLastPg;  /* Current last page of segment */
  2809   2814       int nPad;                       /* Bytes of padding required */
  2810   2815       u8 aSz[3];
  2811   2816   
  2812   2817       iLast2 = (1 + iLast/pFS->szSector) * pFS->szSector - 1;
  2813   2818       assert( fsPageToBlock(pFS, iLast)==fsPageToBlock(pFS, iLast2) );
  2814   2819       nPad = (int)(iLast2 - iLast);
  2815   2820   
................................................................................
  2931   2936   int lsmFsSectorSize(FileSystem *pFS){
  2932   2937     return pFS->szSector;
  2933   2938   }
  2934   2939   
  2935   2940   /*
  2936   2941   ** Helper function for lsmInfoArrayStructure().
  2937   2942   */
  2938         -static Segment *startsWith(Segment *pRun, Pgno iFirst){
         2943  +static Segment *startsWith(Segment *pRun, LsmPgno iFirst){
  2939   2944     return (iFirst==pRun->iFirst) ? pRun : 0;
  2940   2945   }
  2941   2946   
  2942   2947   /*
  2943   2948   ** Return the segment that starts with page iFirst, if any. If no such segment
  2944   2949   ** can be found, return NULL.
  2945   2950   */
  2946         -static Segment *findSegment(Snapshot *pWorker, Pgno iFirst){
         2951  +static Segment *findSegment(Snapshot *pWorker, LsmPgno iFirst){
  2947   2952     Level *pLvl;                    /* Used to iterate through db levels */
  2948   2953     Segment *pSeg = 0;              /* Pointer to segment to return */
  2949   2954   
  2950   2955     for(pLvl=lsmDbSnapshotLevel(pWorker); pLvl && pSeg==0; pLvl=pLvl->pNext){
  2951   2956       if( 0==(pSeg = startsWith(&pLvl->lhs, iFirst)) ){
  2952   2957         int i;
  2953   2958         for(i=0; i<pLvl->nRight; i++){
................................................................................
  2966   2971   ** eventually free the string using lsmFree().
  2967   2972   **
  2968   2973   ** If an error occurs, *pzOut is set to NULL and an LSM error code returned.
  2969   2974   */
  2970   2975   int lsmInfoArrayStructure(
  2971   2976     lsm_db *pDb, 
  2972   2977     int bBlock,                     /* True for block numbers only */
  2973         -  Pgno iFirst,
         2978  +  LsmPgno iFirst,
  2974   2979     char **pzOut
  2975   2980   ){
  2976   2981     int rc = LSM_OK;
  2977   2982     Snapshot *pWorker;              /* Worker snapshot */
  2978   2983     Segment *pArray = 0;            /* Array to report on */
  2979   2984     int bUnlock = 0;
  2980   2985   
................................................................................
  3031   3036     }
  3032   3037     return rc;
  3033   3038   }
  3034   3039   
  3035   3040   int lsmFsSegmentContainsPg(
  3036   3041     FileSystem *pFS, 
  3037   3042     Segment *pSeg, 
  3038         -  Pgno iPg, 
         3043  +  LsmPgno iPg, 
  3039   3044     int *pbRes
  3040   3045   ){
  3041   3046     Redirect *pRedir = pSeg->pRedirect;
  3042   3047     int rc = LSM_OK;
  3043   3048     int iBlk;
  3044   3049     int iLastBlk;
  3045   3050     int iPgBlock;                   /* Block containing page iPg */
................................................................................
  3060   3065   ** This function implements the lsm_info(LSM_INFO_ARRAY_PAGES) request.
  3061   3066   ** If successful, *pzOut is set to point to a nul-terminated string 
  3062   3067   ** containing the array structure and LSM_OK is returned. The caller should
  3063   3068   ** eventually free the string using lsmFree().
  3064   3069   **
  3065   3070   ** If an error occurs, *pzOut is set to NULL and an LSM error code returned.
  3066   3071   */
  3067         -int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut){
         3072  +int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut){
  3068   3073     int rc = LSM_OK;
  3069   3074     Snapshot *pWorker;              /* Worker snapshot */
  3070   3075     Segment *pSeg = 0;              /* Array to report on */
  3071   3076     int bUnlock = 0;
  3072   3077   
  3073   3078     *pzOut = 0;
  3074   3079     if( iFirst==0 ) return LSM_ERROR;
................................................................................
  3293   3298   #ifndef NDEBUG
  3294   3299   /*
  3295   3300   ** Return true if pPg happens to be the last page in segment pSeg. Or false
  3296   3301   ** otherwise. This function is only invoked as part of assert() conditions.
  3297   3302   */
  3298   3303   int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg){
  3299   3304     if( pPg->pFS->pCompress ){
  3300         -    Pgno iNext = 0;
         3305  +    LsmPgno iNext = 0;
  3301   3306       int rc;
  3302   3307       rc = fsNextPageOffset(pPg->pFS, pSeg, pPg->iPg, pPg->nCompress+6, &iNext);
  3303   3308       return (rc!=LSM_OK || iNext==0);
  3304   3309     }
  3305   3310     return (pPg->iPg==pSeg->iLastPg);
  3306   3311   }
  3307   3312   #endif

Changes to ext/lsm1/lsm_main.c.

   579    579       case LSM_INFO_DB_STRUCTURE: {
   580    580         char **pzVal = va_arg(ap, char **);
   581    581         rc = lsmStructList(pDb, pzVal);
   582    582         break;
   583    583       }
   584    584   
   585    585       case LSM_INFO_ARRAY_STRUCTURE: {
   586         -      Pgno pgno = va_arg(ap, Pgno);
          586  +      LsmPgno pgno = va_arg(ap, LsmPgno);
   587    587         char **pzVal = va_arg(ap, char **);
   588    588         rc = lsmInfoArrayStructure(pDb, 0, pgno, pzVal);
   589    589         break;
   590    590       }
   591    591   
   592    592       case LSM_INFO_ARRAY_PAGES: {
   593         -      Pgno pgno = va_arg(ap, Pgno);
          593  +      LsmPgno pgno = va_arg(ap, LsmPgno);
   594    594         char **pzVal = va_arg(ap, char **);
   595    595         rc = lsmInfoArrayPages(pDb, pgno, pzVal);
   596    596         break;
   597    597       }
   598    598   
   599    599       case LSM_INFO_PAGE_HEX_DUMP:
   600    600       case LSM_INFO_PAGE_ASCII_DUMP: {
   601         -      Pgno pgno = va_arg(ap, Pgno);
          601  +      LsmPgno pgno = va_arg(ap, LsmPgno);
   602    602         char **pzVal = va_arg(ap, char **);
   603    603         int bUnlock = 0;
   604    604         rc = infoGetWorker(pDb, 0, &bUnlock);
   605    605         if( rc==LSM_OK ){
   606    606           int bHex = (eParam==LSM_INFO_PAGE_HEX_DUMP);
   607    607           rc = lsmInfoPageDump(pDb, pgno, bHex, pzVal);
   608    608         }
................................................................................
   679    679       int pgsz = lsmFsPageSize(pDb->pFS);
   680    680       int nQuant = LSM_AUTOWORK_QUANT * pgsz;
   681    681       int nBefore;
   682    682       int nAfter;
   683    683       int nDiff;
   684    684   
   685    685       if( nQuant>pDb->nTreeLimit ){
   686         -      nQuant = pDb->nTreeLimit;
          686  +      nQuant = LSM_MAX(pDb->nTreeLimit, pgsz);
   687    687       }
   688    688   
   689    689       nBefore = lsmTreeSize(pDb);
   690    690       if( bDeleteRange ){
   691    691         rc = lsmTreeDelete(pDb, (void *)pKey, nKey, (void *)pVal, nVal);
   692    692       }else{
   693    693         rc = lsmTreeInsert(pDb, (void *)pKey, nKey, (void *)pVal, nVal);

Changes to ext/lsm1/lsm_sorted.c.

   100    100   
   101    101   
   102    102   #ifndef LSM_SEGMENTPTR_FREE_THRESHOLD
   103    103   # define LSM_SEGMENTPTR_FREE_THRESHOLD 1024
   104    104   #endif
   105    105   
   106    106   typedef struct SegmentPtr SegmentPtr;
   107         -typedef struct Blob Blob;
          107  +typedef struct LsmBlob LsmBlob;
   108    108   
   109         -struct Blob {
          109  +struct LsmBlob {
   110    110     lsm_env *pEnv;
   111    111     void *pData;
   112    112     int nData;
   113    113     int nAlloc;
   114    114   };
   115    115   
   116    116   /*
................................................................................
   125    125     Level *pLevel;                /* Level object segment is part of */
   126    126     Segment *pSeg;                /* Segment to access */
   127    127   
   128    128     /* Current page. See segmentPtrLoadPage(). */
   129    129     Page *pPg;                    /* Current page */
   130    130     u16 flags;                    /* Copy of page flags field */
   131    131     int nCell;                    /* Number of cells on pPg */
   132         -  Pgno iPtr;                    /* Base cascade pointer */
          132  +  LsmPgno iPtr;                 /* Base cascade pointer */
   133    133   
   134    134     /* Current cell. See segmentPtrLoadCell() */
   135    135     int iCell;                    /* Current record within page pPg */
   136    136     int eType;                    /* Type of current record */
   137         -  Pgno iPgPtr;                  /* Cascade pointer offset */
          137  +  LsmPgno iPgPtr;               /* Cascade pointer offset */
   138    138     void *pKey; int nKey;         /* Key associated with current record */
   139    139     void *pVal; int nVal;         /* Current record value (eType==WRITE only) */
   140    140   
   141    141     /* Blobs used to allocate buffers for pKey and pVal as required */
   142         -  Blob blob1;
   143         -  Blob blob2;
          142  +  LsmBlob blob1;
          143  +  LsmBlob blob2;
   144    144   };
   145    145   
   146    146   /*
   147    147   ** Used to iterate through the keys stored in a b-tree hierarchy from start
   148    148   ** to finish. Only First() and Next() operations are required.
   149    149   **
   150    150   **   btreeCursorNew()
................................................................................
   167    167     int iPg;                        /* Current entry in aPg[]. -1 -> EOF. */
   168    168     BtreePg *aPg;                   /* Pages from root to current location */
   169    169   
   170    170     /* Cache of current entry. pKey==0 for EOF. */
   171    171     void *pKey;
   172    172     int nKey;
   173    173     int eType;
   174         -  Pgno iPtr;
          174  +  LsmPgno iPtr;
   175    175   
   176    176     /* Storage for key, if not local */
   177         -  Blob blob;
          177  +  LsmBlob blob;
   178    178   };
   179    179   
   180    180   
   181    181   /*
   182    182   ** A cursor used for merged searches or iterations through up to one
   183    183   ** Tree structure and any number of sorted files.
   184    184   **
................................................................................
   199    199   */
   200    200   struct MultiCursor {
   201    201     lsm_db *pDb;                    /* Connection that owns this cursor */
   202    202     MultiCursor *pNext;             /* Next cursor owned by connection pDb */
   203    203     int flags;                      /* Mask of CURSOR_XXX flags */
   204    204   
   205    205     int eType;                      /* Cache of current key type */
   206         -  Blob key;                       /* Cache of current key (or NULL) */
   207         -  Blob val;                       /* Cache of current value */
          206  +  LsmBlob key;                    /* Cache of current key (or NULL) */
          207  +  LsmBlob val;                    /* Cache of current value */
   208    208   
   209    209     /* All the component cursors: */
   210    210     TreeCursor *apTreeCsr[2];       /* Up to two tree cursors */
   211    211     int iFree;                      /* Next element of free-list (-ve for eof) */
   212    212     SegmentPtr *aPtr;               /* Array of segment pointers */
   213    213     int nPtr;                       /* Size of array aPtr[] */
   214    214     BtreeCursor *pBtCsr;            /* b-tree cursor (db writes only) */
................................................................................
   217    217     int nTree;                      /* Size of aTree[] array */
   218    218     int *aTree;                     /* Array of comparison results */
   219    219   
   220    220     /* Used by cursors flushing the in-memory tree only */
   221    221     void *pSystemVal;               /* Pointer to buffer to free */
   222    222   
   223    223     /* Used by worker cursors only */
   224         -  Pgno *pPrevMergePtr;
          224  +  LsmPgno *pPrevMergePtr;
   225    225   };
   226    226   
   227    227   /*
   228    228   ** The following constants are used to assign integers to each component
   229    229   ** cursor of a multi-cursor.
   230    230   */
   231    231   #define CURSOR_DATA_TREE0     0   /* Current tree cursor (apTreeCsr[0]) */
................................................................................
   291    291     lsm_db *pDb;                    /* Database handle */
   292    292     Level *pLevel;                  /* Worker snapshot Level being merged */
   293    293     MultiCursor *pCsr;              /* Cursor to read new segment contents from */
   294    294     int bFlush;                     /* True if this is an in-memory tree flush */
   295    295     Hierarchy hier;                 /* B-tree hierarchy under construction */
   296    296     Page *pPage;                    /* Current output page */
   297    297     int nWork;                      /* Number of calls to mergeWorkerNextPage() */
   298         -  Pgno *aGobble;                  /* Gobble point for each input segment */
          298  +  LsmPgno *aGobble;               /* Gobble point for each input segment */
   299    299   
   300         -  Pgno iIndirect;
          300  +  LsmPgno iIndirect;
   301    301     struct SavedPgno {
   302         -    Pgno iPgno;
          302  +    LsmPgno iPgno;
   303    303       int bStore;
   304    304     } aSave[2];
   305    305   };
   306    306   
   307    307   #ifdef LSM_DEBUG_EXPENSIVE
   308    308   static int assertPointersOk(lsm_db *, Segment *, Segment *, int);
   309    309   static int assertBtreeOk(lsm_db *, Segment *);
................................................................................
   367    367     aOut[3] = (u8)((nVal>>32) & 0xFF);
   368    368     aOut[4] = (u8)((nVal>>24) & 0xFF);
   369    369     aOut[5] = (u8)((nVal>>16) & 0xFF);
   370    370     aOut[6] = (u8)((nVal>> 8) & 0xFF);
   371    371     aOut[7] = (u8)((nVal    ) & 0xFF);
   372    372   }
   373    373   
   374         -static int sortedBlobGrow(lsm_env *pEnv, Blob *pBlob, int nData){
          374  +static int sortedBlobGrow(lsm_env *pEnv, LsmBlob *pBlob, int nData){
   375    375     assert( pBlob->pEnv==pEnv || (pBlob->pEnv==0 && pBlob->pData==0) );
   376    376     if( pBlob->nAlloc<nData ){
   377    377       pBlob->pData = lsmReallocOrFree(pEnv, pBlob->pData, nData);
   378    378       if( !pBlob->pData ) return LSM_NOMEM_BKPT;
   379    379       pBlob->nAlloc = nData;
   380    380       pBlob->pEnv = pEnv;
   381    381     }
   382    382     return LSM_OK;
   383    383   }
   384    384   
   385         -static int sortedBlobSet(lsm_env *pEnv, Blob *pBlob, void *pData, int nData){
          385  +static int sortedBlobSet(lsm_env *pEnv, LsmBlob *pBlob, void *pData, int nData){
   386    386     if( sortedBlobGrow(pEnv, pBlob, nData) ) return LSM_NOMEM;
   387    387     memcpy(pBlob->pData, pData, nData);
   388    388     pBlob->nData = nData;
   389    389     return LSM_OK;
   390    390   }
   391    391   
   392    392   #if 0
   393         -static int sortedBlobCopy(Blob *pDest, Blob *pSrc){
          393  +static int sortedBlobCopy(LsmBlob *pDest, LsmBlob *pSrc){
   394    394     return sortedBlobSet(pDest, pSrc->pData, pSrc->nData);
   395    395   }
   396    396   #endif
   397    397   
   398         -static void sortedBlobFree(Blob *pBlob){
          398  +static void sortedBlobFree(LsmBlob *pBlob){
   399    399     assert( pBlob->pEnv || pBlob->pData==0 );
   400    400     if( pBlob->pData ) lsmFree(pBlob->pEnv, pBlob->pData);
   401         -  memset(pBlob, 0, sizeof(Blob));
          401  +  memset(pBlob, 0, sizeof(LsmBlob));
   402    402   }
   403    403   
   404    404   static int sortedReadData(
   405    405     Segment *pSeg,
   406    406     Page *pPg,
   407    407     int iOff,
   408    408     int nByte,
   409    409     void **ppData,
   410         -  Blob *pBlob
          410  +  LsmBlob *pBlob
   411    411   ){
   412    412     int rc = LSM_OK;
   413    413     int iEnd;
   414    414     int nData;
   415    415     int nCell;
   416    416     u8 *aData;
   417    417   
................................................................................
   477    477     return rc;
   478    478   }
   479    479   
   480    480   static int pageGetNRec(u8 *aData, int nData){
   481    481     return (int)lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]);
   482    482   }
   483    483   
   484         -static Pgno pageGetPtr(u8 *aData, int nData){
   485         -  return (Pgno)lsmGetU64(&aData[SEGMENT_POINTER_OFFSET(nData)]);
          484  +static LsmPgno pageGetPtr(u8 *aData, int nData){
          485  +  return (LsmPgno)lsmGetU64(&aData[SEGMENT_POINTER_OFFSET(nData)]);
   486    486   }
   487    487   
   488    488   static int pageGetFlags(u8 *aData, int nData){
   489    489     return (int)lsmGetU16(&aData[SEGMENT_FLAGS_OFFSET(nData)]);
   490    490   }
   491    491   
   492    492   static u8 *pageGetCell(u8 *aData, int nData, int iCell){
................................................................................
   502    502     return pageGetNRec(aData, nData);
   503    503   }
   504    504   
   505    505   /*
   506    506   ** Return the decoded (possibly relative) pointer value stored in cell 
   507    507   ** iCell from page aData/nData.
   508    508   */
   509         -static Pgno pageGetRecordPtr(u8 *aData, int nData, int iCell){
   510         -  Pgno iRet;                      /* Return value */
          509  +static LsmPgno pageGetRecordPtr(u8 *aData, int nData, int iCell){
          510  +  LsmPgno iRet;                   /* Return value */
   511    511     u8 *aCell;                      /* Pointer to cell iCell */
   512    512   
   513    513     assert( iCell<pageGetNRec(aData, nData) && iCell>=0 );
   514    514     aCell = pageGetCell(aData, nData, iCell);
   515    515     lsmVarintGet64(&aCell[1], &iRet);
   516    516     return iRet;
   517    517   }
................................................................................
   518    518   
   519    519   static u8 *pageGetKey(
   520    520     Segment *pSeg,                  /* Segment pPg belongs to */
   521    521     Page *pPg,                      /* Page to read from */
   522    522     int iCell,                      /* Index of cell on page to read */
   523    523     int *piTopic,                   /* OUT: Topic associated with this key */
   524    524     int *pnKey,                     /* OUT: Size of key in bytes */
   525         -  Blob *pBlob                     /* If required, use this for dynamic memory */
          525  +  LsmBlob *pBlob                  /* If required, use this for dynamic memory */
   526    526   ){
   527    527     u8 *pKey;
   528    528     int nDummy;
   529    529     int eType;
   530    530     u8 *aData;
   531    531     int nData;
   532    532   
................................................................................
   550    550   
   551    551   static int pageGetKeyCopy(
   552    552     lsm_env *pEnv,                  /* Environment handle */
   553    553     Segment *pSeg,                  /* Segment pPg belongs to */
   554    554     Page *pPg,                      /* Page to read from */
   555    555     int iCell,                      /* Index of cell on page to read */
   556    556     int *piTopic,                   /* OUT: Topic associated with this key */
   557         -  Blob *pBlob                     /* If required, use this for dynamic memory */
          557  +  LsmBlob *pBlob                  /* If required, use this for dynamic memory */
   558    558   ){
   559    559     int rc = LSM_OK;
   560    560     int nKey;
   561    561     u8 *aKey;
   562    562   
   563    563     aKey = pageGetKey(pSeg, pPg, iCell, piTopic, &nKey, pBlob);
   564    564     assert( (void *)aKey!=pBlob->pData || nKey==pBlob->nData );
................................................................................
   565    565     if( (void *)aKey!=pBlob->pData ){
   566    566       rc = sortedBlobSet(pEnv, pBlob, aKey, nKey);
   567    567     }
   568    568   
   569    569     return rc;
   570    570   }
   571    571   
   572         -static Pgno pageGetBtreeRef(Page *pPg, int iKey){
   573         -  Pgno iRef;
          572  +static LsmPgno pageGetBtreeRef(Page *pPg, int iKey){
          573  +  LsmPgno iRef;
   574    574     u8 *aData;
   575    575     int nData;
   576    576     u8 *aCell;
   577    577   
   578    578     aData = fsPageData(pPg, &nData);
   579    579     aCell = pageGetCell(aData, nData, iKey);
   580    580     assert( aCell[0]==0 );
................................................................................
   588    588   #define GETVARINT64(a, i) (((i)=((u8*)(a))[0])<=240?1:lsmVarintGet64((a), &(i)))
   589    589   #define GETVARINT32(a, i) (((i)=((u8*)(a))[0])<=240?1:lsmVarintGet32((a), &(i)))
   590    590   
   591    591   static int pageGetBtreeKey(
   592    592     Segment *pSeg,                  /* Segment page pPg belongs to */
   593    593     Page *pPg,
   594    594     int iKey, 
   595         -  Pgno *piPtr, 
          595  +  LsmPgno *piPtr, 
   596    596     int *piTopic, 
   597    597     void **ppKey,
   598    598     int *pnKey,
   599         -  Blob *pBlob
          599  +  LsmBlob *pBlob
   600    600   ){
   601    601     u8 *aData;
   602    602     int nData;
   603    603     u8 *aCell;
   604    604     int eType;
   605    605   
   606    606     aData = fsPageData(pPg, &nData);
................................................................................
   609    609   
   610    610     aCell = pageGetCell(aData, nData, iKey);
   611    611     eType = *aCell++;
   612    612     aCell += GETVARINT64(aCell, *piPtr);
   613    613   
   614    614     if( eType==0 ){
   615    615       int rc;
   616         -    Pgno iRef;                  /* Page number of referenced page */
          616  +    LsmPgno iRef;               /* Page number of referenced page */
   617    617       Page *pRef;
   618    618       aCell += GETVARINT64(aCell, iRef);
   619    619       rc = lsmFsDbPageGet(lsmPageFS(pPg), pSeg, iRef, &pRef);
   620    620       if( rc!=LSM_OK ) return rc;
   621    621       pageGetKeyCopy(lsmPageEnv(pPg), pSeg, pRef, 0, &eType, pBlob);
   622    622       lsmFsPageRelease(pRef);
   623    623       *ppKey = pBlob->pData;
................................................................................
   634    634   static int btreeCursorLoadKey(BtreeCursor *pCsr){
   635    635     int rc = LSM_OK;
   636    636     if( pCsr->iPg<0 ){
   637    637       pCsr->pKey = 0;
   638    638       pCsr->nKey = 0;
   639    639       pCsr->eType = 0;
   640    640     }else{
   641         -    Pgno dummy;
          641  +    LsmPgno dummy;
   642    642       int iPg = pCsr->iPg;
   643    643       int iCell = pCsr->aPg[iPg].iCell;
   644    644       while( iCell<0 && (--iPg)>=0 ){
   645    645         iCell = pCsr->aPg[iPg].iCell-1;
   646    646       }
   647    647       if( iPg<0 || iCell<0 ) return LSM_CORRUPT_BKPT;
   648    648   
................................................................................
   679    679     assert( pCsr->iPg==pCsr->nDepth-1 );
   680    680   
   681    681     aData = fsPageData(pPg->pPage, &nData);
   682    682     nCell = pageGetNRec(aData, nData);
   683    683     assert( pPg->iCell<=nCell );
   684    684     pPg->iCell++;
   685    685     if( pPg->iCell==nCell ){
   686         -    Pgno iLoad;
          686  +    LsmPgno iLoad;
   687    687   
   688    688       /* Up to parent. */
   689    689       lsmFsPageRelease(pPg->pPage);
   690    690       pPg->pPage = 0;
   691    691       pCsr->iPg--;
   692    692       while( pCsr->iPg>=0 ){
   693    693         pPg = &pCsr->aPg[pCsr->iPg];
................................................................................
   838    838     MergeInput *p
   839    839   ){
   840    840     int rc = LSM_OK;
   841    841   
   842    842     if( p->iPg ){
   843    843       lsm_env *pEnv = lsmFsEnv(pCsr->pFS);
   844    844       int iCell;                    /* Current cell number on leaf page */
   845         -    Pgno iLeaf;                   /* Page number of current leaf page */
          845  +    LsmPgno iLeaf;                /* Page number of current leaf page */
   846    846       int nDepth;                   /* Depth of b-tree structure */
   847    847       Segment *pSeg = pCsr->pSeg;
   848    848   
   849    849       /* Decode the MergeInput structure */
   850    850       iLeaf = p->iPg;
   851    851       nDepth = (p->iCell & 0x00FF);
   852    852       iCell = (p->iCell >> 8) - 1;
................................................................................
   862    862         pCsr->nDepth = nDepth;
   863    863         pCsr->aPg[pCsr->iPg].iCell = iCell;
   864    864         rc = lsmFsDbPageGet(pCsr->pFS, pSeg, iLeaf, pp);
   865    865       }
   866    866   
   867    867       /* Populate any other aPg[] array entries */
   868    868       if( rc==LSM_OK && nDepth>1 ){
   869         -      Blob blob = {0,0,0};
          869  +      LsmBlob blob = {0,0,0};
   870    870         void *pSeek;
   871    871         int nSeek;
   872    872         int iTopicSeek;
   873    873         int iPg = 0;
   874    874         int iLoad = (int)pSeg->iRoot;
   875    875         Page *pPg = pCsr->aPg[nDepth-1].pPage;
   876    876    
................................................................................
   879    879           ** In this case, set the iTopicSeek/pSeek/nSeek key to a value
   880    880           ** greater than any real key.  */
   881    881           assert( iCell==-1 );
   882    882           iTopicSeek = 1000;
   883    883           pSeek = 0;
   884    884           nSeek = 0;
   885    885         }else{
   886         -        Pgno dummy;
          886  +        LsmPgno dummy;
   887    887           rc = pageGetBtreeKey(pSeg, pPg,
   888    888               0, &dummy, &iTopicSeek, &pSeek, &nSeek, &pCsr->blob
   889    889           );
   890    890         }
   891    891   
   892    892         do {
   893    893           Page *pPg2;
................................................................................
   908    908             iMax = iCell2-1;
   909    909             iMin = 0;
   910    910   
   911    911             while( iMax>=iMin ){
   912    912               int iTry = (iMin+iMax)/2;
   913    913               void *pKey; int nKey;         /* Key for cell iTry */
   914    914               int iTopic;                   /* Topic for key pKeyT/nKeyT */
   915         -            Pgno iPtr;                    /* Pointer for cell iTry */
          915  +            LsmPgno iPtr;                 /* Pointer for cell iTry */
   916    916               int res;                      /* (pSeek - pKeyT) */
   917    917   
   918    918               rc = pageGetBtreeKey(
   919    919                   pSeg, pPg2, iTry, &iPtr, &iTopic, &pKey, &nKey, &blob
   920    920               );
   921    921               if( rc!=LSM_OK ) break;
   922    922   
................................................................................
   951    951         u8 *aData;
   952    952         int nData;
   953    953   
   954    954         pBtreePg = &pCsr->aPg[pCsr->iPg];
   955    955         aData = fsPageData(pBtreePg->pPage, &nData);
   956    956         pCsr->iPtr = btreeCursorPtr(aData, nData, pBtreePg->iCell+1);
   957    957         if( pBtreePg->iCell<0 ){
   958         -        Pgno dummy;
          958  +        LsmPgno dummy;
   959    959           int i;
   960    960           for(i=pCsr->iPg-1; i>=0; i--){
   961    961             if( pCsr->aPg[i].iCell>0 ) break;
   962    962           }
   963    963           assert( i>=0 );
   964    964           rc = pageGetBtreeKey(pSeg,
   965    965               pCsr->aPg[i].pPage, pCsr->aPg[i].iCell-1,
................................................................................
  1026   1026   }
  1027   1027   
  1028   1028   static int segmentPtrReadData(
  1029   1029     SegmentPtr *pPtr,
  1030   1030     int iOff,
  1031   1031     int nByte,
  1032   1032     void **ppData,
  1033         -  Blob *pBlob
         1033  +  LsmBlob *pBlob
  1034   1034   ){
  1035   1035     return sortedReadData(pPtr->pSeg, pPtr->pPg, iOff, nByte, ppData, pBlob);
  1036   1036   }
  1037   1037   
  1038   1038   static int segmentPtrNextPage(
  1039   1039     SegmentPtr *pPtr,              /* Load page into this SegmentPtr object */
  1040   1040     int eDir                       /* +1 for next(), -1 for prev() */
................................................................................
  1119   1119   
  1120   1120     pSeg = sortedSplitkeySegment(pLevel);
  1121   1121     if( rc==LSM_OK ){
  1122   1122       rc = lsmFsDbPageGet(pDb->pFS, pSeg, pMerge->splitkey.iPg, &pPg);
  1123   1123     }
  1124   1124     if( rc==LSM_OK ){
  1125   1125       int iTopic;
  1126         -    Blob blob = {0, 0, 0, 0};
         1126  +    LsmBlob blob = {0, 0, 0, 0};
  1127   1127       u8 *aData;
  1128   1128       int nData;
  1129   1129     
  1130   1130       aData = lsmFsPageData(pPg, &nData);
  1131   1131       if( pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG ){
  1132   1132         void *pKey;
  1133   1133         int nKey;
  1134         -      Pgno dummy;
         1134  +      LsmPgno dummy;
  1135   1135         rc = pageGetBtreeKey(pSeg,
  1136   1136             pPg, pMerge->splitkey.iCell, &dummy, &iTopic, &pKey, &nKey, &blob
  1137   1137         );
  1138   1138         if( rc==LSM_OK && blob.pData!=pKey ){
  1139   1139           rc = sortedBlobSet(pEnv, &blob, pKey, nKey);
  1140   1140         }
  1141   1141       }else{
................................................................................
  1338   1338   */
  1339   1339   static int assertKeyLocation(
  1340   1340     MultiCursor *pCsr, 
  1341   1341     SegmentPtr *pPtr, 
  1342   1342     void *pKey, int nKey
  1343   1343   ){
  1344   1344     lsm_env *pEnv = lsmFsEnv(pCsr->pDb->pFS);
  1345         -  Blob blob = {0, 0, 0};
         1345  +  LsmBlob blob = {0, 0, 0};
  1346   1346     int eDir;
  1347   1347     int iTopic = 0;                 /* TODO: Fix me */
  1348   1348   
  1349   1349     for(eDir=-1; eDir<=1; eDir+=2){
  1350   1350       Page *pTest = pPtr->pPg;
  1351   1351   
  1352   1352       lsmFsPageRef(pTest);
................................................................................
  1484   1484     return rc;
  1485   1485   }
  1486   1486   
  1487   1487   static int ptrFwdPointer(
  1488   1488     Page *pPage,
  1489   1489     int iCell,
  1490   1490     Segment *pSeg,
  1491         -  Pgno *piPtr,
         1491  +  LsmPgno *piPtr,
  1492   1492     int *pbFound
  1493   1493   ){
  1494   1494     Page *pPg = pPage;
  1495   1495     int iFirst = iCell;
  1496   1496     int rc = LSM_OK;
  1497   1497   
  1498   1498     do {
................................................................................
  1569   1569   **   much better if the multi-cursor could do this lazily - only seek to the
  1570   1570   **   level (N+1) page after the user has moved the cursor on level N passed
  1571   1571   **   the big range-delete.
  1572   1572   */
  1573   1573   static int segmentPtrFwdPointer(
  1574   1574     MultiCursor *pCsr,              /* Multi-cursor pPtr belongs to */
  1575   1575     SegmentPtr *pPtr,               /* Segment-pointer to extract FC ptr from */
  1576         -  Pgno *piPtr                     /* OUT: FC pointer value */
         1576  +  LsmPgno *piPtr                  /* OUT: FC pointer value */
  1577   1577   ){
  1578   1578     Level *pLvl = pPtr->pLevel;
  1579   1579     Level *pNext = pLvl->pNext;
  1580   1580     Page *pPg = pPtr->pPg;
  1581   1581     int rc;
  1582   1582     int bFound;
  1583         -  Pgno iOut = 0;
         1583  +  LsmPgno iOut = 0;
  1584   1584   
  1585   1585     if( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[pLvl->nRight-1] ){
  1586   1586       if( pNext==0 
  1587   1587           || (pNext->nRight==0 && pNext->lhs.iRoot)
  1588   1588           || (pNext->nRight!=0 && pNext->aRhs[0].iRoot)
  1589   1589         ){
  1590   1590         /* Do nothing. The pointer will not be used anyway. */
................................................................................
  1637   1637     int *pbStop
  1638   1638   ){
  1639   1639     int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp;
  1640   1640     int res = 0;                        /* Result of comparison operation */
  1641   1641     int rc = LSM_OK;
  1642   1642     int iMin;
  1643   1643     int iMax;
  1644         -  Pgno iPtrOut = 0;
         1644  +  LsmPgno iPtrOut = 0;
  1645   1645   
  1646   1646     /* If the current page contains an oversized entry, then there are no
  1647   1647     ** pointers to one or more of the subsequent pages in the sorted run.
  1648   1648     ** The following call ensures that the segment-ptr points to the correct 
  1649   1649     ** page in this case.  */
  1650   1650     rc = segmentPtrSearchOversized(pCsr, pPtr, iTopic, pKey, nKey);
  1651   1651     iPtrOut = pPtr->iPtr;
................................................................................
  1764   1764   }
  1765   1765   
  1766   1766   static int seekInBtree(
  1767   1767     MultiCursor *pCsr,              /* Multi-cursor object */
  1768   1768     Segment *pSeg,                  /* Seek within this segment */
  1769   1769     int iTopic,
  1770   1770     void *pKey, int nKey,           /* Key to seek to */
  1771         -  Pgno *aPg,                      /* OUT: Page numbers */
         1771  +  LsmPgno *aPg,                   /* OUT: Page numbers */
  1772   1772     Page **ppPg                     /* OUT: Leaf (sorted-run) page reference */
  1773   1773   ){
  1774   1774     int i = 0;
  1775   1775     int rc;
  1776   1776     int iPg;
  1777   1777     Page *pPg = 0;
  1778         -  Blob blob = {0, 0, 0};
         1778  +  LsmBlob blob = {0, 0, 0};
  1779   1779   
  1780   1780     iPg = (int)pSeg->iRoot;
  1781   1781     do {
  1782         -    Pgno *piFirst = 0;
         1782  +    LsmPgno *piFirst = 0;
  1783   1783       if( aPg ){
  1784   1784         aPg[i++] = iPg;
  1785   1785         piFirst = &aPg[i];
  1786   1786       }
  1787   1787   
  1788   1788       rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, iPg, &pPg);
  1789   1789       assert( rc==LSM_OK || pPg==0 );
................................................................................
  1804   1804   
  1805   1805         iMin = 0;
  1806   1806         iMax = nRec-1;
  1807   1807         while( iMax>=iMin ){
  1808   1808           int iTry = (iMin+iMax)/2;
  1809   1809           void *pKeyT; int nKeyT;       /* Key for cell iTry */
  1810   1810           int iTopicT;                  /* Topic for key pKeyT/nKeyT */
  1811         -        Pgno iPtr;                    /* Pointer associated with cell iTry */
         1811  +        LsmPgno iPtr;                 /* Pointer associated with cell iTry */
  1812   1812           int res;                      /* (pKey - pKeyT) */
  1813   1813   
  1814   1814           rc = pageGetBtreeKey(
  1815   1815               pSeg, pPg, iTry, &iPtr, &iTopicT, &pKeyT, &nKeyT, &blob
  1816   1816           );
  1817   1817           if( rc!=LSM_OK ) break;
  1818   1818           if( piFirst && pKeyT==blob.pData ){
................................................................................
  1895   1895   */
  1896   1896   static int seekInLevel(
  1897   1897     MultiCursor *pCsr,              /* Sorted cursor object to seek */
  1898   1898     SegmentPtr *aPtr,               /* Pointer to array of (nRhs+1) SPs */
  1899   1899     int eSeek,                      /* Search bias - see above */
  1900   1900     int iTopic,                     /* Key topic to search for */
  1901   1901     void *pKey, int nKey,           /* Key to search for */
  1902         -  Pgno *piPgno,                   /* IN/OUT: fraction cascade pointer (or 0) */
         1902  +  LsmPgno *piPgno,                /* IN/OUT: fraction cascade pointer (or 0) */
  1903   1903     int *pbStop                     /* OUT: See above */
  1904   1904   ){
  1905   1905     Level *pLvl = aPtr[0].pLevel;   /* Level to seek within */
  1906   1906     int rc = LSM_OK;                /* Return code */
  1907   1907     int iOut = 0;                   /* Pointer to return to caller */
  1908   1908     int res = -1;                   /* Result of xCmp(pKey, split) */
  1909   1909     int nRhs = pLvl->nRight;        /* Number of right-hand-side segments */
................................................................................
  3051   3051     void *pKey, int nKey, 
  3052   3052     int eSeek
  3053   3053   ){
  3054   3054     int eESeek = eSeek;             /* Effective eSeek parameter */
  3055   3055     int bStop = 0;                  /* Set to true to halt search operation */
  3056   3056     int rc = LSM_OK;                /* Return code */
  3057   3057     int iPtr = 0;                   /* Used to iterate through pCsr->aPtr[] */
  3058         -  Pgno iPgno = 0;                 /* FC pointer value */
         3058  +  LsmPgno iPgno = 0;              /* FC pointer value */
  3059   3059   
  3060   3060     assert( pCsr->apTreeCsr[0]==0 || iTopic==0 );
  3061   3061     assert( pCsr->apTreeCsr[1]==0 || iTopic==0 );
  3062   3062   
  3063   3063     if( eESeek==LSM_SEEK_LEFAST ) eESeek = LSM_SEEK_LE;
  3064   3064   
  3065   3065     assert( eESeek==LSM_SEEK_EQ || eESeek==LSM_SEEK_LE || eESeek==LSM_SEEK_GE );
................................................................................
  3533   3533   ** differences are:
  3534   3534   **
  3535   3535   **   1. The record format is (usually, see below) as follows:
  3536   3536   **
  3537   3537   **         + Type byte (always SORTED_SEPARATOR or SORTED_SYSTEM_SEPARATOR),
  3538   3538   **         + Absolute pointer value (varint),
  3539   3539   **         + Number of bytes in key (varint),
  3540         -**         + Blob containing key data.
         3540  +**         + LsmBlob containing key data.
  3541   3541   **
  3542   3542   **   2. All pointer values are stored as absolute values (not offsets 
  3543   3543   **      relative to the footer pointer value).
  3544   3544   **
  3545   3545   **   3. Each pointer that is part of a record points to a page that 
  3546   3546   **      contains keys smaller than the records key (note: not "equal to or
  3547   3547   **      smaller than - smaller than").
................................................................................
  3567   3567   **
  3568   3568   ** See function seekInBtree() for the code that traverses b-tree pages.
  3569   3569   */
  3570   3570   
  3571   3571   static int mergeWorkerBtreeWrite(
  3572   3572     MergeWorker *pMW,
  3573   3573     u8 eType,
  3574         -  Pgno iPtr,
  3575         -  Pgno iKeyPg,
         3574  +  LsmPgno iPtr,
         3575  +  LsmPgno iKeyPg,
  3576   3576     void *pKey,
  3577   3577     int nKey
  3578   3578   ){
  3579   3579     Hierarchy *p = &pMW->hier;
  3580   3580     lsm_db *pDb = pMW->pDb;         /* Database handle */
  3581   3581     int rc = LSM_OK;                /* Return Code */
  3582   3582     int iLevel;                     /* Level of b-tree hierachy to write to */
................................................................................
  3678   3678   
  3679   3679     return rc;
  3680   3680   }
  3681   3681   
  3682   3682   static int mergeWorkerBtreeIndirect(MergeWorker *pMW){
  3683   3683     int rc = LSM_OK;
  3684   3684     if( pMW->iIndirect ){
  3685         -    Pgno iKeyPg = pMW->aSave[1].iPgno;
         3685  +    LsmPgno iKeyPg = pMW->aSave[1].iPgno;
  3686   3686       rc = mergeWorkerBtreeWrite(pMW, 0, pMW->iIndirect, iKeyPg, 0, 0);
  3687   3687       pMW->iIndirect = 0;
  3688   3688     }
  3689   3689     return rc;
  3690   3690   }
  3691   3691   
  3692   3692   /*
................................................................................
  3699   3699   static int mergeWorkerPushHierarchy(
  3700   3700     MergeWorker *pMW,               /* Merge worker object */
  3701   3701     int iTopic,                     /* Topic value for this key */
  3702   3702     void *pKey,                     /* Pointer to key buffer */
  3703   3703     int nKey                        /* Size of pKey buffer in bytes */
  3704   3704   ){
  3705   3705     int rc = LSM_OK;                /* Return Code */
  3706         -  Pgno iPtr;                      /* Pointer value to accompany pKey/nKey */
         3706  +  LsmPgno iPtr;                   /* Pointer value to accompany pKey/nKey */
  3707   3707   
  3708   3708     assert( pMW->aSave[0].bStore==0 );
  3709   3709     assert( pMW->aSave[1].bStore==0 );
  3710   3710     rc = mergeWorkerBtreeIndirect(pMW);
  3711   3711   
  3712   3712     /* Obtain the absolute pointer value to store along with the key in the
  3713   3713     ** page body. This pointer points to a page that contains keys that are
................................................................................
  3730   3730   }
  3731   3731   
  3732   3732   static int mergeWorkerFinishHierarchy(
  3733   3733     MergeWorker *pMW                /* Merge worker object */
  3734   3734   ){
  3735   3735     int i;                          /* Used to loop through apHier[] */
  3736   3736     int rc = LSM_OK;                /* Return code */
  3737         -  Pgno iPtr;                      /* New right-hand-child pointer value */
         3737  +  LsmPgno iPtr;                   /* New right-hand-child pointer value */
  3738   3738   
  3739   3739     iPtr = pMW->aSave[0].iPgno;
  3740   3740     for(i=0; i<pMW->hier.nHier && rc==LSM_OK; i++){
  3741   3741       Page *pPg = pMW->hier.apHier[i];
  3742   3742       int nData;                    /* Size of aData[] in bytes */
  3743   3743       u8 *aData;                    /* Page data for pPg */
  3744   3744   
................................................................................
  3826   3826   ** zero records. The flags field is cleared. The page footer pointer field
  3827   3827   ** is set to iFPtr.
  3828   3828   **
  3829   3829   ** If successful, LSM_OK is returned. Otherwise, an error code.
  3830   3830   */
  3831   3831   static int mergeWorkerNextPage(
  3832   3832     MergeWorker *pMW,               /* Merge worker object to append page to */
  3833         -  Pgno iFPtr                      /* Pointer value for footer of new page */
         3833  +  LsmPgno iFPtr                   /* Pointer value for footer of new page */
  3834   3834   ){
  3835   3835     int rc = LSM_OK;                /* Return code */
  3836   3836     Page *pNext = 0;                /* New page appended to run */
  3837   3837     lsm_db *pDb = pMW->pDb;         /* Database handle */
  3838   3838   
  3839   3839     rc = lsmFsSortedAppend(pDb->pFS, pDb->pWorker, pMW->pLevel, 0, &pNext);
  3840   3840     assert( rc || pMW->pLevel->lhs.iFirst>0 || pMW->pDb->compress.xCompress );
................................................................................
  4214   4214   
  4215   4215   static int mergeWorkerStep(MergeWorker *pMW){
  4216   4216     lsm_db *pDb = pMW->pDb;       /* Database handle */
  4217   4217     MultiCursor *pCsr;            /* Cursor to read input data from */
  4218   4218     int rc = LSM_OK;              /* Return code */
  4219   4219     int eType;                    /* SORTED_SEPARATOR, WRITE or DELETE */
  4220   4220     void *pKey; int nKey;         /* Key */
  4221         -  Pgno iPtr;
         4221  +  LsmPgno iPtr;
  4222   4222     int iVal;
  4223   4223   
  4224   4224     pCsr = pMW->pCsr;
  4225   4225   
  4226   4226     /* Pull the next record out of the source cursor. */
  4227   4227     lsmMCursorKey(pCsr, &pKey, &nKey);
  4228   4228     eType = pCsr->eType;
................................................................................
  4367   4367         multiCursorIgnoreDelete(pCsr);
  4368   4368       }
  4369   4369     }
  4370   4370   
  4371   4371     if( rc!=LSM_OK ){
  4372   4372       lsmMCursorClose(pCsr, 0);
  4373   4373     }else{
  4374         -    Pgno iLeftPtr = 0;
         4374  +    LsmPgno iLeftPtr = 0;
  4375   4375       Merge merge;                  /* Merge object used to create new level */
  4376   4376       MergeWorker mergeworker;      /* MergeWorker object for the same purpose */
  4377   4377   
  4378   4378       memset(&merge, 0, sizeof(Merge));
  4379   4379       memset(&mergeworker, 0, sizeof(MergeWorker));
  4380   4380   
  4381   4381       pNew->pMerge = &merge;
................................................................................
  4544   4544     assert( pDb->pWorker );
  4545   4545     assert( pLevel->pMerge );
  4546   4546     assert( pLevel->nRight>0 );
  4547   4547   
  4548   4548     memset(pMW, 0, sizeof(MergeWorker));
  4549   4549     pMW->pDb = pDb;
  4550   4550     pMW->pLevel = pLevel;
  4551         -  pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(Pgno) * pLevel->nRight, &rc);
         4551  +  pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*pLevel->nRight,&rc);
  4552   4552   
  4553   4553     /* Create a multi-cursor to read the data to write to the new
  4554   4554     ** segment. The new segment contains:
  4555   4555     **
  4556   4556     **   1. Records from LHS of each of the nMerge levels being merged.
  4557   4557     **   2. Separators from either the last level being merged, or the
  4558   4558     **      separators attached to the LHS of the following level, or neither.
................................................................................
  4626   4626     lsm_db *pDb,                    /* Worker connection */
  4627   4627     MultiCursor *pCsr,              /* Multi-cursor being used for a merge */
  4628   4628     int iGobble                     /* pCsr->aPtr[] entry to operate on */
  4629   4629   ){
  4630   4630     int rc = LSM_OK;
  4631   4631     if( rtTopic(pCsr->eType)==0 ){
  4632   4632       Segment *pSeg = pCsr->aPtr[iGobble].pSeg;
  4633         -    Pgno *aPg;
         4633  +    LsmPgno *aPg;
  4634   4634       int nPg;
  4635   4635   
  4636   4636       /* Seek from the root of the b-tree to the segment leaf that may contain
  4637   4637       ** a key equal to the one multi-cursor currently points to. Record the
  4638   4638       ** page number of each b-tree page and the leaf. The segment may be
  4639   4639       ** gobbled up to (but not including) the first of these page numbers.
  4640   4640       */
  4641   4641       assert( pSeg->iRoot>0 );
  4642         -    aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(Pgno)*32, &rc);
         4642  +    aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*32, &rc);
  4643   4643       if( rc==LSM_OK ){
  4644   4644         rc = seekInBtree(pCsr, pSeg, 
  4645   4645             rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, aPg, 0
  4646   4646         ); 
  4647   4647       }
  4648   4648   
  4649   4649       if( rc==LSM_OK ){
................................................................................
  5248   5248       nRem -= nPg;
  5249   5249       if( nPg ) bDirty = 1;
  5250   5250     }
  5251   5251   
  5252   5252     /* If the in-memory part of the free-list is too large, write a new 
  5253   5253     ** top-level containing just the in-memory free-list entries to disk. */
  5254   5254     if( rc==LSM_OK && pDb->pWorker->freelist.nEntry > pDb->nMaxFreelist ){
  5255         -    int nPg = 0;
  5256   5255       while( rc==LSM_OK && lsmDatabaseFull(pDb) ){
         5256  +      int nPg = 0;
  5257   5257         rc = sortedWork(pDb, 16, nMerge, 1, &nPg);
  5258   5258         nRem -= nPg;
  5259   5259       }
  5260   5260       if( rc==LSM_OK ){
  5261   5261         rc = sortedNewFreelistOnly(pDb);
  5262   5262       }
  5263         -    nRem -= nPg;
  5264         -    if( nPg ) bDirty = 1;
         5263  +    bDirty = 1;
  5265   5264     }
  5266   5265   
  5267   5266     if( rc==LSM_OK ){
  5268   5267       *pnWrite = (nMax - nRem);
  5269   5268       *pbCkpt = (bCkpt && nRem<=0);
  5270   5269       if( nMerge==1 && pDb->nAutockpt>0 && *pnWrite>0
  5271   5270        && pWorker->pLevel 
................................................................................
  5462   5461   /*
  5463   5462   ** Return a string representation of the segment passed as the only argument.
  5464   5463   ** Space for the returned string is allocated using lsmMalloc(), and should
  5465   5464   ** be freed by the caller using lsmFree().
  5466   5465   */
  5467   5466   static char *segToString(lsm_env *pEnv, Segment *pSeg, int nMin){
  5468   5467     int nSize = pSeg->nSize;
  5469         -  Pgno iRoot = pSeg->iRoot;
  5470         -  Pgno iFirst = pSeg->iFirst;
  5471         -  Pgno iLast = pSeg->iLastPg;
         5468  +  LsmPgno iRoot = pSeg->iRoot;
         5469  +  LsmPgno iFirst = pSeg->iFirst;
         5470  +  LsmPgno iLast = pSeg->iLastPg;
  5472   5471     char *z;
  5473   5472   
  5474   5473     char *z1;
  5475   5474     char *z2;
  5476   5475     int nPad;
  5477   5476   
  5478   5477     z1 = lsmMallocPrintf(pEnv, "%d.%d", iFirst, iLast);
................................................................................
  5523   5522       aBuf[0] = '\0';
  5524   5523     }
  5525   5524   
  5526   5525     return i;
  5527   5526   }
  5528   5527   
  5529   5528   void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){
  5530         -  Blob blob = {0, 0, 0};         /* Blob used for keys */
         5529  +  LsmBlob blob = {0, 0, 0};       /* LsmBlob used for keys */
  5531   5530     LsmString s;
  5532   5531     int i;
  5533   5532   
  5534   5533     int nRec;
  5535   5534     int iPtr;
  5536   5535     int flags;
  5537   5536     u8 *aData;
................................................................................
  5559   5558   
  5560   5559       aCell = pageGetCell(aData, nData, i);
  5561   5560       eType = *aCell++;
  5562   5561       assert( (flags & SEGMENT_BTREE_FLAG) || eType!=0 );
  5563   5562       aCell += lsmVarintGet32(aCell, &iPgPtr);
  5564   5563   
  5565   5564       if( eType==0 ){
  5566         -      Pgno iRef;                  /* Page number of referenced page */
         5565  +      LsmPgno iRef;               /* Page number of referenced page */
  5567   5566         aCell += lsmVarintGet64(aCell, &iRef);
  5568   5567         lsmFsDbPageGet(pDb->pFS, pRun, iRef, &pRef);
  5569   5568         aKey = pageGetKey(pRun, pRef, 0, &iTopic, &nKey, &blob);
  5570   5569       }else{
  5571   5570         aCell += lsmVarintGet32(aCell, &nKey);
  5572   5571         if( rtIsWrite(eType) ) aCell += lsmVarintGet32(aCell, &nVal);
  5573   5572         sortedReadData(0, pPg, (aCell-aData), nKey+nVal, (void **)&aKey, &blob);
................................................................................
  5603   5602     int bIndirect,                  /* True to follow indirect refs */
  5604   5603     Page *pPg,
  5605   5604     int iCell,
  5606   5605     int *peType,
  5607   5606     int *piPgPtr,
  5608   5607     u8 **paKey, int *pnKey,
  5609   5608     u8 **paVal, int *pnVal,
  5610         -  Blob *pBlob
         5609  +  LsmBlob *pBlob
  5611   5610   ){
  5612   5611     u8 *aData; int nData;           /* Page data */
  5613   5612     u8 *aKey; int nKey = 0;         /* Key */
  5614   5613     u8 *aVal = 0; int nVal = 0;     /* Value */
  5615   5614     int eType;
  5616   5615     int iPgPtr;
  5617   5616     Page *pRef = 0;                 /* Pointer to page iRef */
................................................................................
  5621   5620   
  5622   5621     aCell = pageGetCell(aData, nData, iCell);
  5623   5622     eType = *aCell++;
  5624   5623     aCell += lsmVarintGet32(aCell, &iPgPtr);
  5625   5624   
  5626   5625     if( eType==0 ){
  5627   5626       int dummy;
  5628         -    Pgno iRef;                  /* Page number of referenced page */
         5627  +    LsmPgno iRef;                 /* Page number of referenced page */
  5629   5628       aCell += lsmVarintGet64(aCell, &iRef);
  5630   5629       if( bIndirect ){
  5631   5630         lsmFsDbPageGet(pDb->pFS, pSeg, iRef, &pRef);
  5632   5631         pageGetKeyCopy(pDb->pEnv, pSeg, pRef, 0, &dummy, pBlob);
  5633   5632         aKey = (u8 *)pBlob->pData;
  5634   5633         nKey = pBlob->nData;
  5635   5634         lsmFsPageRelease(pRef);
................................................................................
  5667   5666   #define INFO_PAGE_DUMP_DATA     0x01
  5668   5667   #define INFO_PAGE_DUMP_VALUES   0x02
  5669   5668   #define INFO_PAGE_DUMP_HEX      0x04
  5670   5669   #define INFO_PAGE_DUMP_INDIRECT 0x08
  5671   5670   
  5672   5671   static int infoPageDump(
  5673   5672     lsm_db *pDb,                    /* Database handle */
  5674         -  Pgno iPg,                       /* Page number of page to dump */
         5673  +  LsmPgno iPg,                    /* Page number of page to dump */
  5675   5674     int flags,
  5676   5675     char **pzOut                    /* OUT: lsmMalloc'd string */
  5677   5676   ){
  5678   5677     int rc = LSM_OK;                /* Return code */
  5679   5678     Page *pPg = 0;                  /* Handle for page iPg */
  5680   5679     int i, j;                       /* Loop counters */
  5681   5680     const int perLine = 16;         /* Bytes per line in the raw hex dump */
................................................................................
  5708   5707     ** to pass a NULL in place of the segment pointer as the second argument
  5709   5708     ** to lsmFsDbPageGet() here.  */
  5710   5709     if( rc==LSM_OK ){
  5711   5710       rc = lsmFsDbPageGet(pDb->pFS, 0, iPg, &pPg);
  5712   5711     }
  5713   5712   
  5714   5713     if( rc==LSM_OK ){
  5715         -    Blob blob = {0, 0, 0, 0};
         5714  +    LsmBlob blob = {0, 0, 0, 0};
  5716   5715       int nKeyWidth = 0;
  5717   5716       LsmString str;
  5718   5717       int nRec;
  5719   5718       int iPtr;
  5720   5719       int flags2;
  5721   5720       int iCell;
  5722   5721       u8 *aData; int nData;         /* Page data and size thereof */
................................................................................
  5743   5742       if( bHex ) nKeyWidth = nKeyWidth * 2;
  5744   5743   
  5745   5744       for(iCell=0; iCell<nRec; iCell++){
  5746   5745         u8 *aKey; int nKey = 0;       /* Key */
  5747   5746         u8 *aVal; int nVal = 0;       /* Value */
  5748   5747         int iPgPtr;
  5749   5748         int eType;
  5750         -      Pgno iAbsPtr;
         5749  +      LsmPgno iAbsPtr;
  5751   5750         char zFlags[8];
  5752   5751   
  5753   5752         infoCellDump(pDb, pSeg, bIndirect, pPg, iCell, &eType, &iPgPtr,
  5754   5753             &aKey, &nKey, &aVal, &nVal, &blob
  5755   5754         );
  5756   5755         iAbsPtr = iPgPtr + ((flags2 & SEGMENT_BTREE_FLAG) ? 0 : iPtr);
  5757   5756   
................................................................................
  5809   5808     }
  5810   5809   
  5811   5810     return rc;
  5812   5811   }
  5813   5812   
  5814   5813   int lsmInfoPageDump(
  5815   5814     lsm_db *pDb,                    /* Database handle */
  5816         -  Pgno iPg,                       /* Page number of page to dump */
         5815  +  LsmPgno iPg,                    /* Page number of page to dump */
  5817   5816     int bHex,                       /* True to output key/value in hex form */
  5818   5817     char **pzOut                    /* OUT: lsmMalloc'd string */
  5819   5818   ){
  5820   5819     int flags = INFO_PAGE_DUMP_DATA | INFO_PAGE_DUMP_VALUES;
  5821   5820     if( bHex ) flags |= INFO_PAGE_DUMP_HEX;
  5822   5821     return infoPageDump(pDb, iPg, flags, pzOut);
  5823   5822   }
................................................................................
  5985   5984     iHdr = SEGMENT_EOF(nOrig, nEntry);
  5986   5985     memmove(&aData[iHdr + (nData-nOrig)], &aData[iHdr], nOrig-iHdr);
  5987   5986   }
  5988   5987   
  5989   5988   #ifdef LSM_DEBUG_EXPENSIVE
  5990   5989   static void assertRunInOrder(lsm_db *pDb, Segment *pSeg){
  5991   5990     Page *pPg = 0;
  5992         -  Blob blob1 = {0, 0, 0, 0};
  5993         -  Blob blob2 = {0, 0, 0, 0};
         5991  +  LsmBlob blob1 = {0, 0, 0, 0};
         5992  +  LsmBlob blob2 = {0, 0, 0, 0};
  5994   5993   
  5995   5994     lsmFsDbPageGet(pDb->pFS, pSeg, pSeg->iFirst, &pPg);
  5996   5995     while( pPg ){
  5997   5996       u8 *aData; int nData;
  5998   5997       Page *pNext;
  5999   5998   
  6000   5999       aData = lsmFsPageData(pPg, &nData);
................................................................................
  6048   6047     Segment *pOne,                  /* Segment containing pointers */
  6049   6048     Segment *pTwo,                  /* Segment containing pointer targets */
  6050   6049     int bRhs                        /* True if pTwo may have been Gobble()d */
  6051   6050   ){
  6052   6051     int rc = LSM_OK;                /* Error code */
  6053   6052     SegmentPtr ptr1;                /* Iterates through pOne */
  6054   6053     SegmentPtr ptr2;                /* Iterates through pTwo */
  6055         -  Pgno iPrev;
         6054  +  LsmPgno iPrev;
  6056   6055   
  6057   6056     assert( pOne && pTwo );
  6058   6057   
  6059   6058     memset(&ptr1, 0, sizeof(ptr1));
  6060   6059     memset(&ptr2, 0, sizeof(ptr1));
  6061   6060     ptr1.pSeg = pOne;
  6062   6061     ptr2.pSeg = pTwo;
................................................................................
  6071   6070     }
  6072   6071   
  6073   6072     if( rc==LSM_OK && ptr1.nCell>0 ){
  6074   6073       rc = segmentPtrLoadCell(&ptr1, 0);
  6075   6074     }
  6076   6075         
  6077   6076     while( rc==LSM_OK && ptr2.pPg ){
  6078         -    Pgno iThis;
         6077  +    LsmPgno iThis;
  6079   6078   
  6080   6079       /* Advance to the next page of segment pTwo that contains at least
  6081   6080       ** one cell. Break out of the loop if the iterator reaches EOF.  */
  6082   6081       do{
  6083   6082         rc = segmentPtrNextPage(&ptr2, 1);
  6084   6083         assert( rc==LSM_OK );
  6085   6084       }while( rc==LSM_OK && ptr2.pPg && ptr2.nCell==0 );
................................................................................
  6133   6132   */
  6134   6133   static int assertBtreeOk(
  6135   6134     lsm_db *pDb,
  6136   6135     Segment *pSeg
  6137   6136   ){
  6138   6137     int rc = LSM_OK;                /* Return code */
  6139   6138     if( pSeg->iRoot ){
  6140         -    Blob blob = {0, 0, 0};        /* Buffer used to cache overflow keys */
         6139  +    LsmBlob blob = {0, 0, 0};     /* Buffer used to cache overflow keys */
  6141   6140       FileSystem *pFS = pDb->pFS;   /* File system to read from */
  6142   6141       Page *pPg = 0;                /* Main run page */
  6143   6142       BtreeCursor *pCsr = 0;        /* Btree cursor */
  6144   6143   
  6145   6144       rc = btreeCursorNew(pDb, pSeg, &pCsr);
  6146   6145       if( rc==LSM_OK ){
  6147   6146         rc = btreeCursorFirst(pCsr);

Added ext/lsm1/tool/mklsm1c.tcl.

            1  +#!/bin/sh
            2  +# restart with tclsh \
            3  +exec tclsh "$0" "$@"
            4  +
            5  +set srcdir [file dirname [file dirname [info script]]]
            6  +set G(src) [string map [list %dir% $srcdir] {
            7  +  %dir%/lsm.h
            8  +  %dir%/lsmInt.h
            9  +  %dir%/lsm_vtab.c
           10  +  %dir%/lsm_ckpt.c
           11  +  %dir%/lsm_file.c
           12  +  %dir%/lsm_log.c
           13  +  %dir%/lsm_main.c
           14  +  %dir%/lsm_mem.c
           15  +  %dir%/lsm_mutex.c
           16  +  %dir%/lsm_shared.c
           17  +  %dir%/lsm_sorted.c
           18  +  %dir%/lsm_str.c
           19  +  %dir%/lsm_tree.c
           20  +  %dir%/lsm_unix.c
           21  +  %dir%/lsm_varint.c
           22  +  %dir%/lsm_win32.c
           23  +}]
           24  +
           25  +set G(hdr) {
           26  +
           27  +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) 
           28  +
           29  +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
           30  +# define NDEBUG 1
           31  +#endif
           32  +#if defined(NDEBUG) && defined(SQLITE_DEBUG)
           33  +# undef NDEBUG
           34  +#endif
           35  +
           36  +}
           37  +
           38  +set G(footer) {
           39  +    
           40  +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) */
           41  +}
           42  +
           43  +#-------------------------------------------------------------------------
           44  +# Read and return the entire contents of text file $zFile from disk.
           45  +#
           46  +proc readfile {zFile} {
           47  +  set fd [open $zFile]
           48  +  set data [read $fd]
           49  +  close $fd
           50  +  return $data
           51  +}
           52  +
           53  +proc lsm1c_init {zOut} {
           54  +  global G
           55  +  set G(fd) stdout
           56  +  set G(fd) [open $zOut w]
           57  +
           58  +  puts -nonewline $G(fd) $G(hdr)
           59  +}
           60  +
           61  +proc lsm1c_printfile {zIn} {
           62  +  global G
           63  +  set data [readfile $zIn]
           64  +  set zTail [file tail $zIn]
           65  +  puts $G(fd) "#line 1 \"$zTail\""
           66  +
           67  +  foreach line [split $data "\n"] {
           68  +    if {[regexp {^# *include.*lsm} $line]} {
           69  +      set line "/* $line */"
           70  +    } elseif { [regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?lsm[^_]} $line] } {
           71  +      set line "static $line"
           72  +    }
           73  +    puts $G(fd) $line
           74  +  }
           75  +}
           76  +
           77  +proc lsm1c_close {} {
           78  +  global G
           79  +  puts -nonewline $G(fd) $G(footer)
           80  +  if {$G(fd)!="stdout"} {
           81  +    close $G(fd)
           82  +  }
           83  +}
           84  +
           85  +
           86  +lsm1c_init lsm1.c
           87  +foreach f $G(src) { lsm1c_printfile $f }
           88  +lsm1c_close

Changes to ext/misc/README.md.

    10     10   as follows:
    11     11   
    12     12     *  **carray.c** &mdash;  This module implements the
    13     13        [carray](https://www.sqlite.org/carray.html) table-valued function.
    14     14        It is a good example of how to go about implementing a custom
    15     15        [table-valued function](https://www.sqlite.org/vtab.html#tabfunc2).
    16     16   
           17  +  *  **csv.c** &mdash;  A [virtual table](https://sqlite.org/vtab.html)
           18  +     for reading 
           19  +     [Comma-Separated-Value (CSV) files](https://en.wikipedia.org/wiki/Comma-separated_values).
           20  +
    17     21     *  **dbdump.c** &mdash;  This is not actually a loadable extension, but
    18     22        rather a library that implements an approximate equivalent to the
    19     23        ".dump" command of the
    20     24        [command-line shell](https://www.sqlite.org/cli.html).
           25  +
           26  +  *  **json1.c** &mdash;  Various SQL functions and table-valued functions
           27  +     for processing JSON.  This extension is already built into the
           28  +     [SQLite amalgamation](https://sqlite.org/amalgamation.html).  See
           29  +     <https://sqlite.org/json1.html> for additional information.
    21     30   
    22     31     *  **memvfs.c** &mdash;  This file implements a custom
    23     32        [VFS](https://www.sqlite.org/vfs.html) that stores an entire database
    24     33        file in a single block of RAM.  It serves as a good example of how
    25     34        to implement a simple custom VFS.
    26     35   
    27     36     *  **rot13.c** &mdash;  This file implements the very simple rot13()
................................................................................
    34     43   
    35     44     *  **shathree.c** &mdash;  An implementation of the sha3() and
    36     45        sha3_query() SQL functions.  The file is named "shathree.c" instead
    37     46        of "sha3.c" because the default entry point names in SQLite are based
    38     47        on the source filename with digits removed, so if we used the name
    39     48        "sha3.c" then the entry point would conflict with the prior "sha1.c"
    40     49        extension.
           50  +
           51  +  *  **unionvtab.c** &mdash; Implementation of the unionvtab and
           52  +     [swarmvtab](https://sqlite.org/swarmvtab.html) virtual tables.
           53  +     These virtual tables allow a single
           54  +     large table to be spread out across multiple database files.  In the
           55  +     case of swarmvtab, the individual database files can be attached on
           56  +     demand.
           57  +
           58  +  *  **zipfile.c** &mdash;  A [virtual table](https://sqlite.org/vtab.html)
           59  +     that can read and write a 
           60  +     [ZIP archive](https://en.wikipedia.org/wiki/Zip_%28file_format%29).

Added ext/misc/appendvfs.c.

            1  +/*
            2  +** 2017-10-20
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** This file implements a VFS shim that allows an SQLite database to be
           14  +** appended onto the end of some other file, such as an executable.
           15  +**
           16  +** A special record must appear at the end of the file that identifies the
           17  +** file as an appended database and provides an offset to page 1.  For
           18  +** best performance page 1 should be located at a disk page boundary, though
           19  +** that is not required.
           20  +**
           21  +** When opening a database using this VFS, the connection might treat
           22  +** the file as an ordinary SQLite database, or it might treat is as a
           23  +** database appended onto some other file.  Here are the rules:
           24  +**
           25  +**  (1)  When opening a new empty file, that file is treated as an ordinary
           26  +**       database.
           27  +**
           28  +**  (2)  When opening a file that begins with the standard SQLite prefix
           29  +**       string "SQLite format 3", that file is treated as an ordinary
           30  +**       database.
           31  +**
           32  +**  (3)  When opening a file that ends with the appendvfs trailer string
           33  +**       "Start-Of-SQLite3-NNNNNNNN" that file is treated as an appended
           34  +**       database.
           35  +**
           36  +**  (4)  If none of the above apply and the SQLITE_OPEN_CREATE flag is
           37  +**       set, then a new database is appended to the already existing file.
           38  +**
           39  +**  (5)  Otherwise, SQLITE_CANTOPEN is returned.
           40  +**
           41  +** To avoid unnecessary complications with the PENDING_BYTE, the size of
           42  +** the file containing the database is limited to 1GB.  This VFS will refuse
           43  +** to read or write past the 1GB mark.  This restriction might be lifted in
           44  +** future versions.  For now, if you need a large database, then keep the
           45  +** database in a separate file.
           46  +**
           47  +** If the file being opened is not an appended database, then this shim is
           48  +** a pass-through into the default underlying VFS.
           49  +**/
           50  +#include "sqlite3ext.h"
           51  +SQLITE_EXTENSION_INIT1
           52  +#include <string.h>
           53  +#include <assert.h>
           54  +
           55  +/* The append mark at the end of the database is:
           56  +**
           57  +**     Start-Of-SQLite3-NNNNNNNN
           58  +**     123456789 123456789 12345
           59  +**
           60  +** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is
           61  +** the offset to page 1.
           62  +*/
           63  +#define APND_MARK_PREFIX     "Start-Of-SQLite3-"
           64  +#define APND_MARK_PREFIX_SZ  17
           65  +#define APND_MARK_SIZE       25
           66  +
           67  +/*
           68  +** Maximum size of the combined prefix + database + append-mark.  This
           69  +** must be less than 0x40000000 to avoid locking issues on Windows.
           70  +*/
           71  +#define APND_MAX_SIZE  (65536*15259)
           72  +
           73  +/*
           74  +** Forward declaration of objects used by this utility
           75  +*/
           76  +typedef struct sqlite3_vfs ApndVfs;
           77  +typedef struct ApndFile ApndFile;
           78  +
           79  +/* Access to a lower-level VFS that (might) implement dynamic loading,
           80  +** access to randomness, etc.
           81  +*/
           82  +#define ORIGVFS(p)  ((sqlite3_vfs*)((p)->pAppData))
           83  +#define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1))
           84  +
           85  +/* An open file */
           86  +struct ApndFile {
           87  +  sqlite3_file base;              /* IO methods */
           88  +  sqlite3_int64 iPgOne;           /* File offset to page 1 */
           89  +  sqlite3_int64 iMark;            /* Start of the append-mark */
           90  +};
           91  +
           92  +/*
           93  +** Methods for ApndFile
           94  +*/
           95  +static int apndClose(sqlite3_file*);
           96  +static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
           97  +static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
           98  +static int apndTruncate(sqlite3_file*, sqlite3_int64 size);
           99  +static int apndSync(sqlite3_file*, int flags);
          100  +static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize);
          101  +static int apndLock(sqlite3_file*, int);
          102  +static int apndUnlock(sqlite3_file*, int);
          103  +static int apndCheckReservedLock(sqlite3_file*, int *pResOut);
          104  +static int apndFileControl(sqlite3_file*, int op, void *pArg);
          105  +static int apndSectorSize(sqlite3_file*);
          106  +static int apndDeviceCharacteristics(sqlite3_file*);
          107  +static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**);
          108  +static int apndShmLock(sqlite3_file*, int offset, int n, int flags);
          109  +static void apndShmBarrier(sqlite3_file*);
          110  +static int apndShmUnmap(sqlite3_file*, int deleteFlag);
          111  +static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp);
          112  +static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
          113  +
          114  +/*
          115  +** Methods for ApndVfs
          116  +*/
          117  +static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
          118  +static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir);
          119  +static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *);
          120  +static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut);
          121  +static void *apndDlOpen(sqlite3_vfs*, const char *zFilename);
          122  +static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg);
          123  +static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void);
          124  +static void apndDlClose(sqlite3_vfs*, void*);
          125  +static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut);
          126  +static int apndSleep(sqlite3_vfs*, int microseconds);
          127  +static int apndCurrentTime(sqlite3_vfs*, double*);
          128  +static int apndGetLastError(sqlite3_vfs*, int, char *);
          129  +static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
          130  +static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr);
          131  +static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z);
          132  +static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName);
          133  +
          134  +static sqlite3_vfs apnd_vfs = {
          135  +  3,                            /* iVersion (set when registered) */
          136  +  0,                            /* szOsFile (set when registered) */
          137  +  1024,                         /* mxPathname */
          138  +  0,                            /* pNext */
          139  +  "apndvfs",                    /* zName */
          140  +  0,                            /* pAppData (set when registered) */ 
          141  +  apndOpen,                     /* xOpen */
          142  +  apndDelete,                   /* xDelete */
          143  +  apndAccess,                   /* xAccess */
          144  +  apndFullPathname,             /* xFullPathname */
          145  +  apndDlOpen,                   /* xDlOpen */
          146  +  apndDlError,                  /* xDlError */
          147  +  apndDlSym,                    /* xDlSym */
          148  +  apndDlClose,                  /* xDlClose */
          149  +  apndRandomness,               /* xRandomness */
          150  +  apndSleep,                    /* xSleep */
          151  +  apndCurrentTime,              /* xCurrentTime */
          152  +  apndGetLastError,             /* xGetLastError */
          153  +  apndCurrentTimeInt64,         /* xCurrentTimeInt64 */
          154  +  apndSetSystemCall,            /* xSetSystemCall */
          155  +  apndGetSystemCall,            /* xGetSystemCall */
          156  +  apndNextSystemCall            /* xNextSystemCall */
          157  +};
          158  +
          159  +static const sqlite3_io_methods apnd_io_methods = {
          160  +  3,                              /* iVersion */
          161  +  apndClose,                      /* xClose */
          162  +  apndRead,                       /* xRead */
          163  +  apndWrite,                      /* xWrite */
          164  +  apndTruncate,                   /* xTruncate */
          165  +  apndSync,                       /* xSync */
          166  +  apndFileSize,                   /* xFileSize */
          167  +  apndLock,                       /* xLock */
          168  +  apndUnlock,                     /* xUnlock */
          169  +  apndCheckReservedLock,          /* xCheckReservedLock */
          170  +  apndFileControl,                /* xFileControl */
          171  +  apndSectorSize,                 /* xSectorSize */
          172  +  apndDeviceCharacteristics,      /* xDeviceCharacteristics */
          173  +  apndShmMap,                     /* xShmMap */
          174  +  apndShmLock,                    /* xShmLock */
          175  +  apndShmBarrier,                 /* xShmBarrier */
          176  +  apndShmUnmap,                   /* xShmUnmap */
          177  +  apndFetch,                      /* xFetch */
          178  +  apndUnfetch                     /* xUnfetch */
          179  +};
          180  +
          181  +
          182  +
          183  +/*
          184  +** Close an apnd-file.
          185  +*/
          186  +static int apndClose(sqlite3_file *pFile){
          187  +  pFile = ORIGFILE(pFile);
          188  +  return pFile->pMethods->xClose(pFile);
          189  +}
          190  +
          191  +/*
          192  +** Read data from an apnd-file.
          193  +*/
          194  +static int apndRead(
          195  +  sqlite3_file *pFile, 
          196  +  void *zBuf, 
          197  +  int iAmt, 
          198  +  sqlite_int64 iOfst
          199  +){
          200  +  ApndFile *p = (ApndFile *)pFile;
          201  +  pFile = ORIGFILE(pFile);
          202  +  return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne);
          203  +}
          204  +
          205  +/*
          206  +** Add the append-mark onto the end of the file.
          207  +*/
          208  +static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){
          209  +  int i;
          210  +  unsigned char a[APND_MARK_SIZE];
          211  +  memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ);
          212  +  for(i=0; i<8; i++){
          213  +    a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff;
          214  +  }
          215  +  return pFile->pMethods->xWrite(pFile, a, APND_MARK_SIZE, p->iMark);
          216  +}
          217  +
          218  +/*
          219  +** Write data to an apnd-file.
          220  +*/
          221  +static int apndWrite(
          222  +  sqlite3_file *pFile,
          223  +  const void *zBuf,
          224  +  int iAmt,
          225  +  sqlite_int64 iOfst
          226  +){
          227  +  int rc;
          228  +  ApndFile *p = (ApndFile *)pFile;
          229  +  pFile = ORIGFILE(pFile);
          230  +  if( iOfst+iAmt>=APND_MAX_SIZE ) return SQLITE_FULL;
          231  +  rc = pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne);
          232  +  if( rc==SQLITE_OK &&  iOfst + iAmt + p->iPgOne > p->iMark ){
          233  +    sqlite3_int64 sz = 0;
          234  +    rc = pFile->pMethods->xFileSize(pFile, &sz);
          235  +    if( rc==SQLITE_OK ){
          236  +      p->iMark = sz - APND_MARK_SIZE;
          237  +      if( iOfst + iAmt + p->iPgOne > p->iMark ){
          238  +        p->iMark = p->iPgOne + iOfst + iAmt;
          239  +        rc = apndWriteMark(p, pFile);
          240  +      }
          241  +    }
          242  +  }
          243  +  return rc;
          244  +}
          245  +
          246  +/*
          247  +** Truncate an apnd-file.
          248  +*/
          249  +static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){
          250  +  int rc;
          251  +  ApndFile *p = (ApndFile *)pFile;
          252  +  pFile = ORIGFILE(pFile);
          253  +  rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE);
          254  +  if( rc==SQLITE_OK ){
          255  +    p->iMark = p->iPgOne+size;
          256  +    rc = apndWriteMark(p, pFile);
          257  +  }
          258  +  return rc;
          259  +}
          260  +
          261  +/*
          262  +** Sync an apnd-file.
          263  +*/
          264  +static int apndSync(sqlite3_file *pFile, int flags){
          265  +  pFile = ORIGFILE(pFile);
          266  +  return pFile->pMethods->xSync(pFile, flags);
          267  +}
          268  +
          269  +/*
          270  +** Return the current file-size of an apnd-file.
          271  +*/
          272  +static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
          273  +  ApndFile *p = (ApndFile *)pFile;
          274  +  int rc;
          275  +  pFile = ORIGFILE(p);
          276  +  rc = pFile->pMethods->xFileSize(pFile, pSize);
          277  +  if( rc==SQLITE_OK && p->iPgOne ){
          278  +    *pSize -= p->iPgOne + APND_MARK_SIZE;
          279  +  }
          280  +  return rc;
          281  +}
          282  +
          283  +/*
          284  +** Lock an apnd-file.
          285  +*/
          286  +static int apndLock(sqlite3_file *pFile, int eLock){
          287  +  pFile = ORIGFILE(pFile);
          288  +  return pFile->pMethods->xLock(pFile, eLock);
          289  +}
          290  +
          291  +/*
          292  +** Unlock an apnd-file.
          293  +*/
          294  +static int apndUnlock(sqlite3_file *pFile, int eLock){
          295  +  pFile = ORIGFILE(pFile);
          296  +  return pFile->pMethods->xUnlock(pFile, eLock);
          297  +}
          298  +
          299  +/*
          300  +** Check if another file-handle holds a RESERVED lock on an apnd-file.
          301  +*/
          302  +static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){
          303  +  pFile = ORIGFILE(pFile);
          304  +  return pFile->pMethods->xCheckReservedLock(pFile, pResOut);
          305  +}
          306  +
          307  +/*
          308  +** File control method. For custom operations on an apnd-file.
          309  +*/
          310  +static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){
          311  +  ApndFile *p = (ApndFile *)pFile;
          312  +  int rc;
          313  +  pFile = ORIGFILE(pFile);
          314  +  rc = pFile->pMethods->xFileControl(pFile, op, pArg);
          315  +  if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){
          316  +    *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", p->iPgOne, *(char**)pArg);
          317  +  }
          318  +  return rc;
          319  +}
          320  +
          321  +/*
          322  +** Return the sector-size in bytes for an apnd-file.
          323  +*/
          324  +static int apndSectorSize(sqlite3_file *pFile){
          325  +  pFile = ORIGFILE(pFile);
          326  +  return pFile->pMethods->xSectorSize(pFile);
          327  +}
          328  +
          329  +/*
          330  +** Return the device characteristic flags supported by an apnd-file.
          331  +*/
          332  +static int apndDeviceCharacteristics(sqlite3_file *pFile){
          333  +  pFile = ORIGFILE(pFile);
          334  +  return pFile->pMethods->xDeviceCharacteristics(pFile);
          335  +}
          336  +
          337  +/* Create a shared memory file mapping */
          338  +static int apndShmMap(
          339  +  sqlite3_file *pFile,
          340  +  int iPg,
          341  +  int pgsz,
          342  +  int bExtend,
          343  +  void volatile **pp
          344  +){
          345  +  pFile = ORIGFILE(pFile);
          346  +  return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp);
          347  +}
          348  +
          349  +/* Perform locking on a shared-memory segment */
          350  +static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){
          351  +  pFile = ORIGFILE(pFile);
          352  +  return pFile->pMethods->xShmLock(pFile,offset,n,flags);
          353  +}
          354  +
          355  +/* Memory barrier operation on shared memory */
          356  +static void apndShmBarrier(sqlite3_file *pFile){
          357  +  pFile = ORIGFILE(pFile);
          358  +  pFile->pMethods->xShmBarrier(pFile);
          359  +}
          360  +
          361  +/* Unmap a shared memory segment */
          362  +static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){
          363  +  pFile = ORIGFILE(pFile);
          364  +  return pFile->pMethods->xShmUnmap(pFile,deleteFlag);
          365  +}
          366  +
          367  +/* Fetch a page of a memory-mapped file */
          368  +static int apndFetch(
          369  +  sqlite3_file *pFile,
          370  +  sqlite3_int64 iOfst,
          371  +  int iAmt,
          372  +  void **pp
          373  +){
          374  +  ApndFile *p = (ApndFile *)pFile;
          375  +  pFile = ORIGFILE(pFile);
          376  +  return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp);
          377  +}
          378  +
          379  +/* Release a memory-mapped page */
          380  +static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
          381  +  ApndFile *p = (ApndFile *)pFile;
          382  +  pFile = ORIGFILE(pFile);
          383  +  return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage);
          384  +}
          385  +
          386  +/*
          387  +** Check to see if the file is an ordinary SQLite database file.
          388  +*/
          389  +static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){
          390  +  int rc;
          391  +  char zHdr[16];
          392  +  static const char aSqliteHdr[] = "SQLite format 3";
          393  +  if( sz<512 ) return 0;
          394  +  rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0);
          395  +  if( rc ) return 0;
          396  +  return memcmp(zHdr, aSqliteHdr, sizeof(zHdr))==0;
          397  +}
          398  +
          399  +/*
          400  +** Try to read the append-mark off the end of a file.  Return the
          401  +** start of the appended database if the append-mark is present.  If
          402  +** there is no append-mark, return -1;
          403  +*/
          404  +static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){
          405  +  int rc, i;
          406  +  sqlite3_int64 iMark;
          407  +  unsigned char a[APND_MARK_SIZE];
          408  +
          409  +  if( sz<=APND_MARK_SIZE ) return -1;
          410  +  rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE);
          411  +  if( rc ) return -1;
          412  +  if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1;
          413  +  iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ]&0x7f))<<56;
          414  +  for(i=1; i<8; i++){    
          415  +    iMark += (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<<(56-8*i);
          416  +  }
          417  +  return iMark;
          418  +}
          419  +
          420  +/*
          421  +** Open an apnd file handle.
          422  +*/
          423  +static int apndOpen(
          424  +  sqlite3_vfs *pVfs,
          425  +  const char *zName,
          426  +  sqlite3_file *pFile,
          427  +  int flags,
          428  +  int *pOutFlags
          429  +){
          430  +  ApndFile *p;
          431  +  sqlite3_file *pSubFile;
          432  +  sqlite3_vfs *pSubVfs;
          433  +  int rc;
          434  +  sqlite3_int64 sz;
          435  +  pSubVfs = ORIGVFS(pVfs);
          436  +  if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){
          437  +    return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags);
          438  +  }
          439  +  p = (ApndFile*)pFile;
          440  +  memset(p, 0, sizeof(*p));
          441  +  pSubFile = ORIGFILE(pFile);
          442  +  p->base.pMethods = &apnd_io_methods;
          443  +  rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags);
          444  +  if( rc ) goto apnd_open_done;
          445  +  rc = pSubFile->pMethods->xFileSize(pSubFile, &sz);
          446  +  if( rc ){
          447  +    pSubFile->pMethods->xClose(pSubFile);
          448  +    goto apnd_open_done;
          449  +  }
          450  +  if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){
          451  +    memmove(pFile, pSubFile, pSubVfs->szOsFile);
          452  +    return SQLITE_OK;
          453  +  }
          454  +  p->iMark = 0;
          455  +  p->iPgOne = apndReadMark(sz, pFile);
          456  +  if( p->iPgOne>0 ){
          457  +    return SQLITE_OK;
          458  +  }
          459  +  if( (flags & SQLITE_OPEN_CREATE)==0 ){
          460  +    pSubFile->pMethods->xClose(pSubFile);
          461  +    rc = SQLITE_CANTOPEN;
          462  +  }
          463  +  p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff;
          464  +apnd_open_done:
          465  +  if( rc ) pFile->pMethods = 0;
          466  +  return rc;
          467  +}
          468  +
          469  +/*
          470  +** All other VFS methods are pass-thrus.
          471  +*/
          472  +static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
          473  +  return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync);
          474  +}
          475  +static int apndAccess(
          476  +  sqlite3_vfs *pVfs, 
          477  +  const char *zPath, 
          478  +  int flags, 
          479  +  int *pResOut
          480  +){
          481  +  return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut);
          482  +}
          483  +static int apndFullPathname(
          484  +  sqlite3_vfs *pVfs, 
          485  +  const char *zPath, 
          486  +  int nOut, 
          487  +  char *zOut
          488  +){
          489  +  return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut);
          490  +}
          491  +static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){
          492  +  return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath);
          493  +}
          494  +static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
          495  +  ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg);
          496  +}
          497  +static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){
          498  +  return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym);
          499  +}
          500  +static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){
          501  +  ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle);
          502  +}
          503  +static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
          504  +  return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut);
          505  +}
          506  +static int apndSleep(sqlite3_vfs *pVfs, int nMicro){
          507  +  return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro);
          508  +}
          509  +static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
          510  +  return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut);
          511  +}
          512  +static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){
          513  +  return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b);
          514  +}
          515  +static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
          516  +  return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p);
          517  +}
          518  +static int apndSetSystemCall(
          519  +  sqlite3_vfs *pVfs,
          520  +  const char *zName,
          521  +  sqlite3_syscall_ptr pCall
          522  +){
          523  +  return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall);
          524  +}
          525  +static sqlite3_syscall_ptr apndGetSystemCall(
          526  +  sqlite3_vfs *pVfs,
          527  +  const char *zName
          528  +){
          529  +  return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName);
          530  +}
          531  +static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){
          532  +  return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName);
          533  +}
          534  +
          535  +  
          536  +#ifdef _WIN32
          537  +__declspec(dllexport)
          538  +#endif
          539  +/* 
          540  +** This routine is called when the extension is loaded.
          541  +** Register the new VFS.
          542  +*/
          543  +int sqlite3_appendvfs_init(
          544  +  sqlite3 *db, 
          545  +  char **pzErrMsg, 
          546  +  const sqlite3_api_routines *pApi
          547  +){
          548  +  int rc = SQLITE_OK;
          549  +  sqlite3_vfs *pOrig;
          550  +  SQLITE_EXTENSION_INIT2(pApi);
          551  +  pOrig = sqlite3_vfs_find(0);
          552  +  apnd_vfs.iVersion = pOrig->iVersion;
          553  +  apnd_vfs.pAppData = pOrig;
          554  +  apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile);
          555  +  rc = sqlite3_vfs_register(&apnd_vfs, 0);
          556  +#ifdef APPENDVFS_TEST
          557  +  if( rc==SQLITE_OK ){
          558  +    rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister);
          559  +  }
          560  +#endif
          561  +  if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
          562  +  return rc;
          563  +}

Changes to ext/misc/compress.c.

    23     23   **
    24     24   ** The output is a BLOB that begins with a variable-length integer that
    25     25   ** is the input size in bytes (the size of X before compression).  The
    26     26   ** variable-length integer is implemented as 1 to 5 bytes.  There are
    27     27   ** seven bits per integer stored in the lower seven bits of each byte.
    28     28   ** More significant bits occur first.  The most significant bit (0x80)
    29     29   ** is a flag to indicate the end of the integer.
           30  +**
           31  +** This function, SQLAR, and ZIP all use the same "deflate" compression
           32  +** algorithm, but each is subtly different:
           33  +**
           34  +**   *  ZIP uses raw deflate.
           35  +**
           36  +**   *  SQLAR uses the "zlib format" which is raw deflate with a two-byte
           37  +**      algorithm-identification header and a four-byte checksum at the end.
           38  +**
           39  +**   *  This utility uses the "zlib format" like SQLAR, but adds the variable-
           40  +**      length integer uncompressed size value at the beginning.
           41  +**
           42  +** This function might be extended in the future to support compression
           43  +** formats other than deflate, by providing a different algorithm-id
           44  +** mark following the variable-length integer size parameter.
    30     45   */
    31     46   static void compressFunc(
    32     47     sqlite3_context *context,
    33     48     int argc,
    34     49     sqlite3_value **argv
    35     50   ){
    36     51     const unsigned char *pIn;

Changes to ext/misc/fileio.c.

     7      7   **    May you do good and not evil.
     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   ******************************************************************************
    12     12   **
    13     13   ** This SQLite extension implements SQL functions readfile() and
    14         -** writefile().
           14  +** writefile(), and eponymous virtual type "fsdir".
           15  +**
           16  +** WRITEFILE(FILE, DATA [, MODE [, MTIME]]):
           17  +**
           18  +**   If neither of the optional arguments is present, then this UDF
           19  +**   function writes blob DATA to file FILE. If successful, the number
           20  +**   of bytes written is returned. If an error occurs, NULL is returned.
           21  +**
           22  +**   If the first option argument - MODE - is present, then it must
           23  +**   be passed an integer value that corresponds to a POSIX mode
           24  +**   value (file type + permissions, as returned in the stat.st_mode
           25  +**   field by the stat() system call). Three types of files may
           26  +**   be written/created:
           27  +**
           28  +**     regular files:  (mode & 0170000)==0100000
           29  +**     symbolic links: (mode & 0170000)==0120000
           30  +**     directories:    (mode & 0170000)==0040000
           31  +**
           32  +**   For a directory, the DATA is ignored. For a symbolic link, it is
           33  +**   interpreted as text and used as the target of the link. For a
           34  +**   regular file, it is interpreted as a blob and written into the
           35  +**   named file. Regardless of the type of file, its permissions are
           36  +**   set to (mode & 0777) before returning.
           37  +**
           38  +**   If the optional MTIME argument is present, then it is interpreted
           39  +**   as an integer - the number of seconds since the unix epoch. The
           40  +**   modification-time of the target file is set to this value before
           41  +**   returning.
           42  +**
           43  +**   If three or more arguments are passed to this function and an
           44  +**   error is encountered, an exception is raised.
           45  +**
           46  +** READFILE(FILE):
           47  +**
           48  +**   Read and return the contents of file FILE (type blob) from disk.
           49  +**
           50  +** FSDIR:
           51  +**
           52  +**   Used as follows:
           53  +**
           54  +**     SELECT * FROM fsdir($path [, $dir]);
           55  +**
           56  +**   Parameter $path is an absolute or relative pathname. If the file that it
           57  +**   refers to does not exist, it is an error. If the path refers to a regular
           58  +**   file or symbolic link, it returns a single row. Or, if the path refers
           59  +**   to a directory, it returns one row for the directory, and one row for each
           60  +**   file within the hierarchy rooted at $path.
           61  +**
           62  +**   Each row has the following columns:
           63  +**
           64  +**     name:  Path to file or directory (text value).
           65  +**     mode:  Value of stat.st_mode for directory entry (an integer).
           66  +**     mtime: Value of stat.st_mtime for directory entry (an integer).
           67  +**     data:  For a regular file, a blob containing the file data. For a
           68  +**            symlink, a text value containing the text of the link. For a
           69  +**            directory, NULL.
           70  +**
           71  +**   If a non-NULL value is specified for the optional $dir parameter and
           72  +**   $path is a relative path, then $path is interpreted relative to $dir. 
           73  +**   And the paths returned in the "name" column of the table are also 
           74  +**   relative to directory $dir.
    15     75   */
    16     76   #include "sqlite3ext.h"
    17     77   SQLITE_EXTENSION_INIT1
    18     78   #include <stdio.h>
           79  +#include <string.h>
           80  +#include <assert.h>
           81  +
           82  +#include <sys/types.h>
           83  +#include <sys/stat.h>
           84  +#include <fcntl.h>
           85  +#if !defined(_WIN32) && !defined(WIN32)
           86  +#  include <unistd.h>
           87  +#  include <dirent.h>
           88  +#  include <utime.h>
           89  +#  include <sys/time.h>
           90  +#else
           91  +#  include "windows.h"
           92  +#  include <io.h>
           93  +#  include <direct.h>
           94  +#  include "test_windirent.h"
           95  +#  define dirent DIRENT
           96  +#  ifndef stat
           97  +#    define stat _stat
           98  +#  endif
           99  +#  define mkdir(path,mode) _mkdir(path)
          100  +#  define lstat(path,buf) stat(path,buf)
          101  +#endif
          102  +#include <time.h>
          103  +#include <errno.h>
          104  +
          105  +
          106  +#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)"
          107  +
          108  +/*
          109  +** Set the result stored by context ctx to a blob containing the 
          110  +** contents of file zName.
          111  +*/
          112  +static void readFileContents(sqlite3_context *ctx, const char *zName){
          113  +  FILE *in;
          114  +  long nIn;
          115  +  void *pBuf;
          116  +
          117  +  in = fopen(zName, "rb");
          118  +  if( in==0 ) return;
          119  +  fseek(in, 0, SEEK_END);
          120  +  nIn = ftell(in);
          121  +  rewind(in);
          122  +  pBuf = sqlite3_malloc( nIn );
          123  +  if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
          124  +    sqlite3_result_blob(ctx, pBuf, nIn, sqlite3_free);
          125  +  }else{
          126  +    sqlite3_free(pBuf);
          127  +  }
          128  +  fclose(in);
          129  +}
    19    130   
    20    131   /*
    21    132   ** Implementation of the "readfile(X)" SQL function.  The entire content
    22    133   ** of the file named X is read and returned as a BLOB.  NULL is returned
    23    134   ** if the file does not exist or is unreadable.
    24    135   */
    25    136   static void readfileFunc(
    26    137     sqlite3_context *context,
    27    138     int argc,
    28    139     sqlite3_value **argv
    29    140   ){
    30    141     const char *zName;
    31         -  FILE *in;
    32         -  long nIn;
    33         -  void *pBuf;
    34         -
    35    142     (void)(argc);  /* Unused parameter */
    36    143     zName = (const char*)sqlite3_value_text(argv[0]);
    37    144     if( zName==0 ) return;
    38         -  in = fopen(zName, "rb");
    39         -  if( in==0 ) return;
    40         -  fseek(in, 0, SEEK_END);
    41         -  nIn = ftell(in);
    42         -  rewind(in);
    43         -  pBuf = sqlite3_malloc( nIn );
    44         -  if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
    45         -    sqlite3_result_blob(context, pBuf, nIn, sqlite3_free);
          145  +  readFileContents(context, zName);
          146  +}
          147  +
          148  +/*
          149  +** Set the error message contained in context ctx to the results of
          150  +** vprintf(zFmt, ...).
          151  +*/
          152  +static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){
          153  +  char *zMsg = 0;
          154  +  va_list ap;
          155  +  va_start(ap, zFmt);
          156  +  zMsg = sqlite3_vmprintf(zFmt, ap);
          157  +  sqlite3_result_error(ctx, zMsg, -1);
          158  +  sqlite3_free(zMsg);
          159  +  va_end(ap);
          160  +}
          161  +
          162  +/*
          163  +** Argument zFile is the name of a file that will be created and/or written
          164  +** by SQL function writefile(). This function ensures that the directory
          165  +** zFile will be written to exists, creating it if required. The permissions
          166  +** for any path components created by this function are set to (mode&0777).
          167  +**
          168  +** If an OOM condition is encountered, SQLITE_NOMEM is returned. Otherwise,
          169  +** SQLITE_OK is returned if the directory is successfully created, or
          170  +** SQLITE_ERROR otherwise.
          171  +*/
          172  +static int makeDirectory(
          173  +  const char *zFile,
          174  +  mode_t mode
          175  +){
          176  +  char *zCopy = sqlite3_mprintf("%s", zFile);
          177  +  int rc = SQLITE_OK;
          178  +
          179  +  if( zCopy==0 ){
          180  +    rc = SQLITE_NOMEM;
    46    181     }else{
    47         -    sqlite3_free(pBuf);
    48         -  }
    49         -  fclose(in);
    50         -}
    51         -
    52         -/*
    53         -** Implementation of the "writefile(X,Y)" SQL function.  The argument Y
    54         -** is written into file X.  The number of bytes written is returned.  Or
    55         -** NULL is returned if something goes wrong, such as being unable to open
    56         -** file X for writing.
          182  +    int nCopy = (int)strlen(zCopy);
          183  +    int i = 1;
          184  +
          185  +    while( rc==SQLITE_OK ){
          186  +      struct stat sStat;
          187  +      int rc2;
          188  +
          189  +      for(; zCopy[i]!='/' && i<nCopy; i++);
          190  +      if( i==nCopy ) break;
          191  +      zCopy[i] = '\0';
          192  +
          193  +      rc2 = stat(zCopy, &sStat);
          194  +      if( rc2!=0 ){
          195  +        if( mkdir(zCopy, mode & 0777) ) rc = SQLITE_ERROR;
          196  +      }else{
          197  +        if( !S_ISDIR(sStat.st_mode) ) rc = SQLITE_ERROR;
          198  +      }
          199  +      zCopy[i] = '/';
          200  +      i++;
          201  +    }
          202  +
          203  +    sqlite3_free(zCopy);
          204  +  }
          205  +
          206  +  return rc;
          207  +}
          208  +
          209  +/*
          210  +** This function does the work for the writefile() UDF. Refer to 
          211  +** header comments at the top of this file for details.
          212  +*/
          213  +static int writeFile(
          214  +  sqlite3_context *pCtx,          /* Context to return bytes written in */
          215  +  const char *zFile,              /* File to write */
          216  +  sqlite3_value *pData,           /* Data to write */
          217  +  mode_t mode,                    /* MODE parameter passed to writefile() */
          218  +  sqlite3_int64 mtime             /* MTIME parameter (or -1 to not set time) */
          219  +){
          220  +#if !defined(_WIN32) && !defined(WIN32)
          221  +  if( S_ISLNK(mode) ){
          222  +    const char *zTo = (const char*)sqlite3_value_text(pData);
          223  +    if( symlink(zTo, zFile)<0 ) return 1;
          224  +  }else
          225  +#endif
          226  +  {
          227  +    if( S_ISDIR(mode) ){
          228  +      if( mkdir(zFile, mode) ){
          229  +        /* The mkdir() call to create the directory failed. This might not
          230  +        ** be an error though - if there is already a directory at the same
          231  +        ** path and either the permissions already match or can be changed
          232  +        ** to do so using chmod(), it is not an error.  */
          233  +        struct stat sStat;
          234  +        if( errno!=EEXIST
          235  +         || 0!=stat(zFile, &sStat)
          236  +         || !S_ISDIR(sStat.st_mode)
          237  +         || ((sStat.st_mode&0777)!=(mode&0777) && 0!=chmod(zFile, mode&0777))
          238  +        ){
          239  +          return 1;
          240  +        }
          241  +      }
          242  +    }else{
          243  +      sqlite3_int64 nWrite = 0;
          244  +      const char *z;
          245  +      int rc = 0;
          246  +      FILE *out = fopen(zFile, "wb");
          247  +      if( out==0 ) return 1;
          248  +      z = (const char*)sqlite3_value_blob(pData);
          249  +      if( z ){
          250  +        sqlite3_int64 n = fwrite(z, 1, sqlite3_value_bytes(pData), out);
          251  +        nWrite = sqlite3_value_bytes(pData);
          252  +        if( nWrite!=n ){
          253  +          rc = 1;
          254  +        }
          255  +      }
          256  +      fclose(out);
          257  +      if( rc==0 && mode && chmod(zFile, mode & 0777) ){
          258  +        rc = 1;
          259  +      }
          260  +      if( rc ) return 2;
          261  +      sqlite3_result_int64(pCtx, nWrite);
          262  +    }
          263  +  }
          264  +
          265  +  if( mtime>=0 ){
          266  +#if defined(_WIN32)
          267  +    /* Windows */
          268  +    FILETIME lastAccess;
          269  +    FILETIME lastWrite;
          270  +    SYSTEMTIME currentTime;
          271  +    LONGLONG intervals;
          272  +    HANDLE hFile;
          273  +    GetSystemTime(&currentTime);
          274  +    SystemTimeToFileTime(&currentTime, &lastAccess);
          275  +    intervals = Int32x32To64(mtime, 10000000) + 116444736000000000;
          276  +    lastWrite.dwLowDateTime = (DWORD)intervals;
          277  +    lastWrite.dwHighDateTime = intervals >> 32;
          278  +    hFile = CreateFile(
          279  +      zFile, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING,
          280  +      FILE_FLAG_BACKUP_SEMANTICS, NULL
          281  +    );
          282  +    if( hFile!=INVALID_HANDLE_VALUE ){
          283  +      BOOL bResult = SetFileTime(hFile, NULL, &lastAccess, &lastWrite);
          284  +      CloseHandle(hFile);
          285  +      return !bResult;
          286  +    }else{
          287  +      return 1;
          288  +    }
          289  +#elif defined(AT_FDCWD) && 0 /* utimensat() is not univerally available */
          290  +    /* Recent unix */
          291  +    struct timespec times[2];
          292  +    times[0].tv_nsec = times[1].tv_nsec = 0;
          293  +    times[0].tv_sec = time(0);
          294  +    times[1].tv_sec = mtime;
          295  +    if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){
          296  +      return 1;
          297  +    }
          298  +#else
          299  +    /* Legacy unix */
          300  +    struct timeval times[2];
          301  +    times[0].tv_usec = times[1].tv_usec = 0;
          302  +    times[0].tv_sec = time(0);
          303  +    times[1].tv_sec = mtime;
          304  +    if( utimes(zFile, times) ){
          305  +      return 1;
          306  +    }
          307  +#endif
          308  +  }
          309  +
          310  +  return 0;
          311  +}
          312  +
          313  +/*
          314  +** Implementation of the "writefile(W,X[,Y[,Z]]])" SQL function.  
          315  +** Refer to header comments at the top of this file for details.
    57    316   */
    58    317   static void writefileFunc(
    59    318     sqlite3_context *context,
    60    319     int argc,
    61    320     sqlite3_value **argv
    62    321   ){
    63         -  FILE *out;
    64         -  const char *z;
    65         -  sqlite3_int64 rc;
    66    322     const char *zFile;
          323  +  mode_t mode = 0;
          324  +  int res;
          325  +  sqlite3_int64 mtime = -1;
    67    326   
    68         -  (void)(argc);  /* Unused parameter */
          327  +  if( argc<2 || argc>4 ){
          328  +    sqlite3_result_error(context, 
          329  +        "wrong number of arguments to function writefile()", -1
          330  +    );
          331  +    return;
          332  +  }
          333  +
    69    334     zFile = (const char*)sqlite3_value_text(argv[0]);
    70    335     if( zFile==0 ) return;
    71         -  out = fopen(zFile, "wb");
    72         -  if( out==0 ) return;
    73         -  z = (const char*)sqlite3_value_blob(argv[1]);
    74         -  if( z==0 ){
    75         -    rc = 0;
          336  +  if( argc>=3 ){
          337  +    mode = (mode_t)sqlite3_value_int(argv[2]);
          338  +  }
          339  +  if( argc==4 ){
          340  +    mtime = sqlite3_value_int64(argv[3]);
          341  +  }
          342  +
          343  +  res = writeFile(context, zFile, argv[1], mode, mtime);
          344  +  if( res==1 && errno==ENOENT ){
          345  +    if( makeDirectory(zFile, mode)==SQLITE_OK ){
          346  +      res = writeFile(context, zFile, argv[1], mode, mtime);
          347  +    }
          348  +  }
          349  +
          350  +  if( argc>2 && res!=0 ){
          351  +    if( S_ISLNK(mode) ){
          352  +      ctxErrorMsg(context, "failed to create symlink: %s", zFile);
          353  +    }else if( S_ISDIR(mode) ){
          354  +      ctxErrorMsg(context, "failed to create directory: %s", zFile);
          355  +    }else{
          356  +      ctxErrorMsg(context, "failed to write file: %s", zFile);
          357  +    }
          358  +  }
          359  +}
          360  +
          361  +/*
          362  +** SQL function:   lsmode(MODE)
          363  +**
          364  +** Given a numberic st_mode from stat(), convert it into a human-readable
          365  +** text string in the style of "ls -l".
          366  +*/
          367  +static void lsModeFunc(
          368  +  sqlite3_context *context,
          369  +  int argc,
          370  +  sqlite3_value **argv
          371  +){
          372  +  int i;
          373  +  int iMode = sqlite3_value_int(argv[0]);
          374  +  char z[16];
          375  +  if( S_ISLNK(iMode) ){
          376  +    z[0] = 'l';
          377  +  }else if( S_ISREG(iMode) ){
          378  +    z[0] = '-';
          379  +  }else if( S_ISDIR(iMode) ){
          380  +    z[0] = 'd';
          381  +  }else{
          382  +    z[0] = '?';
          383  +  }
          384  +  for(i=0; i<3; i++){
          385  +    int m = (iMode >> ((2-i)*3));
          386  +    char *a = &z[1 + i*3];
          387  +    a[0] = (m & 0x4) ? 'r' : '-';
          388  +    a[1] = (m & 0x2) ? 'w' : '-';
          389  +    a[2] = (m & 0x1) ? 'x' : '-';
          390  +  }
          391  +  z[10] = '\0';
          392  +  sqlite3_result_text(context, z, -1, SQLITE_TRANSIENT);
          393  +}
          394  +
          395  +#ifndef SQLITE_OMIT_VIRTUALTABLE
          396  +
          397  +/* 
          398  +** Cursor type for recursively iterating through a directory structure.
          399  +*/
          400  +typedef struct fsdir_cursor fsdir_cursor;
          401  +typedef struct FsdirLevel FsdirLevel;
          402  +
          403  +struct FsdirLevel {
          404  +  DIR *pDir;                 /* From opendir() */
          405  +  char *zDir;                /* Name of directory (nul-terminated) */
          406  +};
          407  +
          408  +struct fsdir_cursor {
          409  +  sqlite3_vtab_cursor base;  /* Base class - must be first */
          410  +
          411  +  int nLvl;                  /* Number of entries in aLvl[] array */
          412  +  int iLvl;                  /* Index of current entry */
          413  +  FsdirLevel *aLvl;          /* Hierarchy of directories being traversed */
          414  +
          415  +  const char *zBase;
          416  +  int nBase;
          417  +
          418  +  struct stat sStat;         /* Current lstat() results */
          419  +  char *zPath;               /* Path to current entry */
          420  +  sqlite3_int64 iRowid;      /* Current rowid */
          421  +};
          422  +
          423  +typedef struct fsdir_tab fsdir_tab;
          424  +struct fsdir_tab {
          425  +  sqlite3_vtab base;         /* Base class - must be first */
          426  +};
          427  +
          428  +/*
          429  +** Construct a new fsdir virtual table object.
          430  +*/
          431  +static int fsdirConnect(
          432  +  sqlite3 *db,
          433  +  void *pAux,
          434  +  int argc, const char *const*argv,
          435  +  sqlite3_vtab **ppVtab,
          436  +  char **pzErr
          437  +){
          438  +  fsdir_tab *pNew = 0;
          439  +  int rc;
          440  +
          441  +  rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA);
          442  +  if( rc==SQLITE_OK ){
          443  +    pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) );
          444  +    if( pNew==0 ) return SQLITE_NOMEM;
          445  +    memset(pNew, 0, sizeof(*pNew));
          446  +  }
          447  +  *ppVtab = (sqlite3_vtab*)pNew;
          448  +  return rc;
          449  +}
          450  +
          451  +/*
          452  +** This method is the destructor for fsdir vtab objects.
          453  +*/
          454  +static int fsdirDisconnect(sqlite3_vtab *pVtab){
          455  +  sqlite3_free(pVtab);
          456  +  return SQLITE_OK;
          457  +}
          458  +
          459  +/*
          460  +** Constructor for a new fsdir_cursor object.
          461  +*/
          462  +static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
          463  +  fsdir_cursor *pCur;
          464  +  pCur = sqlite3_malloc( sizeof(*pCur) );
          465  +  if( pCur==0 ) return SQLITE_NOMEM;
          466  +  memset(pCur, 0, sizeof(*pCur));
          467  +  pCur->iLvl = -1;
          468  +  *ppCursor = &pCur->base;
          469  +  return SQLITE_OK;
          470  +}
          471  +
          472  +/*
          473  +** Reset a cursor back to the state it was in when first returned
          474  +** by fsdirOpen().
          475  +*/
          476  +static void fsdirResetCursor(fsdir_cursor *pCur){
          477  +  int i;
          478  +  for(i=0; i<=pCur->iLvl; i++){
          479  +    FsdirLevel *pLvl = &pCur->aLvl[i];
          480  +    if( pLvl->pDir ) closedir(pLvl->pDir);
          481  +    sqlite3_free(pLvl->zDir);
          482  +  }
          483  +  sqlite3_free(pCur->zPath);
          484  +  pCur->aLvl = 0;
          485  +  pCur->zPath = 0;
          486  +  pCur->zBase = 0;
          487  +  pCur->nBase = 0;
          488  +  pCur->iLvl = -1;
          489  +  pCur->iRowid = 1;
          490  +}
          491  +
          492  +/*
          493  +** Destructor for an fsdir_cursor.
          494  +*/
          495  +static int fsdirClose(sqlite3_vtab_cursor *cur){
          496  +  fsdir_cursor *pCur = (fsdir_cursor*)cur;
          497  +
          498  +  fsdirResetCursor(pCur);
          499  +  sqlite3_free(pCur->aLvl);
          500  +  sqlite3_free(pCur);
          501  +  return SQLITE_OK;
          502  +}
          503  +
          504  +/*
          505  +** Set the error message for the virtual table associated with cursor
          506  +** pCur to the results of vprintf(zFmt, ...).
          507  +*/
          508  +static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){
          509  +  va_list ap;
          510  +  va_start(ap, zFmt);
          511  +  pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
          512  +  va_end(ap);
          513  +}
          514  +
          515  +
          516  +/*
          517  +** Advance an fsdir_cursor to its next row of output.
          518  +*/
          519  +static int fsdirNext(sqlite3_vtab_cursor *cur){
          520  +  fsdir_cursor *pCur = (fsdir_cursor*)cur;
          521  +  mode_t m = pCur->sStat.st_mode;
          522  +
          523  +  pCur->iRowid++;
          524  +  if( S_ISDIR(m) ){
          525  +    /* Descend into this directory */
          526  +    int iNew = pCur->iLvl + 1;
          527  +    FsdirLevel *pLvl;
          528  +    if( iNew>=pCur->nLvl ){
          529  +      int nNew = iNew+1;
          530  +      int nByte = nNew*sizeof(FsdirLevel);
          531  +      FsdirLevel *aNew = (FsdirLevel*)sqlite3_realloc(pCur->aLvl, nByte);
          532  +      if( aNew==0 ) return SQLITE_NOMEM;
          533  +      memset(&aNew[pCur->nLvl], 0, sizeof(FsdirLevel)*(nNew-pCur->nLvl));
          534  +      pCur->aLvl = aNew;
          535  +      pCur->nLvl = nNew;
          536  +    }
          537  +    pCur->iLvl = iNew;
          538  +    pLvl = &pCur->aLvl[iNew];
          539  +    
          540  +    pLvl->zDir = pCur->zPath;
          541  +    pCur->zPath = 0;
          542  +    pLvl->pDir = opendir(pLvl->zDir);
          543  +    if( pLvl->pDir==0 ){
          544  +      fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath);
          545  +      return SQLITE_ERROR;
          546  +    }
          547  +  }
          548  +
          549  +  while( pCur->iLvl>=0 ){
          550  +    FsdirLevel *pLvl = &pCur->aLvl[pCur->iLvl];
          551  +    struct dirent *pEntry = readdir(pLvl->pDir);
          552  +    if( pEntry ){
          553  +      if( pEntry->d_name[0]=='.' ){
          554  +       if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue;
          555  +       if( pEntry->d_name[1]=='\0' ) continue;
          556  +      }
          557  +      sqlite3_free(pCur->zPath);
          558  +      pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name);
          559  +      if( pCur->zPath==0 ) return SQLITE_NOMEM;
          560  +      if( lstat(pCur->zPath, &pCur->sStat) ){
          561  +        fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
          562  +        return SQLITE_ERROR;
          563  +      }
          564  +      return SQLITE_OK;
          565  +    }
          566  +    closedir(pLvl->pDir);
          567  +    sqlite3_free(pLvl->zDir);
          568  +    pLvl->pDir = 0;
          569  +    pLvl->zDir = 0;
          570  +    pCur->iLvl--;
          571  +  }
          572  +
          573  +  /* EOF */
          574  +  sqlite3_free(pCur->zPath);
          575  +  pCur->zPath = 0;
          576  +  return SQLITE_OK;
          577  +}
          578  +
          579  +/*
          580  +** Return values of columns for the row at which the series_cursor
          581  +** is currently pointing.
          582  +*/
          583  +static int fsdirColumn(
          584  +  sqlite3_vtab_cursor *cur,   /* The cursor */
          585  +  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
          586  +  int i                       /* Which column to return */
          587  +){
          588  +  fsdir_cursor *pCur = (fsdir_cursor*)cur;
          589  +  switch( i ){
          590  +    case 0: { /* name */
          591  +      sqlite3_result_text(ctx, &pCur->zPath[pCur->nBase], -1, SQLITE_TRANSIENT);
          592  +      break;
          593  +    }
          594  +
          595  +    case 1: /* mode */
          596  +      sqlite3_result_int64(ctx, pCur->sStat.st_mode);
          597  +      break;
          598  +
          599  +    case 2: /* mtime */
          600  +      sqlite3_result_int64(ctx, pCur->sStat.st_mtime);
          601  +      break;
          602  +
          603  +    case 3: { /* data */
          604  +      mode_t m = pCur->sStat.st_mode;
          605  +      if( S_ISDIR(m) ){
          606  +        sqlite3_result_null(ctx);
          607  +#if !defined(_WIN32) && !defined(WIN32)
          608  +      }else if( S_ISLNK(m) ){
          609  +        char aStatic[64];
          610  +        char *aBuf = aStatic;
          611  +        int nBuf = 64;
          612  +        int n;
          613  +
          614  +        while( 1 ){
          615  +          n = readlink(pCur->zPath, aBuf, nBuf);
          616  +          if( n<nBuf ) break;
          617  +          if( aBuf!=aStatic ) sqlite3_free(aBuf);
          618  +          nBuf = nBuf*2;
          619  +          aBuf = sqlite3_malloc(nBuf);
          620  +          if( aBuf==0 ){
          621  +            sqlite3_result_error_nomem(ctx);
          622  +            return SQLITE_NOMEM;
          623  +          }
          624  +        }
          625  +
          626  +        sqlite3_result_text(ctx, aBuf, n, SQLITE_TRANSIENT);
          627  +        if( aBuf!=aStatic ) sqlite3_free(aBuf);
          628  +#endif
          629  +      }else{
          630  +        readFileContents(ctx, pCur->zPath);
          631  +      }
          632  +    }
          633  +  }
          634  +  return SQLITE_OK;
          635  +}
          636  +
          637  +/*
          638  +** Return the rowid for the current row. In this implementation, the
          639  +** first row returned is assigned rowid value 1, and each subsequent
          640  +** row a value 1 more than that of the previous.
          641  +*/
          642  +static int fsdirRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
          643  +  fsdir_cursor *pCur = (fsdir_cursor*)cur;
          644  +  *pRowid = pCur->iRowid;
          645  +  return SQLITE_OK;
          646  +}
          647  +
          648  +/*
          649  +** Return TRUE if the cursor has been moved off of the last
          650  +** row of output.
          651  +*/
          652  +static int fsdirEof(sqlite3_vtab_cursor *cur){
          653  +  fsdir_cursor *pCur = (fsdir_cursor*)cur;
          654  +  return (pCur->zPath==0);
          655  +}
          656  +
          657  +/*
          658  +** xFilter callback.
          659  +*/
          660  +static int fsdirFilter(
          661  +  sqlite3_vtab_cursor *cur, 
          662  +  int idxNum, const char *idxStr,
          663  +  int argc, sqlite3_value **argv
          664  +){
          665  +  const char *zDir = 0;
          666  +  fsdir_cursor *pCur = (fsdir_cursor*)cur;
          667  +
          668  +  fsdirResetCursor(pCur);
          669  +
          670  +  if( idxNum==0 ){
          671  +    fsdirSetErrmsg(pCur, "table function fsdir requires an argument");
          672  +    return SQLITE_ERROR;
          673  +  }
          674  +
          675  +  assert( argc==idxNum && (argc==1 || argc==2) );
          676  +  zDir = (const char*)sqlite3_value_text(argv[0]);
          677  +  if( zDir==0 ){
          678  +    fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument");
          679  +    return SQLITE_ERROR;
          680  +  }
          681  +  if( argc==2 ){
          682  +    pCur->zBase = (const char*)sqlite3_value_text(argv[1]);
          683  +  }
          684  +  if( pCur->zBase ){
          685  +    pCur->nBase = (int)strlen(pCur->zBase)+1;
          686  +    pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zBase, zDir);
          687  +  }else{
          688  +    pCur->zPath = sqlite3_mprintf("%s", zDir);
          689  +  }
          690  +
          691  +  if( pCur->zPath==0 ){
          692  +    return SQLITE_NOMEM;
          693  +  }
          694  +  if( lstat(pCur->zPath, &pCur->sStat) ){
          695  +    fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
          696  +    return SQLITE_ERROR;
          697  +  }
          698  +
          699  +  return SQLITE_OK;
          700  +}
          701  +
          702  +/*
          703  +** SQLite will invoke this method one or more times while planning a query
          704  +** that uses the generate_series virtual table.  This routine needs to create
          705  +** a query plan for each invocation and compute an estimated cost for that
          706  +** plan.
          707  +**
          708  +** In this implementation idxNum is used to represent the
          709  +** query plan.  idxStr is unused.
          710  +**
          711  +** The query plan is represented by bits in idxNum:
          712  +**
          713  +**  (1)  start = $value  -- constraint exists
          714  +**  (2)  stop = $value   -- constraint exists
          715  +**  (4)  step = $value   -- constraint exists
          716  +**  (8)  output in descending order
          717  +*/
          718  +static int fsdirBestIndex(
          719  +  sqlite3_vtab *tab,
          720  +  sqlite3_index_info *pIdxInfo
          721  +){
          722  +  int i;                 /* Loop over constraints */
          723  +  int idx4 = -1;
          724  +  int idx5 = -1;
          725  +
          726  +  const struct sqlite3_index_constraint *pConstraint;
          727  +  pConstraint = pIdxInfo->aConstraint;
          728  +  for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
          729  +    if( pConstraint->usable==0 ) continue;
          730  +    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
          731  +    if( pConstraint->iColumn==4 ) idx4 = i;
          732  +    if( pConstraint->iColumn==5 ) idx5 = i;
          733  +  }
          734  +
          735  +  if( idx4<0 ){
          736  +    pIdxInfo->idxNum = 0;
          737  +    pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50);
    76    738     }else{
    77         -    rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out);
          739  +    pIdxInfo->aConstraintUsage[idx4].omit = 1;
          740  +    pIdxInfo->aConstraintUsage[idx4].argvIndex = 1;
          741  +    if( idx5>=0 ){
          742  +      pIdxInfo->aConstraintUsage[idx5].omit = 1;
          743  +      pIdxInfo->aConstraintUsage[idx5].argvIndex = 2;
          744  +      pIdxInfo->idxNum = 2;
          745  +      pIdxInfo->estimatedCost = 10.0;
          746  +    }else{
          747  +      pIdxInfo->idxNum = 1;
          748  +      pIdxInfo->estimatedCost = 100.0;
          749  +    }
    78    750     }
    79         -  fclose(out);
    80         -  sqlite3_result_int64(context, rc);
          751  +
          752  +  return SQLITE_OK;
    81    753   }
    82    754   
          755  +/*
          756  +** Register the "fsdir" virtual table.
          757  +*/
          758  +static int fsdirRegister(sqlite3 *db){
          759  +  static sqlite3_module fsdirModule = {
          760  +    0,                         /* iVersion */
          761  +    0,                         /* xCreate */
          762  +    fsdirConnect,              /* xConnect */
          763  +    fsdirBestIndex,            /* xBestIndex */
          764  +    fsdirDisconnect,           /* xDisconnect */
          765  +    0,                         /* xDestroy */
          766  +    fsdirOpen,                 /* xOpen - open a cursor */
          767  +    fsdirClose,                /* xClose - close a cursor */
          768  +    fsdirFilter,               /* xFilter - configure scan constraints */
          769  +    fsdirNext,                 /* xNext - advance a cursor */
          770  +    fsdirEof,                  /* xEof - check for end of scan */
          771  +    fsdirColumn,               /* xColumn - read data */
          772  +    fsdirRowid,                /* xRowid - read data */
          773  +    0,                         /* xUpdate */
          774  +    0,                         /* xBegin */
          775  +    0,                         /* xSync */
          776  +    0,                         /* xCommit */
          777  +    0,                         /* xRollback */
          778  +    0,                         /* xFindMethod */
          779  +    0,                         /* xRename */
          780  +  };
          781  +
................................................................................
          782  +  int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0);
          783  +  return rc;
          784  +}
          785  +#else         /* SQLITE_OMIT_VIRTUALTABLE */
          786  +# define fsdirRegister(x) SQLITE_OK
          787  +#endif
    83    788   
    84    789   #ifdef _WIN32
    85    790   __declspec(dllexport)
    86    791   #endif
    87    792   int sqlite3_fileio_init(
    88    793     sqlite3 *db, 
    89    794     char **pzErrMsg, 
................................................................................
    91    796   ){
    92    797     int rc = SQLITE_OK;
    93    798     SQLITE_EXTENSION_INIT2(pApi);
    94    799     (void)pzErrMsg;  /* Unused parameter */
    95    800     rc = sqlite3_create_function(db, "readfile", 1, SQLITE_UTF8, 0,
    96    801                                  readfileFunc, 0, 0);
    97    802     if( rc==SQLITE_OK ){
    98         -    rc = sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0,
          803  +    rc = sqlite3_create_function(db, "writefile", -1, SQLITE_UTF8, 0,
    99    804                                    writefileFunc, 0, 0);
   100    805     }
          806  +  if( rc==SQLITE_OK ){
          807  +    rc = sqlite3_create_function(db, "lsmode", 1, SQLITE_UTF8, 0,
          808  +                                 lsModeFunc, 0, 0);
          809  +  }
          810  +  if( rc==SQLITE_OK ){
          811  +    rc = fsdirRegister(db);
          812  +  }
   101    813     return rc;
   102    814   }

Changes to ext/misc/memvfs.c.

     6      6   **
     7      7   **    May you do good and not evil.
     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   ******************************************************************************
    12     12   **
    13         -** This is an in-memory read-only VFS implementation.  The application
    14         -** supplies a block of memory which is the database file, and this VFS
    15         -** uses that block of memory.
           13  +** This is an in-memory VFS implementation.  The application supplies
           14  +** a chunk of memory to hold the database file.
    16     15   **
    17         -** Because there is no place to store journals and no good way to lock
    18         -** the "file", this VFS is read-only.
           16  +** Because there is place to store a rollback or wal journal, the database
           17  +** must use one of journal_mode=MEMORY or journal_mode=NONE.
    19     18   **
    20     19   ** USAGE:
    21     20   **
    22         -**    sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336", &db,
    23         -**                    SQLITE_OPEN_READONLY | SQLITE_OPEN_URI,
           21  +**    sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336&max=65536", &db,
           22  +**                    SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI,
    24     23   **                    "memvfs");
    25     24   **
    26         -** The ptr= and sz= query parameters are required or the open will fail.
    27         -** The ptr= parameter gives the memory address of the buffer holding the
    28         -** read-only database and sz= gives the size of the database.  The parameter
    29         -** values may be in hexadecimal or decimal.  The filename is ignored.
           25  +** These are the query parameters:
           26  +**
           27  +**    ptr=          The address of the memory buffer that holds the database.
           28  +**
           29  +**    sz=           The current size the database file
           30  +**
           31  +**    maxsz=        The maximum size of the database.  In other words, the
           32  +**                  amount of space allocated for the ptr= buffer.
           33  +**
           34  +**    freeonclose=  If true, then sqlite3_free() is called on the ptr=
           35  +**                  value when the connection closes.
           36  +**
           37  +** The ptr= and sz= query parameters are required.  If maxsz= is omitted,
           38  +** then it defaults to the sz= value.  Parameter values can be in either
           39  +** decimal or hexadecimal.  The filename in the URI is ignored.
    30     40   */
    31     41   #include <sqlite3ext.h>
    32     42   SQLITE_EXTENSION_INIT1
    33     43   #include <string.h>
    34     44   #include <assert.h>
    35     45   
    36     46   
................................................................................
    45     55   */
    46     56   #define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData))
    47     57   
    48     58   /* An open file */
    49     59   struct MemFile {
    50     60     sqlite3_file base;              /* IO methods */
    51     61     sqlite3_int64 sz;               /* Size of the file */
           62  +  sqlite3_int64 szMax;            /* Space allocated to aData */
    52     63     unsigned char *aData;           /* content of the file */
           64  +  int bFreeOnClose;               /* Invoke sqlite3_free() on aData at close */
    53     65   };
    54     66   
    55     67   /*
    56     68   ** Methods for MemFile
    57     69   */
    58     70   static int memClose(sqlite3_file*);
    59     71   static int memRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
................................................................................
   140    152   /*
   141    153   ** Close an mem-file.
   142    154   **
   143    155   ** The pData pointer is owned by the application, so there is nothing
   144    156   ** to free.
   145    157   */
   146    158   static int memClose(sqlite3_file *pFile){
          159  +  MemFile *p = (MemFile *)pFile;
          160  +  if( p->bFreeOnClose ) sqlite3_free(p->aData);
   147    161     return SQLITE_OK;
   148    162   }
   149    163   
   150    164   /*
   151    165   ** Read data from an mem-file.
   152    166   */
   153    167   static int memRead(
................................................................................
   166    180   */
   167    181   static int memWrite(
   168    182     sqlite3_file *pFile,
   169    183     const void *z,
   170    184     int iAmt,
   171    185     sqlite_int64 iOfst
   172    186   ){
   173         -  return SQLITE_READONLY;
          187  +  MemFile *p = (MemFile *)pFile;
          188  +  if( iOfst+iAmt>p->sz ){
          189  +    if( iOfst+iAmt>p->szMax ) return SQLITE_FULL;
          190  +    if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz);
          191  +    p->sz = iOfst+iAmt;
          192  +  }
          193  +  memcpy(p->aData+iOfst, z, iAmt);
          194  +  return SQLITE_OK;
   174    195   }
   175    196   
   176    197   /*
   177    198   ** Truncate an mem-file.
   178    199   */
   179    200   static int memTruncate(sqlite3_file *pFile, sqlite_int64 size){
   180         -  return SQLITE_READONLY;
          201  +  MemFile *p = (MemFile *)pFile;
          202  +  if( size>p->sz ){
          203  +    if( size>p->szMax ) return SQLITE_FULL;
          204  +    memset(p->aData+p->sz, 0, size-p->sz);
          205  +  }
          206  +  p->sz = size; 
          207  +  return SQLITE_OK;
   181    208   }
   182    209   
   183    210   /*
   184    211   ** Sync an mem-file.
   185    212   */
   186    213   static int memSync(sqlite3_file *pFile, int flags){
   187         -  return SQLITE_READONLY;
          214  +  return SQLITE_OK;
   188    215   }
   189    216   
   190    217   /*
   191    218   ** Return the current file-size of an mem-file.
   192    219   */
   193    220   static int memFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
   194    221     MemFile *p = (MemFile *)pFile;
................................................................................
   196    223     return SQLITE_OK;
   197    224   }
   198    225   
   199    226   /*
   200    227   ** Lock an mem-file.
   201    228   */
   202    229   static int memLock(sqlite3_file *pFile, int eLock){
   203         -  return SQLITE_READONLY;
          230  +  return SQLITE_OK;
   204    231   }
   205    232   
   206    233   /*
   207    234   ** Unlock an mem-file.
   208    235   */
   209    236   static int memUnlock(sqlite3_file *pFile, int eLock){
   210    237     return SQLITE_OK;
................................................................................
   238    265     return 1024;
   239    266   }
   240    267   
   241    268   /*
   242    269   ** Return the device characteristic flags supported by an mem-file.
   243    270   */
   244    271   static int memDeviceCharacteristics(sqlite3_file *pFile){
   245         -  return SQLITE_IOCAP_IMMUTABLE;
          272  +  return SQLITE_IOCAP_ATOMIC | 
          273  +         SQLITE_IOCAP_POWERSAFE_OVERWRITE |
          274  +         SQLITE_IOCAP_SAFE_APPEND |
          275  +         SQLITE_IOCAP_SEQUENTIAL;
   246    276   }
   247    277   
   248    278   /* Create a shared memory file mapping */
   249    279   static int memShmMap(
   250    280     sqlite3_file *pFile,
   251    281     int iPg,
   252    282     int pgsz,
   253    283     int bExtend,
   254    284     void volatile **pp
   255    285   ){
   256         -  return SQLITE_READONLY;
          286  +  return SQLITE_IOERR_SHMMAP;
   257    287   }
   258    288   
   259    289   /* Perform locking on a shared-memory segment */
   260    290   static int memShmLock(sqlite3_file *pFile, int offset, int n, int flags){
   261         -  return SQLITE_READONLY;
          291  +  return SQLITE_IOERR_SHMLOCK;
   262    292   }
   263    293   
   264    294   /* Memory barrier operation on shared memory */
   265    295   static void memShmBarrier(sqlite3_file *pFile){
   266    296     return;
   267    297   }
   268    298   
................................................................................
   301    331     MemFile *p = (MemFile*)pFile;
   302    332     memset(p, 0, sizeof(*p));
   303    333     if( (flags & SQLITE_OPEN_MAIN_DB)==0 ) return SQLITE_CANTOPEN;
   304    334     p->aData = (unsigned char*)sqlite3_uri_int64(zName,"ptr",0);
   305    335     if( p->aData==0 ) return SQLITE_CANTOPEN;
   306    336     p->sz = sqlite3_uri_int64(zName,"sz",0);
   307    337     if( p->sz<0 ) return SQLITE_CANTOPEN;
          338  +  p->szMax = sqlite3_uri_int64(zName,"max",p->sz);
          339  +  if( p->szMax<p->sz ) return SQLITE_CANTOPEN;
          340  +  p->bFreeOnClose = sqlite3_uri_boolean(zName,"freeonclose",0);
   308    341     pFile->pMethods = &mem_io_methods;
   309    342     return SQLITE_OK;
   310    343   }
   311    344   
   312    345   /*
   313    346   ** Delete the file located at zPath. If the dirSync argument is true,
   314    347   ** ensure the file-system modifications are synced to disk before
   315    348   ** returning.
   316    349   */
   317    350   static int memDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
   318         -  return SQLITE_READONLY;
          351  +  return SQLITE_IOERR_DELETE;
   319    352   }
   320    353   
   321    354   /*
   322    355   ** Test for access permissions. Return true if the requested permission
   323    356   ** is available, or false otherwise.
   324    357   */
   325    358   static int memAccess(
   326    359     sqlite3_vfs *pVfs, 
   327    360     const char *zPath, 
   328    361     int flags, 
   329    362     int *pResOut
   330    363   ){
   331         -  /* The spec says there are three possible values for flags.  But only
   332         -  ** two of them are actually used */
   333         -  assert( flags==SQLITE_ACCESS_EXISTS || flags==SQLITE_ACCESS_READWRITE );
   334         -  if( flags==SQLITE_ACCESS_READWRITE ){
   335         -    *pResOut = 0;
   336         -  }else{
   337         -    *pResOut = 1;
   338         -  }
          364  +  *pResOut = 0;
   339    365     return SQLITE_OK;
   340    366   }
   341    367   
   342    368   /*
   343    369   ** Populate buffer zOut with the full canonical pathname corresponding
   344    370   ** to the pathname in zPath. zOut is guaranteed to point to a buffer
   345    371   ** of at least (INST_MAX_PATHNAME+1) bytes.
................................................................................
   412    438   }
   413    439   static int memCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
   414    440     return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p);
   415    441   }
   416    442   
   417    443   #ifdef MEMVFS_TEST
   418    444   /*
   419         -**       memload(FILENAME)
          445  +**       memvfs_from_file(FILENAME, MAXSIZE)
   420    446   **
   421    447   ** This an SQL function used to help in testing the memvfs VFS.  The
   422    448   ** function reads the content of a file into memory and then returns
   423         -** a string that gives the locate and size of the in-memory buffer.
          449  +** a URI that can be handed to ATTACH to attach the memory buffer as
          450  +** a database.  Example:
          451  +**
          452  +**       ATTACH memvfs_from_file('test.db',1048576) AS inmem;
          453  +**
          454  +** The optional MAXSIZE argument gives the size of the memory allocation
          455  +** used to hold the database.  If omitted, it defaults to the size of the
          456  +** file on disk.
   424    457   */
   425    458   #include <stdio.h>
   426         -static void memvfsMemloadFunc(
          459  +static void memvfsFromFileFunc(
   427    460     sqlite3_context *context,
   428    461     int argc,
   429    462     sqlite3_value **argv
   430    463   ){
   431    464     unsigned char *p;
   432    465     sqlite3_int64 sz;
          466  +  sqlite3_int64 szMax;
   433    467     FILE *in;
   434    468     const char *zFilename = (const char*)sqlite3_value_text(argv[0]);
   435         -  char zReturn[100];
          469  +  char *zUri;
   436    470   
   437    471     if( zFilename==0 ) return;
   438    472     in = fopen(zFilename, "rb");
   439    473     if( in==0 ) return;
   440    474     fseek(in, 0, SEEK_END);
   441         -  sz = ftell(in);
          475  +  szMax = sz = ftell(in);
   442    476     rewind(in);
   443         -  p = sqlite3_malloc( sz );
          477  +  if( argc>=2 ){
          478  +    szMax = sqlite3_value_int64(argv[1]);
          479  +    if( szMax<sz ) szMax = sz;
          480  +  }
          481  +  p = sqlite3_malloc64( szMax );
   444    482     if( p==0 ){
   445    483       fclose(in);
   446    484       sqlite3_result_error_nomem(context);
   447    485       return;
   448    486     }
   449    487     fread(p, sz, 1, in);
   450    488     fclose(in);
   451         -  sqlite3_snprintf(sizeof(zReturn),zReturn,"ptr=%lld&sz=%lld",
   452         -                   (sqlite3_int64)p, sz);
   453         -  sqlite3_result_text(context, zReturn, -1, SQLITE_TRANSIENT);
          489  +  zUri = sqlite3_mprintf(
          490  +           "file:/mem?vfs=memvfs&ptr=%lld&sz=%lld&max=%lld&freeonclose=1",
          491  +                         (sqlite3_int64)p, sz, szMax);
          492  +  sqlite3_result_text(context, zUri, -1, sqlite3_free);
          493  +}
          494  +#endif /* MEMVFS_TEST */
          495  +
          496  +#ifdef MEMVFS_TEST
          497  +/*
          498  +**       memvfs_to_file(SCHEMA, FILENAME)
          499  +**
          500  +** The schema identified by SCHEMA must be a memvfs database.  Write
          501  +** the content of this database into FILENAME.
          502  +*/
          503  +static void memvfsToFileFunc(
          504  +  sqlite3_context *context,
          505  +  int argc,
          506  +  sqlite3_value **argv
          507  +){
          508  +  MemFile *p = 0;
          509  +  FILE *out;
          510  +  int rc;
          511  +  sqlite3 *db = sqlite3_context_db_handle(context);
          512  +  sqlite3_vfs *pVfs = 0;
          513  +  const char *zSchema = (const char*)sqlite3_value_text(argv[0]);
          514  +  const char *zFilename = (const char*)sqlite3_value_text(argv[1]);
          515  +
          516  +  if( zFilename==0 ) return;
          517  +  out = fopen(zFilename, "wb");
          518  +  if( out==0 ) return;
          519  +  rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_VFS_POINTER, &pVfs);
          520  +  if( rc || pVfs==0 ) return;
          521  +  if( strcmp(pVfs->zName,"memvfs")!=0 ) return;
          522  +  rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_FILE_POINTER, &p);
          523  +  if( rc ) return;
          524  +  fwrite(p->aData, 1, (size_t)p->sz, out);
          525  +  fclose(out);
   454    526   }
          527  +#endif /* MEMVFS_TEST */
          528  +
          529  +#ifdef MEMVFS_TEST
   455    530   /* Called for each new database connection */
   456    531   static int memvfsRegister(
   457    532     sqlite3 *db,
   458         -  const char **pzErrMsg,
          533  +  char **pzErrMsg,
   459    534     const struct sqlite3_api_routines *pThunk
   460    535   ){
   461         -  return sqlite3_create_function(db, "memload", 1, SQLITE_UTF8, 0,
   462         -                                 memvfsMemloadFunc, 0, 0);
          536  +  sqlite3_create_function(db, "memvfs_from_file", 1, SQLITE_UTF8, 0,
          537  +                          memvfsFromFileFunc, 0, 0);
          538  +  sqlite3_create_function(db, "memvfs_from_file", 2, SQLITE_UTF8, 0,
          539  +                          memvfsFromFileFunc, 0, 0);
          540  +  sqlite3_create_function(db, "memvfs_to_file", 2, SQLITE_UTF8, 0,
          541  +                          memvfsToFileFunc, 0, 0);
          542  +  return SQLITE_OK;
   463    543   }
   464    544   #endif /* MEMVFS_TEST */
   465    545   
   466    546     
   467    547   #ifdef _WIN32
   468    548   __declspec(dllexport)
   469    549   #endif
................................................................................
   481    561     mem_vfs.pAppData = sqlite3_vfs_find(0);
   482    562     mem_vfs.szOsFile = sizeof(MemFile);
   483    563     rc = sqlite3_vfs_register(&mem_vfs, 1);
   484    564   #ifdef MEMVFS_TEST
   485    565     if( rc==SQLITE_OK ){
   486    566       rc = sqlite3_auto_extension((void(*)(void))memvfsRegister);
   487    567     }
          568  +  if( rc==SQLITE_OK ){
          569  +    rc = memvfsRegister(db, pzErrMsg, pApi);
          570  +  }
   488    571   #endif
   489    572     if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
   490    573     return rc;
   491    574   }

Changes to ext/misc/rot13.c.

    43     43     sqlite3_context *context,
    44     44     int argc,
    45     45     sqlite3_value **argv
    46     46   ){
    47     47     const unsigned char *zIn;
    48     48     int nIn;
    49     49     unsigned char *zOut;
    50         -  char *zToFree = 0;
           50  +  unsigned char *zToFree = 0;
    51     51     int i;
    52         -  char zTemp[100];
           52  +  unsigned char zTemp[100];
    53     53     assert( argc==1 );
    54     54     if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return;
    55     55     zIn = (const unsigned char*)sqlite3_value_text(argv[0]);
    56     56     nIn = sqlite3_value_bytes(argv[0]);
    57     57     if( nIn<sizeof(zTemp)-1 ){
    58     58       zOut = zTemp;
    59     59     }else{
    60         -    zOut = zToFree = sqlite3_malloc( nIn+1 );
           60  +    zOut = zToFree = (unsigned char*)sqlite3_malloc64( nIn+1 );
    61     61       if( zOut==0 ){
    62     62         sqlite3_result_error_nomem(context);
    63     63         return;
    64     64       }
    65     65     }
    66     66     for(i=0; i<nIn; i++) zOut[i] = rot13(zIn[i]);
    67     67     zOut[i] = 0;

Changes to ext/misc/shathree.c.

    74     74   };
    75     75   
    76     76   /*
    77     77   ** A single step of the Keccak mixing function for a 1600-bit state
    78     78   */
    79     79   static void KeccakF1600Step(SHA3Context *p){
    80     80     int i;
    81         -  u64 B0, B1, B2, B3, B4;
    82         -  u64 C0, C1, C2, C3, C4;
    83         -  u64 D0, D1, D2, D3, D4;
           81  +  u64 b0, b1, b2, b3, b4;
           82  +  u64 c0, c1, c2, c3, c4;
           83  +  u64 d0, d1, d2, d3, d4;
    84     84     static const u64 RC[] = {
    85     85       0x0000000000000001ULL,  0x0000000000008082ULL,
    86     86       0x800000000000808aULL,  0x8000000080008000ULL,
    87     87       0x000000000000808bULL,  0x0000000080000001ULL,
    88     88       0x8000000080008081ULL,  0x8000000000008009ULL,
    89     89       0x000000000000008aULL,  0x0000000000000088ULL,
    90     90       0x0000000080008009ULL,  0x000000008000000aULL,
................................................................................
    91     91       0x000000008000808bULL,  0x800000000000008bULL,
    92     92       0x8000000000008089ULL,  0x8000000000008003ULL,
    93     93       0x8000000000008002ULL,  0x8000000000000080ULL,
    94     94       0x000000000000800aULL,  0x800000008000000aULL,
    95     95       0x8000000080008081ULL,  0x8000000000008080ULL,
    96     96       0x0000000080000001ULL,  0x8000000080008008ULL
    97     97     };
    98         -# define A00 (p->u.s[0])
    99         -# define A01 (p->u.s[1])
   100         -# define A02 (p->u.s[2])
   101         -# define A03 (p->u.s[3])
   102         -# define A04 (p->u.s[4])
   103         -# define A10 (p->u.s[5])
   104         -# define A11 (p->u.s[6])
   105         -# define A12 (p->u.s[7])
   106         -# define A13 (p->u.s[8])
   107         -# define A14 (p->u.s[9])
   108         -# define A20 (p->u.s[10])
   109         -# define A21 (p->u.s[11])
   110         -# define A22 (p->u.s[12])
   111         -# define A23 (p->u.s[13])
   112         -# define A24 (p->u.s[14])
   113         -# define A30 (p->u.s[15])
   114         -# define A31 (p->u.s[16])
   115         -# define A32 (p->u.s[17])
   116         -# define A33 (p->u.s[18])
   117         -# define A34 (p->u.s[19])
   118         -# define A40 (p->u.s[20])
   119         -# define A41 (p->u.s[21])
   120         -# define A42 (p->u.s[22])
   121         -# define A43 (p->u.s[23])
   122         -# define A44 (p->u.s[24])
           98  +# define a00 (p->u.s[0])
           99  +# define a01 (p->u.s[1])
          100  +# define a02 (p->u.s[2])
          101  +# define a03 (p->u.s[3])
          102  +# define a04 (p->u.s[4])
          103  +# define a10 (p->u.s[5])
          104  +# define a11 (p->u.s[6])
          105  +# define a12 (p->u.s[7])
          106  +# define a13 (p->u.s[8])
          107  +# define a14 (p->u.s[9])
          108  +# define a20 (p->u.s[10])
          109  +# define a21 (p->u.s[11])
          110  +# define a22 (p->u.s[12])
          111  +# define a23 (p->u.s[13])
          112  +# define a24 (p->u.s[14])
          113  +# define a30 (p->u.s[15])
          114  +# define a31 (p->u.s[16])
          115  +# define a32 (p->u.s[17])
          116  +# define a33 (p->u.s[18])
          117  +# define a34 (p->u.s[19])
          118  +# define a40 (p->u.s[20])
          119  +# define a41 (p->u.s[21])
          120  +# define a42 (p->u.s[22])
          121  +# define a43 (p->u.s[23])
          122  +# define a44 (p->u.s[24])
   123    123   # define ROL64(a,x) ((a<<x)|(a>>(64-x)))
   124    124   
   125    125     for(i=0; i<24; i+=4){
   126         -    C0 = A00^A10^A20^A30^A40;
   127         -    C1 = A01^A11^A21^A31^A41;
   128         -    C2 = A02^A12^A22^A32^A42;
   129         -    C3 = A03^A13^A23^A33^A43;
   130         -    C4 = A04^A14^A24^A34^A44;
   131         -    D0 = C4^ROL64(C1, 1);
   132         -    D1 = C0^ROL64(C2, 1);
   133         -    D2 = C1^ROL64(C3, 1);
   134         -    D3 = C2^ROL64(C4, 1);
   135         -    D4 = C3^ROL64(C0, 1);
   136         -
   137         -    B0 = (A00^D0);
   138         -    B1 = ROL64((A11^D1), 44);
   139         -    B2 = ROL64((A22^D2), 43);
   140         -    B3 = ROL64((A33^D3), 21);
   141         -    B4 = ROL64((A44^D4), 14);
   142         -    A00 =   B0 ^((~B1)&  B2 );
   143         -    A00 ^= RC[i];
   144         -    A11 =   B1 ^((~B2)&  B3 );
   145         -    A22 =   B2 ^((~B3)&  B4 );
   146         -    A33 =   B3 ^((~B4)&  B0 );
   147         -    A44 =   B4 ^((~B0)&  B1 );
   148         -
   149         -    B2 = ROL64((A20^D0), 3);
   150         -    B3 = ROL64((A31^D1), 45);
   151         -    B4 = ROL64((A42^D2), 61);
   152         -    B0 = ROL64((A03^D3), 28);
   153         -    B1 = ROL64((A14^D4), 20);
   154         -    A20 =   B0 ^((~B1)&  B2 );
   155         -    A31 =   B1 ^((~B2)&  B3 );
   156         -    A42 =   B2 ^((~B3)&  B4 );
   157         -    A03 =   B3 ^((~B4)&  B0 );
   158         -    A14 =   B4 ^((~B0)&  B1 );
   159         -
   160         -    B4 = ROL64((A40^D0), 18);
   161         -    B0 = ROL64((A01^D1), 1);
   162         -    B1 = ROL64((A12^D2), 6);
   163         -    B2 = ROL64((A23^D3), 25);
   164         -    B3 = ROL64((A34^D4), 8);
   165         -    A40 =   B0 ^((~B1)&  B2 );
   166         -    A01 =   B1 ^((~B2)&  B3 );
   167         -    A12 =   B2 ^((~B3)&  B4 );
   168         -    A23 =   B3 ^((~B4)&  B0 );
   169         -    A34 =   B4 ^((~B0)&  B1 );
   170         -
   171         -    B1 = ROL64((A10^D0), 36);
   172         -    B2 = ROL64((A21^D1), 10);
   173         -    B3 = ROL64((A32^D2), 15);
   174         -    B4 = ROL64((A43^D3), 56);
   175         -    B0 = ROL64((A04^D4), 27);
   176         -    A10 =   B0 ^((~B1)&  B2 );
   177         -    A21 =   B1 ^((~B2)&  B3 );
   178         -    A32 =   B2 ^((~B3)&  B4 );
   179         -    A43 =   B3 ^((~B4)&  B0 );
   180         -    A04 =   B4 ^((~B0)&  B1 );
   181         -
   182         -    B3 = ROL64((A30^D0), 41);
   183         -    B4 = ROL64((A41^D1), 2);
   184         -    B0 = ROL64((A02^D2), 62);
   185         -    B1 = ROL64((A13^D3), 55);
   186         -    B2 = ROL64((A24^D4), 39);
   187         -    A30 =   B0 ^((~B1)&  B2 );
   188         -    A41 =   B1 ^((~B2)&  B3 );
   189         -    A02 =   B2 ^((~B3)&  B4 );
   190         -    A13 =   B3 ^((~B4)&  B0 );
   191         -    A24 =   B4 ^((~B0)&  B1 );
          126  +    c0 = a00^a10^a20^a30^a40;
          127  +    c1 = a01^a11^a21^a31^a41;
          128  +    c2 = a02^a12^a22^a32^a42;
          129  +    c3 = a03^a13^a23^a33^a43;
          130  +    c4 = a04^a14^a24^a34^a44;
          131  +    d0 = c4^ROL64(c1, 1);
          132  +    d1 = c0^ROL64(c2, 1);
          133  +    d2 = c1^ROL64(c3, 1);
          134  +    d3 = c2^ROL64(c4, 1);
          135  +    d4 = c3^ROL64(c0, 1);
          136  +
          137  +    b0 = (a00^d0);
          138  +    b1 = ROL64((a11^d1), 44);
          139  +    b2 = ROL64((a22^d2), 43);
          140  +    b3 = ROL64((a33^d3), 21);
          141  +    b4 = ROL64((a44^d4), 14);
          142  +    a00 =   b0 ^((~b1)&  b2 );
          143  +    a00 ^= RC[i];
          144  +    a11 =   b1 ^((~b2)&  b3 );
          145  +    a22 =   b2 ^((~b3)&  b4 );
          146  +    a33 =   b3 ^((~b4)&  b0 );
          147  +    a44 =   b4 ^((~b0)&  b1 );
          148  +
          149  +    b2 = ROL64((a20^d0), 3);
          150  +    b3 = ROL64((a31^d1), 45);
          151  +    b4 = ROL64((a42^d2), 61);
          152  +    b0 = ROL64((a03^d3), 28);
          153  +    b1 = ROL64((a14^d4), 20);
          154  +    a20 =   b0 ^((~b1)&  b2 );
          155  +    a31 =   b1 ^((~b2)&  b3 );
          156  +    a42 =   b2 ^((~b3)&  b4 );
          157  +    a03 =   b3 ^((~b4)&  b0 );
          158  +    a14 =   b4 ^((~b0)&  b1 );
          159  +
          160  +    b4 = ROL64((a40^d0), 18);
          161  +    b0 = ROL64((a01^d1), 1);
          162  +    b1 = ROL64((a12^d2), 6);
          163  +    b2 = ROL64((a23^d3), 25);
          164  +    b3 = ROL64((a34^d4), 8);
          165  +    a40 =   b0 ^((~b1)&  b2 );
          166  +    a01 =   b1 ^((~b2)&  b3 );
          167  +    a12 =   b2 ^((~b3)&  b4 );
          168  +    a23 =   b3 ^((~b4)&  b0 );
          169  +    a34 =   b4 ^((~b0)&  b1 );
          170  +
          171  +    b1 = ROL64((a10^d0), 36);
          172  +    b2 = ROL64((a21^d1), 10);
          173  +    b3 = ROL64((a32^d2), 15);
          174  +    b4 = ROL64((a43^d3), 56);
          175  +    b0 = ROL64((a04^d4), 27);
          176  +    a10 =   b0 ^((~b1)&  b2 );
          177  +    a21 =   b1 ^((~b2)&  b3 );
          178  +    a32 =   b2 ^((~b3)&  b4 );
          179  +    a43 =   b3 ^((~b4)&  b0 );
          180  +    a04 =   b4 ^((~b0)&  b1 );
          181  +
          182  +    b3 = ROL64((a30^d0), 41);
          183  +    b4 = ROL64((a41^d1), 2);
          184  +    b0 = ROL64((a02^d2), 62);
          185  +    b1 = ROL64((a13^d3), 55);
          186  +    b2 = ROL64((a24^d4), 39);
          187  +    a30 =   b0 ^((~b1)&  b2 );
          188  +    a41 =   b1 ^((~b2)&  b3 );
          189  +    a02 =   b2 ^((~b3)&  b4 );
          190  +    a13 =   b3 ^((~b4)&  b0 );
          191  +    a24 =   b4 ^((~b0)&  b1 );
          192  +
          193  +    c0 = a00^a20^a40^a10^a30;
          194  +    c1 = a11^a31^a01^a21^a41;
          195  +    c2 = a22^a42^a12^a32^a02;
          196  +    c3 = a33^a03^a23^a43^a13;
          197  +    c4 = a44^a14^a34^a04^a24;
          198  +    d0 = c4^ROL64(c1, 1);
          199  +    d1 = c0^ROL64(c2, 1);
          200  +    d2 = c1^ROL64(c3, 1);
          201  +    d3 = c2^ROL64(c4, 1);
          202  +    d4 = c3^ROL64(c0, 1);
          203  +
          204  +    b0 = (a00^d0);
          205  +    b1 = ROL64((a31^d1), 44);
          206  +    b2 = ROL64((a12^d2), 43);
          207  +    b3 = ROL64((a43^d3), 21);
          208  +    b4 = ROL64((a24^d4), 14);
          209  +    a00 =   b0 ^((~b1)&  b2 );
          210  +    a00 ^= RC[i+1];
          211  +    a31 =   b1 ^((~b2)&  b3 );
          212  +    a12 =   b2 ^((~b3)&  b4 );
          213  +    a43 =   b3 ^((~b4)&  b0 );
          214  +    a24 =   b4 ^((~b0)&  b1 );
          215  +
          216  +    b2 = ROL64((a40^d0), 3);
          217  +    b3 = ROL64((a21^d1), 45);
          218  +    b4 = ROL64((a02^d2), 61);
          219  +    b0 = ROL64((a33^d3), 28);
          220  +    b1 = ROL64((a14^d4), 20);
          221  +    a40 =   b0 ^((~b1)&  b2 );
          222  +    a21 =   b1 ^((~b2)&  b3 );
          223  +    a02 =   b2 ^((~b3)&  b4 );
          224  +    a33 =   b3 ^((~b4)&  b0 );
          225  +    a14 =   b4 ^((~b0)&  b1 );
          226  +
          227  +    b4 = ROL64((a30^d0), 18);
          228  +    b0 = ROL64((a11^d1), 1);
          229  +    b1 = ROL64((a42^d2), 6);
          230  +    b2 = ROL64((a23^d3), 25);
          231  +    b3 = ROL64((a04^d4), 8);
          232  +    a30 =   b0 ^((~b1)&  b2 );
          233  +    a11 =   b1 ^((~b2)&  b3 );
          234  +    a42 =   b2 ^((~b3)&  b4 );
          235  +    a23 =   b3 ^((~b4)&  b0 );
          236  +    a04 =   b4 ^((~b0)&  b1 );
          237  +
          238  +    b1 = ROL64((a20^d0), 36);
          239  +    b2 = ROL64((a01^d1), 10);
          240  +    b3 = ROL64((a32^d2), 15);
          241  +    b4 = ROL64((a13^d3), 56);
          242  +    b0 = ROL64((a44^d4), 27);
          243  +    a20 =   b0 ^((~b1)&  b2 );
          244  +    a01 =   b1 ^((~b2)&  b3 );
          245  +    a32 =   b2 ^((~b3)&  b4 );
          246  +    a13 =   b3 ^((~b4)&  b0 );
          247  +    a44 =   b4 ^((~b0)&  b1 );
          248  +
          249  +    b3 = ROL64((a10^d0), 41);
          250  +    b4 = ROL64((a41^d1), 2);
          251  +    b0 = ROL64((a22^d2), 62);
          252  +    b1 = ROL64((a03^d3), 55);
          253  +    b2 = ROL64((a34^d4), 39);
          254  +    a10 =   b0 ^((~b1)&  b2 );
          255  +    a41 =   b1 ^((~b2)&  b3 );
          256  +    a22 =   b2 ^((~b3)&  b4 );
          257  +    a03 =   b3 ^((~b4)&  b0 );
          258  +    a34 =   b4 ^((~b0)&  b1 );
          259  +
          260  +    c0 = a00^a40^a30^a20^a10;
          261  +    c1 = a31^a21^a11^a01^a41;
          262  +    c2 = a12^a02^a42^a32^a22;
          263  +    c3 = a43^a33^a23^a13^a03;
          264  +    c4 = a24^a14^a04^a44^a34;
          265  +    d0 = c4^ROL64(c1, 1);
          266  +    d1 = c0^ROL64(c2, 1);
          267  +    d2 = c1^ROL64(c3, 1);
          268  +    d3 = c2^ROL64(c4, 1);
          269  +    d4 = c3^ROL64(c0, 1);
          270  +
          271  +    b0 = (a00^d0);
          272  +    b1 = ROL64((a21^d1), 44);
          273  +    b2 = ROL64((a42^d2), 43);
          274  +    b3 = ROL64((a13^d3), 21);
          275  +    b4 = ROL64((a34^d4), 14);
          276  +    a00 =   b0 ^((~b1)&  b2 );
          277  +    a00 ^= RC[i+2];
          278  +    a21 =   b1 ^((~b2)&  b3 );
          279  +    a42 =   b2 ^((~b3)&  b4 );
          280  +    a13 =   b3 ^((~b4)&  b0 );
          281  +    a34 =   b4 ^((~b0)&  b1 );
          282  +
          283  +    b2 = ROL64((a30^d0), 3);
          284  +    b3 = ROL64((a01^d1), 45);
          285  +    b4 = ROL64((a22^d2), 61);
          286  +    b0 = ROL64((a43^d3), 28);
          287  +    b1 = ROL64((a14^d4), 20);
          288  +    a30 =   b0 ^((~b1)&  b2 );
          289  +    a01 =   b1 ^((~b2)&  b3 );
          290  +    a22 =   b2 ^((~b3)&  b4 );
          291  +    a43 =   b3 ^((~b4)&  b0 );
          292  +    a14 =   b4 ^((~b0)&  b1 );
          293  +
          294  +    b4 = ROL64((a10^d0), 18);
          295  +    b0 = ROL64((a31^d1), 1);
          296  +    b1 = ROL64((a02^d2), 6);
          297  +    b2 = ROL64((a23^d3), 25);
          298  +    b3 = ROL64((a44^d4), 8);
          299  +    a10 =   b0 ^((~b1)&  b2 );
          300  +    a31 =   b1 ^((~b2)&  b3 );
          301  +    a02 =   b2 ^((~b3)&  b4 );
          302  +    a23 =   b3 ^((~b4)&  b0 );
          303  +    a44 =   b4 ^((~b0)&  b1 );
          304  +
          305  +    b1 = ROL64((a40^d0), 36);
          306  +    b2 = ROL64((a11^d1), 10);
          307  +    b3 = ROL64((a32^d2), 15);
          308  +    b4 = ROL64((a03^d3), 56);
          309  +    b0 = ROL64((a24^d4), 27);
          310  +    a40 =   b0 ^((~b1)&  b2 );
          311  +    a11 =   b1 ^((~b2)&  b3 );
          312  +    a32 =   b2 ^((~b3)&  b4 );
          313  +    a03 =   b3 ^((~b4)&  b0 );
          314  +    a24 =   b4 ^((~b0)&  b1 );
          315  +
          316  +    b3 = ROL64((a20^d0), 41);
          317  +    b4 = ROL64((a41^d1), 2);
          318  +    b0 = ROL64((a12^d2), 62);
          319  +    b1 = ROL64((a33^d3), 55);
          320  +    b2 = ROL64((a04^d4), 39);
          321  +    a20 =   b0 ^((~b1)&  b2 );
          322  +    a41 =   b1 ^((~b2)&  b3 );
          323  +    a12 =   b2 ^((~b3)&  b4 );
          324  +    a33 =   b3 ^((~b4)&  b0 );
          325  +    a04 =   b4 ^((~b0)&  b1 );
          326  +
          327  +    c0 = a00^a30^a10^a40^a20;
          328  +    c1 = a21^a01^a31^a11^a41;
          329  +    c2 = a42^a22^a02^a32^a12;
          330  +    c3 = a13^a43^a23^a03^a33;
          331  +    c4 = a34^a14^a44^a24^a04;
          332  +    d0 = c4^ROL64(c1, 1);
          333  +    d1 = c0^ROL64(c2, 1);
          334  +    d2 = c1^ROL64(c3, 1);
          335  +    d3 = c2^ROL64(c4, 1);
          336  +    d4 = c3^ROL64(c0, 1);
          337  +
          338  +    b0 = (a00^d0);
          339  +    b1 = ROL64((a01^d1), 44);
          340  +    b2 = ROL64((a02^d2), 43);
          341  +    b3 = ROL64((a03^d3), 21);
          342  +    b4 = ROL64((a04^d4), 14);
          343  +    a00 =   b0 ^((~b1)&  b2 );
          344  +    a00 ^= RC[i+3];
          345  +    a01 =   b1 ^((~b2)&  b3 );
          346  +    a02 =   b2 ^((~b3)&  b4 );
          347  +    a03 =   b3 ^((~b4)&  b0 );
          348  +    a04 =   b4 ^((~b0)&  b1 );
          349  +
          350  +    b2 = ROL64((a10^d0), 3);
          351  +    b3 = ROL64((a11^d1), 45);
          352  +    b4 = ROL64((a12^d2), 61);
          353  +    b0 = ROL64((a13^d3), 28);
          354  +    b1 = ROL64((a14^d4), 20);
          355  +    a10 =   b0 ^((~b1)&  b2 );
          356  +    a11 =   b1 ^((~b2)&  b3 );
          357  +    a12 =   b2 ^((~b3)&  b4 );
          358  +    a13 =   b3 ^((~b4)&  b0 );
          359  +    a14 =   b4 ^((~b0)&  b1 );
          360  +
          361  +    b4 = ROL64((a20^d0), 18);
          362  +    b0 = ROL64((a21^d1), 1);
          363  +    b1 = ROL64((a22^d2), 6);
          364  +    b2 = ROL64((a23^d3), 25);
          365  +    b3 = ROL64((a24^d4), 8);
          366  +    a20 =   b0 ^((~b1)&  b2 );
          367  +    a21 =   b1 ^((~b2)&  b3 );
          368  +    a22 =   b2 ^((~b3)&  b4 );
          369  +    a23 =   b3 ^((~b4)&  b0 );
          370  +    a24 =   b4 ^((~b0)&  b1 );
   192    371   
   193         -    C0 = A00^A20^A40^A10^A30;
   194         -    C1 = A11^A31^A01^A21^A41;
   195         -    C2 = A22^A42^A12^A32^A02;
   196         -    C3 = A33^A03^A23^A43^A13;
   197         -    C4 = A44^A14^A34^A04^A24;
   198         -    D0 = C4^ROL64(C1, 1);
   199         -    D1 = C0^ROL64(C2, 1);
   200         -    D2 = C1^ROL64(C3, 1);
   201         -    D3 = C2^ROL64(C4, 1);
   202         -    D4 = C3^ROL64(C0, 1);
   203         -
   204         -    B0 = (A00^D0);
   205         -    B1 = ROL64((A31^D1), 44);
   206         -    B2 = ROL64((A12^D2), 43);
   207         -    B3 = ROL64((A43^D3), 21);
   208         -    B4 = ROL64((A24^D4), 14);
   209         -    A00 =   B0 ^((~B1)&  B2 );
   210         -    A00 ^= RC[i+1];
   211         -    A31 =   B1 ^((~B2)&  B3 );
   212         -    A12 =   B2 ^((~B3)&  B4 );
   213         -    A43 =   B3 ^((~B4)&  B0 );
   214         -    A24 =   B4 ^((~B0)&  B1 );
   215         -
   216         -    B2 = ROL64((A40^D0), 3);
   217         -    B3 = ROL64((A21^D1), 45);
   218         -    B4 = ROL64((A02^D2), 61);
   219         -    B0 = ROL64((A33^D3), 28);
   220         -    B1 = ROL64((A14^D4), 20);
   221         -    A40 =   B0 ^((~B1)&  B2 );
   222         -    A21 =   B1 ^((~B2)&  B3 );
   223         -    A02 =   B2 ^((~B3)&  B4 );
   224         -    A33 =   B3 ^((~B4)&  B0 );
   225         -    A14 =   B4 ^((~B0)&  B1 );
   226         -
   227         -    B4 = ROL64((A30^D0), 18);
   228         -    B0 = ROL64((A11^D1), 1);
   229         -    B1 = ROL64((A42^D2), 6);
   230         -    B2 = ROL64((A23^D3), 25);
   231         -    B3 = ROL64((A04^D4), 8);
   232         -    A30 =   B0 ^((~B1)&  B2 );
   233         -    A11 =   B1 ^((~B2)&  B3 );
   234         -    A42 =   B2 ^((~B3)&  B4 );
   235         -    A23 =   B3 ^((~B4)&  B0 );
   236         -    A04 =   B4 ^((~B0)&  B1 );
   237         -
   238         -    B1 = ROL64((A20^D0), 36);
   239         -    B2 = ROL64((A01^D1), 10);
   240         -    B3 = ROL64((A32^D2), 15);
   241         -    B4 = ROL64((A13^D3), 56);
   242         -    B0 = ROL64((A44^D4), 27);
   243         -    A20 =   B0 ^((~B1)&  B2 );
   244         -    A01 =   B1 ^((~B2)&  B3 );
   245         -    A32 =   B2 ^((~B3)&  B4 );
   246         -    A13 =   B3 ^((~B4)&  B0 );
   247         -    A44 =   B4 ^((~B0)&  B1 );
   248         -
   249         -    B3 = ROL64((A10^D0), 41);
   250         -    B4 = ROL64((A41^D1), 2);
   251         -    B0 = ROL64((A22^D2), 62);
   252         -    B1 = ROL64((A03^D3), 55);
   253         -    B2 = ROL64((A34^D4), 39);
   254         -    A10 =   B0 ^((~B1)&  B2 );
   255         -    A41 =   B1 ^((~B2)&  B3 );
   256         -    A22 =   B2 ^((~B3)&  B4 );
   257         -    A03 =   B3 ^((~B4)&  B0 );
   258         -    A34 =   B4 ^((~B0)&  B1 );
          372  +    b1 = ROL64((a30^d0), 36);
          373  +    b2 = ROL64((a31^d1), 10);
          374  +    b3 = ROL64((a32^d2), 15);
          375  +    b4 = ROL64((a33^d3), 56);
          376  +    b0 = ROL64((a34^d4), 27);
          377  +    a30 =   b0 ^((~b1)&  b2 );
          378  +    a31 =   b1 ^((~b2)&  b3 );
          379  +    a32 =   b2 ^((~b3)&  b4 );
          380  +    a33 =   b3 ^((~b4)&  b0 );
          381  +    a34 =   b4 ^((~b0)&  b1 );
   259    382   
   260         -    C0 = A00^A40^A30^A20^A10;
   261         -    C1 = A31^A21^A11^A01^A41;
   262         -    C2 = A12^A02^A42^A32^A22;
   263         -    C3 = A43^A33^A23^A13^A03;
   264         -    C4 = A24^A14^A04^A44^A34;
   265         -    D0 = C4^ROL64(C1, 1);
   266         -    D1 = C0^ROL64(C2, 1);
   267         -    D2 = C1^ROL64(C3, 1);
   268         -    D3 = C2^ROL64(C4, 1);
   269         -    D4 = C3^ROL64(C0, 1);
   270         -
   271         -    B0 = (A00^D0);
   272         -    B1 = ROL64((A21^D1), 44);
   273         -    B2 = ROL64((A42^D2), 43);
   274         -    B3 = ROL64((A13^D3), 21);
   275         -    B4 = ROL64((A34^D4), 14);
   276         -    A00 =   B0 ^((~B1)&  B2 );
   277         -    A00 ^= RC[i+2];
   278         -    A21 =   B1 ^((~B2)&  B3 );
   279         -    A42 =   B2 ^((~B3)&  B4 );
   280         -    A13 =   B3 ^((~B4)&  B0 );
   281         -    A34 =   B4 ^((~B0)&  B1 );
   282         -
   283         -    B2 = ROL64((A30^D0), 3);
   284         -    B3 = ROL64((A01^D1), 45);
   285         -    B4 = ROL64((A22^D2), 61);
   286         -    B0 = ROL64((A43^D3), 28);
   287         -    B1 = ROL64((A14^D4), 20);
   288         -    A30 =   B0 ^((~B1)&  B2 );
   289         -    A01 =   B1 ^((~B2)&  B3 );
   290         -    A22 =   B2 ^((~B3)&  B4 );
   291         -    A43 =   B3 ^((~B4)&  B0 );
   292         -    A14 =   B4 ^((~B0)&  B1 );
   293         -
   294         -    B4 = ROL64((A10^D0), 18);
   295         -    B0 = ROL64((A31^D1), 1);
   296         -    B1 = ROL64((A02^D2), 6);
   297         -    B2 = ROL64((A23^D3), 25);
   298         -    B3 = ROL64((A44^D4), 8);
   299         -    A10 =   B0 ^((~B1)&  B2 );
   300         -    A31 =   B1 ^((~B2)&  B3 );
   301         -    A02 =   B2 ^((~B3)&  B4 );
   302         -    A23 =   B3 ^((~B4)&  B0 );
   303         -    A44 =   B4 ^((~B0)&  B1 );
   304         -
   305         -    B1 = ROL64((A40^D0), 36);
   306         -    B2 = ROL64((A11^D1), 10);
   307         -    B3 = ROL64((A32^D2), 15);
   308         -    B4 = ROL64((A03^D3), 56);
   309         -    B0 = ROL64((A24^D4), 27);
   310         -    A40 =   B0 ^((~B1)&  B2 );
   311         -    A11 =   B1 ^((~B2)&  B3 );
   312         -    A32 =   B2 ^((~B3)&  B4 );
   313         -    A03 =   B3 ^((~B4)&  B0 );
   314         -    A24 =   B4 ^((~B0)&  B1 );
   315         -
   316         -    B3 = ROL64((A20^D0), 41);
   317         -    B4 = ROL64((A41^D1), 2);
   318         -    B0 = ROL64((A12^D2), 62);
   319         -    B1 = ROL64((A33^D3), 55);
   320         -    B2 = ROL64((A04^D4), 39);
   321         -    A20 =   B0 ^((~B1)&  B2 );
   322         -    A41 =   B1 ^((~B2)&  B3 );
   323         -    A12 =   B2 ^((~B3)&  B4 );
   324         -    A33 =   B3 ^((~B4)&  B0 );
   325         -    A04 =   B4 ^((~B0)&  B1 );
   326         -
   327         -    C0 = A00^A30^A10^A40^A20;
   328         -    C1 = A21^A01^A31^A11^A41;
   329         -    C2 = A42^A22^A02^A32^A12;
   330         -    C3 = A13^A43^A23^A03^A33;
   331         -    C4 = A34^A14^A44^A24^A04;
   332         -    D0 = C4^ROL64(C1, 1);
   333         -    D1 = C0^ROL64(C2, 1);
   334         -    D2 = C1^ROL64(C3, 1);
   335         -    D3 = C2^ROL64(C4, 1);
   336         -    D4 = C3^ROL64(C0, 1);
   337         -
   338         -    B0 = (A00^D0);
   339         -    B1 = ROL64((A01^D1), 44);
   340         -    B2 = ROL64((A02^D2), 43);
   341         -    B3 = ROL64((A03^D3), 21);
   342         -    B4 = ROL64((A04^D4), 14);
   343         -    A00 =   B0 ^((~B1)&  B2 );
   344         -    A00 ^= RC[i+3];
   345         -    A01 =   B1 ^((~B2)&  B3 );
   346         -    A02 =   B2 ^((~B3)&  B4 );
   347         -    A03 =   B3 ^((~B4)&  B0 );
   348         -    A04 =   B4 ^((~B0)&  B1 );
   349         -
   350         -    B2 = ROL64((A10^D0), 3);
   351         -    B3 = ROL64((A11^D1), 45);
   352         -    B4 = ROL64((A12^D2), 61);
   353         -    B0 = ROL64((A13^D3), 28);
   354         -    B1 = ROL64((A14^D4), 20);
   355         -    A10 =   B0 ^((~B1)&  B2 );
   356         -    A11 =   B1 ^((~B2)&  B3 );
   357         -    A12 =   B2 ^((~B3)&  B4 );
   358         -    A13 =   B3 ^((~B4)&  B0 );
   359         -    A14 =   B4 ^((~B0)&  B1 );
   360         -
   361         -    B4 = ROL64((A20^D0), 18);
   362         -    B0 = ROL64((A21^D1), 1);
   363         -    B1 = ROL64((A22^D2), 6);
   364         -    B2 = ROL64((A23^D3), 25);
   365         -    B3 = ROL64((A24^D4), 8);
   366         -    A20 =   B0 ^((~B1)&  B2 );
   367         -    A21 =   B1 ^((~B2)&  B3 );
   368         -    A22 =   B2 ^((~B3)&  B4 );
   369         -    A23 =   B3 ^((~B4)&  B0 );
   370         -    A24 =   B4 ^((~B0)&  B1 );
   371         -
   372         -    B1 = ROL64((A30^D0), 36);
   373         -    B2 = ROL64((A31^D1), 10);
   374         -    B3 = ROL64((A32^D2), 15);
   375         -    B4 = ROL64((A33^D3), 56);
   376         -    B0 = ROL64((A34^D4), 27);
   377         -    A30 =   B0 ^((~B1)&  B2 );
   378         -    A31 =   B1 ^((~B2)&  B3 );
   379         -    A32 =   B2 ^((~B3)&  B4 );
   380         -    A33 =   B3 ^((~B4)&  B0 );
   381         -    A34 =   B4 ^((~B0)&  B1 );
   382         -
   383         -    B3 = ROL64((A40^D0), 41);
   384         -    B4 = ROL64((A41^D1), 2);
   385         -    B0 = ROL64((A42^D2), 62);
   386         -    B1 = ROL64((A43^D3), 55);
   387         -    B2 = ROL64((A44^D4), 39);
   388         -    A40 =   B0 ^((~B1)&  B2 );
   389         -    A41 =   B1 ^((~B2)&  B3 );
   390         -    A42 =   B2 ^((~B3)&  B4 );
   391         -    A43 =   B3 ^((~B4)&  B0 );
   392         -    A44 =   B4 ^((~B0)&  B1 );
          383  +    b3 = ROL64((a40^d0), 41);
          384  +    b4 = ROL64((a41^d1), 2);
          385  +    b0 = ROL64((a42^d2), 62);
          386  +    b1 = ROL64((a43^d3), 55);
          387  +    b2 = ROL64((a44^d4), 39);
          388  +    a40 =   b0 ^((~b1)&  b2 );
          389  +    a41 =   b1 ^((~b2)&  b3 );
          390  +    a42 =   b2 ^((~b3)&  b4 );
          391  +    a43 =   b3 ^((~b4)&  b0 );
          392  +    a44 =   b4 ^((~b0)&  b1 );
   393    393     }
   394    394   }
   395    395   
   396    396   /*
   397    397   ** Initialize a new hash.  iSize determines the size of the hash
   398    398   ** in bits and should be one of 224, 256, 384, or 512.  Or iSize
   399    399   ** can be zero to use the default hash size of 256 bits.

Added ext/misc/sqlar.c.

            1  +/*
            2  +** 2017-12-17
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** Utility functions sqlar_compress() and sqlar_uncompress(). Useful
           14  +** for working with sqlar archives and used by the shell tool's built-in
           15  +** sqlar support.
           16  +*/
           17  +#include "sqlite3ext.h"
           18  +SQLITE_EXTENSION_INIT1
           19  +#include <zlib.h>
           20  +
           21  +/*
           22  +** Implementation of the "sqlar_compress(X)" SQL function.
           23  +**
           24  +** If the type of X is SQLITE_BLOB, and compressing that blob using
           25  +** zlib utility function compress() yields a smaller blob, return the
           26  +** compressed blob. Otherwise, return a copy of X.
           27  +**
           28  +** SQLar uses the "zlib format" for compressed content.  The zlib format
           29  +** contains a two-byte identification header and a four-byte checksum at
           30  +** the end.  This is different from ZIP which uses the raw deflate format.
           31  +**
           32  +** Future enhancements to SQLar might add support for new compression formats.
           33  +** If so, those new formats will be identified by alternative headers in the
           34  +** compressed data.
           35  +*/
           36  +static void sqlarCompressFunc(
           37  +  sqlite3_context *context,
           38  +  int argc,
           39  +  sqlite3_value **argv
           40  +){
           41  +  assert( argc==1 );
           42  +  if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
           43  +    const Bytef *pData = sqlite3_value_blob(argv[0]);
           44  +    uLong nData = sqlite3_value_bytes(argv[0]);
           45  +    uLongf nOut = compressBound(nData);
           46  +    Bytef *pOut;
           47  +
           48  +    pOut = (Bytef*)sqlite3_malloc(nOut);
           49  +    if( pOut==0 ){
           50  +      sqlite3_result_error_nomem(context);
           51  +      return;
           52  +    }else{
           53  +      if( Z_OK!=compress(pOut, &nOut, pData, nData) ){
           54  +        sqlite3_result_error(context, "error in compress()", -1);
           55  +      }else if( nOut<nData ){
           56  +        sqlite3_result_blob(context, pOut, nOut, SQLITE_TRANSIENT);
           57  +      }else{
           58  +        sqlite3_result_value(context, argv[0]);
           59  +      }
           60  +      sqlite3_free(pOut);
           61  +    }
           62  +  }else{
           63  +    sqlite3_result_value(context, argv[0]);
           64  +  }
           65  +}
           66  +
           67  +/*
           68  +** Implementation of the "sqlar_uncompress(X,SZ)" SQL function
           69  +**
           70  +** Parameter SZ is interpreted as an integer. If it is less than or
           71  +** equal to zero, then this function returns a copy of X. Or, if
           72  +** SZ is equal to the size of X when interpreted as a blob, also
           73  +** return a copy of X. Otherwise, decompress blob X using zlib
           74  +** utility function uncompress() and return the results (another
           75  +** blob).
           76  +*/
           77  +static void sqlarUncompressFunc(
           78  +  sqlite3_context *context,
           79  +  int argc,
           80  +  sqlite3_value **argv
           81  +){
           82  +  uLong nData;
           83  +  uLongf sz;
           84  +
           85  +  assert( argc==2 );
           86  +  sz = sqlite3_value_int(argv[1]);
           87  +
           88  +  if( sz<=0 || sz==(nData = sqlite3_value_bytes(argv[0])) ){
           89  +    sqlite3_result_value(context, argv[0]);
           90  +  }else{
           91  +    const Bytef *pData= sqlite3_value_blob(argv[0]);
           92  +    Bytef *pOut = sqlite3_malloc(sz);
           93  +    if( Z_OK!=uncompress(pOut, &sz, pData, nData) ){
           94  +      sqlite3_result_error(context, "error in uncompress()", -1);
           95  +    }else{
           96  +      sqlite3_result_blob(context, pOut, sz, SQLITE_TRANSIENT);
           97  +    }
           98  +    sqlite3_free(pOut);
           99  +  }
          100  +}
          101  +
          102  +
          103  +#ifdef _WIN32
          104  +__declspec(dllexport)
          105  +#endif
          106  +int sqlite3_sqlar_init(
          107  +  sqlite3 *db, 
          108  +  char **pzErrMsg, 
          109  +  const sqlite3_api_routines *pApi
          110  +){
          111  +  int rc = SQLITE_OK;
          112  +  SQLITE_EXTENSION_INIT2(pApi);
          113  +  (void)pzErrMsg;  /* Unused parameter */
          114  +  rc = sqlite3_create_function(db, "sqlar_compress", 1, SQLITE_UTF8, 0,
          115  +                               sqlarCompressFunc, 0, 0);
          116  +  if( rc==SQLITE_OK ){
          117  +    rc = sqlite3_create_function(db, "sqlar_uncompress", 2, SQLITE_UTF8, 0,
          118  +                                 sqlarUncompressFunc, 0, 0);
          119  +  }
          120  +  return rc;
          121  +}

Changes to ext/misc/unionvtab.c.

    51     51   **     3. The smallest rowid in the range of rowids that may be stored in the
    52     52   **        database table (an integer).
    53     53   **
    54     54   **     4. The largest rowid in the range of rowids that may be stored in the
    55     55   **        database table (an integer).
    56     56   **
    57     57   ** SWARMVTAB
           58  +**
           59  +**  LEGACY SYNTAX:
    58     60   **
    59     61   **   A "swarmvtab" virtual table is created similarly to a unionvtab table:
    60     62   **
    61     63   **     CREATE VIRTUAL TABLE <name>
    62     64   **      USING swarmvtab(<sql-statement>, <callback>);
    63     65   **
    64     66   **   The difference is that for a swarmvtab table, the first column returned
    65     67   **   by the <sql statement> must return a path or URI that can be used to open
    66     68   **   the database file containing the source table.  The <callback> option
    67     69   **   is optional.  If included, it is the name of an application-defined
    68     70   **   SQL function that is invoked with the URI of the file, if the file
    69         -**   does not already exist on disk.
           71  +**   does not already exist on disk when required by swarmvtab.
           72  +**
           73  +**  NEW SYNTAX:
           74  +**
           75  +**   Using the new syntax, a swarmvtab table is created with:
           76  +**
           77  +**      CREATE VIRTUAL TABLE <name> USING swarmvtab(
           78  +**        <sql-statement> [, <options>]
           79  +**      );
           80  +**
           81  +**   where valid <options> are:
           82  +**
           83  +**      missing=<udf-function-name>
           84  +**      openclose=<udf-function-name>
           85  +**      maxopen=<integer>
           86  +**      <sql-parameter>=<text-value>
           87  +**
           88  +**   The <sql-statement> must return the same 4 columns as for a swarmvtab
           89  +**   table in legacy mode. However, it may also return a 5th column - the
           90  +**   "context" column. The text value returned in this column is not used
           91  +**   at all by the swarmvtab implementation, except that it is passed as
           92  +**   an additional argument to the two UDF functions that may be invoked
           93  +**   (see below).
           94  +**
           95  +**   The "missing" option, if present, specifies the name of an SQL UDF
           96  +**   function to be invoked if a database file is not already present on
           97  +**   disk when required by swarmvtab. If the <sql-statement> did not provide
           98  +**   a context column, it is invoked as:
           99  +**
          100  +**     SELECT <missing-udf>(<database filename/uri>);
          101  +**
          102  +**   Or, if there was a context column:
          103  +**
          104  +**     SELECT <missing-udf>(<database filename/uri>, <context>);
          105  +**
          106  +**   The "openclose" option may also specify a UDF function. This function
          107  +**   is invoked right before swarmvtab opens a database, and right after
          108  +**   it closes one. The first argument - or first two arguments, if
          109  +**   <sql-statement> supplied the context column - is the same as for
          110  +**   the "missing" UDF. Following this, the UDF is passed integer value
          111  +**   0 before a db is opened, and 1 right after it is closed. If both
          112  +**   a missing and openclose UDF is supplied, the application should expect
          113  +**   the following sequence of calls (for a single database):
          114  +**
          115  +**      SELECT <openclose-udf>(<db filename>, <context>, 0);
          116  +**      if( db not already on disk ){
          117  +**          SELECT <missing-udf>(<db filename>, <context>);
          118  +**      }
          119  +**      ... swarmvtab uses database ...
          120  +**      SELECT <openclose-udf>(<db filename>, <context>, 1);
          121  +**
          122  +**   The "maxopen" option is used to configure the maximum number of
          123  +**   database files swarmvtab will hold open simultaneously (default 9).
          124  +**
          125  +**   If an option name begins with a ":" character, then it is assumed
          126  +**   to be an SQL parameter. In this case, the specified text value is
          127  +**   bound to the same variable of the <sql-statement> before it is 
          128  +**   executed. It is an error of the named SQL parameter does not exist.
          129  +**   For example:
          130  +**
          131  +**     CREATE VIRTUAL TABLE swarm USING swarmvtab(
          132  +**       'SELECT :path || localfile, tbl, min, max FROM swarmdir',
          133  +**       :path='/home/user/databases/'
          134  +**       missing='missing_func'
          135  +**     );
    70    136   */
    71    137   
    72    138   #include "sqlite3ext.h"
    73    139   SQLITE_EXTENSION_INIT1
    74    140   #include <assert.h>
    75    141   #include <string.h>
          142  +#include <stdlib.h>
    76    143   
    77    144   #ifndef SQLITE_OMIT_VIRTUALTABLE
    78    145   
    79    146   /*
    80    147   ** Largest and smallest possible 64-bit signed integers. These macros
    81    148   ** copied from sqliteInt.h.
    82    149   */
................................................................................
   124    191     char *zDb;                      /* Database containing source table */
   125    192     char *zTab;                     /* Source table name */
   126    193     sqlite3_int64 iMin;             /* Minimum rowid */
   127    194     sqlite3_int64 iMax;             /* Maximum rowid */
   128    195   
   129    196     /* Fields used by swarmvtab only */
   130    197     char *zFile;                    /* Database file containing table zTab */
          198  +  char *zContext;                 /* Context string, if any */
   131    199     int nUser;                      /* Current number of users */
   132    200     sqlite3 *db;                    /* Database handle */
   133    201     UnionSrc *pNextClosable;        /* Next in list of closable sources */
   134    202   };
   135    203   
   136    204   /*
   137    205   ** Virtual table  type for union vtab.
................................................................................
   141    209     sqlite3 *db;                    /* Database handle */
   142    210     int bSwarm;                     /* 1 for "swarmvtab", 0 for "unionvtab" */
   143    211     int iPK;                        /* INTEGER PRIMARY KEY column, or -1 */
   144    212     int nSrc;                       /* Number of elements in the aSrc[] array */
   145    213     UnionSrc *aSrc;                 /* Array of source tables, sorted by rowid */
   146    214   
   147    215     /* Used by swarmvtab only */
          216  +  int bHasContext;                /* Has context strings */
   148    217     char *zSourceStr;               /* Expected unionSourceToStr() value */
   149         -  char *zNotFoundCallback;        /* UDF to invoke if file not found on open */
          218  +  sqlite3_stmt *pNotFound;        /* UDF to invoke if file not found on open */
          219  +  sqlite3_stmt *pOpenClose;       /* UDF to invoke on open and close */
          220  +
   150    221     UnionSrc *pClosable;            /* First in list of closable sources */
   151    222     int nOpen;                      /* Current number of open sources */
   152    223     int nMaxOpen;                   /* Maximum number of open sources */
   153    224   };
   154    225   
   155    226   /*
   156    227   ** Virtual table cursor type for union vtab.
................................................................................
   346    417     if( *pRc==SQLITE_OK ){
   347    418       *pRc = rc;
   348    419       if( rc ){
   349    420         *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
   350    421       }
   351    422     }
   352    423   }
          424  +
          425  +/*
          426  +** If an "openclose" UDF was supplied when this virtual table was created,
          427  +** invoke it now. The first argument passed is the name of the database
          428  +** file for source pSrc. The second is integer value bClose.
          429  +**
          430  +** If successful, return SQLITE_OK. Otherwise an SQLite error code. In this
          431  +** case if argument pzErr is not NULL, also set (*pzErr) to an English
          432  +** language error message. The caller is responsible for eventually freeing 
          433  +** any error message using sqlite3_free().
          434  +*/
          435  +static int unionInvokeOpenClose(
          436  +  UnionTab *pTab, 
          437  +  UnionSrc *pSrc, 
          438  +  int bClose,
          439  +  char **pzErr
          440  +){
          441  +  int rc = SQLITE_OK;
          442  +  if( pTab->pOpenClose ){
          443  +    sqlite3_bind_text(pTab->pOpenClose, 1, pSrc->zFile, -1, SQLITE_STATIC);
          444  +    if( pTab->bHasContext ){
          445  +      sqlite3_bind_text(pTab->pOpenClose, 2, pSrc->zContext, -1, SQLITE_STATIC);
          446  +    }
          447  +    sqlite3_bind_int(pTab->pOpenClose, 2+pTab->bHasContext, bClose);
          448  +    sqlite3_step(pTab->pOpenClose);
          449  +    if( SQLITE_OK!=(rc = sqlite3_reset(pTab->pOpenClose)) ){
          450  +      if( pzErr ){
          451  +        *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
          452  +      }
          453  +    }
          454  +  }
          455  +  return rc;
          456  +}
   353    457   
   354    458   /*
   355    459   ** This function is a no-op for unionvtab. For swarmvtab, it attempts to
   356    460   ** close open database files until at most nMax are open. An SQLite error
   357    461   ** code is returned if an error occurs, or SQLITE_OK otherwise.
   358    462   */
   359    463   static void unionCloseSources(UnionTab *pTab, int nMax){
   360    464     while( pTab->pClosable && pTab->nOpen>nMax ){
          465  +    UnionSrc *p;
   361    466       UnionSrc **pp;
   362    467       for(pp=&pTab->pClosable; (*pp)->pNextClosable; pp=&(*pp)->pNextClosable);
   363         -    assert( (*pp)->db );
   364         -    sqlite3_close((*pp)->db);
   365         -    (*pp)->db = 0;
          468  +    p = *pp;
          469  +    assert( p->db );
          470  +    sqlite3_close(p->db);
          471  +    p->db = 0;
   366    472       *pp = 0;
   367    473       pTab->nOpen--;
          474  +    unionInvokeOpenClose(pTab, p, 1, 0);
   368    475     }
   369    476   }
   370    477   
   371    478   /*
   372    479   ** xDisconnect method.
   373    480   */
   374    481   static int unionDisconnect(sqlite3_vtab *pVtab){
   375    482     if( pVtab ){
   376    483       UnionTab *pTab = (UnionTab*)pVtab;
   377    484       int i;
   378    485       for(i=0; i<pTab->nSrc; i++){
   379    486         UnionSrc *pSrc = &pTab->aSrc[i];
          487  +      int bHaveSrcDb = (pSrc->db!=0);
          488  +      sqlite3_close(pSrc->db);
          489  +      if( bHaveSrcDb ){
          490  +        unionInvokeOpenClose(pTab, pSrc, 1, 0);
          491  +      }
   380    492         sqlite3_free(pSrc->zDb);
   381    493         sqlite3_free(pSrc->zTab);
   382    494         sqlite3_free(pSrc->zFile);
   383         -      sqlite3_close(pSrc->db);
          495  +      sqlite3_free(pSrc->zContext);
   384    496       }
          497  +    sqlite3_finalize(pTab->pNotFound);
          498  +    sqlite3_finalize(pTab->pOpenClose);
   385    499       sqlite3_free(pTab->zSourceStr);
   386         -    sqlite3_free(pTab->zNotFoundCallback);
   387    500       sqlite3_free(pTab->aSrc);
   388    501       sqlite3_free(pTab);
   389    502     }
   390    503     return SQLITE_OK;
   391    504   }
   392    505   
   393    506   /*
................................................................................
   492    605       sqlite3_free(z);
   493    606     }
   494    607     sqlite3_free(z0);
   495    608   
   496    609     return rc;
   497    610   }
   498    611   
   499         -
   500    612   /*
   501    613   ** Try to open the swarmvtab database.  If initially unable, invoke the
   502    614   ** not-found callback UDF and then try again.
   503    615   */
   504    616   static int unionOpenDatabaseInner(UnionTab *pTab, UnionSrc *pSrc, char **pzErr){
   505         -  int rc = SQLITE_OK;
   506         -  static const int openFlags = 
   507         -       SQLITE_OPEN_READONLY | SQLITE_OPEN_URI;
          617  +  static const int openFlags = SQLITE_OPEN_READONLY | SQLITE_OPEN_URI;
          618  +  int rc;
          619  +
          620  +  rc = unionInvokeOpenClose(pTab, pSrc, 0, pzErr);
          621  +  if( rc!=SQLITE_OK ) return rc;
          622  +
   508    623     rc = sqlite3_open_v2(pSrc->zFile, &pSrc->db, openFlags, 0);
   509    624     if( rc==SQLITE_OK ) return rc;
   510         -  if( pTab->zNotFoundCallback ){
   511         -    char *zSql = sqlite3_mprintf("SELECT \"%w\"(%Q);",
   512         -                    pTab->zNotFoundCallback, pSrc->zFile);
          625  +  if( pTab->pNotFound ){
   513    626       sqlite3_close(pSrc->db);
   514    627       pSrc->db = 0;
   515         -    if( zSql==0 ){
   516         -      *pzErr = sqlite3_mprintf("out of memory");
   517         -      return SQLITE_NOMEM;
          628  +    sqlite3_bind_text(pTab->pNotFound, 1, pSrc->zFile, -1, SQLITE_STATIC);
          629  +    if( pTab->bHasContext ){
          630  +      sqlite3_bind_text(pTab->pNotFound, 2, pSrc->zContext, -1, SQLITE_STATIC);
   518    631       }
   519         -    rc = sqlite3_exec(pTab->db, zSql, 0, 0, pzErr);
   520         -    sqlite3_free(zSql);
   521         -    if( rc ) return rc;
          632  +    sqlite3_step(pTab->pNotFound);
          633  +    if( SQLITE_OK!=(rc = sqlite3_reset(pTab->pNotFound)) ){
          634  +      *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
          635  +      return rc;
          636  +    }
   522    637       rc = sqlite3_open_v2(pSrc->zFile, &pSrc->db, openFlags, 0);
   523    638     }
   524    639     if( rc!=SQLITE_OK ){
   525    640       *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pSrc->db));
   526    641     }
   527    642     return rc;
   528    643   }
................................................................................
   568    683       if( rc==SQLITE_OK ){
   569    684         pSrc->pNextClosable = pTab->pClosable;
   570    685         pTab->pClosable = pSrc;
   571    686         pTab->nOpen++;
   572    687       }else{
   573    688         sqlite3_close(pSrc->db);
   574    689         pSrc->db = 0;
          690  +      unionInvokeOpenClose(pTab, pSrc, 1, 0);
   575    691       }
   576    692     }
   577    693   
   578    694     return rc;
   579    695   }
   580    696   
   581    697   
................................................................................
   622    738           pTab->pClosable = pSrc;
   623    739         }
   624    740         unionCloseSources(pTab, pTab->nMaxOpen);
   625    741       }
   626    742     }
   627    743     return rc;
   628    744   }
          745  +
          746  +/* 
          747  +** Return true if the argument is a space, tab, CR or LF character.
          748  +*/
          749  +static int union_isspace(char c){
          750  +  return (c==' ' || c=='\n' || c=='\r' || c=='\t');
          751  +}
          752  +
          753  +/* 
          754  +** Return true if the argument is an alphanumeric character in the 
          755  +** ASCII range.
          756  +*/
          757  +static int union_isidchar(char c){
          758  +  return ((c>='a' && c<='z') || (c>='A' && c<'Z') || (c>='0' && c<='9'));
          759  +}
          760  +
          761  +/*
          762  +** This function is called to handle all arguments following the first 
          763  +** (the SQL statement) passed to a swarmvtab (not unionvtab) CREATE 
          764  +** VIRTUAL TABLE statement. It may bind parameters to the SQL statement 
          765  +** or configure members of the UnionTab object passed as the second
          766  +** argument.
          767  +**
          768  +** Refer to header comments at the top of this file for a description
          769  +** of the arguments parsed.
          770  +**
          771  +** This function is a no-op if *pRc is other than SQLITE_OK when it is
          772  +** called. Otherwise, if an error occurs, *pRc is set to an SQLite error
          773  +** code. In this case *pzErr may be set to point to a buffer containing
          774  +** an English language error message. It is the responsibility of the 
          775  +** caller to eventually free the buffer using sqlite3_free().
          776  +*/
          777  +static void unionConfigureVtab(
          778  +  int *pRc,                       /* IN/OUT: Error code */
          779  +  UnionTab *pTab,                 /* Table to configure */
          780  +  sqlite3_stmt *pStmt,            /* SQL statement to find sources */
          781  +  int nArg,                       /* Number of entries in azArg[] array */
          782  +  const char * const *azArg,      /* Array of arguments to consider */
          783  +  char **pzErr                    /* OUT: Error message */
          784  +){
          785  +  int rc = *pRc;
          786  +  int i;
          787  +  if( rc==SQLITE_OK ){
          788  +    pTab->bHasContext = (sqlite3_column_count(pStmt)>4);
          789  +  }
          790  +  for(i=0; rc==SQLITE_OK && i<nArg; i++){
          791  +    char *zArg = unionStrdup(&rc, azArg[i]);
          792  +    if( zArg ){
          793  +      int nOpt = 0;               /* Size of option name in bytes */
          794  +      char *zOpt;                 /* Pointer to option name */
          795  +      char *zVal;                 /* Pointer to value */
          796  +
          797  +      unionDequote(zArg);
          798  +      zOpt = zArg;
          799  +      while( union_isspace(*zOpt) ) zOpt++;
          800  +      zVal = zOpt;
          801  +      if( *zVal==':' ) zVal++;
          802  +      while( union_isidchar(*zVal) ) zVal++;
          803  +      nOpt = (int)(zVal-zOpt);
          804  +
          805  +      while( union_isspace(*zVal) ) zVal++;
          806  +      if( *zVal=='=' ){
          807  +        zOpt[nOpt] = '\0';
          808  +        zVal++;
          809  +        while( union_isspace(*zVal) ) zVal++;
          810  +        zVal = unionStrdup(&rc, zVal);
          811  +        if( zVal ){
          812  +          unionDequote(zVal);
          813  +          if( zOpt[0]==':' ){
          814  +            /* A value to bind to the SQL statement */
          815  +            int iParam = sqlite3_bind_parameter_index(pStmt, zOpt);
          816  +            if( iParam==0 ){
          817  +              *pzErr = sqlite3_mprintf(
          818  +                  "swarmvtab: no such SQL parameter: %s", zOpt
          819  +              );
          820  +              rc = SQLITE_ERROR;
          821  +            }else{
          822  +              rc = sqlite3_bind_text(pStmt, iParam, zVal, -1, SQLITE_TRANSIENT);
          823  +            }
          824  +          }else if( nOpt==7 && 0==sqlite3_strnicmp(zOpt, "maxopen", 7) ){
          825  +            pTab->nMaxOpen = atoi(zVal);
          826  +            if( pTab->nMaxOpen<=0 ){
          827  +              *pzErr = sqlite3_mprintf("swarmvtab: illegal maxopen value");
          828  +              rc = SQLITE_ERROR;
          829  +            }
          830  +          }else if( nOpt==7 && 0==sqlite3_strnicmp(zOpt, "missing", 7) ){
          831  +            if( pTab->pNotFound ){
          832  +              *pzErr = sqlite3_mprintf(
          833  +                  "swarmvtab: duplicate \"missing\" option");
          834  +              rc = SQLITE_ERROR;
          835  +            }else{
          836  +              pTab->pNotFound = unionPreparePrintf(&rc, pzErr, pTab->db,
          837  +                  "SELECT \"%w\"(?%s)", zVal, pTab->bHasContext ? ",?" : ""
          838  +              );
          839  +            }
          840  +          }else if( nOpt==9 && 0==sqlite3_strnicmp(zOpt, "openclose", 9) ){
          841  +            if( pTab->pOpenClose ){
          842  +              *pzErr = sqlite3_mprintf(
          843  +                  "swarmvtab: duplicate \"openclose\" option");
          844  +              rc = SQLITE_ERROR;
          845  +            }else{
          846  +              pTab->pOpenClose = unionPreparePrintf(&rc, pzErr, pTab->db,
          847  +                  "SELECT \"%w\"(?,?%s)", zVal, pTab->bHasContext ? ",?" : ""
          848  +              );
          849  +            }
          850  +          }else{
          851  +            *pzErr = sqlite3_mprintf("swarmvtab: unrecognized option: %s",zOpt);
          852  +            rc = SQLITE_ERROR;
          853  +          }
          854  +          sqlite3_free(zVal);
          855  +        }
          856  +      }else{
          857  +        if( i==0 && nArg==1 ){
          858  +          pTab->pNotFound = unionPreparePrintf(&rc, pzErr, pTab->db,
          859  +              "SELECT \"%w\"(?)", zArg
          860  +          );
          861  +        }else{
          862  +          *pzErr = sqlite3_mprintf( "swarmvtab: parse error: %s", azArg[i]);
          863  +          rc = SQLITE_ERROR;
          864  +        }
          865  +      }
          866  +      sqlite3_free(zArg);
          867  +    }
          868  +  }
          869  +  *pRc = rc;
          870  +}
   629    871   
   630    872   /* 
   631    873   ** xConnect/xCreate method.
   632    874   **
   633    875   ** The argv[] array contains the following:
   634    876   **
   635    877   **   argv[0]   -> module name  ("unionvtab" or "swarmvtab")
................................................................................
   650    892     int bSwarm = (pAux==0 ? 0 : 1);
   651    893     const char *zVtab = (bSwarm ? "swarmvtab" : "unionvtab");
   652    894   
   653    895     if( sqlite3_stricmp("temp", argv[1]) ){
   654    896       /* unionvtab tables may only be created in the temp schema */
   655    897       *pzErr = sqlite3_mprintf("%s tables must be created in TEMP schema", zVtab);
   656    898       rc = SQLITE_ERROR;
   657         -  }else if( argc!=4 && argc!=5 ){
          899  +  }else if( argc<4 || (argc>4 && bSwarm==0) ){
   658    900       *pzErr = sqlite3_mprintf("wrong number of arguments for %s", zVtab);
   659    901       rc = SQLITE_ERROR;
   660    902     }else{
   661    903       int nAlloc = 0;               /* Allocated size of pTab->aSrc[] */
   662    904       sqlite3_stmt *pStmt = 0;      /* Argument statement */
   663    905       char *zArg = unionStrdup(&rc, argv[3]);      /* Copy of argument to CVT */
   664    906   
................................................................................
   669    911       unionDequote(zArg);
   670    912       pStmt = unionPreparePrintf(&rc, pzErr, db, 
   671    913           "SELECT * FROM (%z) ORDER BY 3", zArg
   672    914       );
   673    915   
   674    916       /* Allocate the UnionTab structure */
   675    917       pTab = unionMalloc(&rc, sizeof(UnionTab));
          918  +    if( pTab ){
          919  +      assert( rc==SQLITE_OK );
          920  +      pTab->db = db;
          921  +      pTab->bSwarm = bSwarm;
          922  +      pTab->nMaxOpen = SWARMVTAB_MAX_OPEN;
          923  +    }
          924  +
          925  +    /* Parse other CVT arguments, if any */
          926  +    if( bSwarm ){
          927  +      unionConfigureVtab(&rc, pTab, pStmt, argc-4, &argv[4], pzErr);
          928  +    }
   676    929   
   677    930       /* Iterate through the rows returned by the SQL statement specified
   678    931       ** as an argument to the CREATE VIRTUAL TABLE statement. */
   679    932       while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
   680    933         const char *zDb = (const char*)sqlite3_column_text(pStmt, 0);
   681    934         const char *zTab = (const char*)sqlite3_column_text(pStmt, 1);
   682    935         sqlite3_int64 iMin = sqlite3_column_int64(pStmt, 2);
................................................................................
   711    964           pSrc->iMin = iMin;
   712    965           pSrc->iMax = iMax;
   713    966           if( bSwarm ){
   714    967             pSrc->zFile = unionStrdup(&rc, zDb);
   715    968           }else{
   716    969             pSrc->zDb = unionStrdup(&rc, zDb);
   717    970           }
          971  +        if( pTab->bHasContext ){
          972  +          const char *zContext = (const char*)sqlite3_column_text(pStmt, 4);
          973  +          pSrc->zContext = unionStrdup(&rc, zContext);
          974  +        }
   718    975         }
   719    976       }
   720    977       unionFinalize(&rc, pStmt, pzErr);
   721    978       pStmt = 0;
   722    979   
   723         -    /* Capture the not-found callback UDF name */
   724         -    if( rc==SQLITE_OK && argc>=5 ){
   725         -      pTab->zNotFoundCallback = unionStrdup(&rc, argv[4]);
   726         -      unionDequote(pTab->zNotFoundCallback);
   727         -    }
   728         -
   729    980       /* It is an error if the SELECT statement returned zero rows. If only
   730    981       ** because there is no way to determine the schema of the virtual 
   731    982       ** table in this case.  */
   732    983       if( rc==SQLITE_OK && pTab->nSrc==0 ){
   733    984         *pzErr = sqlite3_mprintf("no source tables configured");
   734    985         rc = SQLITE_ERROR;
   735    986       }
   736    987   
   737    988       /* For unionvtab, verify that all source tables exist and have 
   738    989       ** compatible schemas. For swarmvtab, attach the first database and
   739    990       ** check that the first table is a rowid table only.  */
   740    991       if( rc==SQLITE_OK ){
   741         -      pTab->db = db;
   742         -      pTab->bSwarm = bSwarm;
   743         -      pTab->nMaxOpen = SWARMVTAB_MAX_OPEN;
   744    992         if( bSwarm ){
   745    993           rc = unionOpenDatabase(pTab, 0, pzErr);
   746    994         }else{
   747    995           rc = unionSourceCheck(pTab, pzErr);
   748    996         }
   749    997       }
   750    998   

Added ext/misc/zipfile.c.

            1  +/*
            2  +** 2017-12-26
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** This file implements a virtual table for reading and writing ZIP archive
           14  +** files.
           15  +**
           16  +** Usage example:
           17  +**
           18  +**     SELECT name, sz, datetime(mtime,'unixepoch') FROM zipfile($filename);
           19  +**
           20  +** Current limitations:
           21  +**
           22  +**    *  No support for encryption
           23  +**    *  No support for ZIP archives spanning multiple files
           24  +**    *  No support for zip64 extensions
           25  +**    *  Only the "inflate/deflate" (zlib) compression method is supported
           26  +*/
           27  +#include "sqlite3ext.h"
           28  +SQLITE_EXTENSION_INIT1
           29  +#include <stdio.h>
           30  +#include <string.h>
           31  +#include <assert.h>
           32  +
           33  +#include <sys/types.h>
           34  +#include <sys/stat.h>
           35  +#include <fcntl.h>
           36  +#if !defined(_WIN32) && !defined(WIN32)
           37  +#  include <unistd.h>
           38  +#  include <dirent.h>
           39  +#  include <utime.h>
           40  +#else
           41  +#  include <io.h>
           42  +#endif
           43  +#include <time.h>
           44  +#include <errno.h>
           45  +
           46  +#include <zlib.h>
           47  +
           48  +#ifndef SQLITE_OMIT_VIRTUALTABLE
           49  +
           50  +#ifndef SQLITE_AMALGAMATION
           51  +typedef sqlite3_int64 i64;
           52  +typedef unsigned char u8;
           53  +typedef unsigned short u16;
           54  +typedef unsigned long u32;
           55  +#define MIN(a,b) ((a)<(b) ? (a) : (b))
           56  +#endif
           57  +
           58  +static const char ZIPFILE_SCHEMA[] = 
           59  +  "CREATE TABLE y("
           60  +    "name PRIMARY KEY,"  /* 0: Name of file in zip archive */
           61  +    "mode,"              /* 1: POSIX mode for file */
           62  +    "mtime,"             /* 2: Last modification time (secs since 1970)*/
           63  +    "sz,"                /* 3: Size of object */
           64  +    "rawdata,"           /* 4: Raw data */
           65  +    "data,"              /* 5: Uncompressed data */
           66  +    "method,"            /* 6: Compression method (integer) */
           67  +    "z HIDDEN"           /* 7: Name of zip file */
           68  +  ") WITHOUT ROWID;";
           69  +
           70  +#define ZIPFILE_F_COLUMN_IDX 7    /* Index of column "file" in the above */
           71  +#define ZIPFILE_BUFFER_SIZE (64*1024)
           72  +
           73  +
           74  +/*
           75  +** Magic numbers used to read and write zip files.
           76  +**
           77  +** ZIPFILE_NEWENTRY_MADEBY:
           78  +**   Use this value for the "version-made-by" field in new zip file
           79  +**   entries. The upper byte indicates "unix", and the lower byte 
           80  +**   indicates that the zip file matches pkzip specification 3.0. 
           81  +**   This is what info-zip seems to do.
           82  +**
           83  +** ZIPFILE_NEWENTRY_REQUIRED:
           84  +**   Value for "version-required-to-extract" field of new entries.
           85  +**   Version 2.0 is required to support folders and deflate compression.
           86  +**
           87  +** ZIPFILE_NEWENTRY_FLAGS:
           88  +**   Value for "general-purpose-bit-flags" field of new entries. Bit
           89  +**   11 means "utf-8 filename and comment".
           90  +**
           91  +** ZIPFILE_SIGNATURE_CDS:
           92  +**   First 4 bytes of a valid CDS record.
           93  +**
           94  +** ZIPFILE_SIGNATURE_LFH:
           95  +**   First 4 bytes of a valid LFH record.
           96  +*/
           97  +#define ZIPFILE_EXTRA_TIMESTAMP   0x5455
           98  +#define ZIPFILE_NEWENTRY_MADEBY   ((3<<8) + 30)
           99  +#define ZIPFILE_NEWENTRY_REQUIRED 20
          100  +#define ZIPFILE_NEWENTRY_FLAGS    0x800
          101  +#define ZIPFILE_SIGNATURE_CDS     0x02014b50
          102  +#define ZIPFILE_SIGNATURE_LFH     0x04034b50
          103  +#define ZIPFILE_SIGNATURE_EOCD    0x06054b50
          104  +#define ZIPFILE_LFH_FIXED_SZ      30
          105  +
          106  +/*
          107  +** Set the error message contained in context ctx to the results of
          108  +** vprintf(zFmt, ...).
          109  +*/
          110  +static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){
          111  +  char *zMsg = 0;
          112  +  va_list ap;
          113  +  va_start(ap, zFmt);
          114  +  zMsg = sqlite3_vmprintf(zFmt, ap);
          115  +  sqlite3_result_error(ctx, zMsg, -1);
          116  +  sqlite3_free(zMsg);
          117  +  va_end(ap);
          118  +}
          119  +
          120  +
          121  +/*
          122  +*** 4.3.16  End of central directory record:
          123  +***
          124  +***   end of central dir signature    4 bytes  (0x06054b50)
          125  +***   number of this disk             2 bytes
          126  +***   number of the disk with the
          127  +***   start of the central directory  2 bytes
          128  +***   total number of entries in the
          129  +***   central directory on this disk  2 bytes
          130  +***   total number of entries in
          131  +***   the central directory           2 bytes
          132  +***   size of the central directory   4 bytes
          133  +***   offset of start of central
          134  +***   directory with respect to
          135  +***   the starting disk number        4 bytes
          136  +***   .ZIP file comment length        2 bytes
          137  +***   .ZIP file comment       (variable size)
          138  +*/
          139  +typedef struct ZipfileEOCD ZipfileEOCD;
          140  +struct ZipfileEOCD {
          141  +  u16 iDisk;
          142  +  u16 iFirstDisk;
          143  +  u16 nEntry;
          144  +  u16 nEntryTotal;
          145  +  u32 nSize;
          146  +  u32 iOffset;
          147  +};
          148  +
          149  +/*
          150  +*** 4.3.12  Central directory structure:
          151  +***
          152  +*** ...
          153  +***
          154  +***   central file header signature   4 bytes  (0x02014b50)
          155  +***   version made by                 2 bytes
          156  +***   version needed to extract       2 bytes
          157  +***   general purpose bit flag        2 bytes
          158  +***   compression method              2 bytes
          159  +***   last mod file time              2 bytes
          160  +***   last mod file date              2 bytes
          161  +***   crc-32                          4 bytes
          162  +***   compressed size                 4 bytes
          163  +***   uncompressed size               4 bytes
          164  +***   file name length                2 bytes
          165  +***   extra field length              2 bytes
          166  +***   file comment length             2 bytes
          167  +***   disk number start               2 bytes
          168  +***   internal file attributes        2 bytes
          169  +***   external file attributes        4 bytes
          170  +***   relative offset of local header 4 bytes
          171  +*/
          172  +typedef struct ZipfileCDS ZipfileCDS;
          173  +struct ZipfileCDS {
          174  +  u16 iVersionMadeBy;
          175  +  u16 iVersionExtract;
          176  +  u16 flags;
          177  +  u16 iCompression;
          178  +  u16 mTime;
          179  +  u16 mDate;
          180  +  u32 crc32;
          181  +  u32 szCompressed;
          182  +  u32 szUncompressed;
          183  +  u16 nFile;
          184  +  u16 nExtra;
          185  +  u16 nComment;
          186  +  u16 iDiskStart;
          187  +  u16 iInternalAttr;
          188  +  u32 iExternalAttr;
          189  +  u32 iOffset;
          190  +  char *zFile;                    /* Filename (sqlite3_malloc()) */
          191  +};
          192  +
          193  +/*
          194  +*** 4.3.7  Local file header:
          195  +***
          196  +***   local file header signature     4 bytes  (0x04034b50)
          197  +***   version needed to extract       2 bytes
          198  +***   general purpose bit flag        2 bytes
          199  +***   compression method              2 bytes
          200  +***   last mod file time              2 bytes
          201  +***   last mod file date              2 bytes
          202  +***   crc-32                          4 bytes
          203  +***   compressed size                 4 bytes
          204  +***   uncompressed size               4 bytes
          205  +***   file name length                2 bytes
          206  +***   extra field length              2 bytes
          207  +***   
          208  +*/
          209  +typedef struct ZipfileLFH ZipfileLFH;
          210  +struct ZipfileLFH {
          211  +  u16 iVersionExtract;
          212  +  u16 flags;
          213  +  u16 iCompression;
          214  +  u16 mTime;
          215  +  u16 mDate;
          216  +  u32 crc32;
          217  +  u32 szCompressed;
          218  +  u32 szUncompressed;
          219  +  u16 nFile;
          220  +  u16 nExtra;
          221  +};
          222  +
          223  +typedef struct ZipfileEntry ZipfileEntry;
          224  +struct ZipfileEntry {
          225  +  char *zPath;               /* Path of zipfile entry */
          226  +  u8 *aCdsEntry;             /* Buffer containing entire CDS entry */
          227  +  int nCdsEntry;             /* Size of buffer aCdsEntry[] in bytes */
          228  +  int bDeleted;              /* True if entry has been deleted */
          229  +  ZipfileEntry *pNext;       /* Next element in in-memory CDS */
          230  +};
          231  +
          232  +/* 
          233  +** Cursor type for recursively iterating through a directory structure.
          234  +*/
          235  +typedef struct ZipfileCsr ZipfileCsr;
          236  +struct ZipfileCsr {
          237  +  sqlite3_vtab_cursor base;  /* Base class - must be first */
          238  +  i64 iId;                   /* Cursor ID */
          239  +  int bEof;                  /* True when at EOF */
          240  +
          241  +  /* Used outside of write transactions */
          242  +  FILE *pFile;               /* Zip file */
          243  +  i64 iNextOff;              /* Offset of next record in central directory */
          244  +  ZipfileEOCD eocd;          /* Parse of central directory record */
          245  +
          246  +  /* Used inside write transactions */
          247  +  ZipfileEntry *pCurrent;
          248  +
          249  +  ZipfileCDS cds;            /* Central Directory Structure */
          250  +  ZipfileLFH lfh;            /* Local File Header for current entry */
          251  +  i64 iDataOff;              /* Offset in zipfile to data */
          252  +  u32 mTime;                 /* Extended mtime value */
          253  +  int flags;                 /* Flags byte (see below for bits) */
          254  +  ZipfileCsr *pCsrNext;      /* Next cursor on same virtual table */
          255  +};
          256  +
          257  +/*
          258  +** Values for ZipfileCsr.flags.
          259  +*/
          260  +#define ZIPFILE_MTIME_VALID 0x0001
          261  +
          262  +typedef struct ZipfileTab ZipfileTab;
          263  +struct ZipfileTab {
          264  +  sqlite3_vtab base;         /* Base class - must be first */
          265  +  char *zFile;               /* Zip file this table accesses (may be NULL) */
          266  +  u8 *aBuffer;               /* Temporary buffer used for various tasks */
          267  +
          268  +  ZipfileCsr *pCsrList;      /* List of cursors */
          269  +  i64 iNextCsrid;
          270  +
          271  +  /* The following are used by write transactions only */
          272  +  ZipfileEntry *pFirstEntry; /* Linked list of all files (if pWriteFd!=0) */
          273  +  ZipfileEntry *pLastEntry;  /* Last element in pFirstEntry list */
          274  +  FILE *pWriteFd;            /* File handle open on zip archive */
          275  +  i64 szCurrent;             /* Current size of zip archive */
          276  +  i64 szOrig;                /* Size of archive at start of transaction */
          277  +};
          278  +
          279  +static void zipfileDequote(char *zIn){
          280  +  char q = zIn[0];
          281  +  if( q=='"' || q=='\'' || q=='`' || q=='[' ){
          282  +    char c;
          283  +    int iIn = 1;
          284  +    int iOut = 0;
          285  +    if( q=='[' ) q = ']';
          286  +    while( (c = zIn[iIn++]) ){
          287  +      if( c==q ){
          288  +        if( zIn[iIn++]!=q ) break;
          289  +      }
          290  +      zIn[iOut++] = c;
          291  +    }
          292  +    zIn[iOut] = '\0';
          293  +  }
          294  +}
          295  +
          296  +/*
          297  +** Construct a new ZipfileTab virtual table object.
          298  +** 
          299  +**   argv[0]   -> module name  ("zipfile")
          300  +**   argv[1]   -> database name
          301  +**   argv[2]   -> table name
          302  +**   argv[...] -> "column name" and other module argument fields.
          303  +*/
          304  +static int zipfileConnect(
          305  +  sqlite3 *db,
          306  +  void *pAux,
          307  +  int argc, const char *const*argv,
          308  +  sqlite3_vtab **ppVtab,
          309  +  char **pzErr
          310  +){
          311  +  int nByte = sizeof(ZipfileTab) + ZIPFILE_BUFFER_SIZE;
          312  +  int nFile = 0;
          313  +  const char *zFile = 0;
          314  +  ZipfileTab *pNew = 0;
          315  +  int rc;
          316  +
          317  +  if( argc>3 ){
          318  +    zFile = argv[3];
          319  +    nFile = (int)strlen(zFile)+1;
          320  +  }
          321  +
          322  +  rc = sqlite3_declare_vtab(db, ZIPFILE_SCHEMA);
          323  +  if( rc==SQLITE_OK ){
          324  +    pNew = (ZipfileTab*)sqlite3_malloc(nByte+nFile);
          325  +    if( pNew==0 ) return SQLITE_NOMEM;
          326  +    memset(pNew, 0, nByte+nFile);
          327  +    pNew->aBuffer = (u8*)&pNew[1];
          328  +    if( zFile ){
          329  +      pNew->zFile = (char*)&pNew->aBuffer[ZIPFILE_BUFFER_SIZE];
          330  +      memcpy(pNew->zFile, zFile, nFile);
          331  +      zipfileDequote(pNew->zFile);
          332  +    }
          333  +  }
          334  +  *ppVtab = (sqlite3_vtab*)pNew;
          335  +  return rc;
          336  +}
          337  +
          338  +/*
          339  +** This method is the destructor for zipfile vtab objects.
          340  +*/
          341  +static int zipfileDisconnect(sqlite3_vtab *pVtab){
          342  +  sqlite3_free(pVtab);
          343  +  return SQLITE_OK;
          344  +}
          345  +
          346  +/*
          347  +** Constructor for a new ZipfileCsr object.
          348  +*/
          349  +static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
          350  +  ZipfileTab *pTab = (ZipfileTab*)p;
          351  +  ZipfileCsr *pCsr;
          352  +  pCsr = sqlite3_malloc(sizeof(*pCsr));
          353  +  *ppCsr = (sqlite3_vtab_cursor*)pCsr;
          354  +  if( pCsr==0 ){
          355  +    return SQLITE_NOMEM;
          356  +  }
          357  +  memset(pCsr, 0, sizeof(*pCsr));
          358  +  pCsr->iId = ++pTab->iNextCsrid;
          359  +  pCsr->pCsrNext = pTab->pCsrList;
          360  +  pTab->pCsrList = pCsr;
          361  +  return SQLITE_OK;
          362  +}
          363  +
          364  +/*
          365  +** Reset a cursor back to the state it was in when first returned
          366  +** by zipfileOpen().
          367  +*/
          368  +static void zipfileResetCursor(ZipfileCsr *pCsr){
          369  +  sqlite3_free(pCsr->cds.zFile);
          370  +  pCsr->cds.zFile = 0;
          371  +  pCsr->bEof = 0;
          372  +  if( pCsr->pFile ){
          373  +    fclose(pCsr->pFile);
          374  +    pCsr->pFile = 0;
          375  +  }
          376  +}
          377  +
          378  +/*
          379  +** Destructor for an ZipfileCsr.
          380  +*/
          381  +static int zipfileClose(sqlite3_vtab_cursor *cur){
          382  +  ZipfileCsr *pCsr = (ZipfileCsr*)cur;
          383  +  ZipfileTab *pTab = (ZipfileTab*)(pCsr->base.pVtab);
          384  +  ZipfileCsr **pp;
          385  +  zipfileResetCursor(pCsr);
          386  +
          387  +  /* Remove this cursor from the ZipfileTab.pCsrList list. */
          388  +  for(pp=&pTab->pCsrList; *pp; pp=&((*pp)->pCsrNext)){
          389  +    if( *pp==pCsr ){ 
          390  +      *pp = pCsr->pCsrNext;
          391  +      break;
          392  +    }
          393  +  }
          394  +
          395  +  sqlite3_free(pCsr);
          396  +  return SQLITE_OK;
          397  +}
          398  +
          399  +/*
          400  +** Set the error message for the virtual table associated with cursor
          401  +** pCsr to the results of vprintf(zFmt, ...).
          402  +*/
          403  +static void zipfileSetErrmsg(ZipfileCsr *pCsr, const char *zFmt, ...){
          404  +  va_list ap;
          405  +  va_start(ap, zFmt);
          406  +  pCsr->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
          407  +  va_end(ap);
          408  +}
          409  +
          410  +static int zipfileReadData(
          411  +  FILE *pFile,                    /* Read from this file */
          412  +  u8 *aRead,                      /* Read into this buffer */
          413  +  int nRead,                      /* Number of bytes to read */
          414  +  i64 iOff,                       /* Offset to read from */
          415  +  char **pzErrmsg                 /* OUT: Error message (from sqlite3_malloc) */
          416  +){
          417  +  size_t n;
          418  +  fseek(pFile, (long)iOff, SEEK_SET);
          419  +  n = fread(aRead, 1, nRead, pFile);
          420  +  if( (int)n!=nRead ){
          421  +    *pzErrmsg = sqlite3_mprintf("error in fread()");
          422  +    return SQLITE_ERROR;
          423  +  }
          424  +  return SQLITE_OK;
          425  +}
          426  +
          427  +static int zipfileAppendData(
          428  +  ZipfileTab *pTab,
          429  +  const u8 *aWrite,
          430  +  int nWrite
          431  +){
          432  +  size_t n;
          433  +  fseek(pTab->pWriteFd, (long)pTab->szCurrent, SEEK_SET);
          434  +  n = fwrite(aWrite, 1, nWrite, pTab->pWriteFd);
          435  +  if( (int)n!=nWrite ){
          436  +    pTab->base.zErrMsg = sqlite3_mprintf("error in fwrite()");
          437  +    return SQLITE_ERROR;
          438  +  }
          439  +  pTab->szCurrent += nWrite;
          440  +  return SQLITE_OK;
          441  +}
          442  +
          443  +static u16 zipfileGetU16(const u8 *aBuf){
          444  +  return (aBuf[1] << 8) + aBuf[0];
          445  +}
          446  +static u32 zipfileGetU32(const u8 *aBuf){
          447  +  return ((u32)(aBuf[3]) << 24)
          448  +       + ((u32)(aBuf[2]) << 16)
          449  +       + ((u32)(aBuf[1]) <<  8)
          450  +       + ((u32)(aBuf[0]) <<  0);
          451  +}
          452  +
          453  +static void zipfilePutU16(u8 *aBuf, u16 val){
          454  +  aBuf[0] = val & 0xFF;
          455  +  aBuf[1] = (val>>8) & 0xFF;
          456  +}
          457  +static void zipfilePutU32(u8 *aBuf, u32 val){
          458  +  aBuf[0] = val & 0xFF;
          459  +  aBuf[1] = (val>>8) & 0xFF;
          460  +  aBuf[2] = (val>>16) & 0xFF;
          461  +  aBuf[3] = (val>>24) & 0xFF;
          462  +}
          463  +
          464  +#define zipfileRead32(aBuf) ( aBuf+=4, zipfileGetU32(aBuf-4) )
          465  +#define zipfileRead16(aBuf) ( aBuf+=2, zipfileGetU16(aBuf-2) )
          466  +
          467  +#define zipfileWrite32(aBuf,val) { zipfilePutU32(aBuf,val); aBuf+=4; }
          468  +#define zipfileWrite16(aBuf,val) { zipfilePutU16(aBuf,val); aBuf+=2; }
          469  +
          470  +static u8* zipfileCsrBuffer(ZipfileCsr *pCsr){
          471  +  return ((ZipfileTab*)(pCsr->base.pVtab))->aBuffer;
          472  +}
          473  +
          474  +/*
          475  +** Magic numbers used to read CDS records.
          476  +*/
          477  +#define ZIPFILE_CDS_FIXED_SZ         46
          478  +#define ZIPFILE_CDS_NFILE_OFF        28
          479  +
          480  +/*
          481  +** Decode the CDS record in buffer aBuf into (*pCDS). Return SQLITE_ERROR
          482  +** if the record is not well-formed, or SQLITE_OK otherwise.
          483  +*/
          484  +static int zipfileReadCDS(u8 *aBuf, ZipfileCDS *pCDS){
          485  +  u8 *aRead = aBuf;
          486  +  u32 sig = zipfileRead32(aRead);
          487  +  int rc = SQLITE_OK;
          488  +  if( sig!=ZIPFILE_SIGNATURE_CDS ){
          489  +    rc = SQLITE_ERROR;
          490  +  }else{
          491  +    pCDS->iVersionMadeBy = zipfileRead16(aRead);
          492  +    pCDS->iVersionExtract = zipfileRead16(aRead);
          493  +    pCDS->flags = zipfileRead16(aRead);
          494  +    pCDS->iCompression = zipfileRead16(aRead);
          495  +    pCDS->mTime = zipfileRead16(aRead);
          496  +    pCDS->mDate = zipfileRead16(aRead);
          497  +    pCDS->crc32 = zipfileRead32(aRead);
          498  +    pCDS->szCompressed = zipfileRead32(aRead);
          499  +    pCDS->szUncompressed = zipfileRead32(aRead);
          500  +    assert( aRead==&aBuf[ZIPFILE_CDS_NFILE_OFF] );
          501  +    pCDS->nFile = zipfileRead16(aRead);
          502  +    pCDS->nExtra = zipfileRead16(aRead);
          503  +    pCDS->nComment = zipfileRead16(aRead);
          504  +    pCDS->iDiskStart = zipfileRead16(aRead);
          505  +    pCDS->iInternalAttr = zipfileRead16(aRead);
          506  +    pCDS->iExternalAttr = zipfileRead32(aRead);
          507  +    pCDS->iOffset = zipfileRead32(aRead);
          508  +    assert( aRead==&aBuf[ZIPFILE_CDS_FIXED_SZ] );
          509  +  }
          510  +
          511  +  return rc;
          512  +}
          513  +
          514  +/*
          515  +** Read the CDS record for the current entry from disk into pCsr->cds.
          516  +*/
          517  +static int zipfileCsrReadCDS(ZipfileCsr *pCsr){
          518  +  char **pzErr = &pCsr->base.pVtab->zErrMsg;
          519  +  u8 *aRead;
          520  +  int rc = SQLITE_OK;
          521  +
          522  +  sqlite3_free(pCsr->cds.zFile);
          523  +  pCsr->cds.zFile = 0;
          524  +
          525  +  if( pCsr->pCurrent==0 ){
          526  +    aRead = zipfileCsrBuffer(pCsr);
          527  +    rc = zipfileReadData(
          528  +        pCsr->pFile, aRead, ZIPFILE_CDS_FIXED_SZ, pCsr->iNextOff, pzErr
          529  +    );
          530  +  }else{
          531  +    aRead = pCsr->pCurrent->aCdsEntry;
          532  +  }
          533  +
          534  +  if( rc==SQLITE_OK ){
          535  +    rc = zipfileReadCDS(aRead, &pCsr->cds);
          536  +    if( rc!=SQLITE_OK ){
          537  +      assert( pCsr->pCurrent==0 );
          538  +      zipfileSetErrmsg(pCsr,"failed to read CDS at offset %lld",pCsr->iNextOff);
          539  +    }else{
          540  +      int nRead;
          541  +      if( pCsr->pCurrent==0 ){
          542  +        nRead = pCsr->cds.nFile + pCsr->cds.nExtra;
          543  +        aRead = zipfileCsrBuffer(pCsr);
          544  +        pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ;
          545  +        rc = zipfileReadData(pCsr->pFile, aRead, nRead, pCsr->iNextOff, pzErr);
          546  +      }else{
          547  +        aRead = &aRead[ZIPFILE_CDS_FIXED_SZ];
          548  +      }
          549  +
          550  +      if( rc==SQLITE_OK ){
          551  +        pCsr->cds.zFile = sqlite3_mprintf("%.*s", (int)pCsr->cds.nFile, aRead);
          552  +        pCsr->iNextOff += pCsr->cds.nFile;
          553  +        pCsr->iNextOff += pCsr->cds.nExtra;
          554  +        pCsr->iNextOff += pCsr->cds.nComment;
          555  +      }
          556  +
          557  +      /* Scan the cds.nExtra bytes of "extra" fields for any that can
          558  +      ** be interpreted. The general format of an extra field is:
          559  +      **
          560  +      **   Header ID    2 bytes
          561  +      **   Data Size    2 bytes
          562  +      **   Data         N bytes
          563  +      **
          564  +      */
          565  +      if( rc==SQLITE_OK ){
          566  +        u8 *p = &aRead[pCsr->cds.nFile];
          567  +        u8 *pEnd = &p[pCsr->cds.nExtra];
          568  +
          569  +        while( p<pEnd ){
          570  +          u16 id = zipfileRead16(p);
          571  +          u16 nByte = zipfileRead16(p);
          572  +
          573  +          switch( id ){
          574  +            case ZIPFILE_EXTRA_TIMESTAMP: {
          575  +              u8 b = p[0];
          576  +              if( b & 0x01 ){     /* 0x01 -> modtime is present */
          577  +                pCsr->mTime = zipfileGetU32(&p[1]);
          578  +                pCsr->flags |= ZIPFILE_MTIME_VALID;
          579  +              }
          580  +              break;
          581  +            }
          582  +          }
          583  +
          584  +          p += nByte;
          585  +        }
          586  +      }
          587  +    }
          588  +  }
          589  +
          590  +  return rc;
          591  +}
          592  +
          593  +static FILE *zipfileGetFd(ZipfileCsr *pCsr){
          594  +  if( pCsr->pFile ) return pCsr->pFile;
          595  +  return ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd;
          596  +}
          597  +
          598  +static int zipfileReadLFH(
          599  +  FILE *pFd, 
          600  +  i64 iOffset,
          601  +  u8 *aTmp, 
          602  +  ZipfileLFH *pLFH, 
          603  +  char **pzErr
          604  +){
          605  +  u8 *aRead = aTmp;
          606  +  static const int szFix = ZIPFILE_LFH_FIXED_SZ;
          607  +  int rc;
          608  +
          609  +  rc = zipfileReadData(pFd, aRead, szFix, iOffset, pzErr);
          610  +  if( rc==SQLITE_OK ){
          611  +    u32 sig = zipfileRead32(aRead);
          612  +    if( sig!=ZIPFILE_SIGNATURE_LFH ){
          613  +      *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", (int)iOffset);
          614  +      rc = SQLITE_ERROR;
          615  +    }else{
          616  +      pLFH->iVersionExtract = zipfileRead16(aRead);
          617  +      pLFH->flags = zipfileRead16(aRead);
          618  +      pLFH->iCompression = zipfileRead16(aRead);
          619  +      pLFH->mTime = zipfileRead16(aRead);
          620  +      pLFH->mDate = zipfileRead16(aRead);
          621  +      pLFH->crc32 = zipfileRead32(aRead);
          622  +      pLFH->szCompressed = zipfileRead32(aRead);
          623  +      pLFH->szUncompressed = zipfileRead32(aRead);
          624  +      pLFH->nFile = zipfileRead16(aRead);
          625  +      pLFH->nExtra = zipfileRead16(aRead);
          626  +      assert( aRead==&aTmp[szFix] );
          627  +    }
          628  +  }
          629  +  return rc;
          630  +}
          631  +
          632  +static int zipfileCsrReadLFH(ZipfileCsr *pCsr){
          633  +  FILE *pFile = zipfileGetFd(pCsr);
          634  +  char **pzErr = &pCsr->base.pVtab->zErrMsg;
          635  +  u8 *aRead = zipfileCsrBuffer(pCsr);
          636  +  int rc = zipfileReadLFH(pFile, pCsr->cds.iOffset, aRead, &pCsr->lfh, pzErr);
          637  +  pCsr->iDataOff =  pCsr->cds.iOffset + ZIPFILE_LFH_FIXED_SZ;
          638  +  pCsr->iDataOff += pCsr->lfh.nFile+pCsr->lfh.nExtra;
          639  +  return rc;
          640  +}
          641  +
          642  +
          643  +/*
          644  +** Advance an ZipfileCsr to its next row of output.
          645  +*/
          646  +static int zipfileNext(sqlite3_vtab_cursor *cur){
          647  +  ZipfileCsr *pCsr = (ZipfileCsr*)cur;
          648  +  int rc = SQLITE_OK;
          649  +  pCsr->flags = 0;
          650  +
          651  +  if( pCsr->pCurrent==0 ){
          652  +    i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize;
          653  +    if( pCsr->iNextOff>=iEof ){
          654  +      pCsr->bEof = 1;
          655  +    }
          656  +  }else{
          657  +    assert( pCsr->pFile==0 );
          658  +    do {
          659  +      pCsr->pCurrent = pCsr->pCurrent->pNext;
          660  +    }while( pCsr->pCurrent && pCsr->pCurrent->bDeleted );
          661  +    if( pCsr->pCurrent==0 ){
          662  +      pCsr->bEof = 1;
          663  +    }
          664  +  }
          665  +
          666  +  if( pCsr->bEof==0 ){
          667  +    rc = zipfileCsrReadCDS(pCsr);
          668  +    if( rc==SQLITE_OK ){
          669  +      rc = zipfileCsrReadLFH(pCsr);
          670  +    }
          671  +  }
          672  +
          673  +  return rc;
          674  +}
          675  +
          676  +/*
          677  +** "Standard" MS-DOS time format:
          678  +**
          679  +**   File modification time:
          680  +**     Bits 00-04: seconds divided by 2
          681  +**     Bits 05-10: minute
          682  +**     Bits 11-15: hour
          683  +**   File modification date:
          684  +**     Bits 00-04: day
          685  +**     Bits 05-08: month (1-12)
          686  +**     Bits 09-15: years from 1980 
          687  +*/
          688  +static time_t zipfileMtime(ZipfileCsr *pCsr){
          689  +  struct tm t;
          690  +  memset(&t, 0, sizeof(t));
          691  +  t.tm_sec = (pCsr->cds.mTime & 0x1F)*2;
          692  +  t.tm_min = (pCsr->cds.mTime >> 5) & 0x2F;
          693  +  t.tm_hour = (pCsr->cds.mTime >> 11) & 0x1F;
          694  +
          695  +  t.tm_mday = (pCsr->cds.mDate & 0x1F);
          696  +  t.tm_mon = ((pCsr->cds.mDate >> 5) & 0x0F) - 1;
          697  +  t.tm_year = 80 + ((pCsr->cds.mDate >> 9) & 0x7F);
          698  +
          699  +  return mktime(&t);
          700  +}
          701  +
          702  +static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mTime){
          703  +  time_t t = (time_t)mTime;
          704  +  struct tm res;
          705  +
          706  +#if !defined(_WIN32) && !defined(WIN32)
          707  +  localtime_r(&t, &res);
          708  +#else
          709  +  memcpy(&res, localtime(&t), sizeof(struct tm));
          710  +#endif
          711  +
          712  +  pCds->mTime = (u16)(
          713  +    (res.tm_sec / 2) + 
          714  +    (res.tm_min << 5) +
          715  +    (res.tm_hour << 11));
          716  +
          717  +  pCds->mDate = (u16)(
          718  +    (res.tm_mday-1) +
          719  +    ((res.tm_mon+1) << 5) +
          720  +    ((res.tm_year-80) << 9));
          721  +}
          722  +
          723  +static void zipfileInflate(
          724  +  sqlite3_context *pCtx,          /* Store error here, if any */
          725  +  const u8 *aIn,                  /* Compressed data */
          726  +  int nIn,                        /* Size of buffer aIn[] in bytes */
          727  +  int nOut                        /* Expected output size */
          728  +){
          729  +  u8 *aRes = sqlite3_malloc(nOut);
          730  +  if( aRes==0 ){
          731  +    sqlite3_result_error_nomem(pCtx);
          732  +  }else{
          733  +    int err;
          734  +    z_stream str;
          735  +    memset(&str, 0, sizeof(str));
          736  +
          737  +    str.next_in = (Byte*)aIn;
          738  +    str.avail_in = nIn;
          739  +    str.next_out = (Byte*)aRes;
          740  +    str.avail_out = nOut;
          741  +
          742  +    err = inflateInit2(&str, -15);
          743  +    if( err!=Z_OK ){
          744  +      zipfileCtxErrorMsg(pCtx, "inflateInit2() failed (%d)", err);
          745  +    }else{
          746  +      err = inflate(&str, Z_NO_FLUSH);
          747  +      if( err!=Z_STREAM_END ){
          748  +        zipfileCtxErrorMsg(pCtx, "inflate() failed (%d)", err);
          749  +      }else{
          750  +        sqlite3_result_blob(pCtx, aRes, nOut, SQLITE_TRANSIENT);
          751  +      }
          752  +    }
          753  +    sqlite3_free(aRes);
          754  +    inflateEnd(&str);
          755  +  }
          756  +}
          757  +
          758  +static int zipfileDeflate(
          759  +  ZipfileTab *pTab,               /* Set error message here */
          760  +  const u8 *aIn, int nIn,         /* Input */
          761  +  u8 **ppOut, int *pnOut          /* Output */
          762  +){
          763  +  int nAlloc = (int)compressBound(nIn);
          764  +  u8 *aOut;
          765  +  int rc = SQLITE_OK;
          766  +
          767  +  aOut = (u8*)sqlite3_malloc(nAlloc);
          768  +  if( aOut==0 ){
          769  +    rc = SQLITE_NOMEM;
          770  +  }else{
          771  +    int res;
          772  +    z_stream str;
          773  +    memset(&str, 0, sizeof(str));
          774  +    str.next_in = (z_const Bytef*)aIn;
          775  +    str.avail_in = nIn;
          776  +    str.next_out = aOut;
          777  +    str.avail_out = nAlloc;
          778  +
          779  +    deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
          780  +    res = deflate(&str, Z_FINISH);
          781  +
          782  +    if( res==Z_STREAM_END ){
          783  +      *ppOut = aOut;
          784  +      *pnOut = (int)str.total_out;
          785  +    }else{
          786  +      sqlite3_free(aOut);
          787  +      pTab->base.zErrMsg = sqlite3_mprintf("zipfile: deflate() error");
          788  +      rc = SQLITE_ERROR;
          789  +    }
          790  +    deflateEnd(&str);
          791  +  }
          792  +
          793  +  return rc;
          794  +}
          795  +
          796  +
          797  +/*
          798  +** Return values of columns for the row at which the series_cursor
          799  +** is currently pointing.
          800  +*/
          801  +static int zipfileColumn(
          802  +  sqlite3_vtab_cursor *cur,   /* The cursor */
          803  +  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
          804  +  int i                       /* Which column to return */
          805  +){
          806  +  ZipfileCsr *pCsr = (ZipfileCsr*)cur;
          807  +  int rc = SQLITE_OK;
          808  +  switch( i ){
          809  +    case 0:   /* name */
          810  +      sqlite3_result_text(ctx, pCsr->cds.zFile, -1, SQLITE_TRANSIENT);
          811  +      break;
          812  +    case 1:   /* mode */
          813  +      /* TODO: Whether or not the following is correct surely depends on
          814  +      ** the platform on which the archive was created.  */
          815  +      sqlite3_result_int(ctx, pCsr->cds.iExternalAttr >> 16);
          816  +      break;
          817  +    case 2: { /* mtime */
          818  +      if( pCsr->flags & ZIPFILE_MTIME_VALID ){
          819  +        sqlite3_result_int64(ctx, pCsr->mTime);
          820  +      }else{
          821  +        sqlite3_result_int64(ctx, zipfileMtime(pCsr));
          822  +      }
          823  +      break;
          824  +    }
          825  +    case 3: { /* sz */
          826  +      if( sqlite3_vtab_nochange(ctx)==0 ){
          827  +        sqlite3_result_int64(ctx, pCsr->cds.szUncompressed);
          828  +      }
          829  +      break;
          830  +    }
          831  +    case 4:   /* rawdata */
          832  +      if( sqlite3_vtab_nochange(ctx) ) break;
          833  +    case 5: { /* data */
          834  +      if( i==4 || pCsr->cds.iCompression==0 || pCsr->cds.iCompression==8 ){
          835  +        int sz = pCsr->cds.szCompressed;
          836  +        int szFinal = pCsr->cds.szUncompressed;
          837  +        if( szFinal>0 ){
          838  +          u8 *aBuf = sqlite3_malloc(sz);
          839  +          if( aBuf==0 ){
          840  +            rc = SQLITE_NOMEM;
          841  +          }else{
          842  +            FILE *pFile = zipfileGetFd(pCsr);
          843  +            rc = zipfileReadData(pFile, aBuf, sz, pCsr->iDataOff,
          844  +                &pCsr->base.pVtab->zErrMsg
          845  +            );
          846  +          }
          847  +          if( rc==SQLITE_OK ){
          848  +            if( i==5 && pCsr->cds.iCompression ){
          849  +              zipfileInflate(ctx, aBuf, sz, szFinal);
          850  +            }else{
          851  +              sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT);
          852  +            }
          853  +            sqlite3_free(aBuf);
          854  +          }
          855  +        }else{
          856  +          /* Figure out if this is a directory or a zero-sized file. Consider
          857  +          ** it to be a directory either if the mode suggests so, or if
          858  +          ** the final character in the name is '/'.  */
          859  +          u32 mode = pCsr->cds.iExternalAttr >> 16;
          860  +          if( !(mode & S_IFDIR) && pCsr->cds.zFile[pCsr->cds.nFile-1]!='/' ){
          861  +            sqlite3_result_blob(ctx, "", 0, SQLITE_STATIC);
          862  +          }
          863  +        }
          864  +      }
          865  +      break;
          866  +    }
          867  +    case 6:   /* method */
          868  +      sqlite3_result_int(ctx, pCsr->cds.iCompression);
          869  +      break;
          870  +    case 7:   /* z */
          871  +      sqlite3_result_int64(ctx, pCsr->iId);
          872  +      break;
          873  +  }
          874  +
          875  +  return rc;
          876  +}
          877  +
          878  +/*
          879  +** Return the rowid for the current row.
          880  +*/
          881  +static int zipfileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
          882  +  assert( 0 );
          883  +  return SQLITE_OK;
          884  +}
          885  +
          886  +/*
          887  +** Return TRUE if the cursor has been moved off of the last
          888  +** row of output.
          889  +*/
          890  +static int zipfileEof(sqlite3_vtab_cursor *cur){
          891  +  ZipfileCsr *pCsr = (ZipfileCsr*)cur;
          892  +  return pCsr->bEof;
          893  +}
          894  +
          895  +/*
          896  +*/
          897  +static int zipfileReadEOCD(
          898  +  ZipfileTab *pTab,               /* Return errors here */
          899  +  FILE *pFile,                    /* Read from this file */
          900  +  ZipfileEOCD *pEOCD              /* Object to populate */
          901  +){
          902  +  u8 *aRead = pTab->aBuffer;      /* Temporary buffer */
          903  +  i64 szFile;                     /* Total size of file in bytes */
          904  +  int nRead;                      /* Bytes to read from file */
          905  +  i64 iOff;                       /* Offset to read from */
          906  +  int rc;
          907  +
          908  +  fseek(pFile, 0, SEEK_END);
          909  +  szFile = (i64)ftell(pFile);
          910  +  if( szFile==0 ){
          911  +    memset(pEOCD, 0, sizeof(ZipfileEOCD));
          912  +    return SQLITE_OK;
          913  +  }
          914  +  nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE));
          915  +  iOff = szFile - nRead;
          916  +
          917  +  rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg);
          918  +  if( rc==SQLITE_OK ){
          919  +    int i;
          920  +
          921  +    /* Scan backwards looking for the signature bytes */
          922  +    for(i=nRead-20; i>=0; i--){
          923  +      if( aRead[i]==0x50 && aRead[i+1]==0x4b 
          924  +       && aRead[i+2]==0x05 && aRead[i+3]==0x06 
          925  +      ){
          926  +        break;
          927  +      }
          928  +    }
          929  +    if( i<0 ){
          930  +      pTab->base.zErrMsg = sqlite3_mprintf(
          931  +          "cannot find end of central directory record"
          932  +      );
          933  +      return SQLITE_ERROR;
          934  +    }
          935  +
          936  +    aRead += i+4;
          937  +    pEOCD->iDisk = zipfileRead16(aRead);
          938  +    pEOCD->iFirstDisk = zipfileRead16(aRead);
          939  +    pEOCD->nEntry = zipfileRead16(aRead);
          940  +    pEOCD->nEntryTotal = zipfileRead16(aRead);
          941  +    pEOCD->nSize = zipfileRead32(aRead);
          942  +    pEOCD->iOffset = zipfileRead32(aRead);
          943  +
          944  +#if 0
          945  +    printf("iDisk=%d  iFirstDisk=%d  nEntry=%d  "
          946  +           "nEntryTotal=%d  nSize=%d  iOffset=%d", 
          947  +           (int)pEOCD->iDisk, (int)pEOCD->iFirstDisk, (int)pEOCD->nEntry,
          948  +           (int)pEOCD->nEntryTotal, (int)pEOCD->nSize, (int)pEOCD->iOffset
          949  +    );
          950  +#endif
          951  +  }
          952  +
          953  +  return SQLITE_OK;
          954  +}
          955  +
          956  +/*
          957  +** xFilter callback.
          958  +*/
          959  +static int zipfileFilter(
          960  +  sqlite3_vtab_cursor *cur, 
          961  +  int idxNum, const char *idxStr,
          962  +  int argc, sqlite3_value **argv
          963  +){
          964  +  ZipfileTab *pTab = (ZipfileTab*)cur->pVtab;
          965  +  ZipfileCsr *pCsr = (ZipfileCsr*)cur;
          966  +  const char *zFile;              /* Zip file to scan */
          967  +  int rc = SQLITE_OK;             /* Return Code */
          968  +
          969  +  zipfileResetCursor(pCsr);
          970  +
          971  +  if( pTab->zFile ){
          972  +    zFile = pTab->zFile;
          973  +  }else if( idxNum==0 ){
          974  +    /* Error. This is an eponymous virtual table and the user has not 
          975  +    ** supplied a file name. */
          976  +    zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument");
          977  +    return SQLITE_ERROR;
          978  +  }else{
          979  +    zFile = (const char*)sqlite3_value_text(argv[0]);
          980  +  }
          981  +
          982  +  if( pTab->pWriteFd==0 ){
          983  +    pCsr->pFile = fopen(zFile, "rb");
          984  +    if( pCsr->pFile==0 ){
          985  +      zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile);
          986  +      rc = SQLITE_ERROR;
          987  +    }else{
          988  +      rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd);
          989  +      if( rc==SQLITE_OK ){
          990  +        if( pCsr->eocd.nEntry==0 ){
          991  +          pCsr->bEof = 1;
          992  +        }else{
          993  +          pCsr->iNextOff = pCsr->eocd.iOffset;
          994  +          rc = zipfileNext(cur);
          995  +        }
          996  +      }
          997  +    }
          998  +  }else{
          999  +    ZipfileEntry e;
         1000  +    memset(&e, 0, sizeof(e));
         1001  +    e.pNext = pTab->pFirstEntry;
         1002  +    pCsr->pCurrent = &e;
         1003  +    rc = zipfileNext(cur);
         1004  +    assert( pCsr->pCurrent!=&e );
         1005  +  }
         1006  +
         1007  +  return rc;
         1008  +}
         1009  +
         1010  +/*
         1011  +** xBestIndex callback.
         1012  +*/
         1013  +static int zipfileBestIndex(
         1014  +  sqlite3_vtab *tab,
         1015  +  sqlite3_index_info *pIdxInfo
         1016  +){
         1017  +  int i;
         1018  +
         1019  +  for(i=0; i<pIdxInfo->nConstraint; i++){
         1020  +    const struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
         1021  +    if( pCons->usable==0 ) continue;
         1022  +    if( pCons->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
         1023  +    if( pCons->iColumn!=ZIPFILE_F_COLUMN_IDX ) continue;
         1024  +    break;
         1025  +  }
         1026  +
         1027  +  if( i<pIdxInfo->nConstraint ){
         1028  +    pIdxInfo->aConstraintUsage[i].argvIndex = 1;
         1029  +    pIdxInfo->aConstraintUsage[i].omit = 1;
         1030  +    pIdxInfo->estimatedCost = 1000.0;
         1031  +    pIdxInfo->idxNum = 1;
         1032  +  }else{
         1033  +    pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50);
         1034  +    pIdxInfo->idxNum = 0;
         1035  +  }
         1036  +
         1037  +  return SQLITE_OK;
         1038  +}
         1039  +
         1040  +/*
         1041  +** Add object pNew to the end of the linked list that begins at
         1042  +** ZipfileTab.pFirstEntry and ends with pLastEntry.
         1043  +*/
         1044  +static void zipfileAddEntry(
         1045  +  ZipfileTab *pTab, 
         1046  +  ZipfileEntry *pBefore, 
         1047  +  ZipfileEntry *pNew
         1048  +){
         1049  +  assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) );
         1050  +  assert( pNew->pNext==0 );
         1051  +  if( pBefore==0 ){
         1052  +    if( pTab->pFirstEntry==0 ){
         1053  +      pTab->pFirstEntry = pTab->pLastEntry = pNew;
         1054  +    }else{
         1055  +      assert( pTab->pLastEntry->pNext==0 );
         1056  +      pTab->pLastEntry->pNext = pNew;
         1057  +      pTab->pLastEntry = pNew;
         1058  +    }
         1059  +  }else{
         1060  +    ZipfileEntry **pp;
         1061  +    for(pp=&pTab->pFirstEntry; *pp!=pBefore; pp=&((*pp)->pNext));
         1062  +    pNew->pNext = pBefore;
         1063  +    *pp = pNew;
         1064  +  }
         1065  +}
         1066  +
         1067  +static int zipfileLoadDirectory(ZipfileTab *pTab){
         1068  +  ZipfileEOCD eocd;
         1069  +  int rc;
         1070  +
         1071  +  rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd);
         1072  +  if( rc==SQLITE_OK && eocd.nEntry>0 ){
         1073  +    int i;
         1074  +    int iOff = 0;
         1075  +    u8 *aBuf = sqlite3_malloc(eocd.nSize);
         1076  +    if( aBuf==0 ){
         1077  +      rc = SQLITE_NOMEM;
         1078  +    }else{
         1079  +      rc = zipfileReadData(
         1080  +          pTab->pWriteFd, aBuf, eocd.nSize, eocd.iOffset, &pTab->base.zErrMsg
         1081  +      );
         1082  +    }
         1083  +
         1084  +    for(i=0; rc==SQLITE_OK && i<eocd.nEntry; i++){
         1085  +      u16 nFile;
         1086  +      u16 nExtra;
         1087  +      u16 nComment;
         1088  +      ZipfileEntry *pNew;
         1089  +      u8 *aRec = &aBuf[iOff];
         1090  +
         1091  +      nFile = zipfileGetU16(&aRec[ZIPFILE_CDS_NFILE_OFF]);
         1092  +      nExtra = zipfileGetU16(&aRec[ZIPFILE_CDS_NFILE_OFF+2]);
         1093  +      nComment = zipfileGetU16(&aRec[ZIPFILE_CDS_NFILE_OFF+4]);
         1094  +
         1095  +      pNew = sqlite3_malloc(
         1096  +          sizeof(ZipfileEntry) 
         1097  +        + nFile+1 
         1098  +        + ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment
         1099  +      );
         1100  +      if( pNew==0 ){
         1101  +        rc = SQLITE_NOMEM;
         1102  +      }else{
         1103  +        memset(pNew, 0, sizeof(ZipfileEntry));
         1104  +        pNew->zPath = (char*)&pNew[1];
         1105  +        memcpy(pNew->zPath, &aRec[ZIPFILE_CDS_FIXED_SZ], nFile);
         1106  +        pNew->zPath[nFile] = '\0';
         1107  +        pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1];
         1108  +        pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
         1109  +        memcpy(pNew->aCdsEntry, aRec, pNew->nCdsEntry);
         1110  +        zipfileAddEntry(pTab, 0, pNew);
         1111  +      }
         1112  +
         1113  +      iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
         1114  +    }
         1115  +
         1116  +    sqlite3_free(aBuf);
         1117  +  }
         1118  +
         1119  +  return rc;
         1120  +}
         1121  +
         1122  +static ZipfileEntry *zipfileNewEntry(
         1123  +  ZipfileCDS *pCds,               /* Values for fixed size part of CDS */
         1124  +  const char *zPath,              /* Path for new entry */
         1125  +  int nPath,                      /* strlen(zPath) */
         1126  +  u32 mTime                       /* Modification time (or 0) */
         1127  +){
         1128  +  u8 *aWrite;
         1129  +  ZipfileEntry *pNew;
         1130  +  pCds->nFile = (u16)nPath;
         1131  +  pCds->nExtra = mTime ? 9 : 0;
         1132  +  pNew = (ZipfileEntry*)sqlite3_malloc(
         1133  +    sizeof(ZipfileEntry) + 
         1134  +    nPath+1 + 
         1135  +    ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra
         1136  +  );
         1137  +
         1138  +  if( pNew ){
         1139  +    memset(pNew, 0, sizeof(ZipfileEntry));
         1140  +    pNew->zPath = (char*)&pNew[1];
         1141  +    pNew->aCdsEntry = (u8*)&pNew->zPath[nPath+1];
         1142  +    pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra;
         1143  +    memcpy(pNew->zPath, zPath, nPath+1);
         1144  +
         1145  +    aWrite = pNew->aCdsEntry;
         1146  +    zipfileWrite32(aWrite, ZIPFILE_SIGNATURE_CDS);
         1147  +    zipfileWrite16(aWrite, pCds->iVersionMadeBy);
         1148  +    zipfileWrite16(aWrite, pCds->iVersionExtract);
         1149  +    zipfileWrite16(aWrite, pCds->flags);
         1150  +    zipfileWrite16(aWrite, pCds->iCompression);
         1151  +    zipfileWrite16(aWrite, pCds->mTime);
         1152  +    zipfileWrite16(aWrite, pCds->mDate);
         1153  +    zipfileWrite32(aWrite, pCds->crc32);
         1154  +    zipfileWrite32(aWrite, pCds->szCompressed);
         1155  +    zipfileWrite32(aWrite, pCds->szUncompressed);
         1156  +    zipfileWrite16(aWrite, pCds->nFile);
         1157  +    zipfileWrite16(aWrite, pCds->nExtra);
         1158  +    zipfileWrite16(aWrite, pCds->nComment);      assert( pCds->nComment==0 );
         1159  +    zipfileWrite16(aWrite, pCds->iDiskStart);
         1160  +    zipfileWrite16(aWrite, pCds->iInternalAttr);
         1161  +    zipfileWrite32(aWrite, pCds->iExternalAttr);
         1162  +    zipfileWrite32(aWrite, pCds->iOffset);
         1163  +    assert( aWrite==&pNew->aCdsEntry[ZIPFILE_CDS_FIXED_SZ] );
         1164  +    memcpy(aWrite, zPath, nPath);
         1165  +    if( pCds->nExtra ){
         1166  +      aWrite += nPath;
         1167  +      zipfileWrite16(aWrite, ZIPFILE_EXTRA_TIMESTAMP);
         1168  +      zipfileWrite16(aWrite, 5);
         1169  +      *aWrite++ = 0x01;
         1170  +      zipfileWrite32(aWrite, mTime);
         1171  +    }
         1172  +  }
         1173  +
         1174  +  return pNew;
         1175  +}
         1176  +
         1177  +static int zipfileAppendEntry(
         1178  +  ZipfileTab *pTab,
         1179  +  ZipfileCDS *pCds,
         1180  +  const char *zPath,              /* Path for new entry */
         1181  +  int nPath,                      /* strlen(zPath) */
         1182  +  const u8 *pData,
         1183  +  int nData,
         1184  +  u32 mTime
         1185  +){
         1186  +  u8 *aBuf = pTab->aBuffer;
         1187  +  int rc;
         1188  +
         1189  +  zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_LFH);
         1190  +  zipfileWrite16(aBuf, pCds->iVersionExtract);
         1191  +  zipfileWrite16(aBuf, pCds->flags);
         1192  +  zipfileWrite16(aBuf, pCds->iCompression);
         1193  +  zipfileWrite16(aBuf, pCds->mTime);
         1194  +  zipfileWrite16(aBuf, pCds->mDate);
         1195  +  zipfileWrite32(aBuf, pCds->crc32);
         1196  +  zipfileWrite32(aBuf, pCds->szCompressed);
         1197  +  zipfileWrite32(aBuf, pCds->szUncompressed);
         1198  +  zipfileWrite16(aBuf, (u16)nPath);
         1199  +  zipfileWrite16(aBuf, pCds->nExtra);
         1200  +  assert( aBuf==&pTab->aBuffer[ZIPFILE_LFH_FIXED_SZ] );
         1201  +  rc = zipfileAppendData(pTab, pTab->aBuffer, (int)(aBuf - pTab->aBuffer));
         1202  +  if( rc==SQLITE_OK ){
         1203  +    rc = zipfileAppendData(pTab, (const u8*)zPath, nPath);
         1204  +  }
         1205  +
         1206  +  if( rc==SQLITE_OK && pCds->nExtra ){
         1207  +    aBuf = pTab->aBuffer;
         1208  +    zipfileWrite16(aBuf, ZIPFILE_EXTRA_TIMESTAMP);
         1209  +    zipfileWrite16(aBuf, 5);
         1210  +    *aBuf++ = 0x01;
         1211  +    zipfileWrite32(aBuf, mTime);
         1212  +    rc = zipfileAppendData(pTab, pTab->aBuffer, 9);
         1213  +  }
         1214  +
         1215  +  if( rc==SQLITE_OK ){
         1216  +    rc = zipfileAppendData(pTab, pData, nData);
         1217  +  }
         1218  +
         1219  +  return rc;
         1220  +}
         1221  +
         1222  +static int zipfileGetMode(
         1223  +  ZipfileTab *pTab, 
         1224  +  sqlite3_value *pVal, 
         1225  +  u32 defaultMode,                /* Value to use if pVal IS NULL */
         1226  +  u32 *pMode
         1227  +){
         1228  +  const char *z = (const char*)sqlite3_value_text(pVal);
         1229  +  u32 mode = 0;
         1230  +  if( z==0 ){
         1231  +    mode = defaultMode;
         1232  +  }else if( z[0]>='0' && z[0]<='9' ){
         1233  +    mode = (unsigned int)sqlite3_value_int(pVal);
         1234  +  }else{
         1235  +    const char zTemplate[11] = "-rwxrwxrwx";
         1236  +    int i;
         1237  +    if( strl