/ Check-in [77376b33]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge all the latest trunk changes into the apple-osx branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | apple-osx
Files: files | file ages | folders
SHA1: 77376b332b4460208f837f7bdf6df8c90d5071d5
User & Date: drh 2011-08-02 18:25:20
Context
2011-08-23
18:06
Merge latest trunk changes into the apple-osx branch. check-in: c5f7977b user: dan tags: apple-osx
2011-08-02
18:25
Merge all the latest trunk changes into the apple-osx branch. check-in: 77376b33 user: drh tags: apple-osx
01:57
Make sure IS NOT NULL constraints work on virtual tables. Fix for ticket [6c14288a473ceff]. check-in: a55f4ab9 user: drh tags: trunk
2011-06-25
21:43
Changes to make pragma synchronous sticky when SQLITE_DEFAULT_WAL_SAFETYLEVEL is used check-in: c6158b25 user: adam tags: apple-osx
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   527    527   	$(TCLSH_CMD) $(TOP)/tool/vdbe-compress.tcl <tsrc/vdbe.c >vdbe.new
   528    528   	mv vdbe.new tsrc/vdbe.c
   529    529   	touch .target_source
   530    530   
   531    531   sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl
   532    532   	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl
   533    533   
          534  +sqlite3-all.c:	sqlite3.c $(TOP)/tool/split-sqlite3c.tcl
          535  +	$(TCLSH_CMD) $(TOP)/tool/split-sqlite3c.tcl
          536  +
   534    537   # Rule to build the amalgamation
   535    538   #
   536    539   sqlite3.lo:	sqlite3.c
   537    540   	$(LTCOMPILE) $(TEMP_STORE) -c sqlite3.c
   538    541   
   539    542   # Rules to build the LEMON compiler generator
   540    543   #
................................................................................
   795    798   	cp $(TOP)/src/parse.y .
   796    799   	rm -f parse.h
   797    800   	./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y
   798    801   	mv parse.h parse.h.temp
   799    802   	$(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h
   800    803   
   801    804   sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION
   802         -	tclsh $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
          805  +	$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
   803    806   
   804    807   keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
   805    808   	$(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c
   806    809   	./mkkeywordhash$(BEXE) >keywordhash.h
   807    810   
   808    811   
   809    812   

Changes to Makefile.msc.

    39     39   
    40     40   # Define -DNDEBUG to compile without debugging (i.e., for production usage)
    41     41   # Omitting the define will cause extra debugging code to be inserted and
    42     42   # includes extra comments when "EXPLAIN stmt" is used.
    43     43   #
    44     44   TCC = $(TCC) -DNDEBUG
    45     45   
    46         -# The library that programs using TCL must link against.
           46  +# The locations of the Tcl header and library files.  Also, the library that
           47  +# non-stubs enabled programs using Tcl must link against.  These variables
           48  +# (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment
           49  +# prior to running nmake in order to match the actual installed location and
           50  +# version on this machine.
    47     51   #
    48         -LIBTCL = tcl85.lib
           52  +!if "$(TCLINCDIR)" == ""
    49     53   TCLINCDIR = c:\tcl\include
           54  +!endif
           55  +
           56  +!if "$(TCLLIBDIR)" == ""
    50     57   TCLLIBDIR = c:\tcl\lib
           58  +!endif
           59  +
           60  +!if "$(LIBTCL)" == ""
           61  +LIBTCL = tcl85.lib
           62  +!endif
    51     63   
    52     64   # This is the command to use for tclsh - normally just "tclsh", but we may
    53         -# know the specific version we want to use
           65  +# know the specific version we want to use.  This variable (TCLSH_CMD) may be
           66  +# overridden via the environment prior to running nmake in order to select a
           67  +# specific Tcl shell to use.
    54     68   #
           69  +!if "$(TCLSH_CMD)" == ""
    55     70   TCLSH_CMD = tclsh85
           71  +!endif
    56     72   
    57     73   # Compiler options needed for programs that use the readline() library.
    58     74   #
    59     75   READLINE_FLAGS = -DHAVE_READLINE=0
    60     76   
    61     77   # The library that programs using readline() must link against.
    62     78   #
................................................................................
   114    130   # to deduce the binary type based on the object files.
   115    131   !IF "$(PLATFORM)"!=""
   116    132   LTLINKOPTS = /MACHINE:$(PLATFORM)
   117    133   LTLIBOPTS = /MACHINE:$(PLATFORM)
   118    134   !ENDIF
   119    135   
   120    136   # nawk compatible awk.
   121         -NAWK = .\gawk.exe
          137  +NAWK = gawk.exe
   122    138   
   123    139   # You should not have to change anything below this line
   124    140   ###############################################################################
   125    141   
   126    142   # Object files for the SQLite library (non-amalgamation).
   127    143   #
   128    144   LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
................................................................................
   435    451     $(TOP)\ext\icu\sqliteicu.h
   436    452   EXTHDR = $(EXTHDR) \
   437    453     $(TOP)\ext\rtree\sqlite3rtree.h
   438    454   
   439    455   # This is the default Makefile target.  The objects listed here
   440    456   # are what get build when you type just "make" with no arguments.
   441    457   #
   442         -all:	libsqlite3.lib sqlite3.exe libtclsqlite3.lib
          458  +all:	dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib
   443    459   
   444    460   libsqlite3.lib:	$(LIBOBJ)
   445    461   	$(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS)
   446    462   
   447    463   libtclsqlite3.lib:	tclsqlite.lo libsqlite3.lib
   448    464   	$(LTLIB) $(LTLIBOPTS) /LIBPATH:$(TCLLIBDIR) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCL:tcl=tclstub) $(TLIBS)
   449    465   
................................................................................
   459    475   # all that automatic generation.
   460    476   #
   461    477   .target_source:	$(SRC) $(TOP)\tool\vdbe-compress.tcl
   462    478   	-rmdir /S/Q tsrc
   463    479   	-mkdir tsrc
   464    480   	for %i in ($(SRC)) do copy /Y %i tsrc
   465    481   	del /Q tsrc\sqlite.h.in tsrc\parse.y
   466         -	$(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl <tsrc\vdbe.c >vdbe.new
          482  +	$(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl < tsrc\vdbe.c > vdbe.new
   467    483   	move vdbe.new tsrc\vdbe.c
   468    484   	echo > .target_source
   469    485   
   470    486   sqlite3.c:	.target_source $(TOP)\tool\mksqlite3c.tcl
   471    487   	$(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl
          488  +
          489  +sqlite3-all.c:	sqlite3.c $(TOP)/tool/split-sqlite3c.tcl
          490  +	$(TCLSH_CMD) $(TOP)/tool/split-sqlite3c.tcl
   472    491   
   473    492   # Rule to build the amalgamation
   474    493   #
   475    494   sqlite3.lo:	sqlite3.c
   476    495   	$(LTCOMPILE) -c sqlite3.c
   477    496   
   478    497   # Rules to build the LEMON compiler generator
................................................................................
   716    735   tclsqlite3.exe:	tclsqlite-shell.lo libsqlite3.lib
   717    736   	$(LTLINK) tclsqlite-shell.lo \
   718    737   		/link $(LTLINKOPTS) /LIBPATH:$(TCLLIBDIR) libsqlite3.lib $(LIBTCL)
   719    738   
   720    739   # Rules to build opcodes.c and opcodes.h
   721    740   #
   722    741   opcodes.c:	opcodes.h $(TOP)\mkopcodec.awk
   723         -	$(NAWK) "/#define OP_/ { print }" opcodes.h | sort /+45 | $(NAWK) -f $(TOP)\mkopcodec.awk >opcodes.c
          742  +	$(NAWK) "/#define OP_/ { print }" opcodes.h | sort /+45 | $(NAWK) -f $(TOP)\mkopcodec.awk > opcodes.c
   724    743   
   725    744   opcodes.h:	parse.h $(TOP)\src\vdbe.c $(TOP)\mkopcodeh.awk
   726         -	type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk >opcodes.h
          745  +	type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk > opcodes.h
   727    746   
   728    747   # Rules to build parse.c and parse.h - the outputs of lemon.
   729    748   #
   730    749   parse.h:	parse.c
   731    750   
   732    751   parse.c:	$(TOP)\src\parse.y lemon.exe $(TOP)\addopcodes.awk
   733    752   	del /Q parse.y parse.h parse.h.temp
   734    753   	copy $(TOP)\src\parse.y .
   735    754   	.\lemon.exe $(OPT_FEATURE_FLAGS) $(OPTS) parse.y
   736    755   	move parse.h parse.h.temp
   737         -	$(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp >parse.h
          756  +	$(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp > parse.h
   738    757   
   739    758   sqlite3.h:	$(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION
   740         -	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP) >sqlite3.h
          759  +	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP) > sqlite3.h
   741    760   
   742    761   mkkeywordhash.exe:	$(TOP)\tool\mkkeywordhash.c
   743    762   	$(BCC) -Femkkeywordhash.exe $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)\tool\mkkeywordhash.c
   744    763   
   745    764   keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
   746         -	.\mkkeywordhash.exe >keywordhash.h
          765  +	.\mkkeywordhash.exe > keywordhash.h
   747    766   
   748    767   
   749    768   
   750    769   # Rules to build the extension objects.
   751    770   #
   752    771   icu.lo:	$(TOP)\ext\icu\icu.c $(HDR) $(EXTHDR)
   753    772   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\icu\icu.c
................................................................................
   835    854   	.\testfixture.exe $(TOP)\test\all.test -soak=1
   836    855   
   837    856   test:	testfixture.exe sqlite3.exe
   838    857   	.\testfixture.exe $(TOP)\test\veryquick.test
   839    858   
   840    859   spaceanal_tcl.h:	$(TOP)\tool\spaceanal.tcl
   841    860   	$(NAWK) -f $(TOP)/tool/tostr.awk \
   842         -		$(TOP)\tool\spaceanal.tcl >spaceanal_tcl.h
          861  +		$(TOP)\tool\spaceanal.tcl > spaceanal_tcl.h
   843    862   
   844    863   sqlite3_analyzer.exe:	$(TESTFIXTURE_SRC) spaceanal_tcl.h
   845    864   	$(LTLINK) -DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 \
   846    865   		-DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE \
   847    866   		-DBUILD_sqlite -I$(TCLINCDIR) \
   848    867   		$(TESTFIXTURE_SRC) \
   849    868   		/link $(LTLINKOPTS) /LIBPATH:$(TCLLIBDIR) $(LIBTCL) $(TLIBS)
................................................................................
   862    881   
   863    882   #
   864    883   # Windows section
   865    884   #
   866    885   dll: sqlite3.dll
   867    886   
   868    887   sqlite3.def: libsqlite3.lib
   869         -	echo EXPORTS >sqlite3.def
          888  +	echo EXPORTS > sqlite3.def
   870    889   	dumpbin /all libsqlite3.lib \
   871    890   		| $(NAWK) "/ 1 _sqlite3_/ { sub(/^.* _/,\"\");print }" \
   872         -		| sort >>sqlite3.def
          891  +		| sort >> sqlite3.def
   873    892   
   874    893   sqlite3.dll: $(LIBOBJ) sqlite3.def
   875    894   	link $(LTLINKOPTS) /DLL /DEF:sqlite3.def /OUT:$@ $(LIBOBJ)

Changes to VERSION.

     1         -3.7.7
            1  +3.7.8

Changes to configure.

     1      1   #! /bin/sh
     2      2   # Guess values for system-dependent variables and create Makefiles.
     3         -# Generated by GNU Autoconf 2.62 for sqlite 3.7.7.
            3  +# Generated by GNU Autoconf 2.62 for sqlite 3.7.8.
     4      4   #
     5      5   # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
     6      6   # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
     7      7   # This configure script is free software; the Free Software Foundation
     8      8   # gives unlimited permission to copy, distribute and modify it.
     9      9   ## --------------------- ##
    10     10   ## M4sh Initialization.  ##
................................................................................
   739    739   MFLAGS=
   740    740   MAKEFLAGS=
   741    741   SHELL=${CONFIG_SHELL-/bin/sh}
   742    742   
   743    743   # Identity of this package.
   744    744   PACKAGE_NAME='sqlite'
   745    745   PACKAGE_TARNAME='sqlite'
   746         -PACKAGE_VERSION='3.7.7'
   747         -PACKAGE_STRING='sqlite 3.7.7'
          746  +PACKAGE_VERSION='3.7.8'
          747  +PACKAGE_STRING='sqlite 3.7.8'
   748    748   PACKAGE_BUGREPORT=''
   749    749   
   750    750   # Factoring default headers for most tests.
   751    751   ac_includes_default="\
   752    752   #include <stdio.h>
   753    753   #ifdef HAVE_SYS_TYPES_H
   754    754   # include <sys/types.h>
................................................................................
  1481   1481   #
  1482   1482   # Report the --help message.
  1483   1483   #
  1484   1484   if test "$ac_init_help" = "long"; then
  1485   1485     # Omit some internal or obsolete options to make the list less imposing.
  1486   1486     # This message is too long to be a string in the A/UX 3.1 sh.
  1487   1487     cat <<_ACEOF
  1488         -\`configure' configures sqlite 3.7.7 to adapt to many kinds of systems.
         1488  +\`configure' configures sqlite 3.7.8 to adapt to many kinds of systems.
  1489   1489   
  1490   1490   Usage: $0 [OPTION]... [VAR=VALUE]...
  1491   1491   
  1492   1492   To assign environment variables (e.g., CC, CFLAGS...), specify them as
  1493   1493   VAR=VALUE.  See below for descriptions of some of the useful variables.
  1494   1494   
  1495   1495   Defaults for the options are specified in brackets.
................................................................................
  1546   1546     --build=BUILD     configure for building on BUILD [guessed]
  1547   1547     --host=HOST       cross-compile to build programs to run on HOST [BUILD]
  1548   1548   _ACEOF
  1549   1549   fi
  1550   1550   
  1551   1551   if test -n "$ac_init_help"; then
  1552   1552     case $ac_init_help in
  1553         -     short | recursive ) echo "Configuration of sqlite 3.7.7:";;
         1553  +     short | recursive ) echo "Configuration of sqlite 3.7.8:";;
  1554   1554      esac
  1555   1555     cat <<\_ACEOF
  1556   1556   
  1557   1557   Optional Features:
  1558   1558     --disable-option-checking  ignore unrecognized --enable/--with options
  1559   1559     --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  1560   1560     --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
  1662   1662       cd "$ac_pwd" || { ac_status=$?; break; }
  1663   1663     done
  1664   1664   fi
  1665   1665   
  1666   1666   test -n "$ac_init_help" && exit $ac_status
  1667   1667   if $ac_init_version; then
  1668   1668     cat <<\_ACEOF
  1669         -sqlite configure 3.7.7
         1669  +sqlite configure 3.7.8
  1670   1670   generated by GNU Autoconf 2.62
  1671   1671   
  1672   1672   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
  1673   1673   2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
  1674   1674   This configure script is free software; the Free Software Foundation
  1675   1675   gives unlimited permission to copy, distribute and modify it.
  1676   1676   _ACEOF
  1677   1677     exit
  1678   1678   fi
  1679   1679   cat >config.log <<_ACEOF
  1680   1680   This file contains any messages produced by compilers while
  1681   1681   running configure, to aid debugging if configure makes a mistake.
  1682   1682   
  1683         -It was created by sqlite $as_me 3.7.7, which was
         1683  +It was created by sqlite $as_me 3.7.8, which was
  1684   1684   generated by GNU Autoconf 2.62.  Invocation command line was
  1685   1685   
  1686   1686     $ $0 $@
  1687   1687   
  1688   1688   _ACEOF
  1689   1689   exec 5>>config.log
  1690   1690   {
................................................................................
 14026  14026   
 14027  14027   exec 6>&1
 14028  14028   
 14029  14029   # Save the log message, to keep $[0] and so on meaningful, and to
 14030  14030   # report actual input values of CONFIG_FILES etc. instead of their
 14031  14031   # values after options handling.
 14032  14032   ac_log="
 14033         -This file was extended by sqlite $as_me 3.7.7, which was
        14033  +This file was extended by sqlite $as_me 3.7.8, which was
 14034  14034   generated by GNU Autoconf 2.62.  Invocation command line was
 14035  14035   
 14036  14036     CONFIG_FILES    = $CONFIG_FILES
 14037  14037     CONFIG_HEADERS  = $CONFIG_HEADERS
 14038  14038     CONFIG_LINKS    = $CONFIG_LINKS
 14039  14039     CONFIG_COMMANDS = $CONFIG_COMMANDS
 14040  14040     $ $0 $@
................................................................................
 14079  14079   $config_commands
 14080  14080   
 14081  14081   Report bugs to <bug-autoconf@gnu.org>."
 14082  14082   
 14083  14083   _ACEOF
 14084  14084   cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 14085  14085   ac_cs_version="\\
 14086         -sqlite config.status 3.7.7
        14086  +sqlite config.status 3.7.8
 14087  14087   configured by $0, generated by GNU Autoconf 2.62,
 14088  14088     with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\"
 14089  14089   
 14090  14090   Copyright (C) 2008 Free Software Foundation, Inc.
 14091  14091   This config.status script is free software; the Free Software Foundation
 14092  14092   gives unlimited permission to copy, distribute and modify it."
 14093  14093   

Changes to doc/lemon.html.

   278    278   tokens) and it honors the same commenting conventions as C and C++.</p>
   279    279   
   280    280   <h3>Terminals and Nonterminals</h3>
   281    281   
   282    282   <p>A terminal symbol (token) is any string of alphanumeric
   283    283   and underscore characters
   284    284   that begins with an upper case letter.
   285         -A terminal can contain lower class letters after the first character,
          285  +A terminal can contain lowercase letters after the first character,
   286    286   but the usual convention is to make terminals all upper case.
   287    287   A nonterminal, on the other hand, is any string of alphanumeric
   288    288   and underscore characters than begins with a lower case letter.
   289    289   Again, the usual convention is to make nonterminals use all lower
   290    290   case letters.</p>
   291    291   
   292    292   <p>In Lemon, terminal and nonterminal symbols do not need to 

Changes to ext/fts3/fts3.c.

   308    308   
   309    309   #include "fts3.h"
   310    310   #ifndef SQLITE_CORE 
   311    311   # include "sqlite3ext.h"
   312    312     SQLITE_EXTENSION_INIT1
   313    313   #endif
   314    314   
          315  +static int fts3EvalNext(Fts3Cursor *pCsr);
          316  +static int fts3EvalStart(Fts3Cursor *pCsr);
          317  +static int fts3TermSegReaderCursor(
          318  +    Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **);
          319  +
   315    320   /* 
   316    321   ** Write a 64-bit variable-length integer to memory starting at p[0].
   317    322   ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes.
   318    323   ** The number of bytes written is returned.
   319    324   */
   320    325   int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){
   321    326     unsigned char *q = (unsigned char *) p;
................................................................................
   816    821     for(i=0; i<p->nColumn; i++){
   817    822       fts3Appendf(pRc, &zRet, ",%s(?)", zFunction);
   818    823     }
   819    824     sqlite3_free(zFree);
   820    825     return zRet;
   821    826   }
   822    827   
          828  +/*
          829  +** This function interprets the string at (*pp) as a non-negative integer
          830  +** value. It reads the integer and sets *pnOut to the value read, then 
          831  +** sets *pp to point to the byte immediately following the last byte of
          832  +** the integer value.
          833  +**
          834  +** Only decimal digits ('0'..'9') may be part of an integer value. 
          835  +**
          836  +** If *pp does not being with a decimal digit SQLITE_ERROR is returned and
          837  +** the output value undefined. Otherwise SQLITE_OK is returned.
          838  +**
          839  +** This function is used when parsing the "prefix=" FTS4 parameter.
          840  +*/
   823    841   static int fts3GobbleInt(const char **pp, int *pnOut){
   824         -  const char *p = *pp;
   825         -  int nInt = 0;
          842  +  const char *p = *pp;            /* Iterator pointer */
          843  +  int nInt = 0;                   /* Output value */
          844  +
   826    845     for(p=*pp; p[0]>='0' && p[0]<='9'; p++){
   827    846       nInt = nInt * 10 + (p[0] - '0');
   828    847     }
   829    848     if( p==*pp ) return SQLITE_ERROR;
   830    849     *pnOut = nInt;
   831    850     *pp = p;
   832    851     return SQLITE_OK;
   833    852   }
   834    853   
   835         -
          854  +/*
          855  +** This function is called to allocate an array of Fts3Index structures
          856  +** representing the indexes maintained by the current FTS table. FTS tables
          857  +** always maintain the main "terms" index, but may also maintain one or
          858  +** more "prefix" indexes, depending on the value of the "prefix=" parameter
          859  +** (if any) specified as part of the CREATE VIRTUAL TABLE statement.
          860  +**
          861  +** Argument zParam is passed the value of the "prefix=" option if one was
          862  +** specified, or NULL otherwise.
          863  +**
          864  +** If no error occurs, SQLITE_OK is returned and *apIndex set to point to
          865  +** the allocated array. *pnIndex is set to the number of elements in the
          866  +** array. If an error does occur, an SQLite error code is returned.
          867  +**
          868  +** Regardless of whether or not an error is returned, it is the responsibility
          869  +** of the caller to call sqlite3_free() on the output array to free it.
          870  +*/
   836    871   static int fts3PrefixParameter(
   837    872     const char *zParam,             /* ABC in prefix=ABC parameter to parse */
   838    873     int *pnIndex,                   /* OUT: size of *apIndex[] array */
   839         -  struct Fts3Index **apIndex,     /* OUT: Array of indexes for this table */
   840         -  struct Fts3Index **apFree       /* OUT: Free this with sqlite3_free() */
          874  +  struct Fts3Index **apIndex      /* OUT: Array of indexes for this table */
   841    875   ){
   842         -  struct Fts3Index *aIndex;
   843         -  int nIndex = 1;
          876  +  struct Fts3Index *aIndex;       /* Allocated array */
          877  +  int nIndex = 1;                 /* Number of entries in array */
   844    878   
   845    879     if( zParam && zParam[0] ){
   846    880       const char *p;
   847    881       nIndex++;
   848    882       for(p=zParam; *p; p++){
   849    883         if( *p==',' ) nIndex++;
   850    884       }
   851    885     }
   852    886   
   853    887     aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex);
   854         -  *apIndex = *apFree = aIndex;
          888  +  *apIndex = aIndex;
   855    889     *pnIndex = nIndex;
   856    890     if( !aIndex ){
   857    891       return SQLITE_NOMEM;
   858    892     }
   859    893   
   860    894     memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex);
   861    895     if( zParam ){
................................................................................
   904    938     int nDb;                        /* Bytes required to hold database name */
   905    939     int nName;                      /* Bytes required to hold table name */
   906    940     int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
   907    941     const char **aCol;              /* Array of column names */
   908    942     sqlite3_tokenizer *pTokenizer = 0;        /* Tokenizer for this table */
   909    943   
   910    944     int nIndex;                     /* Size of aIndex[] array */
   911         -  struct Fts3Index *aIndex;       /* Array of indexes for this table */
   912         -  struct Fts3Index *aFree = 0;    /* Free this before returning */
          945  +  struct Fts3Index *aIndex = 0;   /* Array of indexes for this table */
   913    946   
   914    947     /* The results of parsing supported FTS4 key=value options: */
   915    948     int bNoDocsize = 0;             /* True to omit %_docsize table */
   916    949     int bDescIdx = 0;               /* True to store descending indexes */
   917    950     char *zPrefix = 0;              /* Prefix parameter value (or NULL) */
   918    951     char *zCompress = 0;            /* compress=? parameter (or NULL) */
   919    952     char *zUncompress = 0;          /* uncompress=? parameter (or NULL) */
................................................................................
  1042   1075   
  1043   1076     if( pTokenizer==0 ){
  1044   1077       rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
  1045   1078       if( rc!=SQLITE_OK ) goto fts3_init_out;
  1046   1079     }
  1047   1080     assert( pTokenizer );
  1048   1081   
  1049         -  rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree);
         1082  +  rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex);
  1050   1083     if( rc==SQLITE_ERROR ){
  1051   1084       assert( zPrefix );
  1052   1085       *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix);
  1053   1086     }
  1054   1087     if( rc!=SQLITE_OK ) goto fts3_init_out;
  1055   1088   
  1056   1089     /* Allocate and populate the Fts3Table structure. */
................................................................................
  1129   1162     p->nNodeSize = p->nPgsz-35;
  1130   1163   
  1131   1164     /* Declare the table schema to SQLite. */
  1132   1165     fts3DeclareVtab(&rc, p);
  1133   1166   
  1134   1167   fts3_init_out:
  1135   1168     sqlite3_free(zPrefix);
  1136         -  sqlite3_free(aFree);
         1169  +  sqlite3_free(aIndex);
  1137   1170     sqlite3_free(zCompress);
  1138   1171     sqlite3_free(zUncompress);
  1139   1172     sqlite3_free((void *)aCol);
  1140   1173     if( rc!=SQLITE_OK ){
  1141   1174       if( p ){
  1142   1175         fts3DisconnectMethod((sqlite3_vtab *)p);
  1143   1176       }else if( pTokenizer ){
................................................................................
  1720   1753     *p++ = POS_END;
  1721   1754     *pp = p;
  1722   1755     *pp1 = p1 + 1;
  1723   1756     *pp2 = p2 + 1;
  1724   1757   }
  1725   1758   
  1726   1759   /*
  1727         -** nToken==1 searches for adjacent positions.
  1728         -**
  1729   1760   ** This function is used to merge two position lists into one. When it is
  1730   1761   ** called, *pp1 and *pp2 must both point to position lists. A position-list is
  1731   1762   ** the part of a doclist that follows each document id. For example, if a row
  1732   1763   ** contains:
  1733   1764   **
  1734   1765   **     'a b c'|'x y z'|'a b b a'
  1735   1766   **
................................................................................
  1741   1772   ** byte following the 0x00 terminator of their respective position lists.
  1742   1773   **
  1743   1774   ** If isSaveLeft is 0, an entry is added to the output position list for 
  1744   1775   ** each position in *pp2 for which there exists one or more positions in
  1745   1776   ** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e.
  1746   1777   ** when the *pp1 token appears before the *pp2 token, but not more than nToken
  1747   1778   ** slots before it.
         1779  +**
         1780  +** e.g. nToken==1 searches for adjacent positions.
  1748   1781   */
  1749   1782   static int fts3PoslistPhraseMerge(
  1750   1783     char **pp,                      /* IN/OUT: Preallocated output buffer */
  1751   1784     int nToken,                     /* Maximum difference in token positions */
  1752   1785     int isSaveLeft,                 /* Save the left position */
  1753   1786     int isExact,                    /* If *pp1 is exactly nTokens before *pp2 */
  1754   1787     char **pp1,                     /* IN/OUT: Left input list */
................................................................................
  1907   1940       res = 0;
  1908   1941     }
  1909   1942   
  1910   1943     return res;
  1911   1944   }
  1912   1945   
  1913   1946   /* 
  1914         -** A pointer to an instance of this structure is used as the context 
  1915         -** argument to sqlite3Fts3SegReaderIterate()
         1947  +** An instance of this function is used to merge together the (potentially
         1948  +** large number of) doclists for each term that matches a prefix query.
         1949  +** See function fts3TermSelectMerge() for details.
  1916   1950   */
  1917   1951   typedef struct TermSelect TermSelect;
  1918   1952   struct TermSelect {
  1919         -  int isReqPos;
  1920         -  char *aaOutput[16];             /* Malloc'd output buffer */
  1921         -  int anOutput[16];               /* Size of output in bytes */
         1953  +  char *aaOutput[16];             /* Malloc'd output buffers */
         1954  +  int anOutput[16];               /* Size each output buffer in bytes */
  1922   1955   };
  1923   1956   
  1924         -
         1957  +/*
         1958  +** This function is used to read a single varint from a buffer. Parameter
         1959  +** pEnd points 1 byte past the end of the buffer. When this function is
         1960  +** called, if *pp points to pEnd or greater, then the end of the buffer
         1961  +** has been reached. In this case *pp is set to 0 and the function returns.
         1962  +**
         1963  +** If *pp does not point to or past pEnd, then a single varint is read
         1964  +** from *pp. *pp is then set to point 1 byte past the end of the read varint.
         1965  +**
         1966  +** If bDescIdx is false, the value read is added to *pVal before returning.
         1967  +** If it is true, the value read is subtracted from *pVal before this 
         1968  +** function returns.
         1969  +*/
  1925   1970   static void fts3GetDeltaVarint3(
  1926         -  char **pp, 
  1927         -  char *pEnd, 
  1928         -  int bDescIdx,
  1929         -  sqlite3_int64 *pVal
         1971  +  char **pp,                      /* IN/OUT: Point to read varint from */
         1972  +  char *pEnd,                     /* End of buffer */
         1973  +  int bDescIdx,                   /* True if docids are descending */
         1974  +  sqlite3_int64 *pVal             /* IN/OUT: Integer value */
  1930   1975   ){
  1931   1976     if( *pp>=pEnd ){
  1932   1977       *pp = 0;
  1933   1978     }else{
  1934   1979       sqlite3_int64 iVal;
  1935   1980       *pp += sqlite3Fts3GetVarint(*pp, &iVal);
  1936   1981       if( bDescIdx ){
................................................................................
  1937   1982         *pVal -= iVal;
  1938   1983       }else{
  1939   1984         *pVal += iVal;
  1940   1985       }
  1941   1986     }
  1942   1987   }
  1943   1988   
         1989  +/*
         1990  +** This function is used to write a single varint to a buffer. The varint
         1991  +** is written to *pp. Before returning, *pp is set to point 1 byte past the
         1992  +** end of the value written.
         1993  +**
         1994  +** If *pbFirst is zero when this function is called, the value written to
         1995  +** the buffer is that of parameter iVal. 
         1996  +**
         1997  +** If *pbFirst is non-zero when this function is called, then the value 
         1998  +** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal)
         1999  +** (if bDescIdx is non-zero).
         2000  +**
         2001  +** Before returning, this function always sets *pbFirst to 1 and *piPrev
         2002  +** to the value of parameter iVal.
         2003  +*/
  1944   2004   static void fts3PutDeltaVarint3(
  1945   2005     char **pp,                      /* IN/OUT: Output pointer */
  1946   2006     int bDescIdx,                   /* True for descending docids */
  1947   2007     sqlite3_int64 *piPrev,          /* IN/OUT: Previous value written to list */
  1948   2008     int *pbFirst,                   /* IN/OUT: True after first int written */
  1949   2009     sqlite3_int64 iVal              /* Write this value to the list */
  1950   2010   ){
................................................................................
  1957   2017     assert( *pbFirst || *piPrev==0 );
  1958   2018     assert( *pbFirst==0 || iWrite>0 );
  1959   2019     *pp += sqlite3Fts3PutVarint(*pp, iWrite);
  1960   2020     *piPrev = iVal;
  1961   2021     *pbFirst = 1;
  1962   2022   }
  1963   2023   
  1964         -#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2))
  1965   2024   
         2025  +/*
         2026  +** This macro is used by various functions that merge doclists. The two
         2027  +** arguments are 64-bit docid values. If the value of the stack variable
         2028  +** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). 
         2029  +** Otherwise, (i2-i1).
         2030  +**
         2031  +** Using this makes it easier to write code that can merge doclists that are
         2032  +** sorted in either ascending or descending order.
         2033  +*/
         2034  +#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2))
         2035  +
         2036  +/*
         2037  +** This function does an "OR" merge of two doclists (output contains all
         2038  +** positions contained in either argument doclist). If the docids in the 
         2039  +** input doclists are sorted in ascending order, parameter bDescDoclist
         2040  +** should be false. If they are sorted in ascending order, it should be
         2041  +** passed a non-zero value.
         2042  +**
         2043  +** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer
         2044  +** containing the output doclist and SQLITE_OK is returned. In this case
         2045  +** *pnOut is set to the number of bytes in the output doclist.
         2046  +**
         2047  +** If an error occurs, an SQLite error code is returned. The output values
         2048  +** are undefined in this case.
         2049  +*/
  1966   2050   static int fts3DoclistOrMerge(
  1967         -  int bDescIdx,                   /* True if arguments are desc */
         2051  +  int bDescDoclist,               /* True if arguments are desc */
  1968   2052     char *a1, int n1,               /* First doclist */
  1969   2053     char *a2, int n2,               /* Second doclist */
  1970   2054     char **paOut, int *pnOut        /* OUT: Malloc'd doclist */
  1971   2055   ){
  1972   2056     sqlite3_int64 i1 = 0;
  1973   2057     sqlite3_int64 i2 = 0;
  1974   2058     sqlite3_int64 iPrev = 0;
................................................................................
  1985   2069     aOut = sqlite3_malloc(n1+n2);
  1986   2070     if( !aOut ) return SQLITE_NOMEM;
  1987   2071   
  1988   2072     p = aOut;
  1989   2073     fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
  1990   2074     fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
  1991   2075     while( p1 || p2 ){
  1992         -    sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
         2076  +    sqlite3_int64 iDiff = DOCID_CMP(i1, i2);
  1993   2077   
  1994   2078       if( p2 && p1 && iDiff==0 ){
  1995         -      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
         2079  +      fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
  1996   2080         fts3PoslistMerge(&p, &p1, &p2);
  1997         -      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
  1998         -      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
         2081  +      fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
         2082  +      fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
  1999   2083       }else if( !p2 || (p1 && iDiff<0) ){
  2000         -      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
         2084  +      fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
  2001   2085         fts3PoslistCopy(&p, &p1);
  2002         -      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
         2086  +      fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
  2003   2087       }else{
  2004         -      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2);
         2088  +      fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2);
  2005   2089         fts3PoslistCopy(&p, &p2);
  2006         -      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
         2090  +      fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
  2007   2091       }
  2008   2092     }
  2009   2093   
  2010   2094     *paOut = aOut;
  2011   2095     *pnOut = (p-aOut);
  2012   2096     return SQLITE_OK;
  2013   2097   }
  2014   2098   
         2099  +/*
         2100  +** This function does a "phrase" merge of two doclists. In a phrase merge,
         2101  +** the output contains a copy of each position from the right-hand input
         2102  +** doclist for which there is a position in the left-hand input doclist
         2103  +** exactly nDist tokens before it.
         2104  +**
         2105  +** If the docids in the input doclists are sorted in ascending order,
         2106  +** parameter bDescDoclist should be false. If they are sorted in ascending 
         2107  +** order, it should be passed a non-zero value.
         2108  +**
         2109  +** The right-hand input doclist is overwritten by this function.
         2110  +*/
  2015   2111   static void fts3DoclistPhraseMerge(
  2016         -  int bDescIdx,                   /* True if arguments are desc */
         2112  +  int bDescDoclist,               /* True if arguments are desc */
  2017   2113     int nDist,                      /* Distance from left to right (1=adjacent) */
  2018   2114     char *aLeft, int nLeft,         /* Left doclist */
  2019   2115     char *aRight, int *pnRight      /* IN/OUT: Right/output doclist */
  2020   2116   ){
  2021   2117     sqlite3_int64 i1 = 0;
  2022   2118     sqlite3_int64 i2 = 0;
  2023   2119     sqlite3_int64 iPrev = 0;
................................................................................
  2032   2128     assert( nDist>0 );
  2033   2129   
  2034   2130     p = aOut;
  2035   2131     fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
  2036   2132     fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
  2037   2133   
  2038   2134     while( p1 && p2 ){
  2039         -    sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
         2135  +    sqlite3_int64 iDiff = DOCID_CMP(i1, i2);
  2040   2136       if( iDiff==0 ){
  2041   2137         char *pSave = p;
  2042   2138         sqlite3_int64 iPrevSave = iPrev;
  2043   2139         int bFirstOutSave = bFirstOut;
  2044   2140   
  2045         -      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
         2141  +      fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
  2046   2142         if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){
  2047   2143           p = pSave;
  2048   2144           iPrev = iPrevSave;
  2049   2145           bFirstOut = bFirstOutSave;
  2050   2146         }
  2051         -      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
  2052         -      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
         2147  +      fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
         2148  +      fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
  2053   2149       }else if( iDiff<0 ){
  2054   2150         fts3PoslistCopy(0, &p1);
  2055         -      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
         2151  +      fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
  2056   2152       }else{
  2057   2153         fts3PoslistCopy(0, &p2);
  2058         -      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
         2154  +      fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
  2059   2155       }
  2060   2156     }
  2061   2157   
  2062   2158     *pnRight = p - aOut;
  2063   2159   }
  2064   2160   
  2065   2161   
................................................................................
  2068   2164   ** doclist stored in TermSelect.aaOutput[0]. If successful, delete all
  2069   2165   ** other doclists (except the aaOutput[0] one) and return SQLITE_OK.
  2070   2166   **
  2071   2167   ** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
  2072   2168   ** the responsibility of the caller to free any doclists left in the
  2073   2169   ** TermSelect.aaOutput[] array.
  2074   2170   */
  2075         -static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){
         2171  +static int fts3TermSelectFinishMerge(Fts3Table *p, TermSelect *pTS){
  2076   2172     char *aOut = 0;
  2077   2173     int nOut = 0;
  2078   2174     int i;
  2079   2175   
  2080   2176     /* Loop through the doclists in the aaOutput[] array. Merge them all
  2081   2177     ** into a single doclist.
  2082   2178     */
................................................................................
  2109   2205   
  2110   2206     pTS->aaOutput[0] = aOut;
  2111   2207     pTS->anOutput[0] = nOut;
  2112   2208     return SQLITE_OK;
  2113   2209   }
  2114   2210   
  2115   2211   /*
  2116         -** This function is used as the sqlite3Fts3SegReaderIterate() callback when
  2117         -** querying the full-text index for a doclist associated with a term or
  2118         -** term-prefix.
         2212  +** Merge the doclist aDoclist/nDoclist into the TermSelect object passed
         2213  +** as the first argument. The merge is an "OR" merge (see function
         2214  +** fts3DoclistOrMerge() for details).
         2215  +**
         2216  +** This function is called with the doclist for each term that matches
         2217  +** a queried prefix. It merges all these doclists into one, the doclist
         2218  +** for the specified prefix. Since there can be a very large number of
         2219  +** doclists to merge, the merging is done pair-wise using the TermSelect
         2220  +** object.
         2221  +**
         2222  +** This function returns SQLITE_OK if the merge is successful, or an
         2223  +** SQLite error code (SQLITE_NOMEM) if an error occurs.
  2119   2224   */
  2120         -static int fts3TermSelectCb(
  2121         -  Fts3Table *p,                   /* Virtual table object */
  2122         -  void *pContext,                 /* Pointer to TermSelect structure */
  2123         -  char *zTerm,
  2124         -  int nTerm,
  2125         -  char *aDoclist,
  2126         -  int nDoclist
         2225  +static int fts3TermSelectMerge(
         2226  +  Fts3Table *p,                   /* FTS table handle */
         2227  +  TermSelect *pTS,                /* TermSelect object to merge into */
         2228  +  char *aDoclist,                 /* Pointer to doclist */
         2229  +  int nDoclist                    /* Size of aDoclist in bytes */
  2127   2230   ){
  2128         -  TermSelect *pTS = (TermSelect *)pContext;
  2129         -
  2130         -  UNUSED_PARAMETER(p);
  2131         -  UNUSED_PARAMETER(zTerm);
  2132         -  UNUSED_PARAMETER(nTerm);
  2133         -
  2134   2231     if( pTS->aaOutput[0]==0 ){
  2135   2232       /* If this is the first term selected, copy the doclist to the output
  2136   2233       ** buffer using memcpy(). */
  2137   2234       pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
  2138   2235       pTS->anOutput[0] = nDoclist;
  2139   2236       if( pTS->aaOutput[0] ){
  2140   2237         memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
................................................................................
  2197   2294       }
  2198   2295       pCsr->apSegment = apNew;
  2199   2296     }
  2200   2297     pCsr->apSegment[pCsr->nSegment++] = pNew;
  2201   2298     return SQLITE_OK;
  2202   2299   }
  2203   2300   
         2301  +/*
         2302  +** Add seg-reader objects to the Fts3MultiSegReader object passed as the
         2303  +** 8th argument.
         2304  +**
         2305  +** This function returns SQLITE_OK if successful, or an SQLite error code
         2306  +** otherwise.
         2307  +*/
  2204   2308   static int fts3SegReaderCursor(
  2205   2309     Fts3Table *p,                   /* FTS3 table handle */
  2206   2310     int iIndex,                     /* Index to search (from 0 to p->nIndex-1) */
  2207   2311     int iLevel,                     /* Level of segments to scan */
  2208   2312     const char *zTerm,              /* Term to query for */
  2209   2313     int nTerm,                      /* Size of zTerm in bytes */
  2210   2314     int isPrefix,                   /* True for a prefix search */
  2211   2315     int isScan,                     /* True to scan from zTerm to EOF */
  2212         -  Fts3MultiSegReader *pCsr       /* Cursor object to populate */
         2316  +  Fts3MultiSegReader *pCsr        /* Cursor object to populate */
  2213   2317   ){
  2214         -  int rc = SQLITE_OK;
  2215         -  int rc2;
  2216         -  sqlite3_stmt *pStmt = 0;
         2318  +  int rc = SQLITE_OK;             /* Error code */
         2319  +  sqlite3_stmt *pStmt = 0;        /* Statement to iterate through segments */
         2320  +  int rc2;                        /* Result of sqlite3_reset() */
  2217   2321   
  2218   2322     /* If iLevel is less than 0 and this is not a scan, include a seg-reader 
  2219   2323     ** for the pending-terms. If this is a scan, then this call must be being
  2220   2324     ** made by an fts4aux module, not an FTS table. In this case calling
  2221   2325     ** Fts3SegReaderPending might segfault, as the data structures used by 
  2222   2326     ** fts4aux are not completely populated. So it's easiest to filter these
  2223   2327     ** calls out here.  */
................................................................................
  2298   2402     memset(pCsr, 0, sizeof(Fts3MultiSegReader));
  2299   2403   
  2300   2404     return fts3SegReaderCursor(
  2301   2405         p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
  2302   2406     );
  2303   2407   }
  2304   2408   
         2409  +/*
         2410  +** In addition to its current configuration, have the Fts3MultiSegReader
         2411  +** passed as the 4th argument also scan the doclist for term zTerm/nTerm.
         2412  +**
         2413  +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
         2414  +*/
  2305   2415   static int fts3SegReaderCursorAddZero(
  2306         -  Fts3Table *p,
  2307         -  const char *zTerm,
  2308         -  int nTerm,
  2309         -  Fts3MultiSegReader *pCsr
         2416  +  Fts3Table *p,                   /* FTS virtual table handle */
         2417  +  const char *zTerm,              /* Term to scan doclist of */
         2418  +  int nTerm,                      /* Number of bytes in zTerm */
         2419  +  Fts3MultiSegReader *pCsr        /* Fts3MultiSegReader to modify */
  2310   2420   ){
  2311   2421     return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr);
  2312   2422   }
  2313   2423   
  2314         -
  2315         -int sqlite3Fts3TermSegReaderCursor(
         2424  +/*
         2425  +** Open an Fts3MultiSegReader to scan the doclist for term zTerm/nTerm. Or,
         2426  +** if isPrefix is true, to scan the doclist for all terms for which 
         2427  +** zTerm/nTerm is a prefix. If successful, return SQLITE_OK and write
         2428  +** a pointer to the new Fts3MultiSegReader to *ppSegcsr. Otherwise, return
         2429  +** an SQLite error code.
         2430  +**
         2431  +** It is the responsibility of the caller to free this object by eventually
         2432  +** passing it to fts3SegReaderCursorFree() 
         2433  +**
         2434  +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
         2435  +** Output parameter *ppSegcsr is set to 0 if an error occurs.
         2436  +*/
         2437  +static int fts3TermSegReaderCursor(
  2316   2438     Fts3Cursor *pCsr,               /* Virtual table cursor handle */
  2317   2439     const char *zTerm,              /* Term to query for */
  2318   2440     int nTerm,                      /* Size of zTerm in bytes */
  2319   2441     int isPrefix,                   /* True for a prefix search */
  2320   2442     Fts3MultiSegReader **ppSegcsr   /* OUT: Allocated seg-reader cursor */
  2321   2443   ){
  2322         -  Fts3MultiSegReader *pSegcsr;   /* Object to allocate and return */
         2444  +  Fts3MultiSegReader *pSegcsr;    /* Object to allocate and return */
  2323   2445     int rc = SQLITE_NOMEM;          /* Return code */
  2324   2446   
  2325   2447     pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader));
  2326   2448     if( pSegcsr ){
  2327   2449       int i;
  2328   2450       int bFound = 0;               /* True once an index has been found */
  2329   2451       Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
................................................................................
  2359   2481       }
  2360   2482     }
  2361   2483   
  2362   2484     *ppSegcsr = pSegcsr;
  2363   2485     return rc;
  2364   2486   }
  2365   2487   
         2488  +/*
         2489  +** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor().
         2490  +*/
  2366   2491   static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){
  2367   2492     sqlite3Fts3SegReaderFinish(pSegcsr);
  2368   2493     sqlite3_free(pSegcsr);
  2369   2494   }
  2370   2495   
  2371   2496   /*
  2372   2497   ** This function retreives the doclist for the specified term (or term
  2373         -** prefix) from the database. 
  2374         -**
  2375         -** The returned doclist may be in one of two formats, depending on the 
  2376         -** value of parameter isReqPos. If isReqPos is zero, then the doclist is
  2377         -** a sorted list of delta-compressed docids (a bare doclist). If isReqPos
  2378         -** is non-zero, then the returned list is in the same format as is stored 
  2379         -** in the database without the found length specifier at the start of on-disk
  2380         -** doclists.
         2498  +** prefix) from the database.
  2381   2499   */
  2382   2500   static int fts3TermSelect(
  2383   2501     Fts3Table *p,                   /* Virtual table handle */
  2384   2502     Fts3PhraseToken *pTok,          /* Token to query for */
  2385   2503     int iColumn,                    /* Column to query (or -ve for all columns) */
  2386         -  int isReqPos,                   /* True to include position lists in output */
  2387   2504     int *pnOut,                     /* OUT: Size of buffer at *ppOut */
  2388   2505     char **ppOut                    /* OUT: Malloced result buffer */
  2389   2506   ){
  2390   2507     int rc;                         /* Return code */
  2391         -  Fts3MultiSegReader *pSegcsr;   /* Seg-reader cursor for this term */
  2392         -  TermSelect tsc;                 /* Context object for fts3TermSelectCb() */
         2508  +  Fts3MultiSegReader *pSegcsr;    /* Seg-reader cursor for this term */
         2509  +  TermSelect tsc;                 /* Object for pair-wise doclist merging */
  2393   2510     Fts3SegFilter filter;           /* Segment term filter configuration */
  2394   2511   
  2395   2512     pSegcsr = pTok->pSegcsr;
  2396   2513     memset(&tsc, 0, sizeof(TermSelect));
  2397         -  tsc.isReqPos = isReqPos;
  2398   2514   
  2399         -  filter.flags = FTS3_SEGMENT_IGNORE_EMPTY 
         2515  +  filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS
  2400   2516           | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0)
  2401         -        | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
  2402   2517           | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
  2403   2518     filter.iCol = iColumn;
  2404   2519     filter.zTerm = pTok->z;
  2405   2520     filter.nTerm = pTok->n;
  2406   2521   
  2407   2522     rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter);
  2408   2523     while( SQLITE_OK==rc
  2409   2524         && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) 
  2410   2525     ){
  2411         -    rc = fts3TermSelectCb(p, (void *)&tsc, 
  2412         -        pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist
  2413         -    );
         2526  +    rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist);
  2414   2527     }
  2415   2528   
  2416   2529     if( rc==SQLITE_OK ){
  2417         -    rc = fts3TermSelectMerge(p, &tsc);
         2530  +    rc = fts3TermSelectFinishMerge(p, &tsc);
  2418   2531     }
  2419   2532     if( rc==SQLITE_OK ){
  2420   2533       *ppOut = tsc.aaOutput[0];
  2421   2534       *pnOut = tsc.anOutput[0];
  2422   2535     }else{
  2423   2536       int i;
  2424   2537       for(i=0; i<SizeofArray(tsc.aaOutput); i++){
................................................................................
  2436   2549   ** in buffer aList[], size nList bytes.
  2437   2550   **
  2438   2551   ** If the isPoslist argument is true, then it is assumed that the doclist
  2439   2552   ** contains a position-list following each docid. Otherwise, it is assumed
  2440   2553   ** that the doclist is simply a list of docids stored as delta encoded 
  2441   2554   ** varints.
  2442   2555   */
  2443         -static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){
         2556  +static int fts3DoclistCountDocids(char *aList, int nList){
  2444   2557     int nDoc = 0;                   /* Return value */
  2445   2558     if( aList ){
  2446   2559       char *aEnd = &aList[nList];   /* Pointer to one byte after EOF */
  2447   2560       char *p = aList;              /* Cursor */
  2448         -    if( !isPoslist ){
  2449         -      /* The number of docids in the list is the same as the number of 
  2450         -      ** varints. In FTS3 a varint consists of a single byte with the 0x80 
  2451         -      ** bit cleared and zero or more bytes with the 0x80 bit set. So to
  2452         -      ** count the varints in the buffer, just count the number of bytes
  2453         -      ** with the 0x80 bit clear.  */
  2454         -      while( p<aEnd ) nDoc += (((*p++)&0x80)==0);
  2455         -    }else{
  2456         -      while( p<aEnd ){
  2457         -        nDoc++;
  2458         -        while( (*p++)&0x80 );     /* Skip docid varint */
  2459         -        fts3PoslistCopy(0, &p);   /* Skip over position list */
  2460         -      }
         2561  +    while( p<aEnd ){
         2562  +      nDoc++;
         2563  +      while( (*p++)&0x80 );     /* Skip docid varint */
         2564  +      fts3PoslistCopy(0, &p);   /* Skip over position list */
  2461   2565       }
  2462   2566     }
  2463   2567   
  2464   2568     return nDoc;
  2465   2569   }
  2466   2570   
  2467   2571   /*
................................................................................
  2483   2587         pCsr->isEof = 1;
  2484   2588         rc = sqlite3_reset(pCsr->pStmt);
  2485   2589       }else{
  2486   2590         pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
  2487   2591         rc = SQLITE_OK;
  2488   2592       }
  2489   2593     }else{
  2490         -    rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor);
         2594  +    rc = fts3EvalNext((Fts3Cursor *)pCursor);
  2491   2595     }
  2492   2596     assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  2493   2597     return rc;
  2494   2598   }
  2495   2599   
  2496   2600   /*
  2497   2601   ** This is the xFilter interface for the virtual table.  See
................................................................................
  2560   2664         }
  2561   2665         return rc;
  2562   2666       }
  2563   2667   
  2564   2668       rc = sqlite3Fts3ReadLock(p);
  2565   2669       if( rc!=SQLITE_OK ) return rc;
  2566   2670   
  2567         -    rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1);
         2671  +    rc = fts3EvalStart(pCsr);
  2568   2672   
  2569   2673       sqlite3Fts3SegmentsClose(p);
  2570   2674       if( rc!=SQLITE_OK ) return rc;
  2571   2675       pCsr->pNextId = pCsr->aDoclist;
  2572   2676       pCsr->iPrevId = 0;
  2573   2677     }
  2574   2678   
................................................................................
  2967   3071     fts3DbExec(&rc, db,
  2968   3072       "ALTER TABLE %Q.'%q_segdir'   RENAME TO '%q_segdir';",
  2969   3073       p->zDb, p->zName, zName
  2970   3074     );
  2971   3075     return rc;
  2972   3076   }
  2973   3077   
         3078  +/*
         3079  +** The xSavepoint() method.
         3080  +**
         3081  +** Flush the contents of the pending-terms table to disk.
         3082  +*/
  2974   3083   static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  2975   3084     UNUSED_PARAMETER(iSavepoint);
  2976   3085     assert( ((Fts3Table *)pVtab)->inTransaction );
  2977   3086     assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
  2978   3087     TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
  2979   3088     return fts3SyncMethod(pVtab);
  2980   3089   }
         3090  +
         3091  +/*
         3092  +** The xRelease() method.
         3093  +**
         3094  +** This is a no-op.
         3095  +*/
  2981   3096   static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
  2982   3097     TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
  2983   3098     UNUSED_PARAMETER(iSavepoint);
  2984   3099     UNUSED_PARAMETER(pVtab);
  2985   3100     assert( p->inTransaction );
  2986   3101     assert( p->mxSavepoint >= iSavepoint );
  2987   3102     TESTONLY( p->mxSavepoint = iSavepoint-1 );
  2988   3103     return SQLITE_OK;
  2989   3104   }
         3105  +
         3106  +/*
         3107  +** The xRollbackTo() method.
         3108  +**
         3109  +** Discard the contents of the pending terms table.
         3110  +*/
  2990   3111   static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
  2991   3112     Fts3Table *p = (Fts3Table*)pVtab;
  2992   3113     UNUSED_PARAMETER(iSavepoint);
  2993   3114     assert( p->inTransaction );
  2994   3115     assert( p->mxSavepoint >= iSavepoint );
  2995   3116     TESTONLY( p->mxSavepoint = iSavepoint );
  2996   3117     sqlite3Fts3PendingTermsClear(p);
................................................................................
  3131   3252     assert( rc!=SQLITE_OK );
  3132   3253     if( pHash ){
  3133   3254       sqlite3Fts3HashClear(pHash);
  3134   3255       sqlite3_free(pHash);
  3135   3256     }
  3136   3257     return rc;
  3137   3258   }
  3138         -
  3139         -#if !SQLITE_CORE
  3140         -int sqlite3_extension_init(
  3141         -  sqlite3 *db, 
  3142         -  char **pzErrMsg,
  3143         -  const sqlite3_api_routines *pApi
  3144         -){
  3145         -  SQLITE_EXTENSION_INIT2(pApi)
  3146         -  return sqlite3Fts3Init(db);
  3147         -}
  3148         -#endif
  3149         -
  3150   3259   
  3151   3260   /*
  3152   3261   ** Allocate an Fts3MultiSegReader for each token in the expression headed
  3153   3262   ** by pExpr. 
  3154   3263   **
  3155   3264   ** An Fts3SegReader object is a cursor that can seek or scan a range of
  3156   3265   ** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple
................................................................................
  3160   3269   ** If the allocated Fts3MultiSegReader just seeks to a single entry in a
  3161   3270   ** segment b-tree (if the term is not a prefix or it is a prefix for which
  3162   3271   ** there exists prefix b-tree of the right length) then it may be traversed
  3163   3272   ** and merged incrementally. Otherwise, it has to be merged into an in-memory 
  3164   3273   ** doclist and then traversed.
  3165   3274   */
  3166   3275   static void fts3EvalAllocateReaders(
  3167         -  Fts3Cursor *pCsr, 
  3168         -  Fts3Expr *pExpr, 
         3276  +  Fts3Cursor *pCsr,               /* FTS cursor handle */
         3277  +  Fts3Expr *pExpr,                /* Allocate readers for this expression */
  3169   3278     int *pnToken,                   /* OUT: Total number of tokens in phrase. */
  3170   3279     int *pnOr,                      /* OUT: Total number of OR nodes in expr. */
  3171         -  int *pRc
         3280  +  int *pRc                        /* IN/OUT: Error code */
  3172   3281   ){
  3173   3282     if( pExpr && SQLITE_OK==*pRc ){
  3174   3283       if( pExpr->eType==FTSQUERY_PHRASE ){
  3175   3284         int i;
  3176   3285         int nToken = pExpr->pPhrase->nToken;
  3177   3286         *pnToken += nToken;
  3178   3287         for(i=0; i<nToken; i++){
  3179   3288           Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
  3180         -        int rc = sqlite3Fts3TermSegReaderCursor(pCsr, 
         3289  +        int rc = fts3TermSegReaderCursor(pCsr, 
  3181   3290               pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr
  3182   3291           );
  3183   3292           if( rc!=SQLITE_OK ){
  3184   3293             *pRc = rc;
  3185   3294             return;
  3186   3295           }
  3187   3296         }
................................................................................
  3191   3300         *pnOr += (pExpr->eType==FTSQUERY_OR);
  3192   3301         fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc);
  3193   3302         fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc);
  3194   3303       }
  3195   3304     }
  3196   3305   }
  3197   3306   
         3307  +/*
         3308  +** Arguments pList/nList contain the doclist for token iToken of phrase p.
         3309  +** It is merged into the main doclist stored in p->doclist.aAll/nAll.
         3310  +**
         3311  +** This function assumes that pList points to a buffer allocated using
         3312  +** sqlite3_malloc(). This function takes responsibility for eventually
         3313  +** freeing the buffer.
         3314  +*/
  3198   3315   static void fts3EvalPhraseMergeToken(
  3199         -  Fts3Table *pTab,
  3200         -  Fts3Phrase *p,
  3201         -  int iToken,
  3202         -  char *pList,
  3203         -  int nList
         3316  +  Fts3Table *pTab,                /* FTS Table pointer */
         3317  +  Fts3Phrase *p,                  /* Phrase to merge pList/nList into */
         3318  +  int iToken,                     /* Token pList/nList corresponds to */
         3319  +  char *pList,                    /* Pointer to doclist */
         3320  +  int nList                       /* Number of bytes in pList */
  3204   3321   ){
  3205   3322     assert( iToken!=p->iDoclistToken );
  3206   3323   
  3207   3324     if( pList==0 ){
  3208   3325       sqlite3_free(p->doclist.aAll);
  3209   3326       p->doclist.aAll = 0;
  3210   3327       p->doclist.nAll = 0;
................................................................................
  3245   3362       p->doclist.aAll = pRight;
  3246   3363       p->doclist.nAll = nRight;
  3247   3364     }
  3248   3365   
  3249   3366     if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken;
  3250   3367   }
  3251   3368   
         3369  +/*
         3370  +** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist
         3371  +** does not take deferred tokens into account.
         3372  +**
         3373  +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
         3374  +*/
  3252   3375   static int fts3EvalPhraseLoad(
  3253         -  Fts3Cursor *pCsr, 
  3254         -  Fts3Phrase *p
         3376  +  Fts3Cursor *pCsr,               /* FTS Cursor handle */
         3377  +  Fts3Phrase *p                   /* Phrase object */
  3255   3378   ){
  3256   3379     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  3257   3380     int iToken;
  3258   3381     int rc = SQLITE_OK;
  3259   3382   
  3260   3383     for(iToken=0; rc==SQLITE_OK && iToken<p->nToken; iToken++){
  3261   3384       Fts3PhraseToken *pToken = &p->aToken[iToken];
  3262   3385       assert( pToken->pDeferred==0 || pToken->pSegcsr==0 );
  3263   3386   
  3264   3387       if( pToken->pSegcsr ){
  3265   3388         int nThis = 0;
  3266   3389         char *pThis = 0;
  3267         -      rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis);
         3390  +      rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis);
  3268   3391         if( rc==SQLITE_OK ){
  3269   3392           fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis);
  3270   3393         }
  3271   3394       }
  3272   3395       assert( pToken->pSegcsr==0 );
  3273   3396     }
  3274   3397   
  3275   3398     return rc;
  3276   3399   }
  3277   3400   
         3401  +/*
         3402  +** This function is called on each phrase after the position lists for
         3403  +** any deferred tokens have been loaded into memory. It updates the phrases
         3404  +** current position list to include only those positions that are really
         3405  +** instances of the phrase (after considering deferred tokens). If this
         3406  +** means that the phrase does not appear in the current row, doclist.pList
         3407  +** and doclist.nList are both zeroed.
         3408  +**
         3409  +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
         3410  +*/
  3278   3411   static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
  3279         -  int iToken;
  3280         -  int rc = SQLITE_OK;
  3281         -
  3282         -  int nMaxUndeferred = pPhrase->iDoclistToken;
  3283         -  char *aPoslist = 0;
  3284         -  int nPoslist = 0;
  3285         -  int iPrev = -1;
         3412  +  int iToken;                     /* Used to iterate through phrase tokens */
         3413  +  int rc = SQLITE_OK;             /* Return code */
         3414  +  char *aPoslist = 0;             /* Position list for deferred tokens */
         3415  +  int nPoslist = 0;               /* Number of bytes in aPoslist */
         3416  +  int iPrev = -1;                 /* Token number of previous deferred token */
  3286   3417   
  3287   3418     assert( pPhrase->doclist.bFreeList==0 );
  3288   3419   
  3289   3420     for(iToken=0; rc==SQLITE_OK && iToken<pPhrase->nToken; iToken++){
  3290   3421       Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
  3291   3422       Fts3DeferredToken *pDeferred = pToken->pDeferred;
  3292   3423   
................................................................................
  3324   3455           }
  3325   3456         }
  3326   3457         iPrev = iToken;
  3327   3458       }
  3328   3459     }
  3329   3460   
  3330   3461     if( iPrev>=0 ){
         3462  +    int nMaxUndeferred = pPhrase->iDoclistToken;
  3331   3463       if( nMaxUndeferred<0 ){
  3332   3464         pPhrase->doclist.pList = aPoslist;
  3333   3465         pPhrase->doclist.nList = nPoslist;
  3334   3466         pPhrase->doclist.iDocid = pCsr->iPrevId;
  3335   3467         pPhrase->doclist.bFreeList = 1;
  3336   3468       }else{
  3337   3469         int nDistance;
................................................................................
  3372   3504   }
  3373   3505   
  3374   3506   /*
  3375   3507   ** This function is called for each Fts3Phrase in a full-text query 
  3376   3508   ** expression to initialize the mechanism for returning rows. Once this
  3377   3509   ** function has been called successfully on an Fts3Phrase, it may be
  3378   3510   ** used with fts3EvalPhraseNext() to iterate through the matching docids.
         3511  +**
         3512  +** If parameter bOptOk is true, then the phrase may (or may not) use the
         3513  +** incremental loading strategy. Otherwise, the entire doclist is loaded into
         3514  +** memory within this call.
         3515  +**
         3516  +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
  3379   3517   */
  3380   3518   static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
  3381         -  int rc;
         3519  +  int rc;                         /* Error code */
  3382   3520     Fts3PhraseToken *pFirst = &p->aToken[0];
  3383   3521     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  3384   3522   
  3385   3523     if( pCsr->bDesc==pTab->bDescIdx 
  3386   3524      && bOptOk==1 
  3387   3525      && p->nToken==1 
  3388   3526      && pFirst->pSegcsr 
................................................................................
  3402   3540   
  3403   3541     assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr );
  3404   3542     return rc;
  3405   3543   }
  3406   3544   
  3407   3545   /*
  3408   3546   ** This function is used to iterate backwards (from the end to start) 
  3409         -** through doclists.
         3547  +** through doclists. It is used by this module to iterate through phrase
         3548  +** doclists in reverse and by the fts3_write.c module to iterate through
         3549  +** pending-terms lists when writing to databases with "order=desc".
         3550  +**
         3551  +** The doclist may be sorted in ascending (parameter bDescIdx==0) or 
         3552  +** descending (parameter bDescIdx==1) order of docid. Regardless, this
         3553  +** function iterates from the end of the doclist to the beginning.
  3410   3554   */
  3411   3555   void sqlite3Fts3DoclistPrev(
  3412   3556     int bDescIdx,                   /* True if the doclist is desc */
  3413   3557     char *aDoclist,                 /* Pointer to entire doclist */
  3414   3558     int nDoclist,                   /* Length of aDoclist in bytes */
  3415   3559     char **ppIter,                  /* IN/OUT: Iterator pointer */
  3416   3560     sqlite3_int64 *piDocid,         /* IN/OUT: Docid pointer */
................................................................................
  3467   3611   ** SQLITE_OK.
  3468   3612   **
  3469   3613   ** If there is no "next" entry and no error occurs, then *pbEof is set to
  3470   3614   ** 1 before returning. Otherwise, if no error occurs and the iterator is
  3471   3615   ** successfully advanced, *pbEof is set to 0.
  3472   3616   */
  3473   3617   static int fts3EvalPhraseNext(
  3474         -  Fts3Cursor *pCsr, 
  3475         -  Fts3Phrase *p, 
  3476         -  u8 *pbEof
         3618  +  Fts3Cursor *pCsr,               /* FTS Cursor handle */
         3619  +  Fts3Phrase *p,                  /* Phrase object to advance to next docid */
         3620  +  u8 *pbEof                       /* OUT: Set to 1 if EOF */
  3477   3621   ){
  3478   3622     int rc = SQLITE_OK;
  3479   3623     Fts3Doclist *pDL = &p->doclist;
  3480   3624     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  3481   3625   
  3482   3626     if( p->bIncr ){
  3483   3627       assert( p->nToken==1 );
................................................................................
  3515   3659         }
  3516   3660         pDL->pList = pIter;
  3517   3661         fts3PoslistCopy(0, &pIter);
  3518   3662         pDL->nList = (pIter - pDL->pList);
  3519   3663   
  3520   3664         /* pIter now points just past the 0x00 that terminates the position-
  3521   3665         ** list for document pDL->iDocid. However, if this position-list was
  3522         -      ** edited in place by fts3EvalNearTrim2(), then pIter may not actually
         3666  +      ** edited in place by fts3EvalNearTrim(), then pIter may not actually
  3523   3667         ** point to the start of the next docid value. The following line deals
  3524   3668         ** with this case by advancing pIter past the zero-padding added by
  3525         -      ** fts3EvalNearTrim2().  */
         3669  +      ** fts3EvalNearTrim().  */
  3526   3670         while( pIter<pEnd && *pIter==0 ) pIter++;
  3527   3671   
  3528   3672         pDL->pNextDocid = pIter;
  3529   3673         assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter );
  3530   3674         *pbEof = 0;
  3531   3675       }
  3532   3676     }
  3533   3677   
  3534   3678     return rc;
  3535   3679   }
  3536   3680   
         3681  +/*
         3682  +**
         3683  +** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
         3684  +** Otherwise, fts3EvalPhraseStart() is called on all phrases within the
         3685  +** expression. Also the Fts3Expr.bDeferred variable is set to true for any
         3686  +** expressions for which all descendent tokens are deferred.
         3687  +**
         3688  +** If parameter bOptOk is zero, then it is guaranteed that the
         3689  +** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for
         3690  +** each phrase in the expression (subject to deferred token processing).
         3691  +** Or, if bOptOk is non-zero, then one or more tokens within the expression
         3692  +** may be loaded incrementally, meaning doclist.aAll/nAll is not available.
         3693  +**
         3694  +** If an error occurs within this function, *pRc is set to an SQLite error
         3695  +** code before returning.
         3696  +*/
  3537   3697   static void fts3EvalStartReaders(
  3538         -  Fts3Cursor *pCsr, 
  3539         -  Fts3Expr *pExpr, 
  3540         -  int bOptOk,
  3541         -  int *pRc
         3698  +  Fts3Cursor *pCsr,               /* FTS Cursor handle */
         3699  +  Fts3Expr *pExpr,                /* Expression to initialize phrases in */
         3700  +  int bOptOk,                     /* True to enable incremental loading */
         3701  +  int *pRc                        /* IN/OUT: Error code */
  3542   3702   ){
  3543   3703     if( pExpr && SQLITE_OK==*pRc ){
  3544   3704       if( pExpr->eType==FTSQUERY_PHRASE ){
  3545   3705         int i;
  3546   3706         int nToken = pExpr->pPhrase->nToken;
  3547   3707         for(i=0; i<nToken; i++){
  3548   3708           if( pExpr->pPhrase->aToken[i].pDeferred==0 ) break;
................................................................................
  3553   3713         fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc);
  3554   3714         fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc);
  3555   3715         pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred);
  3556   3716       }
  3557   3717     }
  3558   3718   }
  3559   3719   
         3720  +/*
         3721  +** An array of the following structures is assembled as part of the process
         3722  +** of selecting tokens to defer before the query starts executing (as part
         3723  +** of the xFilter() method). There is one element in the array for each
         3724  +** token in the FTS expression.
         3725  +**
         3726  +** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong
         3727  +** to phrases that are connected only by AND and NEAR operators (not OR or
         3728  +** NOT). When determining tokens to defer, each AND/NEAR cluster is considered
         3729  +** separately. The root of a tokens AND/NEAR cluster is stored in 
         3730  +** Fts3TokenAndCost.pRoot.
         3731  +*/
  3560   3732   typedef struct Fts3TokenAndCost Fts3TokenAndCost;
  3561   3733   struct Fts3TokenAndCost {
  3562   3734     Fts3Phrase *pPhrase;            /* The phrase the token belongs to */
  3563   3735     int iToken;                     /* Position of token in phrase */
  3564   3736     Fts3PhraseToken *pToken;        /* The token itself */
  3565         -  Fts3Expr *pRoot; 
  3566         -  int nOvfl;
         3737  +  Fts3Expr *pRoot;                /* Root of NEAR/AND cluster */
         3738  +  int nOvfl;                      /* Number of overflow pages to load doclist */
  3567   3739     int iCol;                       /* The column the token must match */
  3568   3740   };
  3569   3741   
         3742  +/*
         3743  +** This function is used to populate an allocated Fts3TokenAndCost array.
         3744  +**
         3745  +** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
         3746  +** Otherwise, if an error occurs during execution, *pRc is set to an
         3747  +** SQLite error code.
         3748  +*/
  3570   3749   static void fts3EvalTokenCosts(
  3571         -  Fts3Cursor *pCsr, 
  3572         -  Fts3Expr *pRoot, 
  3573         -  Fts3Expr *pExpr, 
  3574         -  Fts3TokenAndCost **ppTC,
  3575         -  Fts3Expr ***ppOr,
  3576         -  int *pRc
         3750  +  Fts3Cursor *pCsr,               /* FTS Cursor handle */
         3751  +  Fts3Expr *pRoot,                /* Root of current AND/NEAR cluster */
         3752  +  Fts3Expr *pExpr,                /* Expression to consider */
         3753  +  Fts3TokenAndCost **ppTC,        /* Write new entries to *(*ppTC)++ */
         3754  +  Fts3Expr ***ppOr,               /* Write new OR root to *(*ppOr)++ */
         3755  +  int *pRc                        /* IN/OUT: Error code */
  3577   3756   ){
  3578   3757     if( *pRc==SQLITE_OK && pExpr ){
  3579   3758       if( pExpr->eType==FTSQUERY_PHRASE ){
  3580   3759         Fts3Phrase *pPhrase = pExpr->pPhrase;
  3581   3760         int i;
  3582   3761         for(i=0; *pRc==SQLITE_OK && i<pPhrase->nToken; i++){
  3583   3762           Fts3TokenAndCost *pTC = (*ppTC)++;
................................................................................
  3601   3780           (*ppOr)++;
  3602   3781         }
  3603   3782         fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
  3604   3783       }
  3605   3784     }
  3606   3785   }
  3607   3786   
         3787  +/*
         3788  +** Determine the average document (row) size in pages. If successful,
         3789  +** write this value to *pnPage and return SQLITE_OK. Otherwise, return
         3790  +** an SQLite error code.
         3791  +**
         3792  +** The average document size in pages is calculated by first calculating 
         3793  +** determining the average size in bytes, B. If B is less than the amount
         3794  +** of data that will fit on a single leaf page of an intkey table in
         3795  +** this database, then the average docsize is 1. Otherwise, it is 1 plus
         3796  +** the number of overflow pages consumed by a record B bytes in size.
         3797  +*/
  3608   3798   static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
  3609   3799     if( pCsr->nRowAvg==0 ){
  3610   3800       /* The average document size, which is required to calculate the cost
  3611         -     ** of each doclist, has not yet been determined. Read the required 
  3612         -     ** data from the %_stat table to calculate it.
  3613         -     **
  3614         -     ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 
  3615         -     ** varints, where nCol is the number of columns in the FTS3 table.
  3616         -     ** The first varint is the number of documents currently stored in
  3617         -     ** the table. The following nCol varints contain the total amount of
  3618         -     ** data stored in all rows of each column of the table, from left
  3619         -     ** to right.
  3620         -     */
         3801  +    ** of each doclist, has not yet been determined. Read the required 
         3802  +    ** data from the %_stat table to calculate it.
         3803  +    **
         3804  +    ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 
         3805  +    ** varints, where nCol is the number of columns in the FTS3 table.
         3806  +    ** The first varint is the number of documents currently stored in
         3807  +    ** the table. The following nCol varints contain the total amount of
         3808  +    ** data stored in all rows of each column of the table, from left
         3809  +    ** to right.
         3810  +    */
  3621   3811       int rc;
  3622   3812       Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
  3623   3813       sqlite3_stmt *pStmt;
  3624   3814       sqlite3_int64 nDoc = 0;
  3625   3815       sqlite3_int64 nByte = 0;
  3626   3816       const char *pEnd;
  3627   3817       const char *a;
................................................................................
  3648   3838       if( rc!=SQLITE_OK ) return rc;
  3649   3839     }
  3650   3840   
  3651   3841     *pnPage = pCsr->nRowAvg;
  3652   3842     return SQLITE_OK;
  3653   3843   }
  3654   3844   
         3845  +/*
         3846  +** This function is called to select the tokens (if any) that will be 
         3847  +** deferred. The array aTC[] has already been populated when this is
         3848  +** called.
         3849  +**
         3850  +** This function is called once for each AND/NEAR cluster in the 
         3851  +** expression. Each invocation determines which tokens to defer within
         3852  +** the cluster with root node pRoot. See comments above the definition
         3853  +** of struct Fts3TokenAndCost for more details.
         3854  +**
         3855  +** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken()
         3856  +** called on each token to defer. Otherwise, an SQLite error code is
         3857  +** returned.
         3858  +*/
  3655   3859   static int fts3EvalSelectDeferred(
  3656         -  Fts3Cursor *pCsr,
  3657         -  Fts3Expr *pRoot,
  3658         -  Fts3TokenAndCost *aTC,
  3659         -  int nTC
         3860  +  Fts3Cursor *pCsr,               /* FTS Cursor handle */
         3861  +  Fts3Expr *pRoot,                /* Consider tokens with this root node */
         3862  +  Fts3TokenAndCost *aTC,          /* Array of expression tokens and costs */
         3863  +  int nTC                         /* Number of entries in aTC[] */
  3660   3864   ){
  3661         -  int nDocSize = 0;
  3662         -  int nDocEst = 0;
  3663         -  int rc = SQLITE_OK;
  3664   3865     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  3665         -  int ii;
         3866  +  int nDocSize = 0;               /* Number of pages per doc loaded */
         3867  +  int rc = SQLITE_OK;             /* Return code */
         3868  +  int ii;                         /* Iterator variable for various purposes */
         3869  +  int nOvfl = 0;                  /* Total overflow pages used by doclists */
         3870  +  int nToken = 0;                 /* Total number of tokens in cluster */
  3666   3871   
  3667         -  int nOvfl = 0;
  3668         -  int nTerm = 0;
         3872  +  int nMinEst = 0;                /* The minimum count for any phrase so far. */
         3873  +  int nLoad4 = 1;                 /* (Phrases that will be loaded)^4. */
  3669   3874   
         3875  +  /* Count the tokens in this AND/NEAR cluster. If none of the doclists
         3876  +  ** associated with the tokens spill onto overflow pages, or if there is
         3877  +  ** only 1 token, exit early. No tokens to defer in this case. */
  3670   3878     for(ii=0; ii<nTC; ii++){
  3671   3879       if( aTC[ii].pRoot==pRoot ){
  3672   3880         nOvfl += aTC[ii].nOvfl;
  3673         -      nTerm++;
         3881  +      nToken++;
  3674   3882       }
  3675   3883     }
  3676         -  if( nOvfl==0 || nTerm<2 ) return SQLITE_OK;
         3884  +  if( nOvfl==0 || nToken<2 ) return SQLITE_OK;
  3677   3885   
         3886  +  /* Obtain the average docsize (in pages). */
  3678   3887     rc = fts3EvalAverageDocsize(pCsr, &nDocSize);
         3888  +  assert( rc!=SQLITE_OK || nDocSize>0 );
  3679   3889   
  3680         -  for(ii=0; ii<nTerm && rc==SQLITE_OK; ii++){
  3681         -    int jj;
  3682         -    Fts3TokenAndCost *pTC = 0;
  3683   3890   
  3684         -    for(jj=0; jj<nTC; jj++){
  3685         -      if( aTC[jj].pToken && aTC[jj].pRoot==pRoot 
  3686         -       && (!pTC || aTC[jj].nOvfl<pTC->nOvfl) 
         3891  +  /* Iterate through all tokens in this AND/NEAR cluster, in ascending order 
         3892  +  ** of the number of overflow pages that will be loaded by the pager layer 
         3893  +  ** to retrieve the entire doclist for the token from the full-text index.
         3894  +  ** Load the doclists for tokens that are either:
         3895  +  **
         3896  +  **   a. The cheapest token in the entire query (i.e. the one visited by the
         3897  +  **      first iteration of this loop), or
         3898  +  **
         3899  +  **   b. Part of a multi-token phrase.
         3900  +  **
         3901  +  ** After each token doclist is loaded, merge it with the others from the
         3902  +  ** same phrase and count the number of documents that the merged doclist
         3903  +  ** contains. Set variable "nMinEst" to the smallest number of documents in 
         3904  +  ** any phrase doclist for which 1 or more token doclists have been loaded.
         3905  +  ** Let nOther be the number of other phrases for which it is certain that
         3906  +  ** one or more tokens will not be deferred.
         3907  +  **
         3908  +  ** Then, for each token, defer it if loading the doclist would result in
         3909  +  ** loading N or more overflow pages into memory, where N is computed as:
         3910  +  **
         3911  +  **    (nMinEst + 4^nOther - 1) / (4^nOther)
         3912  +  */
         3913  +  for(ii=0; ii<nToken && rc==SQLITE_OK; ii++){
         3914  +    int iTC;                      /* Used to iterate through aTC[] array. */
         3915  +    Fts3TokenAndCost *pTC = 0;    /* Set to cheapest remaining token. */
         3916  +
         3917  +    /* Set pTC to point to the cheapest remaining token. */
         3918  +    for(iTC=0; iTC<nTC; iTC++){
         3919  +      if( aTC[iTC].pToken && aTC[iTC].pRoot==pRoot 
         3920  +       && (!pTC || aTC[iTC].nOvfl<pTC->nOvfl) 
  3687   3921         ){
  3688         -        pTC = &aTC[jj];
         3922  +        pTC = &aTC[iTC];
  3689   3923         }
  3690   3924       }
  3691   3925       assert( pTC );
  3692   3926   
  3693         -    /* At this point pTC points to the cheapest remaining token. */
  3694         -    if( ii==0 ){
  3695         -      if( pTC->nOvfl ){
  3696         -        nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10;
  3697         -      }else{
         3927  +    if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){
         3928  +      /* The number of overflow pages to load for this (and therefore all
         3929  +      ** subsequent) tokens is greater than the estimated number of pages 
         3930  +      ** that will be loaded if all subsequent tokens are deferred.
         3931  +      */
         3932  +      Fts3PhraseToken *pToken = pTC->pToken;
         3933  +      rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol);
         3934  +      fts3SegReaderCursorFree(pToken->pSegcsr);
         3935  +      pToken->pSegcsr = 0;
         3936  +    }else{
         3937  +      nLoad4 = nLoad4*4;
         3938  +      if( ii==0 || pTC->pPhrase->nToken>1 ){
         3939  +        /* Either this is the cheapest token in the entire query, or it is
         3940  +        ** part of a multi-token phrase. Either way, the entire doclist will
         3941  +        ** (eventually) be loaded into memory. It may as well be now. */
  3698   3942           Fts3PhraseToken *pToken = pTC->pToken;
  3699   3943           int nList = 0;
  3700   3944           char *pList = 0;
  3701         -        rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList);
         3945  +        rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList);
  3702   3946           assert( rc==SQLITE_OK || pList==0 );
  3703         -
  3704   3947           if( rc==SQLITE_OK ){
  3705         -          nDocEst = fts3DoclistCountDocids(1, pList, nList);
         3948  +          int nCount;
  3706   3949             fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList);
         3950  +          nCount = fts3DoclistCountDocids(
         3951  +              pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll
         3952  +          );
         3953  +          if( ii==0 || nCount<nMinEst ) nMinEst = nCount;
  3707   3954           }
  3708   3955         }
  3709         -    }else{
  3710         -      if( pTC->nOvfl>=(nDocEst*nDocSize) ){
  3711         -        Fts3PhraseToken *pToken = pTC->pToken;
  3712         -        rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol);
  3713         -        fts3SegReaderCursorFree(pToken->pSegcsr);
  3714         -        pToken->pSegcsr = 0;
  3715         -      }
  3716         -      nDocEst = 1 + (nDocEst/4);
  3717   3956       }
  3718   3957       pTC->pToken = 0;
  3719   3958     }
  3720   3959   
  3721   3960     return rc;
  3722   3961   }
  3723   3962   
  3724         -int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){
         3963  +/*
         3964  +** This function is called from within the xFilter method. It initializes
         3965  +** the full-text query currently stored in pCsr->pExpr. To iterate through
         3966  +** the results of a query, the caller does:
         3967  +**
         3968  +**    fts3EvalStart(pCsr);
         3969  +**    while( 1 ){
         3970  +**      fts3EvalNext(pCsr);
         3971  +**      if( pCsr->bEof ) break;
         3972  +**      ... return row pCsr->iPrevId to the caller ...
         3973  +**    }
         3974  +*/
         3975  +static int fts3EvalStart(Fts3Cursor *pCsr){
  3725   3976     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  3726   3977     int rc = SQLITE_OK;
  3727   3978     int nToken = 0;
  3728   3979     int nOr = 0;
  3729   3980   
  3730   3981     /* Allocate a MultiSegReader for each token in the expression. */
  3731         -  fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc);
         3982  +  fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc);
  3732   3983   
  3733         -  /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO:
  3734         -  ** This call will eventually also be responsible for determining which
  3735         -  ** tokens are 'deferred' until the document text is loaded into memory.
  3736         -  **
  3737         -  ** Each token in each phrase is dealt with using one of the following
  3738         -  ** three strategies:
  3739         -  **
  3740         -  **   1. Entire doclist loaded into memory as part of the
  3741         -  **      fts3EvalStartReaders() call.
  3742         -  **
  3743         -  **   2. Doclist loaded into memory incrementally, as part of each
  3744         -  **      sqlite3Fts3EvalNext() call.
  3745         -  **
  3746         -  **   3. Token doclist is never loaded. Instead, documents are loaded into
  3747         -  **      memory and scanned for the token as part of the sqlite3Fts3EvalNext()
  3748         -  **      call. This is known as a "deferred" token.
  3749         -  */
  3750         -
  3751         -  /* If bOptOk is true, check if there are any tokens that should be deferred.
  3752         -  */
  3753         -  if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){
         3984  +  /* Determine which, if any, tokens in the expression should be deferred. */
         3985  +  if( rc==SQLITE_OK && nToken>1 && pTab->bHasStat ){
  3754   3986       Fts3TokenAndCost *aTC;
  3755   3987       Fts3Expr **apOr;
  3756   3988       aTC = (Fts3TokenAndCost *)sqlite3_malloc(
  3757   3989           sizeof(Fts3TokenAndCost) * nToken
  3758   3990         + sizeof(Fts3Expr *) * nOr * 2
  3759   3991       );
  3760   3992       apOr = (Fts3Expr **)&aTC[nToken];
................................................................................
  3762   3994       if( !aTC ){
  3763   3995         rc = SQLITE_NOMEM;
  3764   3996       }else{
  3765   3997         int ii;
  3766   3998         Fts3TokenAndCost *pTC = aTC;
  3767   3999         Fts3Expr **ppOr = apOr;
  3768   4000   
  3769         -      fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc);
         4001  +      fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc);
  3770   4002         nToken = pTC-aTC;
  3771   4003         nOr = ppOr-apOr;
  3772   4004   
  3773   4005         if( rc==SQLITE_OK ){
  3774   4006           rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken);
  3775   4007           for(ii=0; rc==SQLITE_OK && ii<nOr; ii++){
  3776   4008             rc = fts3EvalSelectDeferred(pCsr, apOr[ii], aTC, nToken);
................................................................................
  3777   4009           }
  3778   4010         }
  3779   4011   
  3780   4012         sqlite3_free(aTC);
  3781   4013       }
  3782   4014     }
  3783   4015   
  3784         -  fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc);
         4016  +  fts3EvalStartReaders(pCsr, pCsr->pExpr, 1, &rc);
  3785   4017     return rc;
  3786   4018   }
  3787   4019   
  3788         -static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){
         4020  +/*
         4021  +** Invalidate the current position list for phrase pPhrase.
         4022  +*/
         4023  +static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){
  3789   4024     if( pPhrase->doclist.bFreeList ){
  3790   4025       sqlite3_free(pPhrase->doclist.pList);
  3791   4026     }
  3792   4027     pPhrase->doclist.pList = 0;
  3793   4028     pPhrase->doclist.nList = 0;
  3794   4029     pPhrase->doclist.bFreeList = 0;
  3795   4030   }
  3796   4031   
  3797         -static int fts3EvalNearTrim2(
  3798         -  int nNear,
         4032  +/*
         4033  +** This function is called to edit the position list associated with
         4034  +** the phrase object passed as the fifth argument according to a NEAR
         4035  +** condition. For example:
         4036  +**
         4037  +**     abc NEAR/5 "def ghi"
         4038  +**
         4039  +** Parameter nNear is passed the NEAR distance of the expression (5 in
         4040  +** the example above). When this function is called, *paPoslist points to
         4041  +** the position list, and *pnToken is the number of phrase tokens in, the
         4042  +** phrase on the other side of the NEAR operator to pPhrase. For example,
         4043  +** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to
         4044  +** the position list associated with phrase "abc".
         4045  +**
         4046  +** All positions in the pPhrase position list that are not sufficiently
         4047  +** close to a position in the *paPoslist position list are removed. If this
         4048  +** leaves 0 positions, zero is returned. Otherwise, non-zero.
         4049  +**
         4050  +** Before returning, *paPoslist is set to point to the position lsit 
         4051  +** associated with pPhrase. And *pnToken is set to the number of tokens in
         4052  +** pPhrase.
         4053  +*/
         4054  +static int fts3EvalNearTrim(
         4055  +  int nNear,                      /* NEAR distance. As in "NEAR/nNear". */
  3799   4056     char *aTmp,                     /* Temporary space to use */
  3800   4057     char **paPoslist,               /* IN/OUT: Position list */
  3801   4058     int *pnToken,                   /* IN/OUT: Tokens in phrase of *paPoslist */
  3802   4059     Fts3Phrase *pPhrase             /* The phrase object to trim the doclist of */
  3803   4060   ){
  3804   4061     int nParam1 = nNear + pPhrase->nToken;
  3805   4062     int nParam2 = nNear + *pnToken;
................................................................................
  3823   4080       *paPoslist = pPhrase->doclist.pList;
  3824   4081       *pnToken = pPhrase->nToken;
  3825   4082     }
  3826   4083   
  3827   4084     return res;
  3828   4085   }
  3829   4086   
         4087  +/*
         4088  +** This function is a no-op if *pRc is other than SQLITE_OK when it is called.
         4089  +** Otherwise, it advances the expression passed as the second argument to
         4090  +** point to the next matching row in the database. Expressions iterate through
         4091  +** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero,
         4092  +** or descending if it is non-zero.
         4093  +**
         4094  +** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if
         4095  +** successful, the following variables in pExpr are set:
         4096  +**
         4097  +**   Fts3Expr.bEof                (non-zero if EOF - there is no next row)
         4098  +**   Fts3Expr.iDocid              (valid if bEof==0. The docid of the next row)
         4099  +**
         4100  +** If the expression is of type FTSQUERY_PHRASE, and the expression is not
         4101  +** at EOF, then the following variables are populated with the position list
         4102  +** for the phrase for the visited row:
         4103  +**
         4104  +**   FTs3Expr.pPhrase->doclist.nList        (length of pList in bytes)
         4105  +**   FTs3Expr.pPhrase->doclist.pList        (pointer to position list)
         4106  +**
         4107  +** It says above that this function advances the expression to the next
         4108  +** matching row. This is usually true, but there are the following exceptions:
         4109  +**
         4110  +**   1. Deferred tokens are not taken into account. If a phrase consists
         4111  +**      entirely of deferred tokens, it is assumed to match every row in
         4112  +**      the db. In this case the position-list is not populated at all. 
         4113  +**
         4114  +**      Or, if a phrase contains one or more deferred tokens and one or
         4115  +**      more non-deferred tokens, then the expression is advanced to the 
         4116  +**      next possible match, considering only non-deferred tokens. In other
         4117  +**      words, if the phrase is "A B C", and "B" is deferred, the expression
         4118  +**      is advanced to the next row that contains an instance of "A * C", 
         4119  +**      where "*" may match any single token. The position list in this case
         4120  +**      is populated as for "A * C" before returning.
         4121  +**
         4122  +**   2. NEAR is treated as AND. If the expression is "x NEAR y", it is 
         4123  +**      advanced to point to the next row that matches "x AND y".
         4124  +** 
         4125  +** See fts3EvalTestDeferredAndNear() for details on testing if a row is
         4126  +** really a match, taking into account deferred tokens and NEAR operators.
         4127  +*/
         4128  +static void fts3EvalNextRow(
         4129  +  Fts3Cursor *pCsr,               /* FTS Cursor handle */
         4130  +  Fts3Expr *pExpr,                /* Expr. to advance to next matching row */
         4131  +  int *pRc                        /* IN/OUT: Error code */
         4132  +){
         4133  +  if( *pRc==SQLITE_OK ){
         4134  +    int bDescDoclist = pCsr->bDesc;         /* Used by DOCID_CMP() macro */
         4135  +    assert( pExpr->bEof==0 );
         4136  +    pExpr->bStart = 1;
         4137  +
         4138  +    switch( pExpr->eType ){
         4139  +      case FTSQUERY_NEAR:
         4140  +      case FTSQUERY_AND: {
         4141  +        Fts3Expr *pLeft = pExpr->pLeft;
         4142  +        Fts3Expr *pRight = pExpr->pRight;
         4143  +        assert( !pLeft->bDeferred || !pRight->bDeferred );
         4144  +
         4145  +        if( pLeft->bDeferred ){
         4146  +          /* LHS is entirely deferred. So we assume it matches every row.
         4147  +          ** Advance the RHS iterator to find the next row visited. */
         4148  +          fts3EvalNextRow(pCsr, pRight, pRc);
         4149  +          pExpr->iDocid = pRight->iDocid;
         4150  +          pExpr->bEof = pRight->bEof;
         4151  +        }else if( pRight->bDeferred ){
         4152  +          /* RHS is entirely deferred. So we assume it matches every row.
         4153  +          ** Advance the LHS iterator to find the next row visited. */
         4154  +          fts3EvalNextRow(pCsr, pLeft, pRc);
         4155  +          pExpr->iDocid = pLeft->iDocid;
         4156  +          pExpr->bEof = pLeft->bEof;
         4157  +        }else{
         4158  +          /* Neither the RHS or LHS are deferred. */
         4159  +          fts3EvalNextRow(pCsr, pLeft, pRc);
         4160  +          fts3EvalNextRow(pCsr, pRight, pRc);
         4161  +          while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
         4162  +            sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
         4163  +            if( iDiff==0 ) break;
         4164  +            if( iDiff<0 ){
         4165  +              fts3EvalNextRow(pCsr, pLeft, pRc);
         4166  +            }else{
         4167  +              fts3EvalNextRow(pCsr, pRight, pRc);
         4168  +            }
         4169  +          }
         4170  +          pExpr->iDocid = pLeft->iDocid;
         4171  +          pExpr->bEof = (pLeft->bEof || pRight->bEof);
         4172  +        }
         4173  +        break;
         4174  +      }
         4175  +  
         4176  +      case FTSQUERY_OR: {
         4177  +        Fts3Expr *pLeft = pExpr->pLeft;
         4178  +        Fts3Expr *pRight = pExpr->pRight;
         4179  +        sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
         4180  +
         4181  +        assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
         4182  +        assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );
         4183  +
         4184  +        if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
         4185  +          fts3EvalNextRow(pCsr, pLeft, pRc);
         4186  +        }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
         4187  +          fts3EvalNextRow(pCsr, pRight, pRc);
         4188  +        }else{
         4189  +          fts3EvalNextRow(pCsr, pLeft, pRc);
         4190  +          fts3EvalNextRow(pCsr, pRight, pRc);
         4191  +        }
         4192  +
         4193  +        pExpr->bEof = (pLeft->bEof && pRight->bEof);
         4194  +        iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
         4195  +        if( pRight->bEof || (pLeft->bEof==0 &&  iCmp<0) ){
         4196  +          pExpr->iDocid = pLeft->iDocid;
         4197  +        }else{
         4198  +          pExpr->iDocid = pRight->iDocid;
         4199  +        }
         4200  +
         4201  +        break;
         4202  +      }
         4203  +
         4204  +      case FTSQUERY_NOT: {
         4205  +        Fts3Expr *pLeft = pExpr->pLeft;
         4206  +        Fts3Expr *pRight = pExpr->pRight;
         4207  +
         4208  +        if( pRight->bStart==0 ){
         4209  +          fts3EvalNextRow(pCsr, pRight, pRc);
         4210  +          assert( *pRc!=SQLITE_OK || pRight->bStart );
         4211  +        }
         4212  +
         4213  +        fts3EvalNextRow(pCsr, pLeft, pRc);
         4214  +        if( pLeft->bEof==0 ){
         4215  +          while( !*pRc 
         4216  +              && !pRight->bEof 
         4217  +              && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 
         4218  +          ){
         4219  +            fts3EvalNextRow(pCsr, pRight, pRc);
         4220  +          }
         4221  +        }
         4222  +        pExpr->iDocid = pLeft->iDocid;
         4223  +        pExpr->bEof = pLeft->bEof;
         4224  +        break;
         4225  +      }
         4226  +
         4227  +      default: {
         4228  +        Fts3Phrase *pPhrase = pExpr->pPhrase;
         4229  +        fts3EvalInvalidatePoslist(pPhrase);
         4230  +        *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
         4231  +        pExpr->iDocid = pPhrase->doclist.iDocid;
         4232  +        break;
         4233  +      }
         4234  +    }
         4235  +  }
         4236  +}
         4237  +
         4238  +/*
         4239  +** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR
         4240  +** cluster, then this function returns 1 immediately.
         4241  +**
         4242  +** Otherwise, it checks if the current row really does match the NEAR 
         4243  +** expression, using the data currently stored in the position lists 
         4244  +** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. 
         4245  +**
         4246  +** If the current row is a match, the position list associated with each
         4247  +** phrase in the NEAR expression is edited in place to contain only those
         4248  +** phrase instances sufficiently close to their peers to satisfy all NEAR
         4249  +** constraints. In this case it returns 1. If the NEAR expression does not 
         4250  +** match the current row, 0 is returned. The position lists may or may not
         4251  +** be edited if 0 is returned.
         4252  +*/
  3830   4253   static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
  3831   4254     int res = 1;
  3832   4255   
  3833   4256     /* The following block runs if pExpr is the root of a NEAR query.
  3834   4257     ** For example, the query:
  3835   4258     **
  3836   4259     **         "w" NEAR "x" NEAR "y" NEAR "z"
................................................................................
  3844   4267     **                     |        |
  3845   4268     **                +--NEAR--+   "y"
  3846   4269     **                |        |
  3847   4270     **               "w"      "x"
  3848   4271     **
  3849   4272     ** The right-hand child of a NEAR node is always a phrase. The 
  3850   4273     ** left-hand child may be either a phrase or a NEAR node. There are
  3851         -  ** no exceptions to this.
         4274  +  ** no exceptions to this - it's the way the parser in fts3_expr.c works.
  3852   4275     */
  3853   4276     if( *pRc==SQLITE_OK 
  3854   4277      && pExpr->eType==FTSQUERY_NEAR 
  3855   4278      && pExpr->bEof==0
  3856   4279      && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
  3857   4280     ){
  3858   4281       Fts3Expr *p; 
................................................................................
  3871   4294       }else{
  3872   4295         char *aPoslist = p->pPhrase->doclist.pList;
  3873   4296         int nToken = p->pPhrase->nToken;
  3874   4297   
  3875   4298         for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
  3876   4299           Fts3Phrase *pPhrase = p->pRight->pPhrase;
  3877   4300           int nNear = p->nNear;
  3878         -        res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
         4301  +        res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
  3879   4302         }
  3880   4303     
  3881   4304         aPoslist = pExpr->pRight->pPhrase->doclist.pList;
  3882   4305         nToken = pExpr->pRight->pPhrase->nToken;
  3883   4306         for(p=pExpr->pLeft; p && res; p=p->pLeft){
  3884   4307           int nNear = p->pParent->nNear;
  3885   4308           Fts3Phrase *pPhrase = (
  3886   4309               p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
  3887   4310           );
  3888         -        res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
         4311  +        res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
  3889   4312         }
  3890   4313       }
  3891   4314   
  3892   4315       sqlite3_free(aTmp);
  3893   4316     }
  3894   4317   
  3895   4318     return res;
  3896   4319   }
  3897   4320   
  3898   4321   /*
  3899         -** This macro is used by the fts3EvalNext() function. The two arguments are
  3900         -** 64-bit docid values. If the current query is "ORDER BY docid ASC", then
  3901         -** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then
  3902         -** it returns (i2 - i1). This allows the same code to be used for merging
  3903         -** doclists in ascending or descending order.
         4322  +** This function is a helper function for fts3EvalTestDeferredAndNear().
         4323  +** Assuming no error occurs or has occurred, It returns non-zero if the
         4324  +** expression passed as the second argument matches the row that pCsr 
         4325  +** currently points to, or zero if it does not.
         4326  +**
         4327  +** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
         4328  +** If an error occurs during execution of this function, *pRc is set to 
         4329  +** the appropriate SQLite error code. In this case the returned value is 
         4330  +** undefined.
  3904   4331   */
  3905         -#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2))
  3906         -
  3907         -static void fts3EvalNext(
  3908         -  Fts3Cursor *pCsr, 
  3909         -  Fts3Expr *pExpr, 
  3910         -  int *pRc
         4332  +static int fts3EvalTestExpr(
         4333  +  Fts3Cursor *pCsr,               /* FTS cursor handle */
         4334  +  Fts3Expr *pExpr,                /* Expr to test. May or may not be root. */
         4335  +  int *pRc                        /* IN/OUT: Error code */
  3911   4336   ){
  3912         -  if( *pRc==SQLITE_OK ){
  3913         -    assert( pExpr->bEof==0 );
  3914         -    pExpr->bStart = 1;
  3915         -
  3916         -    switch( pExpr->eType ){
  3917         -      case FTSQUERY_NEAR:
  3918         -      case FTSQUERY_AND: {
  3919         -        Fts3Expr *pLeft = pExpr->pLeft;
  3920         -        Fts3Expr *pRight = pExpr->pRight;
  3921         -        assert( !pLeft->bDeferred || !pRight->bDeferred );
  3922         -        if( pLeft->bDeferred ){
  3923         -          fts3EvalNext(pCsr, pRight, pRc);
  3924         -          pExpr->iDocid = pRight->iDocid;
  3925         -          pExpr->bEof = pRight->bEof;
  3926         -        }else if( pRight->bDeferred ){
  3927         -          fts3EvalNext(pCsr, pLeft, pRc);
  3928         -          pExpr->iDocid = pLeft->iDocid;
  3929         -          pExpr->bEof = pLeft->bEof;
  3930         -        }else{
  3931         -          fts3EvalNext(pCsr, pLeft, pRc);
  3932         -          fts3EvalNext(pCsr, pRight, pRc);
  3933         -
  3934         -          while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
  3935         -            sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
  3936         -            if( iDiff==0 ) break;
  3937         -            if( iDiff<0 ){
  3938         -              fts3EvalNext(pCsr, pLeft, pRc);
  3939         -            }else{
  3940         -              fts3EvalNext(pCsr, pRight, pRc);
  3941         -            }
  3942         -          }
  3943         -
  3944         -          pExpr->iDocid = pLeft->iDocid;
  3945         -          pExpr->bEof = (pLeft->bEof || pRight->bEof);
  3946         -        }
  3947         -        break;
  3948         -      }
  3949         -  
  3950         -      case FTSQUERY_OR: {
  3951         -        Fts3Expr *pLeft = pExpr->pLeft;
  3952         -        Fts3Expr *pRight = pExpr->pRight;
  3953         -        sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
  3954         -
  3955         -        assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
  3956         -        assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );
  3957         -
  3958         -        if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
  3959         -          fts3EvalNext(pCsr, pLeft, pRc);
  3960         -        }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
  3961         -          fts3EvalNext(pCsr, pRight, pRc);
  3962         -        }else{
  3963         -          fts3EvalNext(pCsr, pLeft, pRc);
  3964         -          fts3EvalNext(pCsr, pRight, pRc);
  3965         -        }
  3966         -
  3967         -        pExpr->bEof = (pLeft->bEof && pRight->bEof);
  3968         -        iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
  3969         -        if( pRight->bEof || (pLeft->bEof==0 &&  iCmp<0) ){
  3970         -          pExpr->iDocid = pLeft->iDocid;
  3971         -        }else{
  3972         -          pExpr->iDocid = pRight->iDocid;
  3973         -        }
  3974         -
  3975         -        break;
  3976         -      }
  3977         -
  3978         -      case FTSQUERY_NOT: {
  3979         -        Fts3Expr *pLeft = pExpr->pLeft;
  3980         -        Fts3Expr *pRight = pExpr->pRight;
  3981         -
  3982         -        if( pRight->bStart==0 ){
  3983         -          fts3EvalNext(pCsr, pRight, pRc);
  3984         -          assert( *pRc!=SQLITE_OK || pRight->bStart );
  3985         -        }
  3986         -
  3987         -        fts3EvalNext(pCsr, pLeft, pRc);
  3988         -        if( pLeft->bEof==0 ){
  3989         -          while( !*pRc 
  3990         -              && !pRight->bEof 
  3991         -              && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 
  3992         -          ){
  3993         -            fts3EvalNext(pCsr, pRight, pRc);
  3994         -          }
  3995         -        }
  3996         -        pExpr->iDocid = pLeft->iDocid;
  3997         -        pExpr->bEof = pLeft->bEof;
  3998         -        break;
  3999         -      }
  4000         -
  4001         -      default: {
  4002         -        Fts3Phrase *pPhrase = pExpr->pPhrase;
  4003         -        fts3EvalZeroPoslist(pPhrase);
  4004         -        *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
  4005         -        pExpr->iDocid = pPhrase->doclist.iDocid;
  4006         -        break;
  4007         -      }
  4008         -    }
  4009         -  }
  4010         -}
  4011         -
  4012         -static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){
  4013         -  int bHit = 1;
         4337  +  int bHit = 1;                   /* Return value */
  4014   4338     if( *pRc==SQLITE_OK ){
  4015   4339       switch( pExpr->eType ){
  4016   4340         case FTSQUERY_NEAR:
  4017   4341         case FTSQUERY_AND:
  4018   4342           bHit = (
  4019         -            fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
  4020         -         && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
         4343  +            fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc)
         4344  +         && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc)
  4021   4345            && fts3EvalNearTest(pExpr, pRc)
  4022   4346           );
  4023   4347   
  4024   4348           /* If the NEAR expression does not match any rows, zero the doclist for 
  4025   4349           ** all phrases involved in the NEAR. This is because the snippet(),
  4026   4350           ** offsets() and matchinfo() functions are not supposed to recognize 
  4027   4351           ** any instances of phrases that are part of unmatched NEAR queries. 
................................................................................
  4039   4363           if( bHit==0 
  4040   4364            && pExpr->eType==FTSQUERY_NEAR 
  4041   4365            && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
  4042   4366           ){
  4043   4367             Fts3Expr *p;
  4044   4368             for(p=pExpr; p->pPhrase==0; p=p->pLeft){
  4045   4369               if( p->pRight->iDocid==pCsr->iPrevId ){
  4046         -              fts3EvalZeroPoslist(p->pRight->pPhrase);
         4370  +              fts3EvalInvalidatePoslist(p->pRight->pPhrase);
  4047   4371               }
  4048   4372             }
  4049   4373             if( p->iDocid==pCsr->iPrevId ){
  4050         -            fts3EvalZeroPoslist(p->pPhrase);
         4374  +            fts3EvalInvalidatePoslist(p->pPhrase);
  4051   4375             }
  4052   4376           }
  4053   4377   
  4054   4378           break;
  4055   4379   
  4056   4380         case FTSQUERY_OR: {
  4057         -        int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc);
  4058         -        int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc);
         4381  +        int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc);
         4382  +        int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc);
  4059   4383           bHit = bHit1 || bHit2;
  4060   4384           break;
  4061   4385         }
  4062   4386   
  4063   4387         case FTSQUERY_NOT:
  4064   4388           bHit = (
  4065         -            fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
  4066         -         && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
         4389  +            fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc)
         4390  +         && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc)
  4067   4391           );
  4068   4392           break;
  4069   4393   
  4070   4394         default: {
  4071   4395           if( pCsr->pDeferred 
  4072   4396            && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred)
  4073   4397           ){
  4074   4398             Fts3Phrase *pPhrase = pExpr->pPhrase;
  4075   4399             assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 );
  4076   4400             if( pExpr->bDeferred ){
  4077         -            fts3EvalZeroPoslist(pPhrase);
         4401  +            fts3EvalInvalidatePoslist(pPhrase);
  4078   4402             }
  4079   4403             *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase);
  4080   4404             bHit = (pPhrase->doclist.pList!=0);
  4081   4405             pExpr->iDocid = pCsr->iPrevId;
  4082   4406           }else{
  4083   4407             bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId);
  4084   4408           }
................................................................................
  4086   4410         }
  4087   4411       }
  4088   4412     }
  4089   4413     return bHit;
  4090   4414   }
  4091   4415   
  4092   4416   /*
  4093         -** Return 1 if both of the following are true:
         4417  +** This function is called as the second part of each xNext operation when
         4418  +** iterating through the results of a full-text query. At this point the
         4419  +** cursor points to a row that matches the query expression, with the
         4420  +** following caveats:
         4421  +**
         4422  +**   * Up until this point, "NEAR" operators in the expression have been
         4423  +**     treated as "AND".
         4424  +**
         4425  +**   * Deferred tokens have not yet been considered.
         4426  +**
         4427  +** If *pRc is not SQLITE_OK when this function is called, it immediately
         4428  +** returns 0. Otherwise, it tests whether or not after considering NEAR
         4429  +** operators and deferred tokens the current row is still a match for the
         4430  +** expression. It returns 1 if both of the following are true:
  4094   4431   **
  4095   4432   **   1. *pRc is SQLITE_OK when this function returns, and
  4096   4433   **
  4097   4434   **   2. After scanning the current FTS table row for the deferred tokens,
  4098         -**      it is determined that the row does not match the query.
         4435  +**      it is determined that the row does *not* match the query.
  4099   4436   **
  4100   4437   ** Or, if no error occurs and it seems the current row does match the FTS
  4101   4438   ** query, return 0.
  4102   4439   */
  4103         -static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){
         4440  +static int fts3EvalTestDeferredAndNear(Fts3Cursor *pCsr, int *pRc){
  4104   4441     int rc = *pRc;
  4105   4442     int bMiss = 0;
  4106   4443     if( rc==SQLITE_OK ){
         4444  +
         4445  +    /* If there are one or more deferred tokens, load the current row into
         4446  +    ** memory and scan it to determine the position list for each deferred
         4447  +    ** token. Then, see if this row is really a match, considering deferred
         4448  +    ** tokens and NEAR operators (neither of which were taken into account
         4449  +    ** earlier, by fts3EvalNextRow()). 
         4450  +    */
  4107   4451       if( pCsr->pDeferred ){
  4108   4452         rc = fts3CursorSeek(0, pCsr);
  4109   4453         if( rc==SQLITE_OK ){
  4110   4454           rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
  4111   4455         }
  4112   4456       }
  4113         -    bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc));
         4457  +    bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc));
         4458  +
         4459  +    /* Free the position-lists accumulated for each deferred token above. */
  4114   4460       sqlite3Fts3FreeDeferredDoclists(pCsr);
  4115   4461       *pRc = rc;
  4116   4462     }
  4117   4463     return (rc==SQLITE_OK && bMiss);
  4118   4464   }
  4119   4465   
  4120   4466   /*
  4121   4467   ** Advance to the next document that matches the FTS expression in
  4122   4468   ** Fts3Cursor.pExpr.
  4123   4469   */
  4124         -int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){
         4470  +static int fts3EvalNext(Fts3Cursor *pCsr){
  4125   4471     int rc = SQLITE_OK;             /* Return Code */
  4126   4472     Fts3Expr *pExpr = pCsr->pExpr;
  4127   4473     assert( pCsr->isEof==0 );
  4128   4474     if( pExpr==0 ){
  4129   4475       pCsr->isEof = 1;
  4130   4476     }else{
  4131   4477       do {
  4132   4478         if( pCsr->isRequireSeek==0 ){
  4133   4479           sqlite3_reset(pCsr->pStmt);
  4134   4480         }
  4135   4481         assert( sqlite3_data_count(pCsr->pStmt)==0 );
  4136         -      fts3EvalNext(pCsr, pExpr, &rc);
         4482  +      fts3EvalNextRow(pCsr, pExpr, &rc);
  4137   4483         pCsr->isEof = pExpr->bEof;
  4138   4484         pCsr->isRequireSeek = 1;
  4139   4485         pCsr->isMatchinfoNeeded = 1;
  4140   4486         pCsr->iPrevId = pExpr->iDocid;
  4141         -    }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) );
         4487  +    }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) );
  4142   4488     }
  4143   4489     return rc;
  4144   4490   }
  4145   4491   
  4146   4492   /*
  4147   4493   ** Restart interation for expression pExpr so that the next call to
  4148         -** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental 
         4494  +** fts3EvalNext() visits the first row. Do not allow incremental 
  4149   4495   ** loading or merging of phrase doclists for this iteration.
  4150   4496   **
  4151   4497   ** If *pRc is other than SQLITE_OK when this function is called, it is
  4152   4498   ** a no-op. If an error occurs within this function, *pRc is set to an
  4153   4499   ** SQLite error code before returning.
  4154   4500   */
  4155   4501   static void fts3EvalRestart(
................................................................................
  4157   4503     Fts3Expr *pExpr,
  4158   4504     int *pRc
  4159   4505   ){
  4160   4506     if( pExpr && *pRc==SQLITE_OK ){
  4161   4507       Fts3Phrase *pPhrase = pExpr->pPhrase;
  4162   4508   
  4163   4509       if( pPhrase ){
  4164         -      fts3EvalZeroPoslist(pPhrase);
         4510  +      fts3EvalInvalidatePoslist(pPhrase);
  4165   4511         if( pPhrase->bIncr ){
  4166   4512           assert( pPhrase->nToken==1 );
  4167   4513           assert( pPhrase->aToken[0].pSegcsr );
  4168   4514           sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr);
  4169   4515           *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase);
  4170   4516         }
  4171   4517   
................................................................................
  4273   4619   
  4274   4620         do {
  4275   4621           /* Ensure the %_content statement is reset. */
  4276   4622           if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt);
  4277   4623           assert( sqlite3_data_count(pCsr->pStmt)==0 );
  4278   4624   
  4279   4625           /* Advance to the next document */
  4280         -        fts3EvalNext(pCsr, pRoot, &rc);
         4626  +        fts3EvalNextRow(pCsr, pRoot, &rc);
  4281   4627           pCsr->isEof = pRoot->bEof;
  4282   4628           pCsr->isRequireSeek = 1;
  4283   4629           pCsr->isMatchinfoNeeded = 1;
  4284   4630           pCsr->iPrevId = pRoot->iDocid;
  4285   4631         }while( pCsr->isEof==0 
  4286   4632              && pRoot->eType==FTSQUERY_NEAR 
  4287         -           && fts3EvalLoadDeferred(pCsr, &rc) 
         4633  +           && fts3EvalTestDeferredAndNear(pCsr, &rc) 
  4288   4634         );
  4289   4635   
  4290   4636         if( rc==SQLITE_OK && pCsr->isEof==0 ){
  4291   4637           fts3EvalUpdateCounts(pRoot);
  4292   4638         }
  4293   4639       }
  4294   4640   
................................................................................
  4302   4648         ** order. For this reason, even though it seems more defensive, the 
  4303   4649         ** do loop can not be written:
  4304   4650         **
  4305   4651         **   do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK );
  4306   4652         */
  4307   4653         fts3EvalRestart(pCsr, pRoot, &rc);
  4308   4654         do {
  4309         -        fts3EvalNext(pCsr, pRoot, &rc);
         4655  +        fts3EvalNextRow(pCsr, pRoot, &rc);
  4310   4656           assert( pRoot->bEof==0 );
  4311   4657         }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
  4312         -      fts3EvalLoadDeferred(pCsr, &rc);
         4658  +      fts3EvalTestDeferredAndNear(pCsr, &rc);
  4313   4659       }
  4314   4660     }
  4315   4661     return rc;
  4316   4662   }
  4317   4663   
  4318   4664   /*
  4319   4665   ** This function is used by the matchinfo() module to query a phrase 
................................................................................
  4436   4782   **   * the contents of pPhrase->doclist, and
  4437   4783   **   * any Fts3MultiSegReader objects held by phrase tokens.
  4438   4784   */
  4439   4785   void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){
  4440   4786     if( pPhrase ){
  4441   4787       int i;
  4442   4788       sqlite3_free(pPhrase->doclist.aAll);
  4443         -    fts3EvalZeroPoslist(pPhrase);
         4789  +    fts3EvalInvalidatePoslist(pPhrase);
  4444   4790       memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
  4445   4791       for(i=0; i<pPhrase->nToken; i++){
  4446   4792         fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr);
  4447   4793         pPhrase->aToken[i].pSegcsr = 0;
  4448   4794       }
  4449   4795     }
  4450   4796   }
         4797  +
         4798  +#if !SQLITE_CORE
         4799  +/*
         4800  +** Initialize API pointer table, if required.
         4801  +*/
         4802  +int sqlite3_extension_init(
         4803  +  sqlite3 *db, 
         4804  +  char **pzErrMsg,
         4805  +  const sqlite3_api_routines *pApi
         4806  +){
         4807  +  SQLITE_EXTENSION_INIT2(pApi)
         4808  +  return sqlite3Fts3Init(db);
         4809  +}
         4810  +#endif
  4451   4811   
  4452   4812   #endif

Changes to ext/fts3/fts3Int.h.

    23     23   ** SQLITE_ENABLE_FTS3 macro.  But to avoid confusion we also all
    24     24   ** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3.
    25     25   */
    26     26   #if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3)
    27     27   # define SQLITE_ENABLE_FTS3
    28     28   #endif
    29     29   
    30         -#ifdef SQLITE_ENABLE_FTS3
           30  +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
           31  +
           32  +/* If not building as part of the core, include sqlite3ext.h. */
           33  +#ifndef SQLITE_CORE
           34  +# include "sqlite3ext.h" 
           35  +extern const sqlite3_api_routines *sqlite3_api;
           36  +#endif
           37  +
    31     38   #include "sqlite3.h"
    32     39   #include "fts3_tokenizer.h"
    33     40   #include "fts3_hash.h"
    34     41   
    35     42   /*
    36     43   ** This constant controls how often segments are merged. Once there are
    37     44   ** FTS3_MERGE_COUNT segments of level N, they are merged into a single
................................................................................
   286    293     int nAll;                      /* Size of a[] in bytes */
   287    294     char *pNextDocid;              /* Pointer to next docid */
   288    295   
   289    296     sqlite3_int64 iDocid;          /* Current docid (if pList!=0) */
   290    297     int bFreeList;                 /* True if pList should be sqlite3_free()d */
   291    298     char *pList;                   /* Pointer to position list following iDocid */
   292    299     int nList;                     /* Length of position list */
   293         -} doclist;
          300  +};
   294    301   
   295    302   /*
   296    303   ** A "phrase" is a sequence of one or more tokens that must match in
   297    304   ** sequence.  A single token is the base case and the most common case.
   298    305   ** For a sequence of tokens contained in double-quotes (i.e. "one two three")
   299    306   ** nToken will be the number of tokens in the string.
   300    307   */
................................................................................
   486    493   int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
   487    494   int sqlite3Fts3InitTerm(sqlite3 *db);
   488    495   #endif
   489    496   
   490    497   /* fts3_aux.c */
   491    498   int sqlite3Fts3InitAux(sqlite3 *db);
   492    499   
   493         -int sqlite3Fts3TermSegReaderCursor(
   494         -  Fts3Cursor *pCsr,               /* Virtual table cursor handle */
   495         -  const char *zTerm,              /* Term to query for */
   496         -  int nTerm,                      /* Size of zTerm in bytes */
   497         -  int isPrefix,                   /* True for a prefix search */
   498         -  Fts3MultiSegReader **ppSegcsr   /* OUT: Allocated seg-reader cursor */
   499         -);
   500         -
   501    500   void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
   502    501   
   503         -int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int);
   504         -int sqlite3Fts3EvalNext(Fts3Cursor *pCsr);
   505         -
   506    502   int sqlite3Fts3MsrIncrStart(
   507    503       Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
   508    504   int sqlite3Fts3MsrIncrNext(
   509    505       Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
   510    506   char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); 
   511    507   int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
   512    508   int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
   513    509   
   514    510   int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);
   515    511   
   516         -#endif /* SQLITE_ENABLE_FTS3 */
          512  +#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
   517    513   #endif /* _FTSINT_H */

Changes to ext/fts3/fts3_hash.c.

    26     26   #include "fts3Int.h"
    27     27   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    28     28   
    29     29   #include <assert.h>
    30     30   #include <stdlib.h>
    31     31   #include <string.h>
    32     32   
    33         -#include "sqlite3.h"
    34     33   #include "fts3_hash.h"
    35     34   
    36     35   /*
    37     36   ** Malloc and Free functions
    38     37   */
    39     38   static void *fts3HashMalloc(int n){
    40     39     void *p = sqlite3_malloc(n);

Changes to ext/fts3/fts3_test.c.

    14     14   ** testing. It contains a Tcl command that can be used to test if a document
    15     15   ** matches an FTS NEAR expression.
    16     16   */
    17     17   
    18     18   #include <tcl.h>
    19     19   #include <string.h>
    20     20   #include <assert.h>
           21  +
           22  +#ifdef SQLITE_TEST
    21     23   
    22     24   /* Required so that the "ifdef SQLITE_ENABLE_FTS3" below works */
    23     25   #include "fts3Int.h"
    24     26   
    25     27   #define NM_MAX_TOKEN 12
    26     28   
    27     29   typedef struct NearPhrase NearPhrase;
................................................................................
   315    317   int Sqlitetestfts3_Init(Tcl_Interp *interp){
   316    318     Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);
   317    319     Tcl_CreateObjCommand(interp, 
   318    320         "fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0
   319    321     );
   320    322     return TCL_OK;
   321    323   }
          324  +#endif                  /* ifdef SQLITE_TEST */

Changes to ext/fts3/fts3_tokenizer.c.

    19     19   **
    20     20   **     * The FTS3 module is being built as an extension
    21     21   **       (in which case SQLITE_CORE is not defined), or
    22     22   **
    23     23   **     * The FTS3 module is being built into the core of
    24     24   **       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
    25     25   */
    26         -#include "sqlite3ext.h"
    27         -#ifndef SQLITE_CORE
    28         -  SQLITE_EXTENSION_INIT1
    29         -#endif
    30     26   #include "fts3Int.h"
    31         -
    32     27   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    33     28   
    34     29   #include <assert.h>
    35     30   #include <string.h>
    36     31   
    37     32   /*
    38     33   ** Implementation of the SQL scalar function for accessing the underlying 

Changes to main.mk.

   382    382   
   383    383   sqlite3.c:	target_source $(TOP)/tool/mksqlite3c.tcl
   384    384   	tclsh $(TOP)/tool/mksqlite3c.tcl
   385    385   	echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c
   386    386   	cat sqlite3.c >>tclsqlite3.c
   387    387   	echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c
   388    388   	cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c
          389  +
          390  +sqlite3-all.c:	sqlite3.c $(TOP)/tool/split-sqlite3c.tcl
          391  +	tclsh $(TOP)/tool/split-sqlite3c.tcl
   389    392   
   390    393   fts2amal.c:	target_source $(TOP)/ext/fts2/mkfts2amal.tcl
   391    394   	tclsh $(TOP)/ext/fts2/mkfts2amal.tcl
   392    395   
   393    396   fts3amal.c:	target_source $(TOP)/ext/fts3/mkfts3amal.tcl
   394    397   	tclsh $(TOP)/ext/fts3/mkfts3amal.tcl
   395    398   
................................................................................
   564    567   
   565    568   TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO)
   566    569   $(TEST_EXTENSION): $(TOP)/src/test_loadext.c
   567    570   	$(MKSHLIB) $(TOP)/src/test_loadext.c -o $(TEST_EXTENSION)
   568    571   
   569    572   extensiontest: testfixture$(EXE) $(TEST_EXTENSION)
   570    573   	./testfixture$(EXE) $(TOP)/test/loadext.test
          574  +
          575  +# This target will fail if the SQLite amalgamation contains any exported
          576  +# symbols that do not begin with "sqlite3_". It is run as part of the
          577  +# releasetest.tcl script.
          578  +#
          579  +checksymbols: sqlite3.o
          580  +	nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0
   571    581   
   572    582   
   573    583   # Standard install and cleanup targets
   574    584   #
   575    585   install:	sqlite3 libsqlite3.a sqlite3.h
   576    586   	mv sqlite3 /usr/bin
   577    587   	mv libsqlite3.a /usr/lib

Changes to src/build.c.

  2787   2787       ** the zStmt variable
  2788   2788       */
  2789   2789       if( pStart ){
  2790   2790         assert( pEnd!=0 );
  2791   2791         /* A named index with an explicit CREATE INDEX statement */
  2792   2792         zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s",
  2793   2793           onError==OE_None ? "" : " UNIQUE",
  2794         -        pEnd->z - pName->z + 1,
         2794  +        (int)(pEnd->z - pName->z) + 1,
  2795   2795           pName->z);
  2796   2796       }else{
  2797   2797         /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */
  2798   2798         /* zStmt = sqlite3MPrintf(""); */
  2799   2799         zStmt = 0;
  2800   2800       }
  2801   2801   

Changes to src/date.c.

   423    423   ** routine will always fail.
   424    424   */
   425    425   static int osLocaltime(time_t *t, struct tm *pTm){
   426    426     int rc;
   427    427   #if (!defined(HAVE_LOCALTIME_R) || !HAVE_LOCALTIME_R) \
   428    428         && (!defined(HAVE_LOCALTIME_S) || !HAVE_LOCALTIME_S)
   429    429     struct tm *pX;
          430  +#if SQLITE_THREADSAFE>0
   430    431     sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER);
          432  +#endif
   431    433     sqlite3_mutex_enter(mutex);
   432    434     pX = localtime(t);
   433    435   #ifndef SQLITE_OMIT_BUILTIN_TEST
   434    436     if( sqlite3GlobalConfig.bLocaltimeFault ) pX = 0;
   435    437   #endif
   436    438     if( pX ) *pTm = *pX;
   437    439     sqlite3_mutex_leave(mutex);

Changes to src/delete.c.

   367    367       int iRowSet = ++pParse->nMem;   /* Register for rowset of rows to delete */
   368    368       int iRowid = ++pParse->nMem;    /* Used for storing rowid values. */
   369    369       int regRowid;                   /* Actual register containing rowids */
   370    370   
   371    371       /* Collect rowids of every row to be deleted.
   372    372       */
   373    373       sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet);
   374         -    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,WHERE_DUPLICATES_OK);
          374  +    pWInfo = sqlite3WhereBegin(
          375  +        pParse, pTabList, pWhere, 0, 0, WHERE_DUPLICATES_OK
          376  +    );
   375    377       if( pWInfo==0 ) goto delete_from_cleanup;
   376    378       regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid);
   377    379       sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid);
   378    380       if( db->flags & SQLITE_CountRows ){
   379    381         sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1);
   380    382       }
   381    383       sqlite3WhereEnd(pWInfo);

Changes to src/expr.c.

   898    898       Table *pTab;
   899    899       pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase);
   900    900       pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName);
   901    901       pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias);
   902    902       pNewItem->jointype = pOldItem->jointype;
   903    903       pNewItem->iCursor = pOldItem->iCursor;
   904    904       pNewItem->isPopulated = pOldItem->isPopulated;
          905  +    pNewItem->isCorrelated = pOldItem->isCorrelated;
   905    906       pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex);
   906    907       pNewItem->notIndexed = pOldItem->notIndexed;
   907    908       pNewItem->pIndex = pOldItem->pIndex;
   908    909       pTab = pNewItem->pTab = pOldItem->pTab;
   909    910       if( pTab ){
   910    911         pTab->nRef++;
   911    912       }

Changes to src/fkey.c.

   556    556     sNameContext.pParse = pParse;
   557    557     sqlite3ResolveExprNames(&sNameContext, pWhere);
   558    558   
   559    559     /* Create VDBE to loop through the entries in pSrc that match the WHERE
   560    560     ** clause. If the constraint is not deferred, throw an exception for
   561    561     ** each row found. Otherwise, for deferred constraints, increment the
   562    562     ** deferred constraint counter by nIncr for each row selected.  */
   563         -  pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0);
          563  +  pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0);
   564    564     if( nIncr>0 && pFKey->isDeferred==0 ){
   565    565       sqlite3ParseToplevel(pParse)->mayAbort = 1;
   566    566     }
   567    567     sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr);
   568    568     if( pWInfo ){
   569    569       sqlite3WhereEnd(pWInfo);
   570    570     }

Changes to src/loadext.c.

    80     80   # define sqlite3_progress_handler 0
    81     81   #endif
    82     82   
    83     83   #ifdef SQLITE_OMIT_VIRTUALTABLE
    84     84   # define sqlite3_create_module 0
    85     85   # define sqlite3_create_module_v2 0
    86     86   # define sqlite3_declare_vtab 0
           87  +# define sqlite3_vtab_config 0
           88  +# define sqlite3_vtab_on_conflict 0
    87     89   #endif
    88     90   
    89     91   #ifdef SQLITE_OMIT_SHARED_CACHE
    90     92   # define sqlite3_enable_shared_cache 0
    91     93   #endif
    92     94   
    93     95   #ifdef SQLITE_OMIT_TRACE
................................................................................
   103    105   #ifdef SQLITE_OMIT_INCRBLOB
   104    106   #define sqlite3_bind_zeroblob  0
   105    107   #define sqlite3_blob_bytes     0
   106    108   #define sqlite3_blob_close     0
   107    109   #define sqlite3_blob_open      0
   108    110   #define sqlite3_blob_read      0
   109    111   #define sqlite3_blob_write     0
          112  +#define sqlite3_blob_reopen    0
   110    113   #endif
   111    114   
   112    115   /*
   113    116   ** The following structure contains pointers to all SQLite API routines.
   114    117   ** A pointer to this structure is passed into extensions when they are
   115    118   ** loaded so that the extension can make calls back into the SQLite
   116    119   ** library.
................................................................................
   368    371     sqlite3_wal_checkpoint,
   369    372     sqlite3_wal_hook,
   370    373   #else
   371    374     0,
   372    375     0,
   373    376     0,
   374    377   #endif
          378  +  sqlite3_blob_reopen,
          379  +  sqlite3_vtab_config,
          380  +  sqlite3_vtab_on_conflict,
   375    381   };
   376    382   
   377    383   /*
   378    384   ** Attempt to load an SQLite extension library contained in the file
   379    385   ** zFile.  The entry point is zProc.  zProc may be 0 in which case a
   380    386   ** default entry point name (sqlite3_extension_init) is used.  Use
   381    387   ** of the default name is recommended.

Changes to src/mem3.c.

   429    429   
   430    430   /*
   431    431   ** Free an outstanding memory allocation.
   432    432   **
   433    433   ** This function assumes that the necessary mutexes, if any, are
   434    434   ** already held by the caller. Hence "Unsafe".
   435    435   */
   436         -void memsys3FreeUnsafe(void *pOld){
          436  +static void memsys3FreeUnsafe(void *pOld){
   437    437     Mem3Block *p = (Mem3Block*)pOld;
   438    438     int i;
   439    439     u32 size, x;
   440    440     assert( sqlite3_mutex_held(mem3.mutex) );
   441    441     assert( p>mem3.aPool && p<&mem3.aPool[mem3.nPool] );
   442    442     i = p - mem3.aPool;
   443    443     assert( (mem3.aPool[i-1].u.hdr.size4x&1)==1 );
................................................................................
   504    504     memsys3Leave();
   505    505     return (void*)p; 
   506    506   }
   507    507   
   508    508   /*
   509    509   ** Free memory.
   510    510   */
   511         -void memsys3Free(void *pPrior){
          511  +static void memsys3Free(void *pPrior){
   512    512     assert( pPrior );
   513    513     memsys3Enter();
   514    514     memsys3FreeUnsafe(pPrior);
   515    515     memsys3Leave();
   516    516   }
   517    517   
   518    518   /*
   519    519   ** Change the size of an existing memory allocation
   520    520   */
   521         -void *memsys3Realloc(void *pPrior, int nBytes){
          521  +static void *memsys3Realloc(void *pPrior, int nBytes){
   522    522     int nOld;
   523    523     void *p;
   524    524     if( pPrior==0 ){
   525    525       return sqlite3_malloc(nBytes);
   526    526     }
   527    527     if( nBytes<=0 ){
   528    528       sqlite3_free(pPrior);

Changes to src/os.c.

   134    134     int openFlags;
   135    135     DO_OS_MALLOC_TEST(0);
   136    136     /* 0x87f3f is a mask of SQLITE_OPEN_ flags that are valid to be passed
   137    137     ** down into the VFS layer.  Some SQLITE_OPEN_ flags (for example,
   138    138     ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before
   139    139     ** reaching the VFS. */
   140    140   #if SQLITE_ENABLE_DATA_PROTECTION
   141         -  openFlags = flags & (0x87f3f | SQLITE_OPEN_FILEPROTECTION_MASK);
          141  +  openFlags = flags & (0x87f7f | SQLITE_OPEN_FILEPROTECTION_MASK);
   142    142   #else
   143         -  openFlags = flags & 0x87f3f;
          143  +  openFlags = flags & 0x87f7f;
   144    144   #endif
   145    145     rc = pVfs->xOpen(pVfs, zPath, pFile, openFlags, pFlagsOut);
   146    146     assert( rc==SQLITE_OK || pFile->pMethods==0 );
   147    147     return rc;
   148    148   }
   149    149   int sqlite3OsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
   150    150     return pVfs->xDelete(pVfs, zPath, dirSync);

Changes to src/os_common.h.

    26     26   ** switch.  The following code should catch this problem at compile-time.
    27     27   */
    28     28   #ifdef MEMORY_DEBUG
    29     29   # error "The MEMORY_DEBUG macro is obsolete.  Use SQLITE_DEBUG instead."
    30     30   #endif
    31     31   
    32     32   #ifdef SQLITE_DEBUG
    33         -int sqlite3OSTrace = 0;
    34         -#define OSTRACE(X)          if( sqlite3OSTrace ) sqlite3DebugPrintf X
           33  +# ifndef SQLITE_DEBUG_OS_TRACE
           34  +#   define SQLITE_DEBUG_OS_TRACE 0
           35  +# endif
           36  +  int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE;
           37  +# define OSTRACE(X)          if( sqlite3OSTrace ) sqlite3DebugPrintf X
    35     38   #else
    36         -#define OSTRACE(X)
           39  +# define OSTRACE(X)
    37     40   #endif
    38     41   
    39     42   /*
    40     43   ** Macros for performance tracing.  Normally turned off.  Only works
    41     44   ** on i486 hardware.
    42     45   */
    43     46   #ifdef SQLITE_PERFORMANCE_TRACE

Changes to src/os_unix.c.

   254    254     char aPadding[32];
   255    255   #endif
   256    256   };
   257    257   
   258    258   /*
   259    259   ** Allowed values for the unixFile.ctrlFlags bitmask:
   260    260   */
   261         -#define UNIXFILE_EXCL   0x01     /* Connections from one process only */
   262         -#define UNIXFILE_RDONLY 0x02     /* Connection is read only */
          261  +#define UNIXFILE_EXCL        0x01     /* Connections from one process only */
          262  +#define UNIXFILE_RDONLY      0x02     /* Connection is read only */
          263  +#define UNIXFILE_PERSIST_WAL 0x04     /* Persistent WAL mode */
   263    264   
   264    265   /*
   265    266   ** Include code that is common to all os_*.c files
   266    267   */
   267    268   #include "os_common.h"
   268    269   
   269    270   /*
................................................................................
   960    961     case EIO:
   961    962     case EBADF:
   962    963     case EINVAL:
   963    964     case ENOTCONN:
   964    965     case ENODEV:
   965    966     case ENXIO:
   966    967     case ENOENT:
          968  +#ifdef ESTALE                     /* ESTALE is not defined on Interix systems */
   967    969     case ESTALE:
          970  +#endif
   968    971     case ENOSYS:
   969    972       /* these should force the client to close the file and reconnect */
   970    973       
   971    974     default: 
   972    975       return sqliteIOErr;
   973    976     }
   974    977   }
................................................................................
  3789   3792   ** file-control operation.
  3790   3793   **
  3791   3794   ** If the user has configured a chunk-size for this file, it could be
  3792   3795   ** that the file needs to be extended at this point. Otherwise, the
  3793   3796   ** SQLITE_FCNTL_SIZE_HINT operation is a no-op for Unix.
  3794   3797   */
  3795   3798   static int fcntlSizeHint(unixFile *pFile, i64 nByte){
  3796         -  if( pFile->szChunk ){
         3799  +  { /* preserve indentation of removed "if" */
  3797   3800       i64 nSize;                    /* Required file size */
         3801  +    i64 szChunk;                  /* Chunk size */
  3798   3802       struct stat buf;              /* Used to hold return values of fstat() */
  3799   3803      
  3800   3804       if( osFstat(pFile->h, &buf) ) return SQLITE_IOERR_FSTAT;
  3801   3805   
  3802         -    nSize = ((nByte+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk;
         3806  +    szChunk = pFile->szChunk;
         3807  +    if( szChunk==0 ){
         3808  +      nSize = nByte;
         3809  +    }else{
         3810  +      nSize = ((nByte+szChunk-1) / szChunk) * szChunk;
         3811  +    }
  3803   3812       if( nSize>(i64)buf.st_size ){
  3804   3813   
  3805   3814   #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE
  3806   3815         /* The code below is handling the return value of osFallocate() 
  3807   3816         ** correctly. posix_fallocate() is defined to "returns zero on success, 
  3808   3817         ** or an error number on  failure". See the manpage for details. */
  3809   3818         int err;
................................................................................
  3845   3854   #endif
  3846   3855   static int isProxyLockingMode(unixFile *);
  3847   3856   
  3848   3857   /*
  3849   3858   ** Information and control of an open file handle.
  3850   3859   */
  3851   3860   static int unixFileControl(sqlite3_file *id, int op, void *pArg){
         3861  +  unixFile *pFile = (unixFile*)id;
  3852   3862     switch( op ){
  3853   3863       case SQLITE_FCNTL_LOCKSTATE: {
  3854         -      *(int*)pArg = ((unixFile*)id)->eFileLock;
         3864  +      *(int*)pArg = pFile->eFileLock;
  3855   3865         return SQLITE_OK;
  3856   3866       }
  3857   3867       case SQLITE_LAST_ERRNO: {
  3858         -      *(int*)pArg = ((unixFile*)id)->lastErrno;
         3868  +      *(int*)pArg = pFile->lastErrno;
  3859   3869         return SQLITE_OK;
  3860   3870       }
  3861   3871       case SQLITE_FCNTL_CHUNK_SIZE: {
  3862         -      ((unixFile*)id)->szChunk = *(int *)pArg;
         3872  +      pFile->szChunk = *(int *)pArg;
  3863   3873         return SQLITE_OK;
  3864   3874       }
  3865   3875       case SQLITE_FCNTL_SIZE_HINT: {
  3866         -      return fcntlSizeHint((unixFile *)id, *(i64 *)pArg);
         3876  +      return fcntlSizeHint(pFile, *(i64 *)pArg);
         3877  +    }
         3878  +    case SQLITE_FCNTL_PERSIST_WAL: {
         3879  +      int bPersist = *(int*)pArg;
         3880  +      if( bPersist<0 ){
         3881  +        *(int*)pArg = (pFile->ctrlFlags & UNIXFILE_PERSIST_WAL)!=0;
         3882  +      }else if( bPersist==0 ){
         3883  +        pFile->ctrlFlags &= ~UNIXFILE_PERSIST_WAL;
         3884  +      }else{
         3885  +        pFile->ctrlFlags |= UNIXFILE_PERSIST_WAL;
         3886  +      }
         3887  +      return SQLITE_OK;
  3867   3888       }
  3868   3889   #ifndef NDEBUG
  3869   3890       /* The pager calls this method to signal that it has done
  3870   3891       ** a rollback and that the database is therefore unchanged and
  3871   3892       ** it hence it is OK for the transaction change counter to be
  3872   3893       ** unchanged.
  3873   3894       */
................................................................................
  4334   4355   */
  4335   4356   static void unixShmPurge(unixFile *pFd){
  4336   4357     unixShmNode *p = pFd->pInode->pShmNode;
  4337   4358     assert( unixMutexHeld() );
  4338   4359     if( p && p->nRef==0 ){
  4339   4360       int i;
  4340   4361       assert( p->pInode==pFd->pInode );
  4341         -    if( p->mutex ) sqlite3_mutex_free(p->mutex);
         4362  +    sqlite3_mutex_free(p->mutex);
  4342   4363       for(i=0; i<p->nRegion; i++){
  4343   4364         if( p->h>=0 ){
  4344   4365           munmap(p->apRegion[i], p->szRegion);
  4345   4366         }else{
  4346   4367           sqlite3_free(p->apRegion[i]);
  4347   4368         }
  4348   4369       }
................................................................................
  5467   5488     ** almost certain that an open() call on the same path will also fail.
  5468   5489     ** For this reason, if an error occurs in the stat() call here, it is
  5469   5490     ** ignored and -1 is returned. The caller will try to open a new file
  5470   5491     ** descriptor on the same path, fail, and return an error to SQLite.
  5471   5492     **
  5472   5493     ** Even if a subsequent open() call does succeed, the consequences of
  5473   5494     ** not searching for a resusable file descriptor are not dire.  */
  5474         -  if( 0==stat(zPath, &sStat) ){
         5495  +  if( 0==osStat(zPath, &sStat) ){
  5475   5496       unixInodeInfo *pInode;
  5476   5497   
  5477   5498       unixEnterMutex();
  5478   5499       pInode = inodeList;
  5479   5500       while( pInode && (pInode->fileId.dev!=sStat.st_dev
  5480   5501                        || pInode->fileId.ino!=sStat.st_ino) ){
  5481   5502          pInode = pInode->pNext;
................................................................................
  5545   5566       */
  5546   5567       nDb = sqlite3Strlen30(zPath) - 1; 
  5547   5568       while( nDb>0 && zPath[nDb]!='-' ) nDb--;
  5548   5569       if( nDb==0 ) return SQLITE_OK;
  5549   5570       memcpy(zDb, zPath, nDb);
  5550   5571       zDb[nDb] = '\0';
  5551   5572   
  5552         -    if( 0==stat(zDb, &sStat) ){
         5573  +    if( 0==osStat(zDb, &sStat) ){
  5553   5574         *pMode = sStat.st_mode & 0777;
  5554   5575         *pUid = sStat.st_uid;
  5555   5576         *pGid = sStat.st_gid;
  5556   5577       }else{
  5557   5578         rc = SQLITE_IOERR_FSTAT;
  5558   5579       }
  5559   5580     }else if( flags & SQLITE_OPEN_DELETEONCLOSE ){
................................................................................
  5910   5931   
  5911   5932       default:
  5912   5933         assert(!"Invalid flags argument");
  5913   5934     }
  5914   5935     *pResOut = (osAccess(zPath, amode)==0);
  5915   5936     if( flags==SQLITE_ACCESS_EXISTS && *pResOut ){
  5916   5937       struct stat buf;
  5917         -    if( 0==stat(zPath, &buf) && buf.st_size==0 ){
         5938  +    if( 0==osStat(zPath, &buf) && buf.st_size==0 ){
  5918   5939         *pResOut = 0;
  5919   5940       }
  5920   5941     }
  5921   5942     return SQLITE_OK;
  5922   5943   }
  5923   5944   
  5924   5945   

Changes to src/os_win.c.

    98     98   ** portability layer.
    99     99   */
   100    100   typedef struct winFile winFile;
   101    101   struct winFile {
   102    102     const sqlite3_io_methods *pMethod; /*** Must be first ***/
   103    103     sqlite3_vfs *pVfs;      /* The VFS used to open this file */
   104    104     HANDLE h;               /* Handle for accessing the file */
   105         -  unsigned char locktype; /* Type of lock currently held on this file */
          105  +  u8 locktype;            /* Type of lock currently held on this file */
   106    106     short sharedLockByte;   /* Randomly chosen byte used as a shared lock */
          107  +  u8 bPersistWal;         /* True to persist WAL files */
   107    108     DWORD lastErrno;        /* The Windows errno from the last I/O error */
   108    109     DWORD sectorSize;       /* Sector size of the device file is on */
   109    110     winShm *pShm;           /* Instance of shared memory on this file */
   110    111     const char *zPath;      /* Full pathname of this file */
   111    112     int szChunk;            /* Chunk size configured by FCNTL_CHUNK_SIZE */
   112    113   #if SQLITE_OS_WINCE
   113    114     WCHAR *zDeleteOnClose;  /* Name of file to delete when closing */
................................................................................
   397    398     sqlite3_log(errcode,
   398    399         "os_win.c:%d: (%d) %s(%s) - %s",
   399    400         iLine, iErrno, zFunc, zPath, zMsg
   400    401     );
   401    402   
   402    403     return errcode;
   403    404   }
          405  +
          406  +/*
          407  +** The number of times that a ReadFile(), WriteFile(), and DeleteFile()
          408  +** will be retried following a locking error - probably caused by 
          409  +** antivirus software.  Also the initial delay before the first retry.
          410  +** The delay increases linearly with each retry.
          411  +*/
          412  +#ifndef SQLITE_WIN32_IOERR_RETRY
          413  +# define SQLITE_WIN32_IOERR_RETRY 10
          414  +#endif
          415  +#ifndef SQLITE_WIN32_IOERR_RETRY_DELAY
          416  +# define SQLITE_WIN32_IOERR_RETRY_DELAY 25
          417  +#endif
          418  +static int win32IoerrRetry = SQLITE_WIN32_IOERR_RETRY;
          419  +static int win32IoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY;
          420  +
          421  +/*
          422  +** If a ReadFile() or WriteFile() error occurs, invoke this routine
          423  +** to see if it should be retried.  Return TRUE to retry.  Return FALSE
          424  +** to give up with an error.
          425  +*/
          426  +static int retryIoerr(int *pnRetry){
          427  +  DWORD e;
          428  +  if( *pnRetry>=win32IoerrRetry ){
          429  +    return 0;
          430  +  }
          431  +  e = GetLastError();
          432  +  if( e==ERROR_ACCESS_DENIED ||
          433  +      e==ERROR_LOCK_VIOLATION ||
          434  +      e==ERROR_SHARING_VIOLATION ){
          435  +    Sleep(win32IoerrRetryDelay*(1+*pnRetry));
          436  +    ++*pnRetry;
          437  +    return 1;
          438  +  }
          439  +  return 0;
          440  +}
          441  +
          442  +/*
          443  +** Log a I/O error retry episode.
          444  +*/
          445  +static void logIoerr(int nRetry){
          446  +  if( nRetry ){
          447  +    sqlite3_log(SQLITE_IOERR, 
          448  +      "delayed %dms for lock/sharing conflict",
          449  +      win32IoerrRetryDelay*nRetry*(nRetry+1)/2
          450  +    );
          451  +  }
          452  +}
   404    453   
   405    454   #if SQLITE_OS_WINCE
   406    455   /*************************************************************************
   407    456   ** This section contains code for WinCE only.
   408    457   */
   409    458   /*
   410    459   ** WindowsCE does not have a localtime() function.  So create a
................................................................................
   816    865     sqlite3_file *id,          /* File to read from */
   817    866     void *pBuf,                /* Write content into this buffer */
   818    867     int amt,                   /* Number of bytes to read */
   819    868     sqlite3_int64 offset       /* Begin reading at this offset */
   820    869   ){
   821    870     winFile *pFile = (winFile*)id;  /* file handle */
   822    871     DWORD nRead;                    /* Number of bytes actually read from file */
          872  +  int nRetry = 0;                 /* Number of retrys */
   823    873   
   824    874     assert( id!=0 );
   825    875     SimulateIOError(return SQLITE_IOERR_READ);
   826    876     OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype));
   827    877   
   828    878     if( seekWinFile(pFile, offset) ){
   829    879       return SQLITE_FULL;
   830    880     }
   831         -  if( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){
          881  +  while( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){
          882  +    if( retryIoerr(&nRetry) ) continue;
   832    883       pFile->lastErrno = GetLastError();
   833    884       return winLogError(SQLITE_IOERR_READ, "winRead", pFile->zPath);
   834    885     }
          886  +  logIoerr(nRetry);
   835    887     if( nRead<(DWORD)amt ){
   836    888       /* Unread parts of the buffer must be zero-filled */
   837    889       memset(&((char*)pBuf)[nRead], 0, amt-nRead);
   838    890       return SQLITE_IOERR_SHORT_READ;
   839    891     }
   840    892   
   841    893     return SQLITE_OK;
................................................................................
   849    901     sqlite3_file *id,               /* File to write into */
   850    902     const void *pBuf,               /* The bytes to be written */
   851    903     int amt,                        /* Number of bytes to write */
   852    904     sqlite3_int64 offset            /* Offset into the file to begin writing at */
   853    905   ){
   854    906     int rc;                         /* True if error has occured, else false */
   855    907     winFile *pFile = (winFile*)id;  /* File handle */
          908  +  int nRetry = 0;                 /* Number of retries */
   856    909   
   857    910     assert( amt>0 );
   858    911     assert( pFile );
   859    912     SimulateIOError(return SQLITE_IOERR_WRITE);
   860    913     SimulateDiskfullError(return SQLITE_FULL);
   861    914   
   862    915     OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype));
................................................................................
   863    916   
   864    917     rc = seekWinFile(pFile, offset);
   865    918     if( rc==0 ){
   866    919       u8 *aRem = (u8 *)pBuf;        /* Data yet to be written */
   867    920       int nRem = amt;               /* Number of bytes yet to be written */
   868    921       DWORD nWrite;                 /* Bytes written by each WriteFile() call */
   869    922   
   870         -    while( nRem>0 && WriteFile(pFile->h, aRem, nRem, &nWrite, 0) && nWrite>0 ){
          923  +    while( nRem>0 ){
          924  +      if( !WriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){
          925  +        if( retryIoerr(&nRetry) ) continue;
          926  +        break;
          927  +      }
          928  +      if( nWrite<=0 ) break;
   871    929         aRem += nWrite;
   872    930         nRem -= nWrite;
   873    931       }
   874    932       if( nRem>0 ){
   875    933         pFile->lastErrno = GetLastError();
   876    934         rc = 1;
   877    935       }
................................................................................
   879    937   
   880    938     if( rc ){
   881    939       if(   ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL )
   882    940          || ( pFile->lastErrno==ERROR_DISK_FULL )){
   883    941         return SQLITE_FULL;
   884    942       }
   885    943       return winLogError(SQLITE_IOERR_WRITE, "winWrite", pFile->zPath);
          944  +  }else{
          945  +    logIoerr(nRetry);
   886    946     }
   887    947     return SQLITE_OK;
   888    948   }
   889    949   
   890    950   /*
   891    951   ** Truncate an open file to a specified size
   892    952   */
................................................................................
  1272   1332     return rc;
  1273   1333   }
  1274   1334   
  1275   1335   /*
  1276   1336   ** Control and query of the open file handle.
  1277   1337   */
  1278   1338   static int winFileControl(sqlite3_file *id, int op, void *pArg){
         1339  +  winFile *pFile = (winFile*)id;
  1279   1340     switch( op ){
  1280   1341       case SQLITE_FCNTL_LOCKSTATE: {
  1281         -      *(int*)pArg = ((winFile*)id)->locktype;
         1342  +      *(int*)pArg = pFile->locktype;
  1282   1343         return SQLITE_OK;
  1283   1344       }
  1284   1345       case SQLITE_LAST_ERRNO: {
  1285         -      *(int*)pArg = (int)((winFile*)id)->lastErrno;
         1346  +      *(int*)pArg = (int)pFile->lastErrno;
  1286   1347         return SQLITE_OK;
  1287   1348       }
  1288   1349       case SQLITE_FCNTL_CHUNK_SIZE: {
  1289         -      ((winFile*)id)->szChunk = *(int *)pArg;
         1350  +      pFile->szChunk = *(int *)pArg;
  1290   1351         return SQLITE_OK;
  1291   1352       }
  1292   1353       case SQLITE_FCNTL_SIZE_HINT: {
  1293   1354         sqlite3_int64 sz = *(sqlite3_int64*)pArg;
  1294   1355         SimulateIOErrorBenign(1);
  1295   1356         winTruncate(id, sz);
  1296   1357         SimulateIOErrorBenign(0);
  1297   1358         return SQLITE_OK;
         1359  +    }
         1360  +    case SQLITE_FCNTL_PERSIST_WAL: {
         1361  +      int bPersist = *(int*)pArg;
         1362  +      if( bPersist<0 ){
         1363  +        *(int*)pArg = pFile->bPersistWal;
         1364  +      }else{
         1365  +        pFile->bPersistWal = bPersist!=0;
         1366  +      }
         1367  +      return SQLITE_OK;
  1298   1368       }
  1299   1369       case SQLITE_FCNTL_SYNC_OMITTED: {
  1300   1370         return SQLITE_OK;
         1371  +    }
         1372  +    case SQLITE_FCNTL_WIN32_AV_RETRY: {
         1373  +      int *a = (int*)pArg;
         1374  +      if( a[0]>0 ){
         1375  +        win32IoerrRetry = a[0];
         1376  +      }else{
         1377  +        a[0] = win32IoerrRetry;
         1378  +      }
         1379  +      if( a[1]>0 ){
         1380  +        win32IoerrRetryDelay = a[1];
         1381  +      }else{
         1382  +        a[1] = win32IoerrRetryDelay;
         1383  +      }
         1384  +      return SQLITE_OK;
  1301   1385       }
  1302   1386     }
  1303   1387     return SQLITE_NOTFOUND;
  1304   1388   }
  1305   1389   
  1306   1390   /*
  1307   1391   ** Return the sector size in bytes of the underlying block device for
................................................................................
  2312   2396   ** will open a journal file shortly after it is created in order to do
  2313   2397   ** whatever it does.  While this other process is holding the
  2314   2398   ** file open, we will be unable to delete it.  To work around this
  2315   2399   ** problem, we delay 100 milliseconds and try to delete again.  Up
  2316   2400   ** to MX_DELETION_ATTEMPTs deletion attempts are run before giving
  2317   2401   ** up and returning an error.
  2318   2402   */
  2319         -#define MX_DELETION_ATTEMPTS 5
  2320   2403   static int winDelete(
  2321   2404     sqlite3_vfs *pVfs,          /* Not used on win32 */
  2322   2405     const char *zFilename,      /* Name of file to delete */
  2323   2406     int syncDir                 /* Not used on win32 */
  2324   2407   ){
  2325   2408     int cnt = 0;
  2326         -  DWORD rc;
  2327         -  DWORD error = 0;
         2409  +  int rc;
  2328   2410     void *zConverted;
  2329   2411     UNUSED_PARAMETER(pVfs);
  2330   2412     UNUSED_PARAMETER(syncDir);
  2331   2413   
  2332   2414     SimulateIOError(return SQLITE_IOERR_DELETE);
  2333   2415     zConverted = convertUtf8Filename(zFilename);
  2334   2416     if( zConverted==0 ){
  2335   2417       return SQLITE_NOMEM;
  2336   2418     }
  2337   2419     if( isNT() ){
  2338         -    do{
  2339         -      DeleteFileW(zConverted);
  2340         -    }while(   (   ((rc = GetFileAttributesW(zConverted)) != INVALID_FILE_ATTRIBUTES)
  2341         -               || ((error = GetLastError()) == ERROR_ACCESS_DENIED))
  2342         -           && (++cnt < MX_DELETION_ATTEMPTS)
  2343         -           && (Sleep(100), 1) );
         2420  +    rc = 1;
         2421  +    while( GetFileAttributesW(zConverted)!=INVALID_FILE_ATTRIBUTES &&
         2422  +           (rc = DeleteFileW(zConverted))==0 && retryIoerr(&cnt) ){}
         2423  +    rc = rc ? SQLITE_OK : SQLITE_ERROR;
  2344   2424   /* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. 
  2345   2425   ** Since the ASCII version of these Windows API do not exist for WINCE,
  2346   2426   ** it's important to not reference them for WINCE builds.
  2347   2427   */
  2348   2428   #if SQLITE_OS_WINCE==0
  2349   2429     }else{
  2350         -    do{
  2351         -      DeleteFileA(zConverted);
  2352         -    }while(   (   ((rc = GetFileAttributesA(zConverted)) != INVALID_FILE_ATTRIBUTES)
  2353         -               || ((error = GetLastError()) == ERROR_ACCESS_DENIED))
  2354         -           && (++cnt < MX_DELETION_ATTEMPTS)
  2355         -           && (Sleep(100), 1) );
         2430  +    rc = 1;
         2431  +    while( GetFileAttributesA(zConverted)!=INVALID_FILE_ATTRIBUTES &&
         2432  +           (rc = DeleteFileA(zConverted))==0 && retryIoerr(&cnt) ){}
         2433  +    rc = rc ? SQLITE_OK : SQLITE_ERROR;
  2356   2434   #endif
  2357   2435     }
         2436  +  if( rc ){
         2437  +    rc = winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename);
         2438  +  }else{
         2439  +    logIoerr(cnt);
         2440  +  }
  2358   2441     free(zConverted);
  2359         -  OSTRACE(("DELETE \"%s\" %s\n", zFilename,
  2360         -       ( (rc==INVALID_FILE_ATTRIBUTES) && (error==ERROR_FILE_NOT_FOUND)) ?
  2361         -         "ok" : "failed" ));
  2362         - 
  2363         -  return (   (rc == INVALID_FILE_ATTRIBUTES) 
  2364         -          && (error == ERROR_FILE_NOT_FOUND)) ? SQLITE_OK :
  2365         -                 winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename);
         2442  +  OSTRACE(("DELETE \"%s\" %s\n", zFilename, (rc ? "failed" : "ok" )));
         2443  +  return rc;
  2366   2444   }
  2367   2445   
  2368   2446   /*
  2369   2447   ** Check the existance and status of a file.
  2370   2448   */
  2371   2449   static int winAccess(
  2372   2450     sqlite3_vfs *pVfs,         /* Not used on win32 */
................................................................................
  2381   2459   
  2382   2460     SimulateIOError( return SQLITE_IOERR_ACCESS; );
  2383   2461     zConverted = convertUtf8Filename(zFilename);
  2384   2462     if( zConverted==0 ){
  2385   2463       return SQLITE_NOMEM;
  2386   2464     }
  2387   2465     if( isNT() ){
         2466  +    int cnt = 0;
  2388   2467       WIN32_FILE_ATTRIBUTE_DATA sAttrData;
  2389   2468       memset(&sAttrData, 0, sizeof(sAttrData));
  2390         -    if( GetFileAttributesExW((WCHAR*)zConverted,
         2469  +    while( !(rc = GetFileAttributesExW((WCHAR*)zConverted,
  2391   2470                                GetFileExInfoStandard, 
  2392         -                             &sAttrData) ){
         2471  +                             &sAttrData)) && retryIoerr(&cnt) ){}
         2472  +    if( rc ){
  2393   2473         /* For an SQLITE_ACCESS_EXISTS query, treat a zero-length file
  2394   2474         ** as if it does not exist.
  2395   2475         */
  2396   2476         if(    flags==SQLITE_ACCESS_EXISTS
  2397   2477             && sAttrData.nFileSizeHigh==0 
  2398   2478             && sAttrData.nFileSizeLow==0 ){
  2399   2479           attr = INVALID_FILE_ATTRIBUTES;
  2400   2480         }else{
  2401   2481           attr = sAttrData.dwFileAttributes;
  2402   2482         }
  2403   2483       }else{
         2484  +      logIoerr(cnt);
  2404   2485         if( GetLastError()!=ERROR_FILE_NOT_FOUND ){
  2405   2486           winLogError(SQLITE_IOERR_ACCESS, "winAccess", zFilename);
  2406   2487           free(zConverted);
  2407   2488           return SQLITE_IOERR_ACCESS;
  2408   2489         }else{
  2409   2490           attr = INVALID_FILE_ATTRIBUTES;
  2410   2491         }
................................................................................
  2421   2502     free(zConverted);
  2422   2503     switch( flags ){
  2423   2504       case SQLITE_ACCESS_READ:
  2424   2505       case SQLITE_ACCESS_EXISTS:
  2425   2506         rc = attr!=INVALID_FILE_ATTRIBUTES;
  2426   2507         break;
  2427   2508       case SQLITE_ACCESS_READWRITE:
  2428         -      rc = (attr & FILE_ATTRIBUTE_READONLY)==0;
         2509  +      rc = attr!=INVALID_FILE_ATTRIBUTES &&
         2510  +             (attr & FILE_ATTRIBUTE_READONLY)==0;
  2429   2511         break;
  2430   2512       default:
  2431   2513         assert(!"Invalid flags argument");
  2432   2514     }
  2433   2515     *pResOut = rc;
  2434   2516     return SQLITE_OK;
  2435   2517   }

Changes to src/resolve.c.

   992    992       }
   993    993     
   994    994       /* Recursively resolve names in all subqueries
   995    995       */
   996    996       for(i=0; i<p->pSrc->nSrc; i++){
   997    997         struct SrcList_item *pItem = &p->pSrc->a[i];
   998    998         if( pItem->pSelect ){
          999  +        NameContext *pNC;         /* Used to iterate name contexts */
         1000  +        int nRef = 0;             /* Refcount for pOuterNC and outer contexts */
   999   1001           const char *zSavedContext = pParse->zAuthContext;
         1002  +
         1003  +        /* Count the total number of references to pOuterNC and all of its
         1004  +        ** parent contexts. After resolving references to expressions in
         1005  +        ** pItem->pSelect, check if this value has changed. If so, then
         1006  +        ** SELECT statement pItem->pSelect must be correlated. Set the
         1007  +        ** pItem->isCorrelated flag if this is the case. */
         1008  +        for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef += pNC->nRef;
         1009  +
  1000   1010           if( pItem->zName ) pParse->zAuthContext = pItem->zName;
  1001   1011           sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC);
  1002   1012           pParse->zAuthContext = zSavedContext;
  1003   1013           if( pParse->nErr || db->mallocFailed ) return WRC_Abort;
         1014  +
         1015  +        for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef -= pNC->nRef;
         1016  +        assert( pItem->isCorrelated==0 && nRef<=0 );
         1017  +        pItem->isCorrelated = (nRef!=0);
  1004   1018         }
  1005   1019       }
  1006   1020     
  1007   1021       /* If there are no aggregate functions in the result-set, and no GROUP BY 
  1008   1022       ** expression, do not allow aggregates in any of the other expressions.
  1009   1023       */
  1010   1024       assert( (p->selFlags & SF_Aggregate)==0 );

Changes to src/select.c.

  3717   3717     ExprList *pOrderBy;    /* The ORDER BY clause.  May be NULL */
  3718   3718     ExprList *pGroupBy;    /* The GROUP BY clause.  May be NULL */
  3719   3719     Expr *pHaving;         /* The HAVING clause.  May be NULL */
  3720   3720     int isDistinct;        /* True if the DISTINCT keyword is present */
  3721   3721     int distinct;          /* Table to use for the distinct set */
  3722   3722     int rc = 1;            /* Value to return from this function */
  3723   3723     int addrSortIndex;     /* Address of an OP_OpenEphemeral instruction */
         3724  +  int addrDistinctIndex; /* Address of an OP_OpenEphemeral instruction */
  3724   3725     AggInfo sAggInfo;      /* Information used by aggregate queries */
  3725   3726     int iEnd;              /* Address of the end of the query */
  3726   3727     sqlite3 *db;           /* The database connection */
  3727   3728   
  3728   3729   #ifndef SQLITE_OMIT_EXPLAIN
  3729   3730     int iRestoreSelectId = pParse->iSelectId;
  3730   3731     pParse->iSelectId = pParse->iNextSelectId++;
................................................................................
  3843   3844       }
  3844   3845       rc = multiSelect(pParse, p, pDest);
  3845   3846       explainSetInteger(pParse->iSelectId, iRestoreSelectId);
  3846   3847       return rc;
  3847   3848     }
  3848   3849   #endif
  3849   3850   
  3850         -  /* If possible, rewrite the query to use GROUP BY instead of DISTINCT.
  3851         -  ** GROUP BY might use an index, DISTINCT never does.
  3852         -  */
  3853         -  assert( p->pGroupBy==0 || (p->selFlags & SF_Aggregate)!=0 );
  3854         -  if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ){
  3855         -    p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0);
  3856         -    pGroupBy = p->pGroupBy;
  3857         -    p->selFlags &= ~SF_Distinct;
  3858         -  }
  3859         -
  3860   3851     /* If there is both a GROUP BY and an ORDER BY clause and they are
  3861   3852     ** identical, then disable the ORDER BY clause since the GROUP BY
  3862   3853     ** will cause elements to come out in the correct order.  This is
  3863   3854     ** an optimization - the correct answer should result regardless.
  3864   3855     ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER
  3865   3856     ** to disable this optimization for testing purposes.
  3866   3857     */
  3867   3858     if( sqlite3ExprListCompare(p->pGroupBy, pOrderBy)==0
  3868   3859            && (db->flags & SQLITE_GroupByOrder)==0 ){
  3869   3860       pOrderBy = 0;
  3870   3861     }
         3862  +
         3863  +  /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and 
         3864  +  ** if the select-list is the same as the ORDER BY list, then this query
         3865  +  ** can be rewritten as a GROUP BY. In other words, this:
         3866  +  **
         3867  +  **     SELECT DISTINCT xyz FROM ... ORDER BY xyz
         3868  +  **
         3869  +  ** is transformed to:
         3870  +  **
         3871  +  **     SELECT xyz FROM ... GROUP BY xyz
         3872  +  **
         3873  +  ** The second form is preferred as a single index (or temp-table) may be 
         3874  +  ** used for both the ORDER BY and DISTINCT processing. As originally 
         3875  +  ** written the query must use a temp-table for at least one of the ORDER 
         3876  +  ** BY and DISTINCT, and an index or separate temp-table for the other.
         3877  +  */
         3878  +  if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct 
         3879  +   && sqlite3ExprListCompare(pOrderBy, p->pEList)==0
         3880  +  ){
         3881  +    p->selFlags &= ~SF_Distinct;
         3882  +    p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0);
         3883  +    pGroupBy = p->pGroupBy;
         3884  +    pOrderBy = 0;
         3885  +  }
  3871   3886   
  3872   3887     /* If there is an ORDER BY clause, then this sorting
  3873   3888     ** index might end up being unused if the data can be 
  3874   3889     ** extracted in pre-sorted order.  If that is the case, then the
  3875   3890     ** OP_OpenEphemeral instruction will be changed to an OP_Noop once
  3876   3891     ** we figure out that the sorting index is not needed.  The addrSortIndex
  3877   3892     ** variable is used to facilitate that change.
................................................................................
  3900   3915     p->nSelectRow = (double)LARGEST_INT64;
  3901   3916     computeLimitRegisters(pParse, p, iEnd);
  3902   3917   
  3903   3918     /* Open a virtual index to use for the distinct set.
  3904   3919     */
  3905   3920     if( p->selFlags & SF_Distinct ){
  3906   3921       KeyInfo *pKeyInfo;
  3907         -    assert( isAgg || pGroupBy );
  3908   3922       distinct = pParse->nTab++;
  3909   3923       pKeyInfo = keyInfoFromExprList(pParse, p->pEList);
  3910         -    sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0,
  3911         -                        (char*)pKeyInfo, P4_KEYINFO_HANDOFF);
         3924  +    addrDistinctIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0,
         3925  +        (char*)pKeyInfo, P4_KEYINFO_HANDOFF);
  3912   3926       sqlite3VdbeChangeP5(v, BTREE_UNORDERED);
  3913   3927     }else{
  3914         -    distinct = -1;
         3928  +    distinct = addrDistinctIndex = -1;
  3915   3929     }
  3916   3930   
  3917   3931     /* Aggregate and non-aggregate queries are handled differently */
  3918   3932     if( !isAgg && pGroupBy==0 ){
  3919         -    /* This case is for non-aggregate queries
  3920         -    ** Begin the database scan
  3921         -    */
  3922         -    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0);
         3933  +    ExprList *pDist = (isDistinct ? p->pEList : 0);
         3934  +
         3935  +    /* Begin the database scan. */
         3936  +    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, pDist, 0);
  3923   3937       if( pWInfo==0 ) goto select_end;
  3924   3938       if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut;
  3925   3939   
  3926   3940       /* If sorting index that was created by a prior OP_OpenEphemeral 
  3927   3941       ** instruction ended up not being needed, then change the OP_OpenEphemeral
  3928   3942       ** into an OP_Noop.
  3929   3943       */
  3930   3944       if( addrSortIndex>=0 && pOrderBy==0 ){
  3931   3945         sqlite3VdbeChangeToNoop(v, addrSortIndex, 1);
  3932   3946         p->addrOpenEphm[2] = -1;
  3933   3947       }
  3934   3948   
  3935         -    /* Use the standard inner loop
  3936         -    */
  3937         -    assert(!isDistinct);
  3938         -    selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, -1, pDest,
         3949  +    if( pWInfo->eDistinct ){
         3950  +      VdbeOp *pOp;                /* No longer required OpenEphemeral instr. */
         3951  +     
         3952  +      assert( addrDistinctIndex>0 );
         3953  +      pOp = sqlite3VdbeGetOp(v, addrDistinctIndex);
         3954  +
         3955  +      assert( isDistinct );
         3956  +      assert( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED 
         3957  +           || pWInfo->eDistinct==WHERE_DISTINCT_UNIQUE 
         3958  +      );
         3959  +      distinct = -1;
         3960  +      if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED ){
         3961  +        int iJump;
         3962  +        int iExpr;
         3963  +        int iFlag = ++pParse->nMem;
         3964  +        int iBase = pParse->nMem+1;
         3965  +        int iBase2 = iBase + pEList->nExpr;
         3966  +        pParse->nMem += (pEList->nExpr*2);
         3967  +
         3968  +        /* Change the OP_OpenEphemeral coded earlier to an OP_Integer. The
         3969  +        ** OP_Integer initializes the "first row" flag.  */
         3970  +        pOp->opcode = OP_Integer;
         3971  +        pOp->p1 = 1;
         3972  +        pOp->p2 = iFlag;
         3973  +
         3974  +        sqlite3ExprCodeExprList(pParse, pEList, iBase, 1);
         3975  +        iJump = sqlite3VdbeCurrentAddr(v) + 1 + pEList->nExpr + 1 + 1;
         3976  +        sqlite3VdbeAddOp2(v, OP_If, iFlag, iJump-1);
         3977  +        for(iExpr=0; iExpr<pEList->nExpr; iExpr++){
         3978  +          CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[iExpr].pExpr);
         3979  +          sqlite3VdbeAddOp3(v, OP_Ne, iBase+iExpr, iJump, iBase2+iExpr);
         3980  +          sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ);
         3981  +          sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
         3982  +        }
         3983  +        sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iContinue);
         3984  +
         3985  +        sqlite3VdbeAddOp2(v, OP_Integer, 0, iFlag);
         3986  +        assert( sqlite3VdbeCurrentAddr(v)==iJump );
         3987  +        sqlite3VdbeAddOp3(v, OP_Move, iBase, iBase2, pEList->nExpr);
         3988  +      }else{
         3989  +        pOp->opcode = OP_Noop;
         3990  +      }
         3991  +    }
         3992  +
         3993  +    /* Use the standard inner loop. */
         3994  +    selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, distinct, pDest,
  3939   3995                       pWInfo->iContinue, pWInfo->iBreak);
  3940   3996   
  3941   3997       /* End the database scan loop.
  3942   3998       */
  3943   3999       sqlite3WhereEnd(pWInfo);
  3944   4000     }else{
  3945   4001       /* This is the processing for aggregate queries */
................................................................................
  4041   4097   
  4042   4098         /* Begin a loop that will extract all source rows in GROUP BY order.
  4043   4099         ** This might involve two separate loops with an OP_Sort in between, or
  4044   4100         ** it might be a single loop that uses an index to extract information
  4045   4101         ** in the right order to begin with.
  4046   4102         */
  4047   4103         sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset);
  4048         -      pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0);
         4104  +      pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0, 0);
  4049   4105         if( pWInfo==0 ) goto select_end;
  4050   4106         if( pGroupBy==0 ){
  4051   4107           /* The optimizer is able to deliver rows in group by order so
  4052   4108           ** we do not have to sort.  The OP_OpenEphemeral table will be
  4053   4109           ** cancelled later because we still need to use the pKeyInfo
  4054   4110           */
  4055   4111           pGroupBy = p->pGroupBy;
................................................................................
  4303   4359           }
  4304   4360     
  4305   4361           /* This case runs if the aggregate has no GROUP BY clause.  The
  4306   4362           ** processing is much simpler since there is only a single row
  4307   4363           ** of output.
  4308   4364           */
  4309   4365           resetAccumulator(pParse, &sAggInfo);
  4310         -        pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, flag);
         4366  +        pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, 0, flag);
  4311   4367           if( pWInfo==0 ){
  4312   4368             sqlite3ExprListDelete(db, pDel);
  4313   4369             goto select_end;
  4314   4370           }
  4315   4371           updateAccumulator(pParse, &sAggInfo);
  4316   4372           if( !pMinMax && flag ){
  4317   4373             sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iBreak);

Changes to src/shell.c.

  2628   2628     "   -stats               print memory stats before each finalize\n"
  2629   2629     "   -nullvalue 'text'    set text string for NULL values\n"
  2630   2630     "   -version             show SQLite version\n"
  2631   2631     "   -vfs NAME            use NAME as the default VFS\n"
  2632   2632   #ifdef SQLITE_ENABLE_VFSTRACE
  2633   2633     "   -vfstrace            enable tracing of all VFS calls\n"
  2634   2634   #endif
         2635  +#ifdef SQLITE_ENABLE_MULTIPLEX
         2636  +  "   -multiplex           enable the multiplexor VFS\n"
         2637  +#endif
  2635   2638   ;
  2636   2639   static void usage(int showDetail){
  2637   2640     fprintf(stderr,
  2638   2641         "Usage: %s [OPTIONS] FILENAME [SQL]\n"  
  2639   2642         "FILENAME is the name of an SQLite database. A new database is created\n"
  2640   2643         "if the file does not previously exist.\n", Argv0);
  2641   2644     if( showDetail ){
................................................................................
  2729   2732            const char *zOldVfsName,
  2730   2733            int (*xOut)(const char*,void*),
  2731   2734            void *pOutArg,
  2732   2735            int makeDefault
  2733   2736         );
  2734   2737         vfstrace_register("trace",0,(int(*)(const char*,void*))fputs,stderr,1);
  2735   2738   #endif
         2739  +#ifdef SQLITE_ENABLE_MULTIPLEX
         2740  +    }else if( strcmp(argv[i],"-multiplex")==0 ){
         2741  +      extern int sqlite3_multiple_initialize(const char*,int);
         2742  +      sqlite3_multiplex_initialize(0, 1);
         2743  +#endif
  2736   2744       }else if( strcmp(argv[i],"-vfs")==0 ){
  2737   2745         sqlite3_vfs *pVfs = sqlite3_vfs_find(argv[++i]);
  2738   2746         if( pVfs ){
  2739   2747           sqlite3_vfs_register(pVfs, 1);
  2740   2748         }else{
  2741   2749           fprintf(stderr, "no such VFS: \"%s\"\n", argv[i]);
  2742   2750           exit(1);
................................................................................
  2847   2855         stdin_is_interactive = 1;
  2848   2856       }else if( strcmp(z,"-batch")==0 ){
  2849   2857         stdin_is_interactive = 0;
  2850   2858       }else if( strcmp(z,"-heap")==0 ){
  2851   2859         i++;
  2852   2860       }else if( strcmp(z,"-vfs")==0 ){
  2853   2861         i++;
         2862  +#ifdef SQLITE_ENABLE_VFSTRACE
  2854   2863       }else if( strcmp(z,"-vfstrace")==0 ){
  2855   2864         i++;
         2865  +#endif
         2866  +#ifdef SQLITE_ENABLE_MULTIPLEX
         2867  +    }else if( strcmp(z,"-multiplex")==0 ){
         2868  +      i++;
         2869  +#endif
  2856   2870       }else if( strcmp(z,"-help")==0 || strcmp(z, "--help")==0 ){
  2857   2871         usage(1);
  2858   2872       }else{
  2859   2873         fprintf(stderr,"%s: Error: unknown option: %s\n", Argv0, z);
  2860   2874         fprintf(stderr,"Use -help for a list of options.\n");
  2861   2875         return 1;
  2862   2876       }

Changes to src/sqlite.h.in.

   733    733   ** when the database connection has [PRAGMA synchronous] set to OFF.)^
   734    734   ** Some specialized VFSes need this signal in order to operate correctly
   735    735   ** when [PRAGMA synchronous | PRAGMA synchronous=OFF] is set, but most 
   736    736   ** VFSes do not need this signal and should silently ignore this opcode.
   737    737   ** Applications should not call [sqlite3_file_control()] with this
   738    738   ** opcode as doing so may disrupt the operation of the specialized VFSes
   739    739   ** that do require it.  
          740  +**
          741  +** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic
          742  +** retry counts and intervals for certain disk I/O operations for the
          743  +** windows [VFS] in order to work to provide robustness against
          744  +** anti-virus programs.  By default, the windows VFS will retry file read,
          745  +** file write, and file delete opertions up to 10 times, with a delay
          746  +** of 25 milliseconds before the first retry and with the delay increasing
          747  +** by an additional 25 milliseconds with each subsequent retry.  This
          748  +** opcode allows those to values (10 retries and 25 milliseconds of delay)
          749  +** to be adjusted.  The values are changed for all database connections
          750  +** within the same process.  The argument is a pointer to an array of two
          751  +** integers where the first integer i the new retry count and the second
          752  +** integer is the delay.  If either integer is negative, then the setting
          753  +** is not changed but instead the prior value of that setting is written
          754  +** into the array entry, allowing the current retry settings to be
          755  +** interrogated.  The zDbName parameter is ignored.
          756  +**
          757  +** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the
          758  +** persistent [WAL | Write AHead Log] setting.  By default, the auxiliary
          759  +** write ahead log and shared memory files used for transaction control
          760  +** are automatically deleted when the latest connection to the database
          761  +** closes.  Setting persistent WAL mode causes those files to persist after
          762  +** close.  Persisting the files is useful when other processes that do not
          763  +** have write permission on the directory containing the database file want
          764  +** to read the database file, as the WAL and shared memory files must exist
          765  +** in order for the database to be readable.  The fourth parameter to
          766  +** [sqlite3_file_control()] for this opcode should be a pointer to an integer.
          767  +** That integer is 0 to disable persistent WAL mode or 1 to enable persistent
          768  +** WAL mode.  If the integer is -1, then it is overwritten with the current
          769  +** WAL persistence setting.
          770  +** 
   740    771   */
   741    772   #define SQLITE_FCNTL_LOCKSTATE        1
   742    773   #define SQLITE_GET_LOCKPROXYFILE      2
   743    774   #define SQLITE_SET_LOCKPROXYFILE      3
   744    775   #define SQLITE_LAST_ERRNO             4
   745    776   #define SQLITE_FCNTL_SIZE_HINT        5
   746    777   #define SQLITE_FCNTL_CHUNK_SIZE       6
   747    778   #define SQLITE_FCNTL_FILE_POINTER     7
   748    779   #define SQLITE_FCNTL_SYNC_OMITTED     8
   749         -
          780  +#define SQLITE_FCNTL_WIN32_AV_RETRY   9
          781  +#define SQLITE_FCNTL_PERSIST_WAL     10
   750    782   
   751    783   /*
   752    784   ** CAPI3REF: Mutex Handle
   753    785   **
   754    786   ** The mutex module within SQLite defines [sqlite3_mutex] to be an
   755    787   ** abstract type for a mutex object.  The SQLite core never looks
   756    788   ** at the internal representation of an [sqlite3_mutex].  It only

Changes to src/sqlite3ext.h.

   208    208     const char *(*sourceid)(void);
   209    209     int (*stmt_status)(sqlite3_stmt*,int,int);
   210    210     int (*strnicmp)(const char*,const char*,int);
   211    211     int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*);
   212    212     int (*wal_autocheckpoint)(sqlite3*,int);
   213    213     int (*wal_checkpoint)(sqlite3*,const char*);
   214    214     void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*);
          215  +  int (*blob_reopen)(sqlite3_blob*,sqlite3_int64);
          216  +  int (*vtab_config)(sqlite3*,int op,...);
          217  +  int (*vtab_on_conflict)(sqlite3*);
   215    218   };
   216    219   
   217    220   /*
   218    221   ** The following macros redefine the API routines so that they are
   219    222   ** redirected throught the global sqlite3_api structure.
   220    223   **
   221    224   ** This header file is also used by the loadext.c source file
................................................................................
   408    411   #define sqlite3_sourceid               sqlite3_api->sourceid
   409    412   #define sqlite3_stmt_status            sqlite3_api->stmt_status
   410    413   #define sqlite3_strnicmp               sqlite3_api->strnicmp
   411    414   #define sqlite3_unlock_notify          sqlite3_api->unlock_notify
   412    415   #define sqlite3_wal_autocheckpoint     sqlite3_api->wal_autocheckpoint
   413    416   #define sqlite3_wal_checkpoint         sqlite3_api->wal_checkpoint
   414    417   #define sqlite3_wal_hook               sqlite3_api->wal_hook
          418  +#define sqlite3_blob_reopen            sqlite3_api->blob_reopen
          419  +#define sqlite3_vtab_config            sqlite3_api->vtab_config
          420  +#define sqlite3_vtab_on_conflict       sqlite3_api->vtab_on_conflict
   415    421   #endif /* SQLITE_CORE */
   416    422   
   417    423   #define SQLITE_EXTENSION_INIT1     const sqlite3_api_routines *sqlite3_api = 0;
   418    424   #define SQLITE_EXTENSION_INIT2(v)  sqlite3_api = v;
   419    425   
   420    426   #endif /* _SQLITE3EXT_H_ */

Changes to src/sqliteInt.h.

   962    962   #define SQLITE_ColumnCache    0x02        /* Disable the column cache */
   963    963   #define SQLITE_IndexSort      0x04        /* Disable indexes for sorting */
   964    964   #define SQLITE_IndexSearch    0x08        /* Disable indexes for searching */
   965    965   #define SQLITE_IndexCover     0x10        /* Disable index covering table */
   966    966   #define SQLITE_GroupByOrder   0x20        /* Disable GROUPBY cover of ORDERBY */
   967    967   #define SQLITE_FactorOutConst 0x40        /* Disable factoring out constants */
   968    968   #define SQLITE_IdxRealAsInt   0x80        /* Store REAL as INT in indices */
          969  +#define SQLITE_DistinctOpt    0x80        /* DISTINCT using indexes */
   969    970   #define SQLITE_OptMask        0xff        /* Mask of all disablable opts */
   970    971   
   971    972   /*
   972    973   ** Possible values for the sqlite.magic field.
   973    974   ** The numbers are obtained at random and have no special meaning, other
   974    975   ** than being distinct from one another.
   975    976   */
................................................................................
  1853   1854       char *zName;      /* Name of the table */
  1854   1855       char *zAlias;     /* The "B" part of a "A AS B" phrase.  zName is the "A" */
  1855   1856       Table *pTab;      /* An SQL table corresponding to zName */
  1856   1857       Select *pSelect;  /* A SELECT statement used in place of a table name */
  1857   1858       u8 isPopulated;   /* Temporary table associated with SELECT is populated */
  1858   1859       u8 jointype;      /* Type of join between this able and the previous */
  1859   1860       u8 notIndexed;    /* True if there is a NOT INDEXED clause */
         1861  +    u8 isCorrelated;  /* True if sub-query is correlated */
  1860   1862   #ifndef SQLITE_OMIT_EXPLAIN
  1861   1863       u8 iSelectId;     /* If pSelect!=0, the id of the sub-select in EQP */
  1862   1864   #endif
  1863   1865       int iCursor;      /* The VDBE cursor number used to access this table */
  1864   1866       Expr *pOn;        /* The ON clause of a join */
  1865   1867       IdList *pUsing;   /* The USING clause of a join */
  1866   1868       Bitmask colUsed;  /* Bit N (1<<N) set if column N of pTab is used */
................................................................................
  1972   1974   ** into the second half to give some continuity.
  1973   1975   */
  1974   1976   struct WhereInfo {
  1975   1977     Parse *pParse;       /* Parsing and code generating context */
  1976   1978     u16 wctrlFlags;      /* Flags originally passed to sqlite3WhereBegin() */
  1977   1979     u8 okOnePass;        /* Ok to use one-pass algorithm for UPDATE or DELETE */
  1978   1980     u8 untestedTerms;    /* Not all WHERE terms resolved by outer loop */
         1981  +  u8 eDistinct;
  1979   1982     SrcList *pTabList;             /* List of tables in the join */
  1980   1983     int iTop;                      /* The very beginning of the WHERE loop */
  1981   1984     int iContinue;                 /* Jump here to continue with next record */
  1982   1985     int iBreak;                    /* Jump here to break out of the loop */
  1983   1986     int nLevel;                    /* Number of nested loop */
  1984   1987     struct WhereClause *pWC;       /* Decomposition of the WHERE clause */
  1985   1988     double savedNQueryLoop;        /* pParse->nQueryLoop outside the WHERE loop */
  1986   1989     double nRowOut;                /* Estimated number of output rows */
  1987   1990     WhereLevel a[1];               /* Information about each nest loop in WHERE */
  1988   1991   };
  1989   1992   
         1993  +#define WHERE_DISTINCT_UNIQUE 1
         1994  +#define WHERE_DISTINCT_ORDERED 2
         1995  +
  1990   1996   /*
  1991   1997   ** A NameContext defines a context in which to resolve table and column
  1992   1998   ** names.  The context consists of a list of tables (the pSrcList) field and
  1993   1999   ** a list of named expression (pEList).  The named expression list may
  1994   2000   ** be NULL.  The pSrc corresponds to the FROM clause of a SELECT or
  1995   2001   ** to the table being operated on by INSERT, UPDATE, or DELETE.  The
  1996   2002   ** pEList corresponds to the result set of a SELECT and is NULL for
................................................................................
  2744   2750   int sqlite3IsReadOnly(Parse*, Table*, int);
  2745   2751   void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
  2746   2752   #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY)
  2747   2753   Expr *sqlite3LimitWhere(Parse *, SrcList *, Expr *, ExprList *, Expr *, Expr *, char *);
  2748   2754   #endif
  2749   2755   void sqlite3DeleteFrom(Parse*, SrcList*, Expr*);
  2750   2756   void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int);
  2751         -WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**, u16);
         2757  +WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**,ExprList*,u16);
  2752   2758   void sqlite3WhereEnd(WhereInfo*);
  2753   2759   int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int);
  2754   2760   void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int);
  2755   2761   void sqlite3ExprCodeMove(Parse*, int, int, int);
  2756   2762   void sqlite3ExprCodeCopy(Parse*, int, int, int);
  2757   2763   void sqlite3ExprCacheStore(Parse*, int, int, int);
  2758   2764   void sqlite3ExprCachePush(Parse*);

Changes to src/tclsqlite.c.

   103    103   };
   104    104   
   105    105   typedef struct IncrblobChannel IncrblobChannel;
   106    106   
   107    107   /*
   108    108   ** There is one instance of this structure for each SQLite database
   109    109   ** that has been opened by the SQLite TCL interface.
          110  +**
          111  +** If this module is built with SQLITE_TEST defined (to create the SQLite
          112  +** testfixture executable), then it may be configured to use either
          113  +** sqlite3_prepare_v2() or sqlite3_prepare() to prepare SQL statements.
          114  +** If SqliteDb.bLegacyPrepare is true, sqlite3_prepare() is used.
   110    115   */
   111    116   typedef struct SqliteDb SqliteDb;
   112    117   struct SqliteDb {
   113    118     sqlite3 *db;               /* The "real" database structure. MUST BE FIRST */
   114    119     Tcl_Interp *interp;        /* The interpreter used for this database */
   115    120     char *zBusy;               /* The busy callback routine */
   116    121     char *zCommit;             /* The commit hook callback routine */
................................................................................
   131    136     SqlPreparedStmt *stmtList; /* List of prepared statements*/
   132    137     SqlPreparedStmt *stmtLast; /* Last statement in the list */
   133    138     int maxStmt;               /* The next maximum number of stmtList */
   134    139     int nStmt;                 /* Number of statements in stmtList */
   135    140     IncrblobChannel *pIncrblob;/* Linked list of open incrblob channels */
   136    141     int nStep, nSort, nIndex;  /* Statistics for most recent operation */
   137    142     int nTransaction;          /* Number of nested [transaction] methods */
          143  +#ifdef SQLITE_TEST
          144  +  int bLegacyPrepare;        /* True to use sqlite3_prepare() */
          145  +#endif
   138    146   };
   139    147   
   140    148   struct IncrblobChannel {
   141    149     sqlite3_blob *pBlob;      /* sqlite3 blob handle */
   142    150     SqliteDb *pDb;            /* Associated database connection */
   143    151     int iSeek;                /* Current seek offset */
   144    152     Tcl_Channel channel;      /* Channel identifier */
................................................................................
   424    432     }
   425    433     pNew->interp = pDb->interp;
   426    434     pNew->pScript = 0;
   427    435     pNew->pNext = pDb->pFunc;
   428    436     pDb->pFunc = pNew;
   429    437     return pNew;
   430    438   }
          439  +
          440  +/*
          441  +** Free a single SqlPreparedStmt object.
          442  +*/
          443  +static void dbFreeStmt(SqlPreparedStmt *pStmt){
          444  +#ifdef SQLITE_TEST
          445  +  if( sqlite3_sql(pStmt->pStmt)==0 ){
          446  +    Tcl_Free((char *)pStmt->zSql);
          447  +  }
          448  +#endif
          449  +  sqlite3_finalize(pStmt->pStmt);
          450  +  Tcl_Free((char *)pStmt);
          451  +}
   431    452   
   432    453   /*
   433    454   ** Finalize and free a list of prepared statements
   434    455   */
   435         -static void flushStmtCache( SqliteDb *pDb ){
          456  +static void flushStmtCache(SqliteDb *pDb){
   436    457     SqlPreparedStmt *pPreStmt;
          458  +  SqlPreparedStmt *pNext;
   437    459   
   438         -  while(  pDb->stmtList ){
   439         -    sqlite3_finalize( pDb->stmtList->pStmt );
   440         -    pPreStmt = pDb->stmtList;
   441         -    pDb->stmtList = pDb->stmtList->pNext;
   442         -    Tcl_Free( (char*)pPreStmt );
          460  +  for(pPreStmt = pDb->stmtList; pPreStmt; pPreStmt=pNext){
          461  +    pNext = pPreStmt->pNext;
          462  +    dbFreeStmt(pPreStmt);
   443    463     }
   444    464     pDb->nStmt = 0;
   445    465     pDb->stmtLast = 0;
          466  +  pDb->stmtList = 0;
   446    467   }
   447    468   
   448    469   /*
   449    470   ** TCL calls this procedure when an sqlite3 database command is
   450    471   ** deleted.
   451    472   */
   452    473   static void DbDeleteCmd(void *db){
................................................................................
  1025   1046       }
  1026   1047       sqlite3_exec(pDb->db, "ROLLBACK", 0, 0, 0);
  1027   1048     }
  1028   1049     pDb->disableAuth--;
  1029   1050   
  1030   1051     return rc;
  1031   1052   }
         1053  +
         1054  +/*
         1055  +** Unless SQLITE_TEST is defined, this function is a simple wrapper around
         1056  +** sqlite3_prepare_v2(). If SQLITE_TEST is defined, then it uses either
         1057  +** sqlite3_prepare_v2() or legacy interface sqlite3_prepare(), depending
         1058  +** on whether or not the [db_use_legacy_prepare] command has been used to 
         1059  +** configure the connection.
         1060  +*/
         1061  +static int dbPrepare(
         1062  +  SqliteDb *pDb,                  /* Database object */
         1063  +  const char *zSql,               /* SQL to compile */
         1064  +  sqlite3_stmt **ppStmt,          /* OUT: Prepared statement */
         1065  +  const char **pzOut              /* OUT: Pointer to next SQL statement */
         1066  +){
         1067  +#ifdef SQLITE_TEST
         1068  +  if( pDb->bLegacyPrepare ){
         1069  +    return sqlite3_prepare(pDb->db, zSql, -1, ppStmt, pzOut);
         1070  +  }
         1071  +#endif
         1072  +  return sqlite3_prepare_v2(pDb->db, zSql, -1, ppStmt, pzOut);
         1073  +}
  1032   1074   
  1033   1075   /*
  1034   1076   ** Search the cache for a prepared-statement object that implements the
  1035   1077   ** first SQL statement in the buffer pointed to by parameter zIn. If
  1036   1078   ** no such prepared-statement can be found, allocate and prepare a new
  1037   1079   ** one. In either case, bind the current values of the relevant Tcl
  1038   1080   ** variables to any $var, :var or @var variables in the statement. Before
................................................................................
  1096   1138     }
  1097   1139     
  1098   1140     /* If no prepared statement was found. Compile the SQL text. Also allocate
  1099   1141     ** a new SqlPreparedStmt structure.  */
  1100   1142     if( pPreStmt==0 ){
  1101   1143       int nByte;
  1102   1144   
  1103         -    if( SQLITE_OK!=sqlite3_prepare_v2(pDb->db, zSql, -1, &pStmt, pzOut) ){
         1145  +    if( SQLITE_OK!=dbPrepare(pDb, zSql, &pStmt, pzOut) ){
  1104   1146         Tcl_SetObjResult(interp, dbTextToObj(sqlite3_errmsg(pDb->db)));
  1105   1147         return TCL_ERROR;
  1106   1148       }
  1107   1149       if( pStmt==0 ){
  1108   1150         if( SQLITE_OK!=sqlite3_errcode(pDb->db) ){
  1109   1151           /* A compile-time error in the statement. */
  1110   1152           Tcl_SetObjResult(interp, dbTextToObj(sqlite3_errmsg(pDb->db)));
................................................................................
  1123   1165       pPreStmt = (SqlPreparedStmt*)Tcl_Alloc(nByte);
  1124   1166       memset(pPreStmt, 0, nByte);
  1125   1167   
  1126   1168       pPreStmt->pStmt = pStmt;
  1127   1169       pPreStmt->nSql = (*pzOut - zSql);
  1128   1170       pPreStmt->zSql = sqlite3_sql(pStmt);
  1129   1171       pPreStmt->apParm = (Tcl_Obj **)&pPreStmt[1];
         1172  +#ifdef SQLITE_TEST
         1173  +    if( pPreStmt->zSql==0 ){
         1174  +      char *zCopy = Tcl_Alloc(pPreStmt->nSql + 1);
         1175  +      memcpy(zCopy, zSql, pPreStmt->nSql);
         1176  +      zCopy[pPreStmt->nSql] = '\0';
         1177  +      pPreStmt->zSql = zCopy;
         1178  +    }
         1179  +#endif
  1130   1180     }
  1131   1181     assert( pPreStmt );
  1132   1182     assert( strlen30(pPreStmt->zSql)==pPreStmt->nSql );
  1133   1183     assert( 0==memcmp(pPreStmt->zSql, zSql, pPreStmt->nSql) );
  1134   1184   
  1135   1185     /* Bind values to parameters that begin with $ or : */  
  1136   1186     for(i=1; i<=nVar; i++){
................................................................................
  1176   1226     }
  1177   1227     pPreStmt->nParm = iParm;
  1178   1228     *ppPreStmt = pPreStmt;
  1179   1229   
  1180   1230     return TCL_OK;
  1181   1231   }
  1182   1232   
  1183         -
  1184   1233   /*
  1185   1234   ** Release a statement reference obtained by calling dbPrepareAndBind().
  1186   1235   ** There should be exactly one call to this function for each call to
  1187   1236   ** dbPrepareAndBind().
  1188   1237   **
  1189   1238   ** If the discard parameter is non-zero, then the statement is deleted
  1190   1239   ** immediately. Otherwise it is added to the LRU list and may be returned
................................................................................
  1201   1250     for(i=0; i<pPreStmt->nParm; i++){
  1202   1251       Tcl_DecrRefCount(pPreStmt->apParm[i]);
  1203   1252     }
  1204   1253     pPreStmt->nParm = 0;
  1205   1254   
  1206   1255     if( pDb->maxStmt<=0 || discard ){
  1207   1256       /* If the cache is turned off, deallocated the statement */
  1208         -    sqlite3_finalize(pPreStmt->pStmt);
  1209         -    Tcl_Free((char *)pPreStmt);
         1257  +    dbFreeStmt(pPreStmt);
  1210   1258     }else{
  1211   1259       /* Add the prepared statement to the beginning of the cache list. */
  1212   1260       pPreStmt->pNext = pDb->stmtList;
  1213   1261       pPreStmt->pPrev = 0;
  1214   1262       if( pDb->stmtList ){
  1215   1263        pDb->stmtList->pPrev = pPreStmt;
  1216   1264       }
................................................................................
  1222   1270         assert( pDb->nStmt>0 );
  1223   1271       }
  1224   1272       pDb->nStmt++;
  1225   1273      
  1226   1274       /* If we have too many statement in cache, remove the surplus from 
  1227   1275       ** the end of the cache list.  */
  1228   1276       while( pDb->nStmt>pDb->maxStmt ){
  1229         -      sqlite3_finalize(pDb->stmtLast->pStmt);
  1230         -      pDb->stmtLast = pDb->stmtLast->pPrev;
  1231         -      Tcl_Free((char*)pDb->stmtLast->pNext);
         1277  +      SqlPreparedStmt *pLast = pDb->stmtLast;
         1278  +      pDb->stmtLast = pLast->pPrev;
  1232   1279         pDb->stmtLast->pNext = 0;
  1233   1280         pDb->nStmt--;
         1281  +      dbFreeStmt(pLast);
  1234   1282       }
  1235   1283     }
  1236   1284   }
  1237   1285   
  1238   1286   /*
  1239   1287   ** Structure used with dbEvalXXX() functions:
  1240   1288   **
................................................................................
  1359   1407   ** A return value of TCL_OK means there is a row of data available. The
  1360   1408   ** data may be accessed using dbEvalRowInfo() and dbEvalColumnValue(). This
  1361   1409   ** is analogous to a return of SQLITE_ROW from sqlite3_step(). If TCL_BREAK
  1362   1410   ** is returned, then the SQL script has finished executing and there are
  1363   1411   ** no further rows available. This is similar to SQLITE_DONE.
  1364   1412   */
  1365   1413   static int dbEvalStep(DbEvalContext *p){
         1414  +  const char *zPrevSql = 0;       /* Previous value of p->zSql */
         1415  +
  1366   1416     while( p->zSql[0] || p->pPreStmt ){
  1367   1417       int rc;
  1368   1418       if( p->pPreStmt==0 ){
         1419  +      zPrevSql = (p->zSql==zPrevSql ? 0 : p->zSql);
  1369   1420         rc = dbPrepareAndBind(p->pDb, p->zSql, &p->zSql, &p->pPreStmt);
  1370   1421         if( rc!=TCL_OK ) return rc;
  1371   1422       }else{
  1372   1423         int rcs;
  1373   1424         SqliteDb *pDb = p->pDb;
  1374   1425         SqlPreparedStmt *pPreStmt = p->pPreStmt;
  1375   1426         sqlite3_stmt *pStmt = pPreStmt->pStmt;
................................................................................
  1388   1439         pDb->nIndex = sqlite3_stmt_status(pStmt,SQLITE_STMTSTATUS_AUTOINDEX,1);
  1389   1440         dbReleaseColumnNames(p);
  1390   1441         p->pPreStmt = 0;
  1391   1442   
  1392   1443         if( rcs!=SQLITE_OK ){
  1393   1444           /* If a run-time error occurs, report the error and stop reading
  1394   1445           ** the SQL.  */
  1395         -        Tcl_SetObjResult(pDb->interp, dbTextToObj(sqlite3_errmsg(pDb->db)));
  1396   1446           dbReleaseStmt(pDb, pPreStmt, 1);
         1447  +#if SQLITE_TEST
         1448  +        if( p->pDb->bLegacyPrepare && rcs==SQLITE_SCHEMA && zPrevSql ){
         1449  +          /* If the runtime error was an SQLITE_SCHEMA, and the database
         1450  +          ** handle is configured to use the legacy sqlite3_prepare() 
         1451  +          ** interface, retry prepare()/step() on the same SQL statement.
         1452  +          ** This only happens once. If there is a second SQLITE_SCHEMA
         1453  +          ** error, the error will be returned to the caller. */
         1454  +          p->zSql = zPrevSql;
         1455  +          continue;
         1456  +        }
         1457  +#endif
         1458  +        Tcl_SetObjResult(pDb->interp, dbTextToObj(sqlite3_errmsg(pDb->db)));
  1397   1459           return TCL_ERROR;
  1398   1460         }else{
  1399   1461           dbReleaseStmt(pDb, pPreStmt, 0);
  1400   1462         }
  1401   1463       }
  1402   1464     }
  1403   1465   
................................................................................
  2922   2984         if( Tcl_GetBooleanFromObj(interp, objv[i+1], &b) ) return TCL_ERROR;
  2923   2985         if( b ){
  2924   2986           flags |= SQLITE_OPEN_NOMUTEX;
  2925   2987           flags &= ~SQLITE_OPEN_FULLMUTEX;
  2926   2988         }else{
  2927   2989           flags &= ~SQLITE_OPEN_NOMUTEX;
  2928   2990         }
  2929         -   }else if( strcmp(zArg, "-fullmutex")==0 ){
         2991  +    }else if( strcmp(zArg, "-fullmutex")==0 ){
  2930   2992         int b;
  2931   2993         if( Tcl_GetBooleanFromObj(interp, objv[i+1], &b) ) return TCL_ERROR;
  2932   2994         if( b ){
  2933   2995           flags |= SQLITE_OPEN_FULLMUTEX;
  2934   2996           flags &= ~SQLITE_OPEN_NOMUTEX;
  2935   2997         }else{
  2936   2998           flags &= ~SQLITE_OPEN_FULLMUTEX;
................................................................................
  3523   3585     if( !slave ){
  3524   3586       return TCL_ERROR;
  3525   3587     }
  3526   3588   
  3527   3589     init_all(slave);
  3528   3590     return TCL_OK;
  3529   3591   }
         3592  +
         3593  +/*
         3594  +** Tclcmd: db_use_legacy_prepare DB BOOLEAN
         3595  +**
         3596  +**   The first argument to this command must be a database command created by
         3597  +**   [sqlite3]. If the second argument is true, then the handle is configured
         3598  +**   to use the sqlite3_prepare_v2() function to prepare statements. If it
         3599  +**   is false, sqlite3_prepare().
         3600  +*/
         3601  +static int db_use_legacy_prepare_cmd(
         3602  +  ClientData cd,
         3603  +  Tcl_Interp *interp,
         3604  +  int objc,
         3605  +  Tcl_Obj *CONST objv[]
         3606  +){
         3607  +  Tcl_CmdInfo cmdInfo;
         3608  +  SqliteDb *pDb;
         3609  +  int bPrepare;
         3610  +
         3611  +  if( objc!=3 ){
         3612  +    Tcl_WrongNumArgs(interp, 1, objv, "DB BOOLEAN");
         3613  +    return TCL_ERROR;
         3614  +  }
         3615  +
         3616  +  if( !Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){
         3617  +    Tcl_AppendResult(interp, "no such db: ", Tcl_GetString(objv[1]), (char*)0);
         3618  +    return TCL_ERROR;
         3619  +  }
         3620  +  pDb = (SqliteDb*)cmdInfo.objClientData;
         3621  +  if( Tcl_GetBooleanFromObj(interp, objv[2], &bPrepare) ){
         3622  +    return TCL_ERROR;
         3623  +  }
         3624  +
         3625  +  pDb->bLegacyPrepare = bPrepare;
         3626  +
         3627  +  Tcl_ResetResult(interp);
         3628  +  return TCL_OK;
         3629  +}
  3530   3630   #endif
  3531   3631   
  3532   3632   /*
  3533   3633   ** Configure the interpreter passed as the first argument to have access
  3534   3634   ** to the commands and linked variables that make up:
  3535   3635   **
  3536   3636   **   * the [sqlite3] extension itself, 
................................................................................
  3629   3729       Sqlitetestfuzzer_Init(interp);
  3630   3730       Sqlitetestwholenumber_Init(interp);
  3631   3731   
  3632   3732   #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
  3633   3733       Sqlitetestfts3_Init(interp);
  3634   3734   #endif
  3635   3735   
  3636         -    Tcl_CreateObjCommand(interp,"load_testfixture_extensions",init_all_cmd,0,0);
         3736  +    Tcl_CreateObjCommand(
         3737  +        interp, "load_testfixture_extensions", init_all_cmd, 0, 0
         3738  +    );
         3739  +    Tcl_CreateObjCommand(
         3740  +        interp, "db_use_legacy_prepare", db_use_legacy_prepare_cmd, 0, 0
         3741  +    );
  3637   3742   
  3638   3743   #ifdef SQLITE_SSE
  3639   3744       Sqlitetestsse_Init(interp);
  3640   3745   #endif
  3641   3746     }
  3642   3747   #endif
  3643   3748   }

Changes to src/test1.c.

  5278   5278     }
  5279   5279   #else
  5280   5280     Tcl_SetObjResult(interp, Tcl_NewIntObj(0)); 
  5281   5281   #endif
  5282   5282     
  5283   5283     return TCL_OK;  
  5284   5284   }
         5285  +
         5286  +/*
         5287  +** tclcmd:   file_control_win32_av_retry DB  NRETRY  DELAY
         5288  +**
         5289  +** This TCL command runs the sqlite3_file_control interface with
         5290  +** the SQLITE_FCNTL_WIN32_AV_RETRY opcode.
         5291  +*/
         5292  +static int file_control_win32_av_retry(
         5293  +  ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
         5294  +  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
         5295  +  int objc,              /* Number of arguments */
         5296  +  Tcl_Obj *CONST objv[]  /* Command arguments */
         5297  +){
         5298  +  sqlite3 *db;
         5299  +  int rc;
         5300  +  int a[2];
         5301  +  char z[100];
         5302  +
         5303  +  if( objc!=4 ){
         5304  +    Tcl_AppendResult(interp, "wrong # args: should be \"",
         5305  +        Tcl_GetStringFromObj(objv[0], 0), " DB NRETRY DELAY", 0);
         5306  +    return TCL_ERROR;
         5307  +  }
         5308  +  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
         5309  +    return TCL_ERROR;
         5310  +  }
         5311  +  if( Tcl_GetIntFromObj(interp, objv[2], &a[0]) ) return TCL_ERROR;
         5312  +  if( Tcl_GetIntFromObj(interp, objv[3], &a[1]) ) return TCL_ERROR;
         5313  +  rc = sqlite3_file_control(db, NULL, SQLITE_FCNTL_WIN32_AV_RETRY, (void*)a);
         5314  +  sqlite3_snprintf(sizeof(z), z, "%d %d %d", rc, a[0], a[1]);
         5315  +  Tcl_AppendResult(interp, z, (char*)0);
         5316  +  return TCL_OK;  
         5317  +}
         5318  +
         5319  +/*
         5320  +** tclcmd:   file_control_persist_wal DB PERSIST-FLAG
         5321  +**
         5322  +** This TCL command runs the sqlite3_file_control interface with
         5323  +** the SQLITE_FCNTL_PERSIST_WAL opcode.
         5324  +*/
         5325  +static int file_control_persist_wal(
         5326  +  ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
         5327  +  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
         5328  +  int objc,              /* Number of arguments */
         5329  +  Tcl_Obj *CONST objv[]  /* Command arguments */
         5330  +){
         5331  +  sqlite3 *db;
         5332  +  int rc;
         5333  +  int bPersist;
         5334  +  char z[100];
         5335  +
         5336  +  if( objc!=3 ){
         5337  +    Tcl_AppendResult(interp, "wrong # args: should be \"",
         5338  +        Tcl_GetStringFromObj(objv[0], 0), " DB FLAG", 0);
         5339  +    return TCL_ERROR;
         5340  +  }
         5341  +  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
         5342  +    return TCL_ERROR;
         5343  +  }
         5344  +  if( Tcl_GetIntFromObj(interp, objv[2], &bPersist) ) return TCL_ERROR;
         5345  +  rc = sqlite3_file_control(db, NULL, SQLITE_FCNTL_PERSIST_WAL, (void*)&bPersist);
         5346  +  sqlite3_snprintf(sizeof(z), z, "%d %d", rc, bPersist);
         5347  +  Tcl_AppendResult(interp, z, (char*)0);
         5348  +  return TCL_OK;  
         5349  +}
  5285   5350   
  5286   5351   /*
  5287   5352   ** tclcmd:   sqlite3_vfs_list
  5288   5353   **
  5289   5354   **   Return a tcl list containing the names of all registered vfs's.
  5290   5355   */
  5291   5356   static int vfs_list(
................................................................................
  5756   5821       }
  5757   5822     }
  5758   5823   
  5759   5824     Tcl_ResetResult(interp);
  5760   5825     return TCL_OK;
  5761   5826   }
  5762   5827   
         5828  +#if SQLITE_OS_WIN
         5829  +/*
         5830  +** Information passed from the main thread into the windows file locker
         5831  +** background thread.
         5832  +*/
         5833  +struct win32FileLocker {
         5834  +  HANDLE h;           /* Handle of the file to be locked */
         5835  +  int delay1;         /* Delay before locking */
         5836  +  int delay2;         /* Delay before unlocking */
         5837  +  int ok;             /* Finished ok */
         5838  +  int err;            /* True if an error occurs */
         5839  +};
         5840  +#endif
         5841  +
         5842  +
         5843  +#if SQLITE_OS_WIN
         5844  +/*
         5845  +** The background thread that does file locking.
         5846  +*/
         5847  +static void win32_file_locker(void *pAppData){
         5848  +  struct win32FileLocker *p = (struct win32FileLocker*)pAppData;
         5849  +  if( p->delay1 ) Sleep(p->delay1);
         5850  +  if( LockFile(p->h, 0, 0, 100000000, 0) ){
         5851  +    Sleep(p->delay2);
         5852  +    UnlockFile(p->h, 0, 0, 100000000, 0);
         5853  +    p->ok = 1;
         5854  +  }else{
         5855  +    p->err = 1;
         5856  +  }
         5857  +  CloseHandle(p->h);
         5858  +  p->h = 0;
         5859  +  p->delay1 = 0;
         5860  +  p->delay2 = 0;
         5861  +}
         5862  +#endif
         5863  +
         5864  +#if SQLITE_OS_WIN
         5865  +/*
         5866  +**      lock_win32_file FILENAME DELAY1 DELAY2
         5867  +**
         5868  +** Get an exclusive manditory lock on file for DELAY2 milliseconds.
         5869  +** Wait DELAY1 milliseconds before acquiring the lock.
         5870  +*/
         5871  +static int win32_file_lock(
         5872  +  void * clientData,
         5873  +  Tcl_Interp *interp,
         5874  +  int objc,
         5875  +  Tcl_Obj *CONST objv[]
         5876  +){
         5877  +  static struct win32FileLocker x = { 0, 0, 0 };
         5878  +  const char *zFilename;
         5879  +  int retry = 0;
         5880  +  
         5881  +  if( objc!=4 && objc!=1 ){
         5882  +    Tcl_WrongNumArgs(interp, 1, objv, "FILENAME DELAY1 DELAY2");
         5883  +    return TCL_ERROR;
         5884  +  }
         5885  +  if( objc==1 ){
         5886  +    char zBuf[200];
         5887  +    sqlite3_snprintf(sizeof(zBuf), zBuf, "%d %d %d %d %d",
         5888  +                     x.ok, x.err, x.delay1, x.delay2, x.h);
         5889  +    Tcl_AppendResult(interp, zBuf, (char*)0);
         5890  +    return TCL_OK;
         5891  +  }
         5892  +  while( x.h && retry<30 ){
         5893  +    retry++;
         5894  +    Sleep(100);
         5895  +  }
         5896  +  if( x.h ){
         5897  +    Tcl_AppendResult(interp, "busy", (char*)0);
         5898  +    return TCL_ERROR;
         5899  +  }
         5900  +  if( Tcl_GetIntFromObj(interp, objv[2], &x.delay1) ) return TCL_ERROR;
         5901  +  if( Tcl_GetIntFromObj(interp, objv[3], &x.delay2) ) return TCL_ERROR;
         5902  +  zFilename = Tcl_GetString(objv[1]);
         5903  +  x.h = CreateFile(zFilename, GENERIC_READ|GENERIC_WRITE,
         5904  +              FILE_SHARE_READ|FILE_SHARE_WRITE, 0, OPEN_ALWAYS,
         5905  +              FILE_ATTRIBUTE_NORMAL, 0);
         5906  +  if( !x.h ){
         5907  +    Tcl_AppendResult(interp, "cannot open file: ", zFilename, (char*)0);
         5908  +    return TCL_ERROR;
         5909  +  }
         5910  +  _beginthread(win32_file_locker, 0, (void*)&x);
         5911  +  Sleep(0);
         5912  +  return TCL_OK;
         5913  +}
         5914  +#endif
         5915  +
  5763   5916   
  5764   5917   /*
  5765   5918   **      optimization_control DB OPT BOOLEAN
  5766   5919   **
  5767   5920   ** Enable or disable query optimizations using the sqlite3_test_control()
  5768   5921   ** interface.  Disable if BOOLEAN is false and enable if BOOLEAN is true.
  5769   5922   ** OPT is the name of the optimization to be disabled.
................................................................................
  5937   6090        { "sqlite3_extended_result_codes", test_extended_result_codes, 0},
  5938   6091        { "sqlite3_limit",                 test_limit,                 0},
  5939   6092   
  5940   6093        { "save_prng_state",               save_prng_state,    0 },
  5941   6094        { "restore_prng_state",            restore_prng_state, 0 },
  5942   6095        { "reset_prng_state",              reset_prng_state,   0 },
  5943   6096        { "optimization_control",          optimization_control,0},
         6097  +#if SQLITE_OS_WIN
         6098  +     { "lock_win32_file",               win32_file_lock,    0 },
         6099  +#endif
  5944   6100        { "tcl_objproc",                   runAsObjProc,       0 },
  5945   6101   
  5946   6102        /* sqlite3_column_*() API */
  5947   6103        { "sqlite3_column_count",          test_column_count  ,0 },
  5948   6104        { "sqlite3_data_count",            test_data_count    ,0 },
  5949   6105        { "sqlite3_column_type",           test_column_type   ,0 },
  5950   6106        { "sqlite3_column_blob",           test_column_blob   ,0 },
................................................................................
  5989   6145        { "file_control_lasterrno_test", file_control_lasterrno_test,  0   },
  5990   6146        { "file_control_lockproxy_test", file_control_lockproxy_test,  0   },
  5991   6147   #ifdef __APPLE__
  5992   6148        { "file_control_truncate_test", file_control_truncate_test,  0   },
  5993   6149        { "file_control_replace_test", file_control_replace_test,  0   },
  5994   6150   #endif 
  5995   6151        { "file_control_chunksize_test", file_control_chunksize_test,  0   },
  5996         -     { "file_control_sizehint_test", file_control_sizehint_test,  0   },
         6152  +     { "file_control_sizehint_test",  file_control_sizehint_test,   0   },
         6153  +     { "file_control_win32_av_retry", file_control_win32_av_retry,  0   },
         6154  +     { "file_control_persist_wal",    file_control_persist_wal,     0   },
  5997   6155        { "sqlite3_vfs_list",           vfs_list,     0   },
  5998   6156        { "sqlite3_create_function_v2", test_create_function_v2, 0 },
  5999   6157        { "path_is_local",              path_is_local,  0   },
  6000   6158        { "path_is_dos",                path_is_dos,  0   },
  6001   6159   
  6002   6160        /* Functions from os.h */
  6003   6161   #ifndef SQLITE_OMIT_UTF16

Changes to src/test_config.c.

   593    593     LINKVAR( MAX_LIKE_PATTERN_LENGTH );
   594    594     LINKVAR( MAX_TRIGGER_DEPTH );
   595    595     LINKVAR( DEFAULT_TEMP_CACHE_SIZE );
   596    596     LINKVAR( DEFAULT_CACHE_SIZE );
   597    597     LINKVAR( DEFAULT_PAGE_SIZE );
   598    598     LINKVAR( DEFAULT_FILE_FORMAT );
   599    599     LINKVAR( MAX_ATTACHED );
          600  +  LINKVAR( MAX_DEFAULT_PAGE_SIZE );
   600    601   
   601    602     {
   602    603       static const int cv_TEMP_STORE = SQLITE_TEMP_STORE;
   603    604       Tcl_LinkVar(interp, "TEMP_STORE", (char *)&(cv_TEMP_STORE),
   604    605                   TCL_LINK_INT | TCL_LINK_READ_ONLY);
   605    606     }
   606    607   }

Changes to src/test_multiplex.c.

    41     41   ** The multiplex VFS allows databases up to 32 GiB in size.  But it splits
    42     42   ** the files up into 1 GiB pieces, so that they will work even on filesystems
    43     43   ** that do not support large files.
    44     44   */
    45     45   #include "sqlite3.h"
    46     46   #include <string.h>
    47     47   #include <assert.h>
           48  +#include <stdlib.h>
    48     49   #include "test_multiplex.h"
    49     50   
    50     51   #ifndef SQLITE_CORE
    51     52     #define SQLITE_CORE 1  /* Disable the API redefinition in sqlite3ext.h */
    52     53   #endif
    53     54   #include "sqlite3ext.h"
    54     55   
................................................................................
    74     75   #define sqlite3_mutex_held(X)     ((void)(X),1)
    75     76   #define sqlite3_mutex_notheld(X)  ((void)(X),1)
    76     77   #endif /* SQLITE_THREADSAFE==0 */
    77     78   
    78     79   
    79     80   /************************ Shim Definitions ******************************/
    80     81   
    81         -#define SQLITE_MULTIPLEX_VFS_NAME "multiplex"
           82  +#ifndef SQLITE_MULTIPLEX_VFS_NAME
           83  +# define SQLITE_MULTIPLEX_VFS_NAME "multiplex"
           84  +#endif
    82     85   
    83     86   /* This is the limit on the chunk size.  It may be changed by calling
    84     87   ** the xFileControl() interface.  It will be rounded up to a 
    85         -** multiple of MAX_PAGE_SIZE.  We default it here to 1GB.
           88  +** multiple of MAX_PAGE_SIZE.  We default it here to 2GiB less 64KiB.
    86     89   */
    87         -#define SQLITE_MULTIPLEX_CHUNK_SIZE (MAX_PAGE_SIZE*16384)
           90  +#ifndef SQLITE_MULTIPLEX_CHUNK_SIZE
           91  +# define SQLITE_MULTIPLEX_CHUNK_SIZE 2147418112
           92  +#endif
    88     93   
    89     94   /* Default limit on number of chunks.  Care should be taken
    90     95   ** so that values for chunks numbers fit in the SQLITE_MULTIPLEX_EXT_FMT
    91     96   ** format specifier. It may be changed by calling
    92     97   ** the xFileControl() interface.
    93     98   */
    94         -#define SQLITE_MULTIPLEX_MAX_CHUNKS 32
           99  +#ifndef SQLITE_MULTIPLEX_MAX_CHUNKS
          100  +# define SQLITE_MULTIPLEX_MAX_CHUNKS 32
          101  +#endif
    95    102   
    96    103   /* If SQLITE_MULTIPLEX_EXT_OVWR is defined, the 
    97    104   ** last SQLITE_MULTIPLEX_EXT_SZ characters of the 
    98    105   ** filename will be overwritten, otherwise, the 
    99    106   ** multiplex extension is simply appended to the filename.
   100    107   ** Ex.  (undefined) test.db -> test.db01
   101    108   **      (defined)   test.db -> test.01
................................................................................
   115    122   ** makeup a single SQLite DB file.  This allows the size of the DB
   116    123   ** to exceed the limits imposed by the file system.
   117    124   **
   118    125   ** There is an instance of the following object for each defined multiplex
   119    126   ** group.
   120    127   */
   121    128   struct multiplexGroup {
   122         -  sqlite3_file **pReal;            /* Handles to each chunk */
   123         -  char *bOpen;                     /* array of bools - 0 if chunk not opened */
          129  +  struct multiplexReal {           /* For each chunk */
          130  +    sqlite3_file *p;                  /* Handle for the chunk */
          131  +    char *z;                          /* Name of this chunk */
          132  +  } *aReal;                        /* list of all chunks */
          133  +  int nReal;                       /* Number of chunks */
   124    134     char *zName;                     /* Base filename of this group */
   125    135     int nName;                       /* Length of base filename */
   126    136     int flags;                       /* Flags used for original opening */
   127         -  int nChunkSize;                  /* Chunk size used for this group */
   128         -  int nMaxChunks;                  /* Max number of chunks for this group */
          137  +  unsigned int szChunk;            /* Chunk size used for this group */
   129    138     int bEnabled;                    /* TRUE to use Multiplex VFS for this file */
   130    139     multiplexGroup *pNext, *pPrev;   /* Doubly linked list of all group objects */
   131    140   };
   132    141   
   133    142   /*
   134    143   ** An instance of the following object represents each open connection
   135    144   ** to a file that is multiplex'ed.  This object is a 
................................................................................
   180    189     ** shim, the following mutex must be held.
   181    190     */
   182    191     sqlite3_mutex *pMutex;
   183    192   
   184    193     /* List of multiplexGroup objects.
   185    194     */
   186    195     multiplexGroup *pGroups;
   187         -
   188         -  /* Storage for temp file names.  Allocated during 
   189         -  ** initialization to the max pathname of the underlying VFS.
   190         -  */
   191         -  char *zName;
   192         -
   193    196   } gMultiplex;
   194    197   
   195    198   /************************* Utility Routines *********************************/
   196    199   /*
   197    200   ** Acquire and release the mutex used to serialize access to the
   198    201   ** list of multiplexGroups.
   199    202   */
................................................................................
   261    264       j = multiplexStrlen30(zBuf);
   262    265       if( (j + 8 + 1 + 3 + 1) <= nBuf ){
   263    266         /* Make 3 attempts to generate a unique name. */
   264    267         do {
   265    268           attempts++;
   266    269           sqlite3_randomness(8, &zBuf[j]);
   267    270           for(i=0; i<8; i++){
   268         -          zBuf[j+i] = (char)zChars[ ((unsigned char)zBuf[j+i])%(sizeof(zChars)-1) ];
          271  +          unsigned char uc = (unsigned char)zBuf[j+i];
          272  +          zBuf[j+i] = (char)zChars[uc%(sizeof(zChars)-1)];
   269    273           }
   270    274           memcpy(&zBuf[j+i], ".tmp", 5);
   271    275           rc = pOrigVfs->xAccess(pOrigVfs, zBuf, SQLITE_ACCESS_EXISTS, &exists);
   272    276         } while ( (rc==SQLITE_OK) && exists && (attempts<3) );
   273    277         if( rc==SQLITE_OK && exists ){
   274    278           rc = SQLITE_ERROR;
   275    279         }
   276    280       }
   277    281     }
   278    282   
   279    283     return rc;
   280    284   }
          285  +
          286  +/* Compute the filename for the iChunk-th chunk
          287  +*/
          288  +static int multiplexSubFilename(multiplexGroup *pGroup, int iChunk){
          289  +  if( iChunk>=pGroup->nReal ){
          290  +    struct multiplexReal *p;
          291  +    p = sqlite3_realloc(pGroup->aReal, (iChunk+1)*sizeof(*p));
          292  +    if( p==0 ){
          293  +      return SQLITE_NOMEM;
          294  +    }
          295  +    memset(&p[pGroup->nReal], 0, sizeof(p[0])*(iChunk+1-pGroup->nReal));
          296  +    pGroup->aReal = p;
          297  +    pGroup->nReal = iChunk+1;
          298  +  }
          299  +  if( pGroup->aReal[iChunk].z==0 ){
          300  +    char *z;
          301  +    int n = pGroup->nName;
          302  +    pGroup->aReal[iChunk].z = z = sqlite3_malloc( n+3 );
          303  +    if( z==0 ){
          304  +      return SQLITE_NOMEM;
          305  +    }
          306  +    memcpy(z, pGroup->zName, n+1);
          307  +    if( iChunk>0 ){
          308  +#ifdef SQLITE_ENABLE_8_3_NAMES
          309  +      if( n>3 && z[n-3]=='.' ){
          310  +        n--;
          311  +      }else if( n>4 && z[n-4]=='.' ){
          312  +        n -= 2;
          313  +      }
          314  +#endif
          315  +      sqlite3_snprintf(3,&z[n],"%02d",iChunk);
          316  +    }
          317  +  }
          318  +  return SQLITE_OK;
          319  +}
   281    320   
   282    321   /* Translate an sqlite3_file* that is really a multiplexGroup* into
   283    322   ** the sqlite3_file* for the underlying original VFS.
   284    323   */
   285         -static sqlite3_file *multiplexSubOpen(multiplexConn *pConn, int iChunk, int *rc, int *pOutFlags){
   286         -  multiplexGroup *pGroup = pConn->pGroup;
          324  +static sqlite3_file *multiplexSubOpen(
          325  +  multiplexGroup *pGroup,
          326  +  int iChunk,
          327  +  int *rc,
          328  +  int *pOutFlags
          329  +){
          330  +  sqlite3_file *pSubOpen = 0;
   287    331     sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;        /* Real VFS */
   288         -  if( iChunk<pGroup->nMaxChunks ){
   289         -    sqlite3_file *pSubOpen = pGroup->pReal[iChunk];    /* Real file descriptor */
   290         -    if( !pGroup->bOpen[iChunk] ){
   291         -      memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1);
   292         -      if( iChunk ){
   293         -#ifdef SQLITE_MULTIPLEX_EXT_OVWR
   294         -        sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, SQLITE_MULTIPLEX_EXT_FMT, iChunk);
   295         -#else
   296         -        sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName, SQLITE_MULTIPLEX_EXT_FMT, iChunk);
   297         -#endif
   298         -      }
   299         -      *rc = pOrigVfs->xOpen(pOrigVfs, gMultiplex.zName, pSubOpen, pGroup->flags, pOutFlags);
   300         -      if( *rc==SQLITE_OK ){
   301         -        pGroup->bOpen[iChunk] = -1;
   302         -        return pSubOpen;
   303         -      }
   304         -      return NULL;
          332  +  *rc = multiplexSubFilename(pGroup, iChunk);
          333  +  if( (*rc)==SQLITE_OK && (pSubOpen = pGroup->aReal[iChunk].p)==0 ){
          334  +    pSubOpen = sqlite3_malloc( pOrigVfs->szOsFile );
          335  +    if( pSubOpen==0 ){
          336  +      *rc = SQLITE_NOMEM;
          337  +      return 0;
   305    338       }
   306         -    *rc = SQLITE_OK;
   307         -    return pSubOpen;
          339  +    pGroup->aReal[iChunk].p = pSubOpen;
          340  +    *rc = pOrigVfs->xOpen(pOrigVfs, pGroup->aReal[iChunk].z, pSubOpen,
          341  +                          pGroup->flags, pOutFlags);
          342  +    if( *rc!=SQLITE_OK ){
          343  +      sqlite3_free(pSubOpen);
          344  +      pGroup->aReal[iChunk].p = 0;
          345  +      return 0;
          346  +    }
   308    347     }
   309         -  *rc = SQLITE_FULL;
   310         -  return NULL;
          348  +  return pSubOpen;
   311    349   }
   312    350   
   313    351   /*
   314    352   ** This is the implementation of the multiplex_control() SQL function.
   315    353   */
   316    354   static void multiplexControlFunc(
   317    355     sqlite3_context *context,
................................................................................
   361    399     const sqlite3_api_routines *pApi
   362    400   ){
   363    401     int rc;
   364    402     rc = sqlite3_create_function(db, "multiplex_control", 2, SQLITE_ANY, 
   365    403         0, multiplexControlFunc, 0, 0);
   366    404     return rc;
   367    405   }
          406  +
          407  +/*
          408  +** Close a single sub-file in the connection group.
          409  +*/
          410  +static void multiplexSubClose(
          411  +  multiplexGroup *pGroup,
          412  +  int iChunk,
          413  +  sqlite3_vfs *pOrigVfs
          414  +){
          415  +  sqlite3_file *pSubOpen = pGroup->aReal[iChunk].p;
          416  +  if( pSubOpen ){
          417  +    pSubOpen->pMethods->xClose(pSubOpen);
          418  +    if( pOrigVfs ) pOrigVfs->xDelete(pOrigVfs, pGroup->aReal[iChunk].z, 0);
          419  +    sqlite3_free(pGroup->aReal[iChunk].p);
          420  +  }
          421  +  sqlite3_free(pGroup->aReal[iChunk].z);
          422  +  memset(&pGroup->aReal[iChunk], 0, sizeof(pGroup->aReal[iChunk]));
          423  +}
          424  +
          425  +/*
          426  +** Deallocate memory held by a multiplexGroup
          427  +*/
          428  +static void multiplexFreeComponents(multiplexGroup *pGroup){
          429  +  int i;
          430  +  for(i=0; i<pGroup->nReal; i++){ multiplexSubClose(pGroup, i, 0); }
          431  +  sqlite3_free(pGroup->aReal);
          432  +  pGroup->aReal = 0;
          433  +  pGroup->nReal = 0;
          434  +}
          435  +
   368    436   
   369    437   /************************* VFS Method Wrappers *****************************/
   370    438   
   371    439   /*
   372    440   ** This is the xOpen method used for the "multiplex" VFS.
   373    441   **
   374    442   ** Most of the work is done by the underlying original VFS.  This method
................................................................................
   378    446   static int multiplexOpen(
   379    447     sqlite3_vfs *pVfs,         /* The multiplex VFS */
   380    448     const char *zName,         /* Name of file to be opened */
   381    449     sqlite3_file *pConn,       /* Fill in this file descriptor */
   382    450     int flags,                 /* Flags to control the opening */
   383    451     int *pOutFlags             /* Flags showing results of opening */
   384    452   ){
   385         -  int rc = SQLITE_OK;                            /* Result code */
   386         -  multiplexConn *pMultiplexOpen;                 /* The new multiplex file descriptor */
   387         -  multiplexGroup *pGroup;                        /* Corresponding multiplexGroup object */
   388         -  sqlite3_file *pSubOpen;                        /* Real file descriptor */
          453  +  int rc = SQLITE_OK;                  /* Result code */
          454  +  multiplexConn *pMultiplexOpen;       /* The new multiplex file descriptor */
          455  +  multiplexGroup *pGroup;              /* Corresponding multiplexGroup object */
          456  +  sqlite3_file *pSubOpen = 0;                    /* Real file descriptor */
   389    457     sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;   /* Real VFS */
   390    458     int nName;
   391         -  int i;
   392    459     int sz;
          460  +  char *zToFree = 0;
   393    461   
   394    462     UNUSED_PARAMETER(pVfs);
          463  +  memset(pConn, 0, pVfs->szOsFile);
   395    464   
   396    465     /* We need to create a group structure and manage
   397    466     ** access to this group of files.
   398    467     */
   399    468     multiplexEnter();
   400    469     pMultiplexOpen = (multiplexConn*)pConn;
   401    470   
   402    471     /* If the second argument to this function is NULL, generate a 
   403    472     ** temporary file name to use.  This will be handled by the
   404    473     ** original xOpen method.  We just need to allocate space for
   405    474     ** it.
   406    475     */
   407    476     if( !zName ){
   408         -    rc = multiplexGetTempname(pOrigVfs, pOrigVfs->mxPathname, gMultiplex.zName);
   409         -    zName = gMultiplex.zName;
          477  +    zName = zToFree = sqlite3_malloc( pOrigVfs->mxPathname + 10 );
          478  +    if( zName==0 ){
          479  +      rc = SQLITE_NOMEM;
          480  +    }else{
          481  +      rc = multiplexGetTempname(pOrigVfs, pOrigVfs->mxPathname, zToFree);
          482  +    }
   410    483     }
   411    484   
   412    485     if( rc==SQLITE_OK ){
   413    486       /* allocate space for group */
   414    487       nName = multiplexStrlen30(zName);
   415         -    sz = sizeof(multiplexGroup)                                /* multiplexGroup */
   416         -       + (sizeof(sqlite3_file *)*SQLITE_MULTIPLEX_MAX_CHUNKS)  /* pReal[] */
   417         -       + (pOrigVfs->szOsFile*SQLITE_MULTIPLEX_MAX_CHUNKS)      /* *pReal */
   418         -       + SQLITE_MULTIPLEX_MAX_CHUNKS                           /* bOpen[] */
   419         -       + nName + 1;                                            /* zName */
   420         -#ifndef SQLITE_MULTIPLEX_EXT_OVWR
   421         -    sz += SQLITE_MULTIPLEX_EXT_SZ;
   422         -    assert(nName+SQLITE_MULTIPLEX_EXT_SZ < pOrigVfs->mxPathname);
   423         -#else
   424         -    assert(nName >= SQLITE_MULTIPLEX_EXT_SZ);
   425         -    assert(nName < pOrigVfs->mxPathname);
   426         -#endif
          488  +    sz = sizeof(multiplexGroup)                             /* multiplexGroup */
          489  +       + nName + 1;                                         /* zName */
   427    490       pGroup = sqlite3_malloc( sz );
   428    491       if( pGroup==0 ){
   429         -      rc=SQLITE_NOMEM;
          492  +      rc = SQLITE_NOMEM;
   430    493       }
   431    494     }
   432    495   
   433    496     if( rc==SQLITE_OK ){
   434    497       /* assign pointers to extra space allocated */
   435    498       char *p = (char *)&pGroup[1];
   436    499       pMultiplexOpen->pGroup = pGroup;
   437    500       memset(pGroup, 0, sz);
   438    501       pGroup->bEnabled = -1;
   439         -    pGroup->nChunkSize = SQLITE_MULTIPLEX_CHUNK_SIZE;
   440         -    pGroup->nMaxChunks = SQLITE_MULTIPLEX_MAX_CHUNKS;
   441         -    pGroup->pReal = (sqlite3_file **)p;
   442         -    p += (sizeof(sqlite3_file *)*pGroup->nMaxChunks);
   443         -    for(i=0; i<pGroup->nMaxChunks; i++){
   444         -      pGroup->pReal[i] = (sqlite3_file *)p;
   445         -      p += pOrigVfs->szOsFile;
          502  +    pGroup->szChunk = SQLITE_MULTIPLEX_CHUNK_SIZE;
          503  +    if( flags & SQLITE_OPEN_URI ){
          504  +      const char *zChunkSize;
          505  +      zChunkSize = sqlite3_uri_parameter(zName, "chunksize");
          506  +      if( zChunkSize ){
          507  +        unsigned int n = 0;
          508  +        int i;
          509  +        for(i=0; zChunkSize[i]>='0' && zChunkSize[i]<='9'; i++){
          510  +          n = n*10 + zChunkSize[i] - '0';
          511  +        }
          512  +        if( n>0 ){
          513  +          pGroup->szChunk = (n+0xffff)&~0xffff;
          514  +        }else{
          515  +          /* A zero or negative chunksize disabled the multiplexor */
          516  +          pGroup->bEnabled = 0;
          517  +        }
          518  +      }
   446    519       }
   447         -    /* bOpen[] vals should all be zero from memset above */
   448         -    pGroup->bOpen = p;
   449         -    p += pGroup->nMaxChunks;
   450    520       pGroup->zName = p;
   451    521       /* save off base filename, name length, and original open flags  */
   452    522       memcpy(pGroup->zName, zName, nName+1);
   453    523       pGroup->nName = nName;
   454    524       pGroup->flags = flags;
   455         -    pSubOpen = multiplexSubOpen(pMultiplexOpen, 0, &rc, pOutFlags);
          525  +    rc = multiplexSubFilename(pGroup, 1);
          526  +    if( rc==SQLITE_OK ){
          527  +      pSubOpen = multiplexSubOpen(pGroup, 0, &rc, pOutFlags);
          528  +    }
   456    529       if( pSubOpen ){
   457         -      /* if this file is already larger than chunk size, disable 
   458         -      ** the multiplex feature.
   459         -      */
          530  +      int exists, rc2, rc3;
   460    531         sqlite3_int64 sz;
   461         -      int rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz);
   462         -      if( (rc2==SQLITE_OK) && (sz>pGroup->nChunkSize) ){
   463         -        pGroup->bEnabled = 0;
          532  +
          533  +      rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz);
          534  +      if( rc2==SQLITE_OK ){
          535  +        /* If the first overflow file exists and if the size of the main file
          536  +        ** is different from the chunk size, that means the chunk size is set
          537  +        ** set incorrectly.  So fix it.
          538  +        **
          539  +        ** Or, if the first overflow file does not exist and the main file is
          540  +        ** larger than the chunk size, that means the chunk size is too small.
          541  +        ** But we have no way of determining the intended chunk size, so 
          542  +        ** just disable the multiplexor all togethre.
          543  +        */
          544  +        rc3 = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[1].z,
          545  +            SQLITE_ACCESS_EXISTS, &exists);
          546  +        if( rc3==SQLITE_OK && exists && sz==(sz&0xffff0000) && sz>0
          547  +            && sz!=pGroup->szChunk ){
          548  +          pGroup->szChunk = sz;
          549  +        }else if( rc3==SQLITE_OK && !exists && sz>pGroup->szChunk ){
          550  +          pGroup->bEnabled = 0;
          551  +        }
   464    552         }
          553  +
   465    554         if( pSubOpen->pMethods->iVersion==1 ){
   466    555           pMultiplexOpen->base.pMethods = &gMultiplex.sIoMethodsV1;
   467    556         }else{
   468    557           pMultiplexOpen->base.pMethods = &gMultiplex.sIoMethodsV2;
   469    558         }
   470    559         /* place this group at the head of our list */
   471    560         pGroup->pNext = gMultiplex.pGroups;
   472    561         if( gMultiplex.pGroups ) gMultiplex.pGroups->pPrev = pGroup;
   473    562         gMultiplex.pGroups = pGroup;
   474    563       }else{
          564  +      multiplexFreeComponents(pGroup);
   475    565         sqlite3_free(pGroup);
   476    566       }
   477    567     }
   478    568     multiplexLeave();
          569  +  sqlite3_free(zToFree);
   479    570     return rc;
   480    571   }
   481    572   
   482    573   /*
   483    574   ** This is the xDelete method used for the "multiplex" VFS.
   484         -** It attempts to delete the filename specified, as well
   485         -** as additional files with the SQLITE_MULTIPLEX_EXT_FMT extension.
          575  +** It attempts to delete the filename specified.
   486    576   */
   487    577   static int multiplexDelete(
   488    578     sqlite3_vfs *pVfs,         /* The multiplex VFS */
   489    579     const char *zName,         /* Name of file to delete */
   490    580     int syncDir
   491    581   ){
   492    582     sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;   /* Real VFS */
   493         -  int rc = SQLITE_OK;
   494         -  int nName = multiplexStrlen30(zName);
   495         -  int i;
   496         -
   497         -  UNUSED_PARAMETER(pVfs);
   498         -
   499         -  multiplexEnter();
   500         -  memcpy(gMultiplex.zName, zName, nName+1);
   501         -  for(i=0; i<SQLITE_MULTIPLEX_MAX_CHUNKS; i++){
   502         -    int rc2;
   503         -    int exists = 0;
   504         -    if( i ){
   505         -#ifdef SQLITE_MULTIPLEX_EXT_OVWR
   506         -        sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, 
   507         -            gMultiplex.zName+nName-SQLITE_MULTIPLEX_EXT_SZ, 
   508         -            SQLITE_MULTIPLEX_EXT_FMT, i);
   509         -#else
   510         -        sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, 
   511         -            gMultiplex.zName+nName, 
   512         -            SQLITE_MULTIPLEX_EXT_FMT, i);
   513         -#endif
   514         -    }
   515         -    rc2 = pOrigVfs->xAccess(pOrigVfs, gMultiplex.zName, 
   516         -        SQLITE_ACCESS_EXISTS, &exists);
   517         -    if( rc2==SQLITE_OK && exists ){
   518         -      /* if it exists, delete it */
   519         -      rc2 = pOrigVfs->xDelete(pOrigVfs, gMultiplex.zName, syncDir);
   520         -      if( rc2!=SQLITE_OK ) rc = rc2;
   521         -    }else{
   522         -      /* stop at first "gap" */
   523         -      break;
   524         -    }
   525         -  }
   526         -  multiplexLeave();
   527         -  return rc;
          583  +  return pOrigVfs->xDelete(pOrigVfs, zName, syncDir);
   528    584   }
   529    585   
   530    586   static int multiplexAccess(sqlite3_vfs *a, const char *b, int c, int *d){
   531    587     return gMultiplex.pOrigVfs->xAccess(gMultiplex.pOrigVfs, b, c, d);
   532    588   }
   533    589   static int multiplexFullPathname(sqlite3_vfs *a, const char *b, int c, char *d){
   534    590     return gMultiplex.pOrigVfs->xFullPathname(gMultiplex.pOrigVfs, b, c, d);
................................................................................
   568    624   ** The group structure for this file is unlinked from 
   569    625   ** our list of groups and freed.
   570    626   */
   571    627   static int multiplexClose(sqlite3_file *pConn){
   572    628     multiplexConn *p = (multiplexConn*)pConn;
   573    629     multiplexGroup *pGroup = p->pGroup;
   574    630     int rc = SQLITE_OK;
   575         -  int i;
   576    631     multiplexEnter();
   577         -  /* close any open handles */
   578         -  for(i=0; i<pGroup->nMaxChunks; i++){
   579         -    if( pGroup->bOpen[i] ){
   580         -      sqlite3_file *pSubOpen = pGroup->pReal[i];
   581         -      int rc2 = pSubOpen->pMethods->xClose(pSubOpen);
   582         -      if( rc2!=SQLITE_OK ) rc = rc2;
   583         -      pGroup->bOpen[i] = 0;
   584         -    }
   585         -  }
          632  +  multiplexFreeComponents(pGroup);
   586    633     /* remove from linked list */
   587    634     if( pGroup->pNext ) pGroup->pNext->pPrev = pGroup->pPrev;
   588    635     if( pGroup->pPrev ){
   589    636       pGroup->pPrev->pNext = pGroup->pNext;
   590    637     }else{
   591    638       gMultiplex.pGroups = pGroup->pNext;
   592    639     }
................................................................................
   606    653     sqlite3_int64 iOfst
   607    654   ){
   608    655     multiplexConn *p = (multiplexConn*)pConn;
   609    656     multiplexGroup *pGroup = p->pGroup;
   610    657     int rc = SQLITE_OK;
   611    658     multiplexEnter();
   612    659     if( !pGroup->bEnabled ){
   613         -    sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
   614         -    rc = ( !pSubOpen ) ? SQLITE_IOERR_READ : pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst);
          660  +    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
          661  +    if( pSubOpen==0 ){
          662  +      rc = SQLITE_IOERR_READ;
          663  +    }else{
          664  +      rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst);
          665  +    }
   615    666     }else{
   616    667       while( iAmt > 0 ){
   617         -      int i = (int)(iOfst / pGroup->nChunkSize);
   618         -      sqlite3_file *pSubOpen = multiplexSubOpen(p, i, &rc, NULL);
          668  +      int i = (int)(iOfst / pGroup->szChunk);
          669  +      sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL);
   619    670         if( pSubOpen ){
   620         -        int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - pGroup->nChunkSize;
          671  +        int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) - pGroup->szChunk;
   621    672           if( extra<0 ) extra = 0;
   622    673           iAmt -= extra;
   623         -        rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst % pGroup->nChunkSize);
          674  +        rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt,
          675  +                                       iOfst % pGroup->szChunk);
   624    676           if( rc!=SQLITE_OK ) break;
   625    677           pBuf = (char *)pBuf + iAmt;
   626    678           iOfst += iAmt;
   627    679           iAmt = extra;
   628    680         }else{
   629    681           rc = SQLITE_IOERR_READ;
   630    682           break;
................................................................................
   646    698     sqlite3_int64 iOfst
   647    699   ){
   648    700     multiplexConn *p = (multiplexConn*)pConn;
   649    701     multiplexGroup *pGroup = p->pGroup;
   650    702     int rc = SQLITE_OK;
   651    703     multiplexEnter();
   652    704     if( !pGroup->bEnabled ){
   653         -    sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
   654         -    rc = ( !pSubOpen ) ? SQLITE_IOERR_WRITE : pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst);
          705  +    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
          706  +    if( pSubOpen==0 ){
          707  +      rc = SQLITE_IOERR_WRITE;
          708  +    }else{
          709  +      rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst);
          710  +    }
   655    711     }else{
   656    712       while( iAmt > 0 ){
   657         -      int i = (int)(iOfst / pGroup->nChunkSize);
   658         -      sqlite3_file *pSubOpen = multiplexSubOpen(p, i, &rc, NULL);
          713  +      int i = (int)(iOfst / pGroup->szChunk);
          714  +      sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL);
   659    715         if( pSubOpen ){
   660         -        int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - pGroup->nChunkSize;
          716  +        int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) -
          717  +                    pGroup->szChunk;
   661    718           if( extra<0 ) extra = 0;
   662    719           iAmt -= extra;
   663         -        rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst % pGroup->nChunkSize);
          720  +        rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt,
          721  +                                        iOfst % pGroup->szChunk);
   664    722           if( rc!=SQLITE_OK ) break;
   665    723           pBuf = (char *)pBuf + iAmt;
   666    724           iOfst += iAmt;
   667    725           iAmt = extra;
   668    726         }else{
   669    727           rc = SQLITE_IOERR_WRITE;
   670    728           break;
................................................................................
   681    739   */
   682    740   static int multiplexTruncate(sqlite3_file *pConn, sqlite3_int64 size){
   683    741     multiplexConn *p = (multiplexConn*)pConn;
   684    742     multiplexGroup *pGroup = p->pGroup;
   685    743     int rc = SQLITE_OK;
   686    744     multiplexEnter();
   687    745     if( !pGroup->bEnabled ){
   688         -    sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
   689         -    rc = ( !pSubOpen ) ? SQLITE_IOERR_TRUNCATE : pSubOpen->pMethods->xTruncate(pSubOpen, size);
          746  +    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
          747  +    if( pSubOpen==0 ){
          748  +      rc = SQLITE_IOERR_TRUNCATE;
          749  +    }else{
          750  +      rc = pSubOpen->pMethods->xTruncate(pSubOpen, size);
          751  +    }
   690    752     }else{
   691    753       int rc2;
   692    754       int i;
   693    755       sqlite3_file *pSubOpen;
   694    756       sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;   /* Real VFS */
   695         -    memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1);
   696    757       /* delete the chunks above the truncate limit */
   697         -    for(i=(int)(size / pGroup->nChunkSize)+1; i<pGroup->nMaxChunks; i++){
   698         -      /* close any open chunks before deleting them */
   699         -      if( pGroup->bOpen[i] ){
   700         -        pSubOpen = pGroup->pReal[i];
   701         -        rc2 = pSubOpen->pMethods->xClose(pSubOpen);
   702         -        if( rc2!=SQLITE_OK ) rc = SQLITE_IOERR_TRUNCATE;
   703         -        pGroup->bOpen[i] = 0;
   704         -      }
   705         -#ifdef SQLITE_MULTIPLEX_EXT_OVWR
   706         -      sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, 
   707         -          gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, 
   708         -          SQLITE_MULTIPLEX_EXT_FMT, i);
   709         -#else
   710         -      sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, 
   711         -          gMultiplex.zName+pGroup->nName, 
   712         -          SQLITE_MULTIPLEX_EXT_FMT, i);
   713         -#endif
   714         -      rc2 = pOrigVfs->xDelete(pOrigVfs, gMultiplex.zName, 0);
   715         -      if( rc2!=SQLITE_OK ) rc = SQLITE_IOERR_TRUNCATE;
          758  +    for(i=(int)(size / pGroup->szChunk)+1; i<pGroup->nReal; i++){
          759  +      multiplexSubClose(pGroup, i, pOrigVfs);
   716    760       }
   717         -    pSubOpen = multiplexSubOpen(p, (int)(size / pGroup->nChunkSize), &rc2, NULL);
          761  +    pSubOpen = multiplexSubOpen(pGroup, (int)(size/pGroup->szChunk), &rc2,0);
   718    762       if( pSubOpen ){
   719         -      rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->nChunkSize);
          763  +      rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->szChunk);
   720    764         if( rc2!=SQLITE_OK ) rc = rc2;
   721    765       }else{
   722    766         rc = SQLITE_IOERR_TRUNCATE;
   723    767       }
   724    768     }
   725    769     multiplexLeave();
   726    770     return rc;
................................................................................
   730    774   */
   731    775   static int multiplexSync(sqlite3_file *pConn, int flags){
   732    776     multiplexConn *p = (multiplexConn*)pConn;
   733    777     multiplexGroup *pGroup = p->pGroup;
   734    778     int rc = SQLITE_OK;
   735    779     int i;
   736    780     multiplexEnter();
   737         -  for(i=0; i<pGroup->nMaxChunks; i++){
   738         -    /* if we don't have it open, we don't need to sync it */
   739         -    if( pGroup->bOpen[i] ){
   740         -      sqlite3_file *pSubOpen = pGroup->pReal[i];
          781  +  for(i=0; i<pGroup->nReal; i++){
          782  +    sqlite3_file *pSubOpen = pGroup->aReal[i].p;
          783  +    if( pSubOpen ){
   741    784         int rc2 = pSubOpen->pMethods->xSync(pSubOpen, flags);
   742    785         if( rc2!=SQLITE_OK ) rc = rc2;
   743    786       }
   744    787     }
   745    788     multiplexLeave();
   746    789     return rc;
   747    790   }
................................................................................
   753    796     multiplexConn *p = (multiplexConn*)pConn;
   754    797     multiplexGroup *pGroup = p->pGroup;
   755    798     int rc = SQLITE_OK;
   756    799     int rc2;
   757    800     int i;
   758    801     multiplexEnter();
   759    802     if( !pGroup->bEnabled ){
   760         -    sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
   761         -    rc = ( !pSubOpen ) ? SQLITE_IOERR_FSTAT : pSubOpen->pMethods->xFileSize(pSubOpen, pSize);
          803  +    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
          804  +    if( pSubOpen==0 ){
          805  +      rc = SQLITE_IOERR_FSTAT;
          806  +    }else{
          807  +      rc = pSubOpen->pMethods->xFileSize(pSubOpen, pSize);
          808  +    }
   762    809     }else{
          810  +    sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;
   763    811       *pSize = 0;
   764         -    for(i=0; i<pGroup->nMaxChunks; i++){
   765         -      sqlite3_file *pSubOpen = NULL;
   766         -      /* if not opened already, check to see if the chunk exists */
   767         -      if( pGroup->bOpen[i] ){
   768         -        pSubOpen = pGroup->pReal[i];
          812  +    for(i=0; 1; i++){
          813  +      sqlite3_file *pSubOpen = 0;
          814  +      int exists = 0;
          815  +      rc = multiplexSubFilename(pGroup, i);
          816  +      if( rc ) break;
          817  +      rc2 = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[i].z,
          818  +          SQLITE_ACCESS_EXISTS, &exists);
          819  +      if( rc2==SQLITE_OK && exists){
          820  +        /* if it exists, open it */
          821  +        pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL);
   769    822         }else{
   770         -        sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;   /* Real VFS */
   771         -        int exists = 0;
   772         -        memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1);
   773         -        if( i ){
   774         -#ifdef SQLITE_MULTIPLEX_EXT_OVWR
   775         -          sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, 
   776         -              gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, 
   777         -              SQLITE_MULTIPLEX_EXT_FMT, i);
   778         -#else
   779         -          sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, 
   780         -              gMultiplex.zName+pGroup->nName, 
   781         -              SQLITE_MULTIPLEX_EXT_FMT, i);
   782         -#endif
   783         -        }
   784         -        rc2 = pOrigVfs->xAccess(pOrigVfs, gMultiplex.zName, 
   785         -            SQLITE_ACCESS_EXISTS, &exists);
   786         -        if( rc2==SQLITE_OK && exists){
   787         -          /* if it exists, open it */
   788         -          pSubOpen = multiplexSubOpen(p, i, &rc, NULL);
   789         -        }else{
   790         -          /* stop at first "gap" */
   791         -          break;
   792         -        }
          823  +        /* stop at first "gap" */
          824  +        break;
   793    825         }
   794    826         if( pSubOpen ){
   795    827           sqlite3_int64 sz;
   796    828           rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz);
   797    829           if( rc2!=SQLITE_OK ){
   798    830             rc = rc2;
   799    831           }else{
   800         -          if( sz>pGroup->nChunkSize ){
          832  +          if( sz>pGroup->szChunk ){
   801    833               rc = SQLITE_IOERR_FSTAT;
   802    834             }
   803    835             *pSize += sz;
   804    836           }
   805    837         }else{
   806    838           break;
   807    839         }
................................................................................
   812    844   }
   813    845   
   814    846   /* Pass xLock requests through to the original VFS unchanged.
   815    847   */
   816    848   static int multiplexLock(sqlite3_file *pConn, int lock){
   817    849     multiplexConn *p = (multiplexConn*)pConn;
   818    850     int rc;
   819         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          851  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   820    852     if( pSubOpen ){
   821    853       return pSubOpen->pMethods->xLock(pSubOpen, lock);
   822    854     }
   823    855     return SQLITE_BUSY;
   824    856   }
   825    857   
   826    858   /* Pass xUnlock requests through to the original VFS unchanged.
   827    859   */
   828    860   static int multiplexUnlock(sqlite3_file *pConn, int lock){
   829    861     multiplexConn *p = (multiplexConn*)pConn;
   830    862     int rc;
   831         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          863  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   832    864     if( pSubOpen ){
   833    865       return pSubOpen->pMethods->xUnlock(pSubOpen, lock);
   834    866     }
   835    867     return SQLITE_IOERR_UNLOCK;
   836    868   }
   837    869   
   838    870   /* Pass xCheckReservedLock requests through to the original VFS unchanged.
   839    871   */
   840    872   static int multiplexCheckReservedLock(sqlite3_file *pConn, int *pResOut){
   841    873     multiplexConn *p = (multiplexConn*)pConn;
   842    874     int rc;
   843         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          875  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   844    876     if( pSubOpen ){
   845    877       return pSubOpen->pMethods->xCheckReservedLock(pSubOpen, pResOut);
   846    878     }
   847    879     return SQLITE_IOERR_CHECKRESERVEDLOCK;
   848    880   }
   849    881   
   850    882   /* Pass xFileControl requests through to the original VFS unchanged,
................................................................................
   863    895           int bEnabled = *(int *)pArg;
   864    896           pGroup->bEnabled = bEnabled;
   865    897           rc = SQLITE_OK;
   866    898         }
   867    899         break;
   868    900       case MULTIPLEX_CTRL_SET_CHUNK_SIZE:
   869    901         if( pArg ) {
   870         -        int nChunkSize = *(int *)pArg;
   871         -        if( nChunkSize<1 ){
          902  +        unsigned int szChunk = *(unsigned*)pArg;
          903  +        if( szChunk<1 ){
   872    904             rc = SQLITE_MISUSE;
   873    905           }else{
   874    906             /* Round up to nearest multiple of MAX_PAGE_SIZE. */
   875         -          nChunkSize = (nChunkSize + (MAX_PAGE_SIZE-1));
   876         -          nChunkSize &= ~(MAX_PAGE_SIZE-1);
   877         -          pGroup->nChunkSize = nChunkSize;
          907  +          szChunk = (szChunk + (MAX_PAGE_SIZE-1));
          908  +          szChunk &= ~(MAX_PAGE_SIZE-1);
          909  +          pGroup->szChunk = szChunk;
   878    910             rc = SQLITE_OK;
   879    911           }
   880    912         }
   881    913         break;
   882    914       case MULTIPLEX_CTRL_SET_MAX_CHUNKS:
   883         -      if( pArg ) {
   884         -        int nMaxChunks = *(int *)pArg;
   885         -        if(( nMaxChunks<1 ) || ( nMaxChunks>SQLITE_MULTIPLEX_MAX_CHUNKS )){
   886         -          rc = SQLITE_MISUSE;
   887         -        }else{
   888         -          pGroup->nMaxChunks = nMaxChunks;
   889         -          rc = SQLITE_OK;
   890         -        }
   891         -      }
          915  +      rc = SQLITE_OK;
   892    916         break;
   893    917       case SQLITE_FCNTL_SIZE_HINT:
   894    918       case SQLITE_FCNTL_CHUNK_SIZE:
   895    919         /* no-op these */
   896    920         rc = SQLITE_OK;
   897    921         break;
   898    922       default:
   899         -      pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          923  +      pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
   900    924         if( pSubOpen ){
   901    925           rc = pSubOpen->pMethods->xFileControl(pSubOpen, op, pArg);
   902    926         }
   903    927         break;
   904    928     }
   905    929     return rc;
   906    930   }
   907    931   
   908    932   /* Pass xSectorSize requests through to the original VFS unchanged.
   909    933   */
   910    934   static int multiplexSectorSize(sqlite3_file *pConn){
   911    935     multiplexConn *p = (multiplexConn*)pConn;
   912    936     int rc;
   913         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          937  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   914    938     if( pSubOpen ){
   915    939       return pSubOpen->pMethods->xSectorSize(pSubOpen);
   916    940     }
   917    941     return DEFAULT_SECTOR_SIZE;
   918    942   }
   919    943   
   920    944   /* Pass xDeviceCharacteristics requests through to the original VFS unchanged.
   921    945   */
   922    946   static int multiplexDeviceCharacteristics(sqlite3_file *pConn){
   923    947     multiplexConn *p = (multiplexConn*)pConn;
   924    948     int rc;
   925         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          949  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   926    950     if( pSubOpen ){
   927    951       return pSubOpen->pMethods->xDeviceCharacteristics(pSubOpen);
   928    952     }
   929    953     return 0;
   930    954   }
   931    955   
   932    956   /* Pass xShmMap requests through to the original VFS unchanged.
................................................................................
   936    960     int iRegion,                    /* Region to retrieve */
   937    961     int szRegion,                   /* Size of regions */
   938    962     int bExtend,                    /* True to extend file if necessary */
   939    963     void volatile **pp              /* OUT: Mapped memory */
   940    964   ){
   941    965     multiplexConn *p = (multiplexConn*)pConn;
   942    966     int rc;
   943         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          967  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   944    968     if( pSubOpen ){
   945         -    return pSubOpen->pMethods->xShmMap(pSubOpen, iRegion, szRegion, bExtend, pp);
          969  +    return pSubOpen->pMethods->xShmMap(pSubOpen, iRegion, szRegion, bExtend,pp);
   946    970     }
   947    971     return SQLITE_IOERR;
   948    972   }
   949    973   
   950    974   /* Pass xShmLock requests through to the original VFS unchanged.
   951    975   */
   952    976   static int multiplexShmLock(
................................................................................
   953    977     sqlite3_file *pConn,       /* Database file holding the shared memory */
   954    978     int ofst,                  /* First lock to acquire or release */
   955    979     int n,                     /* Number of locks to acquire or release */
   956    980     int flags                  /* What to do with the lock */
   957    981   ){
   958    982     multiplexConn *p = (multiplexConn*)pConn;
   959    983     int rc;
   960         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          984  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   961    985     if( pSubOpen ){
   962    986       return pSubOpen->pMethods->xShmLock(pSubOpen, ofst, n, flags);
   963    987     }
   964    988     return SQLITE_BUSY;
   965    989   }
   966    990   
   967    991   /* Pass xShmBarrier requests through to the original VFS unchanged.
   968    992   */
   969    993   static void multiplexShmBarrier(sqlite3_file *pConn){
   970    994     multiplexConn *p = (multiplexConn*)pConn;
   971    995     int rc;
   972         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          996  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   973    997     if( pSubOpen ){
   974    998       pSubOpen->pMethods->xShmBarrier(pSubOpen);
   975    999     }
   976   1000   }
   977   1001   
   978   1002   /* Pass xShmUnmap requests through to the original VFS unchanged.
   979   1003   */
   980   1004   static int multiplexShmUnmap(sqlite3_file *pConn, int deleteFlag){
   981   1005     multiplexConn *p = (multiplexConn*)pConn;
   982   1006     int rc;
   983         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
         1007  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   984   1008     if( pSubOpen ){
   985   1009       return pSubOpen->pMethods->xShmUnmap(pSubOpen, deleteFlag);
   986   1010     }
   987   1011     return SQLITE_OK;
   988   1012   }
   989   1013   
   990   1014   /************************** Public Interfaces *****************************/
................................................................................
  1006   1030     pOrigVfs = sqlite3_vfs_find(zOrigVfsName);
  1007   1031     if( pOrigVfs==0 ) return SQLITE_ERROR;
  1008   1032     assert( pOrigVfs!=&gMultiplex.sThisVfs );
  1009   1033     gMultiplex.pMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST);
  1010   1034     if( !gMultiplex.pMutex ){
  1011   1035       return SQLITE_NOMEM;
  1012   1036     }
  1013         -  gMultiplex.zName = sqlite3_malloc(pOrigVfs->mxPathname);
  1014         -  if( !gMultiplex.zName ){
  1015         -    sqlite3_mutex_free(gMultiplex.pMutex);
  1016         -    return SQLITE_NOMEM;
  1017         -  }
  1018   1037     gMultiplex.pGroups = NULL;
  1019   1038     gMultiplex.isInitialized = 1;
  1020   1039     gMultiplex.pOrigVfs = pOrigVfs;
  1021   1040     gMultiplex.sThisVfs = *pOrigVfs;
  1022   1041     gMultiplex.sThisVfs.szOsFile += sizeof(multiplexConn);
  1023   1042     gMultiplex.sThisVfs.zName = SQLITE_MULTIPLEX_VFS_NAME;
  1024   1043     gMultiplex.sThisVfs.xOpen = multiplexOpen;
................................................................................
  1043   1062     gMultiplex.sIoMethodsV1.xSync = multiplexSync;
  1044   1063     gMultiplex.sIoMethodsV1.xFileSize = multiplexFileSize;
  1045   1064     gMultiplex.sIoMethodsV1.xLock = multiplexLock;
  1046   1065     gMultiplex.sIoMethodsV1.xUnlock = multiplexUnlock;
  1047   1066     gMultiplex.sIoMethodsV1.xCheckReservedLock = multiplexCheckReservedLock;
  1048   1067     gMultiplex.sIoMethodsV1.xFileControl = multiplexFileControl;
  1049   1068     gMultiplex.sIoMethodsV1.xSectorSize = multiplexSectorSize;
  1050         -  gMultiplex.sIoMethodsV1.xDeviceCharacteristics = multiplexDeviceCharacteristics;
         1069  +  gMultiplex.sIoMethodsV1.xDeviceCharacteristics =
         1070  +                                            multiplexDeviceCharacteristics;
  1051   1071     gMultiplex.sIoMethodsV2 = gMultiplex.sIoMethodsV1;
  1052   1072     gMultiplex.sIoMethodsV2.iVersion = 2;
  1053   1073     gMultiplex.sIoMethodsV2.xShmMap = multiplexShmMap;
  1054   1074     gMultiplex.sIoMethodsV2.xShmLock = multiplexShmLock;
  1055   1075     gMultiplex.sIoMethodsV2.xShmBarrier = multiplexShmBarrier;
  1056   1076     gMultiplex.sIoMethodsV2.xShmUnmap = multiplexShmUnmap;
  1057   1077     sqlite3_vfs_register(&gMultiplex.sThisVfs, makeDefault);
................................................................................
  1070   1090   ** THIS ROUTINE IS NOT THREADSAFE.  Call this routine exactly once while
  1071   1091   ** shutting down in order to free all remaining multiplex groups.
  1072   1092   */
  1073   1093   int sqlite3_multiplex_shutdown(void){
  1074   1094     if( gMultiplex.isInitialized==0 ) return SQLITE_MISUSE;
  1075   1095     if( gMultiplex.pGroups ) return SQLITE_MISUSE;
  1076   1096     gMultiplex.isInitialized = 0;
  1077         -  sqlite3_free(gMultiplex.zName);
  1078   1097     sqlite3_mutex_free(gMultiplex.pMutex);
  1079   1098     sqlite3_vfs_unregister(&gMultiplex.sThisVfs);
  1080   1099     memset(&gMultiplex, 0, sizeof(gMultiplex));
  1081   1100     return SQLITE_OK;
  1082   1101   }
  1083   1102   
  1084   1103   /***************************** Test Code ***********************************/
................................................................................
  1172   1191             Tcl_NewStringObj(pGroup->zName, -1));
  1173   1192       Tcl_ListObjAppendElement(interp, pGroupTerm,
  1174   1193             Tcl_NewIntObj(pGroup->nName));
  1175   1194       Tcl_ListObjAppendElement(interp, pGroupTerm,
  1176   1195             Tcl_NewIntObj(pGroup->flags));
  1177   1196   
  1178   1197       /* count number of chunks with open handles */
  1179         -    for(i=0; i<pGroup->nMaxChunks; i++){
  1180         -      if( pGroup->bOpen[i] ) nChunks++;
         1198  +    for(i=0; i<pGroup->nReal; i++){
         1199  +      if( pGroup->aReal[i].p!=0 ) nChunks++;
  1181   1200       }
  1182   1201       Tcl_ListObjAppendElement(interp, pGroupTerm,
  1183   1202             Tcl_NewIntObj(nChunks));
  1184   1203   
  1185   1204       Tcl_ListObjAppendElement(interp, pGroupTerm,
  1186         -          Tcl_NewIntObj(pGroup->nChunkSize));
         1205  +          Tcl_NewIntObj(pGroup->szChunk));
  1187   1206       Tcl_ListObjAppendElement(interp, pGroupTerm,
  1188         -          Tcl_NewIntObj(pGroup->nMaxChunks));
         1207  +          Tcl_NewIntObj(pGroup->nReal));
  1189   1208   
  1190   1209       Tcl_ListObjAppendElement(interp, pResult, pGroupTerm);
  1191   1210     }
  1192   1211     multiplexLeave();
  1193   1212     Tcl_SetObjResult(interp, pResult);
  1194   1213     return TCL_OK;
  1195   1214   }

Changes to src/trigger.c.

   113    113     }else{
   114    114       /* Figure out the db that the the trigger will be created in */
   115    115       iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName);
   116    116       if( iDb<0 ){
   117    117         goto trigger_cleanup;
   118    118       }
   119    119     }
          120  +  if( !pTableName || db->mallocFailed ){
          121  +    goto trigger_cleanup;
          122  +  }
          123  +
          124  +  /* A long-standing parser bug is that this syntax was allowed:
          125  +  **
          126  +  **    CREATE TRIGGER attached.demo AFTER INSERT ON attached.tab ....
          127  +  **                                                 ^^^^^^^^
          128  +  **
          129  +  ** To maintain backwards compatibility, ignore the database
          130  +  ** name on pTableName if we are reparsing our of SQLITE_MASTER.
          131  +  */
          132  +  if( db->init.busy && iDb!=1 ){
          133  +    sqlite3DbFree(db, pTableName->a[0].zDatabase);
          134  +    pTableName->a[0].zDatabase = 0;
          135  +  }
   120    136   
   121    137     /* If the trigger name was unqualified, and the table is a temp table,
   122    138     ** then set iDb to 1 to create the trigger in the temporary database.
   123    139     ** If sqlite3SrcListLookup() returns 0, indicating the table does not
   124    140     ** exist, the error is caught by the block below.
   125    141     */
   126         -  if( !pTableName || db->mallocFailed ){
   127         -    goto trigger_cleanup;
   128         -  }
   129    142     pTab = sqlite3SrcListLookup(pParse, pTableName);
   130    143     if( db->init.busy==0 && pName2->n==0 && pTab
   131    144           && pTab->pSchema==db->aDb[1].pSchema ){
   132    145       iDb = 1;
   133    146     }
   134    147   
   135    148     /* Ensure the table name matches database name and that the table exists */

Changes to src/update.c.

   307    307     if( sqlite3ResolveExprNames(&sNC, pWhere) ){
   308    308       goto update_cleanup;
   309    309     }
   310    310   
   311    311     /* Begin the database scan
   312    312     */
   313    313     sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid);
   314         -  pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0, WHERE_ONEPASS_DESIRED);
          314  +  pWInfo = sqlite3WhereBegin(
          315  +      pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED
          316  +  );
   315    317     if( pWInfo==0 ) goto update_cleanup;
   316    318     okOnePass = pWInfo->okOnePass;
   317    319   
   318    320     /* Remember the rowid of every item to be updated.
   319    321     */
   320    322     sqlite3VdbeAddOp2(v, OP_Rowid, iCur, regOldRowid);
   321    323     if( !okOnePass ){

Changes to src/util.c.

  1145   1145     if( x>=0 ) return x;
  1146   1146     if( x==(int)0x80000000 ) return 0x7fffffff;
  1147   1147     return -x;
  1148   1148   }
  1149   1149   
  1150   1150   #ifdef SQLITE_ENABLE_8_3_NAMES
  1151   1151   /*
  1152         -** If SQLITE_ENABLE_8_3_NAME is set at compile-time and if the database
         1152  +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database
  1153   1153   ** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and
  1154   1154   ** if filename in z[] has a suffix (a.k.a. "extension") that is longer than
  1155   1155   ** three characters, then shorten the suffix on z[] to be the last three
  1156   1156   ** characters of the original suffix.
         1157  +**
         1158  +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always
         1159  +** do the suffix shortening regardless of URI parameter.
  1157   1160   **
  1158   1161   ** Examples:
  1159   1162   **
  1160   1163   **     test.db-journal    =>   test.nal
  1161   1164   **     test.db-wal        =>   test.wal
  1162   1165   **     test.db-shm        =>   test.shm
  1163   1166   */
  1164   1167   void sqlite3FileSuffix3(const char *zBaseFilename, char *z){
         1168  +#if SQLITE_ENABLE_8_3_NAMES<2
  1165   1169     const char *zOk;
  1166   1170     zOk = sqlite3_uri_parameter(zBaseFilename, "8_3_names");
  1167         -  if( zOk && sqlite3GetBoolean(zOk) ){
         1171  +  if( zOk && sqlite3GetBoolean(zOk) )
         1172  +#endif
         1173  +  {
  1168   1174       int i, sz;
  1169   1175       sz = sqlite3Strlen30(z);
  1170   1176       for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){}
  1171   1177       if( z[i]=='.' && ALWAYS(sz>i+4) ) memcpy(&z[i+1], &z[sz-3], 4);
  1172   1178     }
  1173   1179   }
  1174   1180   #endif

Changes to src/vdbe.c.

  3168   3168       ** sqlite3BtreeCreateTable() with the BTREE_BLOBKEY flag before
  3169   3169       ** opening it. If a transient table is required, just use the
  3170   3170       ** automatically created table with root-page 1 (an BLOB_INTKEY table).
  3171   3171       */
  3172   3172       if( pOp->p4.pKeyInfo ){
  3173   3173         int pgno;
  3174   3174         assert( pOp->p4type==P4_KEYINFO );
  3175         -      rc = sqlite3BtreeCreateTable(pCx->pBt, &pgno, BTREE_BLOBKEY); 
         3175  +      rc = sqlite3BtreeCreateTable(pCx->pBt, &pgno, BTREE_BLOBKEY | pOp->p5); 
  3176   3176         if( rc==SQLITE_OK ){
  3177   3177           assert( pgno==MASTER_ROOT+1 );
  3178   3178           rc = sqlite3BtreeCursor(pCx->pBt, pgno, 1, 
  3179   3179                                   (KeyInfo*)pOp->p4.z, pCx->pCursor);
  3180   3180           pCx->pKeyInfo = pOp->p4.pKeyInfo;
  3181   3181           pCx->pKeyInfo->enc = ENC(p->db);
  3182   3182         }

Changes to src/vdbeapi.c.

   501    501     }
   502    502     db = v->db;
   503    503     sqlite3_mutex_enter(db->mutex);
   504    504     while( (rc = sqlite3Step(v))==SQLITE_SCHEMA
   505    505            && cnt++ < SQLITE_MAX_SCHEMA_RETRY
   506    506            && (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){
   507    507       sqlite3_reset(pStmt);
   508         -    v->expired = 0;
          508  +    assert( v->expired==0 );
   509    509     }
   510    510     if( rc2!=SQLITE_OK && ALWAYS(v->isPrepareV2) && ALWAYS(db->pErr) ){
   511    511       /* This case occurs after failing to recompile an sql statement. 
   512    512       ** The error message from the SQL compiler has already been loaded 
   513    513       ** into the database handle. This block copies the error message 
   514    514       ** from the database handle into the statement and sets the statement
   515    515       ** program counter to 0 to ensure that when the statement is 

Changes to src/vdbeaux.c.

  1501   1501     p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort);
  1502   1502     if( pParse->explain && nMem<10 ){
  1503   1503       nMem = 10;
  1504   1504     }
  1505   1505     memset(zCsr, 0, zEnd-zCsr);
  1506   1506     zCsr += (zCsr - (u8*)0)&7;
  1507   1507     assert( EIGHT_BYTE_ALIGNMENT(zCsr) );
         1508  +  p->expired = 0;
  1508   1509   
  1509   1510     /* Memory for registers, parameters, cursor, etc, is allocated in two
  1510   1511     ** passes.  On the first pass, we try to reuse unused space at the 
  1511   1512     ** end of the opcode array.  If we are unable to satisfy all memory
  1512   1513     ** requirements by reusing the opcode array tail, then the second
  1513   1514     ** pass will fill in the rest using a fresh allocation.  
  1514   1515     **

Changes to src/wal.c.

  1801   1801       ** the database. In this case checkpoint the database and unlink both
  1802   1802       ** the wal and wal-index files.
  1803   1803       **
  1804   1804       ** The EXCLUSIVE lock is not released before returning.
  1805   1805       */
  1806   1806       rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE);
  1807   1807       if( rc==SQLITE_OK ){
         1808  +      int bPersistWal = -1;
  1808   1809         if( pWal->exclusiveMode==WAL_NORMAL_MODE ){
  1809   1810           pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
  1810   1811         }
  1811   1812         rc = sqlite3WalCheckpoint(
  1812   1813             pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0
  1813   1814         );
  1814         -      if( rc==SQLITE_OK ){
         1815  +      sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersistWal);
         1816  +      if( rc==SQLITE_OK && bPersistWal!=1 ){
  1815   1817           isDelete = 1;
  1816   1818         }
  1817   1819       }
  1818   1820   
  1819   1821       walIndexClose(pWal, isDelete);
  1820   1822       sqlite3OsClose(pWal->pWalFd);
  1821   1823       if( isDelete ){

Changes to src/where.c.

   249    249   #define WHERE_IDX_ONLY     0x00800000  /* Use index only - omit table */
   250    250   #define WHERE_ORDERBY      0x01000000  /* Output will appear in correct order */
   251    251   #define WHERE_REVERSE      0x02000000  /* Scan in reverse order */
   252    252   #define WHERE_UNIQUE       0x04000000  /* Selects no more than one row */
   253    253   #define WHERE_VIRTUALTABLE 0x08000000  /* Use virtual-table processing */
   254    254   #define WHERE_MULTI_OR     0x10000000  /* OR using multiple indices */
   255    255   #define WHERE_TEMP_INDEX   0x20000000  /* Uses an ephemeral index */
          256  +#define WHERE_DISTINCT     0x40000000  /* Correct order for DISTINCT */
   256    257   
   257    258   /*
   258    259   ** Initialize a preallocated WhereClause structure.
   259    260   */
   260    261   static void whereClauseInit(
   261    262     WhereClause *pWC,        /* The WhereClause to be initialized */
   262    263     Parse *pParse,           /* The parsing context */
................................................................................
  1393   1394       if( (exprTableUsage(pMaskSet, pList->a[iFirst++].pExpr)&allowed)!=0 ){
  1394   1395         return 1;
  1395   1396       }
  1396   1397     }
  1397   1398     return 0;
  1398   1399   }
  1399   1400   
         1401  +/*
         1402  +** This function searches the expression list passed as the second argument
         1403  +** for an expression of type TK_COLUMN that refers to the same column and
         1404  +** uses the same collation sequence as the iCol'th column of index pIdx.
         1405  +** Argument iBase is the cursor number used for the table that pIdx refers
         1406  +** to.
         1407  +**
         1408  +** If such an expression is found, its index in pList->a[] is returned. If
         1409  +** no expression is found, -1 is returned.
         1410  +*/
         1411  +static int findIndexCol(
         1412  +  Parse *pParse,                  /* Parse context */
         1413  +  ExprList *pList,                /* Expression list to search */
         1414  +  int iBase,                      /* Cursor for table associated with pIdx */
         1415  +  Index *pIdx,                    /* Index to match column of */
         1416  +  int iCol                        /* Column of index to match */
         1417  +){
         1418  +  int i;
         1419  +  const char *zColl = pIdx->azColl[iCol];
         1420  +
         1421  +  for(i=0; i<pList->nExpr; i++){
         1422  +    Expr *p = pList->a[i].pExpr;
         1423  +    if( p->op==TK_COLUMN
         1424  +     && p->iColumn==pIdx->aiColumn[iCol]
         1425  +     && p->iTable==iBase
         1426  +    ){
         1427  +      CollSeq *pColl = sqlite3ExprCollSeq(pParse, p);
         1428  +      if( ALWAYS(pColl) && 0==sqlite3StrICmp(pColl->zName, zColl) ){
         1429  +        return i;
         1430  +      }
         1431  +    }
         1432  +  }
         1433  +
         1434  +  return -1;
         1435  +}
         1436  +
         1437  +/*
         1438  +** This routine determines if pIdx can be used to assist in processing a
         1439  +** DISTINCT qualifier. In other words, it tests whether or not using this
         1440  +** index for the outer loop guarantees that rows with equal values for
         1441  +** all expressions in the pDistinct list are delivered grouped together.
         1442  +**
         1443  +** For example, the query 
         1444  +**
         1445  +**   SELECT DISTINCT a, b, c FROM tbl WHERE a = ?
         1446  +**
         1447  +** can benefit from any index on columns "b" and "c".
         1448  +*/
         1449  +static int isDistinctIndex(
         1450  +  Parse *pParse,                  /* Parsing context */
         1451  +  WhereClause *pWC,               /* The WHERE clause */
         1452  +  Index *pIdx,                    /* The index being considered */
         1453  +  int base,                       /* Cursor number for the table pIdx is on */
         1454  +  ExprList *pDistinct,            /* The DISTINCT expressions */
         1455  +  int nEqCol                      /* Number of index columns with == */
         1456  +){
         1457  +  Bitmask mask = 0;               /* Mask of unaccounted for pDistinct exprs */
         1458  +  int i;                          /* Iterator variable */
         1459  +
         1460  +  if( pIdx->zName==0 || pDistinct==0 || pDistinct->nExpr>=BMS ) return 0;
         1461  +  testcase( pDistinct->nExpr==BMS-1 );
         1462  +
         1463  +  /* Loop through all the expressions in the distinct list. If any of them
         1464  +  ** are not simple column references, return early. Otherwise, test if the
         1465  +  ** WHERE clause contains a "col=X" clause. If it does, the expression
         1466  +  ** can be ignored. If it does not, and the column does not belong to the
         1467  +  ** same table as index pIdx, return early. Finally, if there is no
         1468  +  ** matching "col=X" expression and the column is on the same table as pIdx,
         1469  +  ** set the corresponding bit in variable mask.
         1470  +  */
         1471  +  for(i=0; i<pDistinct->nExpr; i++){
         1472  +    WhereTerm *pTerm;
         1473  +    Expr *p = pDistinct->a[i].pExpr;
         1474  +    if( p->op!=TK_COLUMN ) return 0;
         1475  +    pTerm = findTerm(pWC, p->iTable, p->iColumn, ~(Bitmask)0, WO_EQ, 0);
         1476  +    if( pTerm ){
         1477  +      Expr *pX = pTerm->pExpr;
         1478  +      CollSeq *p1 = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
         1479  +      CollSeq *p2 = sqlite3ExprCollSeq(pParse, p);
         1480  +      if( p1==p2 ) continue;
         1481  +    }
         1482  +    if( p->iTable!=base ) return 0;
         1483  +    mask |= (((Bitmask)1) << i);
         1484  +  }
         1485  +
         1486  +  for(i=nEqCol; mask && i<pIdx->nColumn; i++){
         1487  +    int iExpr = findIndexCol(pParse, pDistinct, base, pIdx, i);
         1488  +    if( iExpr<0 ) break;
         1489  +    mask &= ~(((Bitmask)1) << iExpr);
         1490  +  }
         1491  +
         1492  +  return (mask==0);
         1493  +}
         1494  +
         1495  +
         1496  +/*
         1497  +** Return true if the DISTINCT expression-list passed as the third argument
         1498  +** is redundant. A DISTINCT list is redundant if the database contains a
         1499  +** UNIQUE index that guarantees that the result of the query will be distinct
         1500  +** anyway.
         1501  +*/
         1502  +static int isDistinctRedundant(
         1503  +  Parse *pParse,
         1504  +  SrcList *pTabList,
         1505  +  WhereClause *pWC,
         1506  +  ExprList *pDistinct
         1507  +){
         1508  +  Table *pTab;
         1509  +  Index *pIdx;
         1510  +  int i;                          
         1511  +  int iBase;
         1512  +
         1513  +  /* If there is more than one table or sub-select in the FROM clause of
         1514  +  ** this query, then it will not be possible to show that the DISTINCT 
         1515  +  ** clause is redundant. */
         1516  +  if( pTabList->nSrc!=1 ) return 0;
         1517  +  iBase = pTabList->a[0].iCursor;
         1518  +  pTab = pTabList->a[0].pTab;
         1519  +
         1520  +  /* If any of the expressions is an IPK column on table iBase, then return 
         1521  +  ** true. Note: The (p->iTable==iBase) part of this test may be false if the
         1522  +  ** current SELECT is a correlated sub-query.
         1523  +  */
         1524  +  for(i=0; i<pDistinct->nExpr; i++){
         1525  +    Expr *p = pDistinct->a[i].pExpr;
         1526  +    if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1;
         1527  +  }
         1528  +
         1529  +  /* Loop through all indices on the table, checking each to see if it makes
         1530  +  ** the DISTINCT qualifier redundant. It does so if:
         1531  +  **
         1532  +  **   1. The index is itself UNIQUE, and
         1533  +  **
         1534  +  **   2. All of the columns in the index are either part of the pDistinct
         1535  +  **      list, or else the WHERE clause contains a term of the form "col=X",
         1536  +  **      where X is a constant value. The collation sequences of the
         1537  +  **      comparison and select-list expressions must match those of the index.
         1538  +  */
         1539  +  for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
         1540  +    if( pIdx->onError==OE_None ) continue;
         1541  +    for(i=0; i<pIdx->nColumn; i++){
         1542  +      int iCol = pIdx->aiColumn[i];
         1543  +      if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) 
         1544  +       && 0>findIndexCol(pParse, pDistinct, iBase, pIdx, i)
         1545  +      ){
         1546  +        break;
         1547  +      }
         1548  +    }
         1549  +    if( i==pIdx->nColumn ){
         1550  +      /* This index implies that the DISTINCT qualifier is redundant. */
         1551  +      return 1;
         1552  +    }
         1553  +  }
         1554  +
         1555  +  return 0;
         1556  +}
  1400   1557   
  1401   1558   /*
  1402   1559   ** This routine decides if pIdx can be used to satisfy the ORDER BY
  1403   1560   ** clause.  If it can, it returns 1.  If pIdx cannot satisfy the
  1404   1561   ** ORDER BY clause, this routine returns 0.
  1405   1562   **
  1406   1563   ** pOrderBy is an ORDER BY clause from a SELECT statement.  pTab is the
................................................................................
  1429   1586   ){
  1430   1587     int i, j;                       /* Loop counters */
  1431   1588     int sortOrder = 0;              /* XOR of index and ORDER BY sort direction */
  1432   1589     int nTerm;                      /* Number of ORDER BY terms */
  1433   1590     struct ExprList_item *pTerm;    /* A term of the ORDER BY clause */
  1434   1591     sqlite3 *db = pParse->db;
  1435   1592   
  1436         -  assert( pOrderBy!=0 );
         1593  +  if( !pOrderBy ) return 0;
         1594  +  if( wsFlags & WHERE_COLUMN_IN ) return 0;
         1595  +  if( pIdx->bUnordered ) return 0;
         1596  +
  1437   1597     nTerm = pOrderBy->nExpr;
  1438   1598     assert( nTerm>0 );
  1439   1599   
  1440   1600     /* Argument pIdx must either point to a 'real' named index structure, 
  1441   1601     ** or an index structure allocated on the stack by bestBtreeIndex() to
  1442   1602     ** represent the rowid index that is part of every table.  */
  1443   1603     assert( pIdx->zName || (pIdx->nColumn==1 && pIdx->aiColumn[0]==-1) );
................................................................................
  1742   1902     double nTableRow;           /* Rows in the input table */
  1743   1903     double logN;                /* log(nTableRow) */
  1744   1904     double costTempIdx;         /* per-query cost of the transient index */
  1745   1905     WhereTerm *pTerm;           /* A single term of the WHERE clause */
  1746   1906     WhereTerm *pWCEnd;          /* End of pWC->a[] */
  1747   1907     Table *pTable;              /* Table tht might be indexed */
  1748   1908   
         1909  +  if( pParse->nQueryLoop<=(double)1 ){
         1910  +    /* There is no point in building an automatic index for a single scan */
         1911  +    return;
         1912  +  }
  1749   1913     if( (pParse->db->flags & SQLITE_AutoIndex)==0 ){
  1750   1914       /* Automatic indices are disabled at run-time */
  1751   1915       return;
  1752   1916     }
  1753   1917     if( (pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 ){
  1754   1918       /* We already have some kind of index in use for this query. */
  1755   1919       return;
  1756   1920     }
  1757   1921     if( pSrc->notIndexed ){
  1758   1922       /* The NOT INDEXED clause appears in the SQL. */
  1759   1923       return;
         1924  +  }
         1925  +  if( pSrc->isCorrelated ){
         1926  +    /* The source is a correlated sub-query. No point in indexing it. */
         1927  +    return;
  1760   1928     }
  1761   1929   
  1762   1930     assert( pParse->nQueryLoop >= (double)1 );
  1763   1931     pTable = pSrc->pTab;
  1764   1932     nTableRow = pTable->nRowEst;
  1765   1933     logN = estLog(nTableRow);
  1766   1934     costTempIdx = 2*logN*(nTableRow/pParse->nQueryLoop + 1);
................................................................................
  1970   2138     ** to this virtual table */
  1971   2139     for(i=nTerm=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
  1972   2140       if( pTerm->leftCursor != pSrc->iCursor ) continue;
  1973   2141       assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 );
  1974   2142       testcase( pTerm->eOperator==WO_IN );
  1975   2143       testcase( pTerm->eOperator==WO_ISNULL );
  1976   2144       if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue;
         2145  +    if( pTerm->wtFlags & TERM_VNULL ) continue;
  1977   2146       nTerm++;
  1978   2147     }
  1979   2148   
  1980   2149     /* If the ORDER BY clause contains only columns in the current 
  1981   2150     ** virtual table then allocate space for the aOrderBy part of
  1982   2151     ** the sqlite3_index_info structure.
  1983   2152     */
................................................................................
  2020   2189   
  2021   2190     for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
  2022   2191       if( pTerm->leftCursor != pSrc->iCursor ) continue;
  2023   2192       assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 );
  2024   2193       testcase( pTerm->eOperator==WO_IN );
  2025   2194       testcase( pTerm->eOperator==WO_ISNULL );
  2026   2195       if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue;
         2196  +    if( pTerm->wtFlags & TERM_VNULL ) continue;
  2027   2197       pIdxCons[j].iColumn = pTerm->u.leftColumn;
  2028   2198       pIdxCons[j].iTermOffset = i;
  2029   2199       pIdxCons[j].op = (u8)pTerm->eOperator;
  2030   2200       /* The direct assignment in the previous line is possible only because
  2031   2201       ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical.  The
  2032   2202       ** following asserts verify this fact. */
  2033   2203       assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ );
................................................................................
  2685   2855   static void bestBtreeIndex(
  2686   2856     Parse *pParse,              /* The parsing context */
  2687   2857     WhereClause *pWC,           /* The WHERE clause */
  2688   2858     struct SrcList_item *pSrc,  /* The FROM clause term to search */
  2689   2859     Bitmask notReady,           /* Mask of cursors not available for indexing */
  2690   2860     Bitmask notValid,           /* Cursors not available for any purpose */
  2691   2861     ExprList *pOrderBy,         /* The ORDER BY clause */
         2862  +  ExprList *pDistinct,        /* The select-list if query is DISTINCT */
  2692   2863     WhereCost *pCost            /* Lowest cost query plan */
  2693   2864   ){
  2694   2865     int iCur = pSrc->iCursor;   /* The cursor of the table to be accessed */
  2695   2866     Index *pProbe;              /* An index we are evaluating */
  2696   2867     Index *pIdx;                /* Copy of pProbe, or zero for IPK index */
  2697   2868     int eqTermMask;             /* Current mask of valid equality operators */
  2698   2869     int idxEqTermMask;          /* Index mask of valid equality operators */
................................................................................
  2825   2996       **             SELECT a, b, c FROM tbl WHERE a = 1;
  2826   2997       */
  2827   2998       int nEq;                      /* Number of == or IN terms matching index */
  2828   2999       int bInEst = 0;               /* True if "x IN (SELECT...)" seen */
  2829   3000       int nInMul = 1;               /* Number of distinct equalities to lookup */
  2830   3001       int estBound = 100;           /* Estimated reduction in search space */
  2831   3002       int nBound = 0;               /* Number of range constraints seen */
  2832         -    int bSort = 0;                /* True if external sort required */
         3003  +    int bSort = !!pOrderBy;       /* True if external sort required */
         3004  +    int bDist = !!pDistinct;      /* True if index cannot help with DISTINCT */
  2833   3005       int bLookup = 0;              /* True if not a covering index */
  2834   3006       WhereTerm *pTerm;             /* A single term of the WHERE clause */
  2835   3007   #ifdef SQLITE_ENABLE_STAT2
  2836   3008       WhereTerm *pFirstTerm = 0;    /* First term matching the index */
  2837   3009   #endif
  2838   3010   
  2839   3011       /* Determine the values of nEq and nInMul */
................................................................................
  2889   3061         }
  2890   3062       }
  2891   3063   
  2892   3064       /* If there is an ORDER BY clause and the index being considered will
  2893   3065       ** naturally scan rows in the required order, set the appropriate flags
  2894   3066       ** in wsFlags. Otherwise, if there is an ORDER BY clause but the index
  2895   3067       ** will scan rows in a different order, set the bSort variable.  */
  2896         -    if( pOrderBy ){
  2897         -      if( (wsFlags & WHERE_COLUMN_IN)==0
  2898         -        && pProbe->bUnordered==0
  2899         -        && isSortingIndex(pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy,
  2900         -                          nEq, wsFlags, &rev)
  2901         -      ){
  2902         -        wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY;
  2903         -        wsFlags |= (rev ? WHERE_REVERSE : 0);
  2904         -      }else{
  2905         -        bSort = 1;
  2906         -      }
         3068  +    if( isSortingIndex(
         3069  +          pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, nEq, wsFlags, &rev)
         3070  +    ){
         3071  +      bSort = 0;
         3072  +      wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY;
         3073  +      wsFlags |= (rev ? WHERE_REVERSE : 0);
         3074  +    }
         3075  +
         3076  +    /* If there is a DISTINCT qualifier and this index will scan rows in
         3077  +    ** order of the DISTINCT expressions, clear bDist and set the appropriate
         3078  +    ** flags in wsFlags. */
         3079  +    if( isDistinctIndex(pParse, pWC, pProbe, iCur, pDistinct, nEq) ){
         3080  +      bDist = 0;
         3081  +      wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT;
  2907   3082       }
  2908   3083   
  2909   3084       /* If currently calculating the cost of using an index (not the IPK
  2910   3085       ** index), determine if all required column data may be obtained without 
  2911   3086       ** using the main table (i.e. if the index is a covering
  2912   3087       ** index for this query). If it is, set the WHERE_IDX_ONLY flag in
  2913   3088       ** wsFlags. Otherwise, set the bLookup variable to true.  */
................................................................................
  2934   3109       nRow = (double)(aiRowEst[nEq] * nInMul);
  2935   3110       if( bInEst && nRow*2>aiRowEst[0] ){
  2936   3111         nRow = aiRowEst[0]/2;
  2937   3112         nInMul = (int)(nRow / aiRowEst[nEq]);
  2938   3113       }
  2939   3114   
  2940   3115   #ifdef SQLITE_ENABLE_STAT2
  2941         -    /* If the constraint is of the form x=VALUE and histogram
         3116  +    /* If the constraint is of the form x=VALUE or x IN (E1,E2,...)
         3117  +    ** and we do not think that values of x are unique and if histogram
  2942   3118       ** data is available for column x, then it might be possible
  2943   3119       ** to get a better estimate on the number of rows based on
  2944   3120       ** VALUE and how common that value is according to the histogram.
  2945   3121       */
  2946         -    if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 ){
         3122  +    if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 && aiRowEst[1]>1 ){
  2947   3123         if( pFirstTerm->eOperator & (WO_EQ|WO_ISNULL) ){
  2948   3124           testcase( pFirstTerm->eOperator==WO_EQ );
  2949   3125           testcase( pFirstTerm->eOperator==WO_ISNULL );
  2950   3126           whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, &nRow);
  2951   3127         }else if( pFirstTerm->eOperator==WO_IN && bInEst==0 ){
  2952   3128           whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, &nRow);
  2953   3129         }
................................................................................
  3016   3192       ** adds C*N*log10(N) to the cost, where N is the number of rows to be 
  3017   3193       ** sorted and C is a factor between 1.95 and 4.3.  We will split the
  3018   3194       ** difference and select C of 3.0.
  3019   3195       */
  3020   3196       if( bSort ){
  3021   3197         cost += nRow*estLog(nRow)*3;
  3022   3198       }
         3199  +    if( bDist ){
         3200  +      cost += nRow*estLog(nRow)*3;
         3201  +    }
  3023   3202   
  3024   3203       /**** Cost of using this index has now been computed ****/
  3025   3204   
  3026   3205       /* If there are additional constraints on this table that cannot
  3027   3206       ** be used with the current index, but which might lower the number
  3028   3207       ** of output rows, adjust the nRow value accordingly.  This only 
  3029   3208       ** matters if the current index is the least costly, so do not bother
................................................................................
  3161   3340       if( p->needToFreeIdxStr ){
  3162   3341         sqlite3_free(p->idxStr);
  3163   3342       }
  3164   3343       sqlite3DbFree(pParse->db, p);
  3165   3344     }else
  3166   3345   #endif
  3167   3346     {
  3168         -    bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost);
         3347  +    bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, 0, pCost);
  3169   3348     }
  3170   3349   }
  3171   3350   
  3172   3351   /*
  3173   3352   ** Disable a term in the WHERE clause.  Except, do not disable the term
  3174   3353   ** if it controls a LEFT OUTER JOIN and it did not originate in the ON
  3175   3354   ** or USING clause of that join.
................................................................................
  4123   4302       iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn);
  4124   4303   
  4125   4304       for(ii=0; ii<pOrWc->nTerm; ii++){
  4126   4305         WhereTerm *pOrTerm = &pOrWc->a[ii];
  4127   4306         if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){
  4128   4307           WhereInfo *pSubWInfo;          /* Info for single OR-term scan */
  4129   4308           /* Loop through table entries that match term pOrTerm. */
  4130         -        pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0,
         4309  +        pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, 0,
  4131   4310                           WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE |
  4132   4311                           WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY);
  4133   4312           if( pSubWInfo ){
  4134   4313             explainOneScan(
  4135   4314                 pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0
  4136   4315             );
  4137   4316             if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){
................................................................................
  4364   4543   ** output order, then the *ppOrderBy is unchanged.
  4365   4544   */
  4366   4545   WhereInfo *sqlite3WhereBegin(
  4367   4546     Parse *pParse,        /* The parser context */
  4368   4547     SrcList *pTabList,    /* A list of all tables to be scanned */
  4369   4548     Expr *pWhere,         /* The WHERE clause */
  4370   4549     ExprList **ppOrderBy, /* An ORDER BY clause, or NULL */
         4550  +  ExprList *pDistinct,  /* The select-list for DISTINCT queries - or NULL */
  4371   4551     u16 wctrlFlags        /* One of the WHERE_* flags defined in sqliteInt.h */
  4372   4552   ){
  4373   4553     int i;                     /* Loop counter */
  4374   4554     int nByteWInfo;            /* Num. bytes allocated for WhereInfo struct */
  4375   4555     int nTabList;              /* Number of elements in pTabList */
  4376   4556     WhereInfo *pWInfo;         /* Will become the return value of this function */
  4377   4557     Vdbe *v = pParse->pVdbe;   /* The virtual database engine */
................................................................................
  4423   4603     pWInfo->pParse = pParse;
  4424   4604     pWInfo->pTabList = pTabList;
  4425   4605     pWInfo->iBreak = sqlite3VdbeMakeLabel(v);
  4426   4606     pWInfo->pWC = pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo];
  4427   4607     pWInfo->wctrlFlags = wctrlFlags;
  4428   4608     pWInfo->savedNQueryLoop = pParse->nQueryLoop;
  4429   4609     pMaskSet = (WhereMaskSet*)&pWC[1];
         4610  +
         4611  +  /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via
         4612  +  ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */
         4613  +  if( db->flags & SQLITE_DistinctOpt ) pDistinct = 0;
  4430   4614   
  4431   4615     /* Split the WHERE clause into separate subexpressions where each
  4432   4616     ** subexpression is separated by an AND operator.
  4433   4617     */
  4434   4618     initMaskSet(pMaskSet);
  4435   4619     whereClauseInit(pWC, pParse, pMaskSet);
  4436   4620     sqlite3ExprCodeConstants(pParse, pWhere);
................................................................................
  4490   4674     ** want to analyze these virtual terms, so start analyzing at the end
  4491   4675     ** and work forward so that the added virtual terms are never processed.
  4492   4676     */
  4493   4677     exprAnalyzeAll(pTabList, pWC);
  4494   4678     if( db->mallocFailed ){
  4495   4679       goto whereBeginError;
  4496   4680     }
         4681  +
         4682  +  /* Check if the DISTINCT qualifier, if there is one, is redundant. 
         4683  +  ** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to
         4684  +  ** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT.
         4685  +  */
         4686  +  if( pDistinct && isDistinctRedundant(pParse, pTabList, pWC, pDistinct) ){
         4687  +    pDistinct = 0;
         4688  +    pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
         4689  +  }
  4497   4690   
  4498   4691     /* Chose the best index to use for each table in the FROM clause.
  4499   4692     **
  4500   4693     ** This loop fills in the following fields:
  4501   4694     **
  4502   4695     **   pWInfo->a[].pIdx      The index to use for this level of the loop.
  4503   4696     **   pWInfo->a[].wsFlags   WHERE_xxx flags associated with pIdx
................................................................................
  4574   4767       notIndexed = 0;
  4575   4768       for(isOptimal=(iFrom<nTabList-1); isOptimal>=0 && bestJ<0; isOptimal--){
  4576   4769         Bitmask mask;             /* Mask of tables not yet ready */
  4577   4770         for(j=iFrom, pTabItem=&pTabList->a[j]; j<nTabList; j++, pTabItem++){
  4578   4771           int doNotReorder;    /* True if this table should not be reordered */
  4579   4772           WhereCost sCost;     /* Cost information from best[Virtual]Index() */
  4580   4773           ExprList *pOrderBy;  /* ORDER BY clause for index to optimize */
         4774  +        ExprList *pDist;     /* DISTINCT clause for index to optimize */
  4581   4775     
  4582   4776           doNotReorder =  (pTabItem->jointype & (JT_LEFT|JT_CROSS))!=0;
  4583   4777           if( j!=iFrom && doNotReorder ) break;
  4584   4778           m = getMask(pMaskSet, pTabItem->iCursor);
  4585   4779           if( (m & notReady)==0 ){
  4586   4780             if( j==iFrom ) iFrom++;
  4587   4781             continue;
  4588   4782           }
  4589   4783           mask = (isOptimal ? m : notReady);
  4590   4784           pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0);
         4785  +        pDist = (i==0 ? pDistinct : 0);
  4591   4786           if( pTabItem->pIndex==0 ) nUnconstrained++;
  4592   4787     
  4593   4788           WHERETRACE(("=== trying table %d with isOptimal=%d ===\n",
  4594   4789                       j, isOptimal));
  4595   4790           assert( pTabItem->pTab );
  4596   4791   #ifndef SQLITE_OMIT_VIRTUALTABLE
  4597   4792           if( IsVirtual(pTabItem->pTab) ){
................................................................................
  4598   4793             sqlite3_index_info **pp = &pWInfo->a[j].pIdxInfo;
  4599   4794             bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy,
  4600   4795                              &sCost, pp);
  4601   4796           }else 
  4602   4797   #endif
  4603   4798           {
  4604   4799             bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy,
  4605         -                         &sCost);
         4800  +              pDist, &sCost);
  4606   4801           }
  4607   4802           assert( isOptimal || (sCost.used&notReady)==0 );
  4608   4803   
  4609   4804           /* If an INDEXED BY clause is present, then the plan must use that
  4610   4805           ** index if it uses any index at all */
  4611   4806           assert( pTabItem->pIndex==0 
  4612   4807                     || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0
................................................................................
  4658   4853       assert( bestJ>=0 );
  4659   4854       assert( notReady & getMask(pMaskSet, pTabList->a[bestJ].iCursor) );
  4660   4855       WHERETRACE(("*** Optimizer selects table %d for loop %d"
  4661   4856                   " with cost=%g and nRow=%g\n",
  4662   4857                   bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow));
  4663   4858       if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){
  4664   4859         *ppOrderBy = 0;
         4860  +    }
         4861  +    if( (bestPlan.plan.wsFlags & WHERE_DISTINCT)!=0 ){
         4862  +      assert( pWInfo->eDistinct==0 );
         4863  +      pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
  4665   4864       }
  4666   4865       andFlags &= bestPlan.plan.wsFlags;
  4667   4866       pLevel->plan = bestPlan.plan;
  4668   4867       testcase( bestPlan.plan.wsFlags & WHERE_INDEXED );
  4669   4868       testcase( bestPlan.plan.wsFlags & WHERE_TEMP_INDEX );
  4670   4869       if( bestPlan.plan.wsFlags & (WHERE_INDEXED|WHERE_TEMP_INDEX) ){
  4671   4870         pLevel->iIdxCur = pParse->nTab++;

Changes to test/all.test.

    34     34   run_test_suite journaltest 
    35     35   run_test_suite inmemory_journal
    36     36   run_test_suite pcache0 
    37     37   run_test_suite pcache10 
    38     38   run_test_suite pcache50 
    39     39   run_test_suite pcache90 
    40     40   run_test_suite pcache100
           41  +run_test_suite prepare
    41     42   
    42     43   if {$::tcl_platform(platform)=="unix"} {
    43     44     ifcapable !default_autovacuum {
    44     45       run_test_suite autovacuum_crash
    45     46     }
    46     47   }
    47     48   
    48     49   finish_test
    49     50   
    50     51   

Changes to test/alter2.test.

   133    133     execsql {
   134    134       SELECT sum(a), c FROM abc GROUP BY c;
   135    135     }
   136    136   } {8 {} 1 10}
   137    137   do_test alter2-1.9 {
   138    138     # ALTER TABLE abc ADD COLUMN d;
   139    139     alter_table abc {CREATE TABLE abc(a, b, c, d);}
          140  +  if {[permutation] == "prepare"} { db cache flush }
   140    141     execsql { SELECT * FROM abc; }
   141    142     execsql {
   142    143       UPDATE abc SET d = 11 WHERE c IS NULL AND a<4;
   143    144       SELECT * FROM abc;
   144    145     }
   145    146   } {1 2 10 {} 3 4 {} 11 5 6 {} {}}
   146    147   do_test alter2-1.10 {

Changes to test/autoindex1.test.

   243    243     1 0 0 {SCAN TABLE sheep AS s (~1000000 rows)} 
   244    244     1 1 1 {SEARCH TABLE flock_owner AS prev USING INDEX sqlite_autoindex_flock_owner_1 (flock_no=? AND owner_change_date<?) (~2 rows)} 
   245    245     1 0 0 {EXECUTE CORRELATED SCALAR SUBQUERY 2} 
   246    246     2 0 0 {SEARCH TABLE flock_owner AS later USING COVERING INDEX sqlite_autoindex_flock_owner_1 (flock_no=? AND owner_change_date>? AND owner_change_date<?) (~1 rows)} 
   247    247     0 0 0 {SCAN TABLE sheep AS x USING INDEX sheep_reg_flock_index (~1000000 rows)} 
   248    248     0 1 1 {SEARCH SUBQUERY 1 AS y USING AUTOMATIC COVERING INDEX (sheep_no=?) (~8 rows)}
   249    249   }
          250  +
          251  +
          252  +do_execsql_test autoindex1-700 {
          253  +  CREATE TABLE t5(a, b, c);
          254  +  EXPLAIN QUERY PLAN SELECT a FROM t5 WHERE b=10 ORDER BY c;
          255  +} {
          256  +  0 0 0 {SCAN TABLE t5 (~100000 rows)} 
          257  +  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
          258  +}
          259  +
   250    260   
   251    261   finish_test

Changes to test/collate5.test.

    53     53       INSERT INTO collate5t1 VALUES('N', NULL);
    54     54     } 
    55     55   } {}
    56     56   do_test collate5-1.1 {
    57     57     execsql {
    58     58       SELECT DISTINCT a FROM collate5t1;
    59     59     }
    60         -} {A B N}
           60  +} {a b n}
    61     61   do_test collate5-1.2 {
    62     62     execsql {
    63     63       SELECT DISTINCT b FROM collate5t1;
    64     64     }
    65         -} {{} Apple apple banana}
           65  +} {apple Apple banana {}}
    66     66   do_test collate5-1.3 {
    67     67     execsql {
    68     68       SELECT DISTINCT a, b FROM collate5t1;
    69     69     }
    70         -} {A Apple a apple B banana N {}}
           70  +} {a apple A Apple b banana n {}}
    71     71   
    72     72   # Ticket #3376
    73     73   #
    74     74   do_test collate5-1.11 {
    75     75     execsql {
    76     76       CREATE TABLE tkt3376(a COLLATE nocase PRIMARY KEY);
    77     77       INSERT INTO tkt3376 VALUES('abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz');

Added test/distinct.test.

            1  +# 2011 July 1
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is the DISTINCT modifier.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +
           18  +set testprefix distinct
           19  +
           20  +
           21  +proc is_distinct_noop {sql} {
           22  +  set sql1 $sql
           23  +  set sql2 [string map {DISTINCT ""} $sql]
           24  +
           25  +  set program1 [list]
           26  +  set program2 [list]
           27  +  db eval "EXPLAIN $sql1" {
           28  +    if {$opcode != "Noop"} { lappend program1 $opcode }
           29  +  }
           30  +  db eval "EXPLAIN $sql2" {
           31  +    if {$opcode != "Noop"} { lappend program2 $opcode }
           32  +  }
           33  +
           34  +  return [expr {$program1==$program2}]
           35  +}
           36  +
           37  +proc do_distinct_noop_test {tn sql} {
           38  +  uplevel [list do_test $tn [list is_distinct_noop $sql] 1]
           39  +}
           40  +proc do_distinct_not_noop_test {tn sql} {
           41  +  uplevel [list do_test $tn [list is_distinct_noop $sql] 0]
           42  +}
           43  +
           44  +proc do_temptables_test {tn sql temptables} {
           45  +  uplevel [list do_test $tn [subst -novar {
           46  +    set ret ""
           47  +    db eval "EXPLAIN [set sql]" {
           48  +      if {$opcode == "OpenEphemeral"} { 
           49  +        if {$p5 != "10" && $p5!="00"} { error "p5 = $p5" }
           50  +        if {$p5 == "10"} {
           51  +          lappend ret hash
           52  +        } else {
           53  +          lappend ret btree
           54  +        }
           55  +      }
           56  +    }
           57  +    set ret
           58  +  }] $temptables]
           59  +}
           60  +
           61  +
           62  +#-------------------------------------------------------------------------
           63  +# The following tests - distinct-1.* - check that the planner correctly 
           64  +# detects cases where a UNIQUE index means that a DISTINCT clause is 
           65  +# redundant. Currently the planner only detects such cases when there
           66  +# is a single table in the FROM clause.
           67  +#
           68  +do_execsql_test 1.0 {
           69  +  CREATE TABLE t1(a, b, c, d);
           70  +  CREATE UNIQUE INDEX i1 ON t1(b, c);
           71  +  CREATE UNIQUE INDEX i2 ON t1(d COLLATE nocase);
           72  +
           73  +  CREATE TABLE t2(x INTEGER PRIMARY KEY, y);
           74  +
           75  +  CREATE TABLE t3(c1 PRIMARY KEY, c2);
           76  +  CREATE INDEX i3 ON t3(c2);
           77  +}
           78  +foreach {tn noop sql} {
           79  +
           80  +  1   1   "SELECT DISTINCT b, c FROM t1"
           81  +  2   1   "SELECT DISTINCT c FROM t1 WHERE b = ?"
           82  +  3   1   "SELECT DISTINCT rowid FROM t1"
           83  +  4   1   "SELECT DISTINCT rowid, a FROM t1"
           84  +  5   1   "SELECT DISTINCT x FROM t2"
           85  +  6   1   "SELECT DISTINCT * FROM t2"
           86  +  7   1   "SELECT DISTINCT * FROM (SELECT * FROM t2)"
           87  +
           88  +  8   1   "SELECT DISTINCT * FROM t1"
           89  +
           90  +  8   0   "SELECT DISTINCT a, b FROM t1"
           91  +
           92  +  9   0   "SELECT DISTINCT c FROM t1 WHERE b IN (1,2)"
           93  +  10  0   "SELECT DISTINCT c FROM t1"
           94  +  11  0   "SELECT DISTINCT b FROM t1"
           95  +
           96  +  12  0   "SELECT DISTINCT a, d FROM t1"
           97  +  13  0   "SELECT DISTINCT a, b, c COLLATE nocase FROM t1"
           98  +  14  1   "SELECT DISTINCT a, d COLLATE nocase FROM t1"
           99  +  15  0   "SELECT DISTINCT a, d COLLATE binary FROM t1"
          100  +  16  1   "SELECT DISTINCT a, b, c COLLATE binary FROM t1"
          101  +
          102  +  16  0   "SELECT DISTINCT t1.rowid FROM t1, t2"
          103  +  17  0   { /* Technically, it would be possible to detect that DISTINCT
          104  +            ** is a no-op in cases like the following. But SQLite does not
          105  +            ** do so. */
          106  +            SELECT DISTINCT t1.rowid FROM t1, t2 WHERE t1.rowid=t2.rowid }
          107  +
          108  +  18  1   "SELECT DISTINCT c1, c2 FROM t3"
          109  +  19  1   "SELECT DISTINCT c1 FROM t3"
          110  +  20  1   "SELECT DISTINCT * FROM t3"
          111  +  21  0   "SELECT DISTINCT c2 FROM t3"
          112  +
          113  +  22  0   "SELECT DISTINCT * FROM (SELECT 1, 2, 3 UNION SELECT 4, 5, 6)"
          114  +  23  1   "SELECT DISTINCT rowid FROM (SELECT 1, 2, 3 UNION SELECT 4, 5, 6)"
          115  +
          116  +  24  0   "SELECT DISTINCT rowid/2 FROM t1"
          117  +  25  1   "SELECT DISTINCT rowid/2, rowid FROM t1"
          118  +  26  1   "SELECT DISTINCT rowid/2, b FROM t1 WHERE c = ?"
          119  +} {
          120  +  if {$noop} {
          121  +    do_distinct_noop_test 1.$tn $sql
          122  +  } else {
          123  +    do_distinct_not_noop_test 1.$tn $sql
          124  +  }
          125  +}
          126  +
          127  +#-------------------------------------------------------------------------
          128  +# The following tests - distinct-2.* - test cases where an index is
          129  +# used to deliver results in order of the DISTINCT expressions. 
          130  +#
          131  +drop_all_tables
          132  +do_execsql_test 2.0 {
          133  +  CREATE TABLE t1(a, b, c);
          134  +
          135  +  CREATE INDEX i1 ON t1(a, b);
          136  +  CREATE INDEX i2 ON t1(b COLLATE nocase, c COLLATE nocase);
          137  +
          138  +  INSERT INTO t1 VALUES('a', 'b', 'c');
          139  +  INSERT INTO t1 VALUES('A', 'B', 'C');
          140  +  INSERT INTO t1 VALUES('a', 'b', 'c');
          141  +  INSERT INTO t1 VALUES('A', 'B', 'C');
          142  +}
          143  +
          144  +foreach {tn sql temptables res} {
          145  +  1   "a, b FROM t1"                                       {}      {A B a b}
          146  +  2   "b, a FROM t1"                                       {}      {B A b a}
          147  +  3   "a, b, c FROM t1"                                    {hash}  {a b c A B C}
          148  +  4   "a, b, c FROM t1 ORDER BY a, b, c"                   {btree} {A B C a b c}
          149  +  5   "b FROM t1 WHERE a = 'a'"                            {}      {b}
          150  +  6   "b FROM t1"                                          {hash}  {b B}
          151  +  7   "a FROM t1"                                          {}      {A a}
          152  +  8   "b COLLATE nocase FROM t1"                           {}      {b}
          153  +  9   "b COLLATE nocase FROM t1 ORDER BY b COLLATE nocase" {}      {B}
          154  +} {
          155  +  do_execsql_test    2.$tn.1 "SELECT DISTINCT $sql" $res
          156  +  do_temptables_test 2.$tn.2 "SELECT DISTINCT $sql" $temptables
          157  +}
          158  +
          159  +do_execsql_test 2.A {
          160  +  SELECT (SELECT DISTINCT o.a FROM t1 AS i) FROM t1 AS o;
          161  +} {a A a A}
          162  +
          163  +
          164  +
          165  +
          166  +finish_test

Changes to test/e_select.test.

  1234   1234   do_select_tests e_select-5 {
  1235   1235     3.1 "SELECT ALL x FROM h2" {One Two Three Four one two three four}
  1236   1236     3.2 "SELECT ALL x FROM h1, h2 ON (x=b)" {One one Four four}
  1237   1237   
  1238   1238     3.1 "SELECT x FROM h2" {One Two Three Four one two three four}
  1239   1239     3.2 "SELECT x FROM h1, h2 ON (x=b)" {One one Four four}
  1240   1240   
  1241         -  4.1 "SELECT DISTINCT x FROM h2" {four one three two}
  1242         -  4.2 "SELECT DISTINCT x FROM h1, h2 ON (x=b)" {four one}
         1241  +  4.1 "SELECT DISTINCT x FROM h2" {One Two Three Four}
         1242  +  4.2 "SELECT DISTINCT x FROM h1, h2 ON (x=b)" {One Four}
  1243   1243   } 
  1244   1244   
  1245   1245   # EVIDENCE-OF: R-02054-15343 For the purposes of detecting duplicate
  1246   1246   # rows, two NULL values are considered to be equal.
  1247   1247   #
  1248   1248   do_select_tests e_select-5.5 {
  1249   1249     1  "SELECT DISTINCT d FROM h3" {{} 2 2,3 2,4 3}
  1250   1250   }
  1251   1251   
  1252   1252   # EVIDENCE-OF: R-58359-52112 The normal rules for selecting a collation
  1253   1253   # sequence to compare text values with apply.
  1254   1254   #
  1255   1255   do_select_tests e_select-5.6 {
  1256         -  1  "SELECT DISTINCT b FROM h1"                  {I IV four i iv one}
  1257         -  2  "SELECT DISTINCT b COLLATE nocase FROM h1"   {four i iv one}
  1258         -  3  "SELECT DISTINCT x FROM h2"                  {four one three two}
         1256  +  1  "SELECT DISTINCT b FROM h1"                  {one I i four IV iv}
         1257  +  2  "SELECT DISTINCT b COLLATE nocase FROM h1"   {one I four IV}
         1258  +  3  "SELECT DISTINCT x FROM h2"                  {One Two Three Four}
  1259   1259     4  "SELECT DISTINCT x COLLATE binary FROM h2"   {
  1260         -    Four One Three Two four one three two
         1260  +    One Two Three Four one two three four
  1261   1261     }
  1262   1262   }
  1263   1263   
  1264   1264   #-------------------------------------------------------------------------
  1265   1265   # The following tests - e_select-7.* - test that statements made to do
  1266   1266   # with compound SELECT statements are correct.
  1267   1267   #

Changes to test/exists.test.

   155    155         sql1 { DROP INDEX IF EXISTS aux.i2 }
   156    156         sql2 { SELECT name FROM aux.sqlite_master WHERE type = 'index' }
   157    157       } {}
   158    158       do_test 3.$tn.2.2 {
   159    159         sql1 { DROP INDEX IF EXISTS i2 }
   160    160         sql2 { CREATE INDEX aux.i2 ON t2(x) }
   161    161         sql1 { DROP INDEX IF EXISTS i2 }
   162         -      sql2 { SELECT name FROM aux.sqlite_master WHERE type = 'index' }
          162  +      sql2 { SELECT * FROM aux.sqlite_master WHERE type = 'index' }
   163    163       } {}
   164    164   
   165    165       # VIEW objects.
   166    166       #
   167    167       do_test 3.$tn.3.1 {
   168    168         sql1 { DROP VIEW IF EXISTS aux.v1 }
   169    169         sql2 { CREATE VIEW aux.v1 AS SELECT * FROM t2 }

Changes to test/fts3auto.test.

   103    103   
   104    104     do_execsql_test $tn$title.4 "
   105    105       SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl 
   106    106       WHERE $tbl MATCH '$match' ORDER BY docid ASC
   107    107     " $matchinfo_asc
   108    108   }
   109    109   
   110         -#    fts3_make_deferrable TABLE TOKEN
          110  +#    fts3_make_deferrable TABLE TOKEN ?NROW?
   111    111   #
   112         -proc fts3_make_deferrable {tbl token} {
          112  +proc fts3_make_deferrable {tbl token {nRow 0}} {
   113    113   
   114    114     set stmt [sqlite3_prepare db "SELECT * FROM $tbl" -1 dummy]
   115    115     set name [sqlite3_column_name $stmt 0]
   116    116     sqlite3_finalize $stmt
   117    117   
   118         -  set nRow [db one "SELECT count(*) FROM $tbl"]
          118  +  if {$nRow==0} {
          119  +    set nRow [db one "SELECT count(*) FROM $tbl"]
          120  +  }
   119    121     set pgsz [db one "PRAGMA page_size"]
   120    122     execsql BEGIN
   121    123     for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} {
   122    124       set doc [string repeat "$token " 100]
   123    125       execsql "INSERT INTO $tbl ($name) VALUES(\$doc)"
   124    126     }
   125    127     execsql "INSERT INTO $tbl ($name) VALUES('aaaaaaa ${token}aaaaa')"
................................................................................
   648    650     do_fts3query_test 6.$tn.2 t1 {b:G AND c:I}
   649    651     do_fts3query_test 6.$tn.3 t1 {b:G NEAR c:I}
   650    652     do_fts3query_test 6.$tn.4 t1 {a:C OR b:G OR c:K OR d:C}
   651    653     do_fts3query_test 6.$tn.5 t1 {a:G OR b:G}
   652    654   
   653    655     catchsql { COMMIT }
   654    656   }
          657  +
          658  +foreach {tn create} {
          659  +  1    "fts4(x)"
          660  +  2    "fts4(x, order=DESC)"
          661  +} {
          662  +  execsql [subst {
          663  +    DROP TABLE IF EXISTS t1;
          664  +    CREATE VIRTUAL TABLE t1 USING $create;
          665  +  }]
          666  +
          667  +  foreach {x} {
          668  +    "F E N O T K X V A X I E X A P G Q V H U"
          669  +    "R V A E T C V Q N I E L O N U G J K L U"
          670  +    "U Y I G W M V F J L X I D C H F P J Q B"
          671  +    "S G D Z X R P G S S Y B K A S G A I L L"
          672  +    "L S I C H T Z S R Q P R N K J X L F M J"
          673  +    "C C C D P X B Z C M A D A C X S B T X V"
          674  +    "W Y J M D R G V R K B X S A W R I T N C"
          675  +    "P K L W T M S P O Y Y V V O E H Q A I R"
          676  +    "C D Y I C Z F H J C O Y A Q F L S B D K"
          677  +    "P G S C Y C Y V I M B D S Z D D Y W I E"
          678  +    "Z K Z U E E S F Y X T U A L W O U J C Q"
          679  +    "P A T Z S W L P L Q V Y Y I P W U X S S"
          680  +    "I U I H U O F Z F R H R F T N D X A G M"
          681  +    "N A B M S H K X S O Y D T X S B R Y H Z"
          682  +    "L U D A S K I L S V Z J P U B E B Y H M"
          683  +  } {
          684  +    execsql { INSERT INTO t1 VALUES($x) }
          685  +  }
          686  +
          687  +  # Add extra documents to the database such that token "B" will be considered
          688  +  # deferrable if considering the other tokens means that 2 or fewer documents
          689  +  # will be loaded into memory.
          690  +  #
          691  +  fts3_make_deferrable t1 B 2
          692  +
          693  +  # B is not deferred in either of the first two tests below, since filtering
          694  +  # on "M" or "D" returns 10 documents or so. But filtering on "M * D" only
          695  +  # returns 2, so B is deferred in this case.
          696  +  #
          697  +  do_fts3query_test 7.$tn.1             t1 {"M B"}
          698  +  do_fts3query_test 7.$tn.2             t1 {"B D"}
          699  +  do_fts3query_test 7.$tn.3 -deferred B t1 {"M B D"}
          700  +}
   655    701   
   656    702   set sqlite_fts3_enable_parentheses $sfep
   657    703   finish_test
   658    704   

Changes to test/func3.test.

    13     13   # with functions created using sqlite3_create_function_v2() is 
    14     14   # correctly invoked.
    15     15   #
    16     16   set testdir [file dirname $argv0]
    17     17   source $testdir/tester.tcl
    18     18   
    19     19   
    20         -do_test func3-1.1 {
    21         -  set destroyed 0
    22         -  proc destroy {} { set ::destroyed 1 }
    23         -  sqlite3_create_function_v2 db f2 -1 any -func f2 -destroy destroy
    24         -  set destroyed
    25         -} 0
    26         -do_test func3-1.2 {
    27         -  sqlite3_create_function_v2 db f2 -1 utf8 -func f2
    28         -  set destroyed
    29         -} 0
    30         -do_test func3-1.3 {
    31         -  sqlite3_create_function_v2 db f2 -1 utf16le -func f2
    32         -  set destroyed
    33         -} 0
    34         -do_test func3-1.4 {
    35         -  sqlite3_create_function_v2 db f2 -1 utf16be -func f2
    36         -  set destroyed
    37         -} 1
           20  +ifcapable utf16 {
           21  +  do_test func3-1.1 {
           22  +    set destroyed 0
           23  +    proc destroy {} { set ::destroyed 1 }
           24  +    sqlite3_create_function_v2 db f2 -1 any -func f2 -destroy destroy
           25  +    set destroyed
           26  +  } 0
           27  +  do_test func3-1.2 {
           28  +    sqlite3_create_function_v2 db f2 -1 utf8 -func f2
           29  +    set destroyed
           30  +  } 0
           31  +  do_test func3-1.3 {
           32  +    sqlite3_create_function_v2 db f2 -1 utf16le -func f2
           33  +    set destroyed
           34  +  } 0
           35  +  do_test func3-1.4 {
           36  +    sqlite3_create_function_v2 db f2 -1 utf16be -func f2
           37  +    set destroyed
           38  +  } 1
           39  +}
    38     40   
    39     41   do_test func3-2.1 {
    40     42     set destroyed 0
    41     43     proc destroy {} { set ::destroyed 1 }
    42     44     sqlite3_create_function_v2 db f3 -1 utf8 -func f3 -destroy destroy
    43     45     set destroyed
    44     46   } 0

Changes to test/fuzzer1.test.

  1372   1372   do_test fuzzer1-2.3 {
  1373   1373     execsql {
  1374   1374       SELECT DISTINCT streetname.n FROM f2, streetname
  1375   1375        WHERE f2.word MATCH 'tayle'
  1376   1376          AND f2.distance<=200
  1377   1377          AND streetname.n>=f2.word AND streetname.n<=(f2.word || x'F7BFBFBF')
  1378   1378     }
  1379         -} {steelewood tallia tallu talwyn taymouth thelema trailer {tyler finley}}
         1379  +} {{tyler finley} trailer taymouth steelewood tallia tallu talwyn thelema}
  1380   1380   
  1381   1381   
  1382   1382   finish_test

Changes to test/incrblob_err.test.

    31     31     set bytes [file size [info script]]
    32     32     execsql {
    33     33       CREATE TABLE blobs(k, v BLOB);
    34     34       INSERT INTO blobs VALUES(1, zeroblob($::bytes));
    35     35     }
    36     36   } -tclbody {
    37     37     set ::blob [db incrblob blobs v 1]
           38  +  fconfigure $::blob -translation binary
    38     39     set rc [catch {puts -nonewline $::blob $::data}]
    39     40     if {$rc} { error "out of memory" }
    40     41   } 
    41     42   
    42     43   do_malloc_test 2 -tclprep {
    43     44     execsql {
    44     45       CREATE TABLE blobs(k, v BLOB);
................................................................................
    67     68     } elseif {$::r ne $::data} {
    68     69       error "Bad data read..."
    69     70     }
    70     71     set rc [catch {close $::blob}]
    71     72     if {$rc} { 
    72     73       error "out of memory" 
    73     74     }
    74         -} 
    75         -
           75  +}
    76     76   
    77     77   do_ioerr_test incrblob_err-4 -cksum 1 -sqlprep {
    78     78     CREATE TABLE blobs(k, v BLOB);
    79     79     INSERT INTO blobs VALUES(1, $::data);
    80     80   } -tclbody {
    81     81     set ::blob [db incrblob blobs v 1]
    82     82     read $::blob
................................................................................
    83     83   }
    84     84   
    85     85   do_ioerr_test incrblob_err-5 -cksum 1 -sqlprep {
    86     86     CREATE TABLE blobs(k, v BLOB);
    87     87     INSERT INTO blobs VALUES(1, zeroblob(length(CAST($::data AS BLOB))));
    88     88   } -tclbody {
    89     89     set ::blob [db incrblob blobs v 1]
           90  +  fconfigure $::blob -translation binary
    90     91     puts -nonewline $::blob $::data
    91     92     close $::blob
    92     93   }
    93     94   
    94     95   do_ioerr_test incrblob_err-6 -cksum 1 -sqlprep {
    95     96     CREATE TABLE blobs(k, v BLOB);
    96     97     INSERT INTO blobs VALUES(1, $::data || $::data || $::data);
    97     98   } -tclbody {
    98     99     set ::blob [db incrblob blobs v 1]
          100  +  fconfigure $::blob -translation binary
    99    101     seek $::blob -20 end
   100    102     puts -nonewline $::blob "12345678900987654321"
   101    103     close $::blob
   102    104   }
   103    105   
   104    106   do_ioerr_test incrblob_err-7 -cksum 1 -sqlprep {
   105    107     PRAGMA auto_vacuum = 1;

Changes to test/insert4.test.

   108    108   #
   109    109   do_test insert4-2.4.1 {
   110    110     execsql {
   111    111       DELETE FROM t3;
   112    112       INSERT INTO t3 SELECT DISTINCT * FROM t2;
   113    113       SELECT * FROM t3;
   114    114     }
   115         -} {1 9 9 1}
          115  +} {9 1 1 9}
   116    116   xferopt_test insert4-2.4.2 0
   117    117   do_test insert4-2.4.3 {
   118    118     catchsql {
   119    119       DELETE FROM t1;
   120    120       INSERT INTO t1 SELECT DISTINCT * FROM t2;
   121    121     }
   122    122   } {1 {constraint failed}}

Changes to test/like.test.

    66     66   } {ABC abc}
    67     67   do_test like-1.4 {
    68     68     execsql {
    69     69       SELECT x FROM t1 WHERE x LIKE 'aBc' ORDER BY 1;
    70     70     }
    71     71   } {ABC abc}
    72     72   do_test like-1.5.1 {
           73  +  # Use sqlite3_exec() to verify fix for ticket [25ee81271091] 2011-06-26
           74  +  sqlite3_exec db {PRAGMA case_sensitive_like=on}
           75  +} {0 {}}
           76  +do_test like-1.5.2 {
    73     77     execsql {
    74         -    PRAGMA case_sensitive_like=on;
    75     78       SELECT x FROM t1 WHERE x LIKE 'abc' ORDER BY 1;
    76     79     }
    77     80   } {abc}
    78         -do_test like-1.5.2 {
           81  +do_test like-1.5.3 {
    79     82     execsql {
    80     83       PRAGMA case_sensitive_like; -- no argument; does not change setting
    81     84       SELECT x FROM t1 WHERE x LIKE 'abc' ORDER BY 1;
    82     85     }
    83     86   } {abc}
    84     87   do_test like-1.6 {
    85     88     execsql {

Changes to test/misc5.test.

   501    501             )    
   502    502             WHERE artist <> '' 
   503    503           )  
   504    504          )       
   505    505         )  
   506    506         ORDER BY LOWER(artist) ASC;
   507    507       }
   508         -  } {one}
          508  +  } {two}
   509    509   }
   510    510   
   511    511   # Ticket #1370.  Do not overwrite small files (less than 1024 bytes)
   512    512   # when trying to open them as a database.
   513    513   #
   514    514   if {[permutation] == ""} {
   515    515     do_test misc5-4.1 {

Changes to test/multiplex.test.

    51     51     list $msg
    52     52   }
    53     53   
    54     54   # This attempts to delete the base file and 
    55     55   # and files with the chunk extension.
    56     56   proc multiplex_delete {name} {
    57     57     global g_max_chunks
           58  +  forcedelete $name
    58     59     for {set i 0} {$i<$g_max_chunks} {incr i} {
    59     60       forcedelete [multiplex_name $name $i]
    60     61       forcedelete [multiplex_name $name-journal $i]
    61     62       forcedelete [multiplex_name $name-wal $i]
    62     63     }
    63     64   }
    64     65   
................................................................................
    80     81   do_test multiplex-1.7 { sqlite3_multiplex_initialize "" 1 }        {SQLITE_OK}
    81     82   do_test multiplex-1.8 { sqlite3_multiplex_shutdown }               {SQLITE_OK}
    82     83   
    83     84   
    84     85   do_test multiplex-1.9.1  { sqlite3_multiplex_initialize "" 1 }     {SQLITE_OK}
    85     86   do_test multiplex-1.9.2  { sqlite3 db test.db }                    {}
    86     87   do_test multiplex-1.9.3  { multiplex_set db main 32768 16 }        {SQLITE_OK}
    87         -do_test multiplex-1.9.4  { multiplex_set db main 32768 -1 }        {SQLITE_MISUSE}
    88         -do_test multiplex-1.9.5  { multiplex_set db main -1 16 }           {SQLITE_MISUSE}
           88  +do_test multiplex-1.9.4  { multiplex_set db main 32768 -1 }        {SQLITE_OK}
    89     89   do_test multiplex-1.9.6  { multiplex_set db main 31 16 }           {SQLITE_OK}
    90         -do_test multiplex-1.9.7  { multiplex_set db main 32768 100 }       {SQLITE_MISUSE}
           90  +do_test multiplex-1.9.7  { multiplex_set db main 32768 100 }       {SQLITE_OK}
    91     91   do_test multiplex-1.9.8  { multiplex_set db main 1073741824 1 }    {SQLITE_OK}
    92     92   do_test multiplex-1.9.9  { db close }                              {}
    93     93   do_test multiplex-1.9.10 { sqlite3_multiplex_shutdown }            {SQLITE_OK}
    94     94   
    95     95   do_test multiplex-1.10.1  { sqlite3_multiplex_initialize "" 1 }                                  {SQLITE_OK}
    96     96   do_test multiplex-1.10.2  { sqlite3 db test.db }                                                 {}
    97     97   do_test multiplex-1.10.3  { lindex [ catchsql { SELECT multiplex_control(2, 32768); } ] 0 }      {0}
    98         -do_test multiplex-1.10.4  { lindex [ catchsql { SELECT multiplex_control(3, -1); } ] 0 }         {1}
    99         -do_test multiplex-1.10.5  { lindex [ catchsql { SELECT multiplex_control(2, -1); } ] 0 }         {1}
           98  +do_test multiplex-1.10.4  { lindex [ catchsql { SELECT multiplex_control(3, -1); } ] 0 }         {0}
   100     99   do_test multiplex-1.10.6  { lindex [ catchsql { SELECT multiplex_control(2, 31); } ] 0 }         {0}
   101         -do_test multiplex-1.10.7  { lindex [ catchsql { SELECT multiplex_control(3, 100); } ] 0 }        {1}
          100  +do_test multiplex-1.10.7  { lindex [ catchsql { SELECT multiplex_control(3, 100); } ] 0 }        {0}
   102    101   do_test multiplex-1.10.8  { lindex [ catchsql { SELECT multiplex_control(2, 1073741824); } ] 0 } {0}
   103    102   do_test multiplex-1.10.9  { db close }                                                           {}
   104    103   do_test multiplex-1.10.10 { sqlite3_multiplex_shutdown }                                         {SQLITE_OK}
   105    104   
   106    105   do_test multiplex-1.11.1  { sqlite3_multiplex_initialize "" 1 }               {SQLITE_OK}
   107    106   do_test multiplex-1.11.2  { sqlite3 db test.db }                              {}
   108    107   do_test multiplex-1.11.3  { sqlite3_multiplex_control db main enable 0  }     {SQLITE_OK}
................................................................................
   148    147   #
   149    148   #   multiplex-2.7.*: Disable/enable tests.
   150    149   #
   151    150   
   152    151   sqlite3_multiplex_initialize "" 1
   153    152   multiplex_set db main 32768 16
   154    153   
          154  +file delete -force test.x
   155    155   do_test multiplex-2.1.2 {
   156         -  sqlite3 db test.db
          156  +  sqlite3 db test.x
   157    157     execsql {
   158    158       PRAGMA page_size=1024;
   159    159       PRAGMA auto_vacuum=OFF;
   160    160       PRAGMA journal_mode=DELETE;
   161    161     }
   162    162     execsql {
   163    163       CREATE TABLE t1(a, b);
   164    164       INSERT INTO t1 VALUES(1, randomblob(1100));
   165    165       INSERT INTO t1 VALUES(2, randomblob(1100));
   166    166     }
   167    167   } {}
   168         -do_test multiplex-2.1.3 { file size [multiplex_name test.db 0] } {4096}
          168  +do_test multiplex-2.1.3 { file size [multiplex_name test.x 0] } {4096}
   169    169   do_test multiplex-2.1.4 {
   170    170     execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) }
   171    171   } {}
   172    172   
   173    173   do_test multiplex-2.2.1 {
   174    174     execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) }
   175    175   } {}
   176         -do_test multiplex-2.2.3 { file size [multiplex_name test.db 0] } {6144}
          176  +do_test multiplex-2.2.3 { file size [multiplex_name test.x 0] } {6144}
   177    177   
   178    178   do_test multiplex-2.3.1 {
   179         -  sqlite3 db2 test2.db
          179  +  sqlite3 db2 test2.x
   180    180     db2 close
   181    181   } {}
   182    182   
   183    183   
   184    184   do_test multiplex-2.4.1 {
   185    185     sqlite3_multiplex_shutdown
   186    186   } {SQLITE_MISUSE}
   187    187   do_test multiplex-2.4.2 {
   188    188     execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) }
   189    189   } {}
   190         -do_test multiplex-2.4.4 { file size [multiplex_name test.db 0] } {7168}
          190  +do_test multiplex-2.4.4 { file size [multiplex_name test.x 0] } {7168}
   191    191   do_test multiplex-2.4.99 {
   192    192     db close
   193    193     sqlite3_multiplex_shutdown
   194    194   } {SQLITE_OK}
   195    195   
   196    196   
   197    197   do_test multiplex-2.5.1 {
   198         -  multiplex_delete test.db
          198  +  multiplex_delete test.x
   199    199     sqlite3_multiplex_initialize "" 1
   200         -  sqlite3 db test.db
          200  +  sqlite3 db test.x
   201    201     multiplex_set db main 4096 16
   202    202   } {SQLITE_OK}
   203    203   
   204    204   do_test multiplex-2.5.2 {
   205    205     execsql {
   206    206       PRAGMA page_size = 1024;
   207    207       PRAGMA journal_mode = delete;
................................................................................
   238    238     db eval {SELECT a,length(b) FROM t1 WHERE a=2}
   239    239   } {2 4000}
   240    240   
   241    241   do_test multiplex-2.5.8 {
   242    242     db eval {SELECT a,length(b) FROM t1 WHERE a=4}
   243    243   } {4 4000}
   244    244   
   245         -do_test multiplex-2.5.9 { file size [multiplex_name test.db 0] } [list $g_chunk_size]
   246         -do_test multiplex-2.5.10 { file size [multiplex_name test.db 1] } [list $g_chunk_size]
          245  +do_test multiplex-2.5.9 { file size [multiplex_name test.x 0] } [list $g_chunk_size]
          246  +do_test multiplex-2.5.10 { file size [multiplex_name test.x 1] } [list $g_chunk_size]
   247    247   
   248    248   do_test multiplex-2.5.99 {
   249    249     db close
   250    250     sqlite3_multiplex_shutdown
   251    251   } {SQLITE_OK}
   252    252   
   253    253   
................................................................................
   525    525   do_faultsim_test multiplex-5.5 -prep {
   526    526     catch { sqlite3_multiplex_shutdown }
   527    527   } -body {
   528    528     sqlite3_multiplex_initialize "" 1
   529    529     multiplex_set db main 32768 16
   530    530   }
   531    531   
   532         -# test that mismatch filesize is detected
   533         -#
   534         -# Do not run this test if $::G(perm:presql) is set. If it is set, then the
   535         -# expected IO error will occur within the Tcl [sqlite3] wrapper, not within
   536         -# the first SQL statement executed below. This breaks the test case.
   537         -#
   538         -if {0==[info exists ::G(perm:presql)] || $::G(perm:presql) == ""} {
   539         -  set all_journal_modes {delete persist truncate memory off}
   540         -  foreach jmode $all_journal_modes {
   541         -    do_test multiplex-5.6.1.$jmode {
   542         -      sqlite3_multiplex_shutdown
   543         -      multiplex_delete test.db
   544         -      sqlite3 db test.db
   545         -      db eval {
   546         -        PRAGMA page_size = 1024;
   547         -        PRAGMA auto_vacuum = off;
   548         -      }
   549         -      db eval "PRAGMA journal_mode = $jmode;"
   550         -    } $jmode
   551         -    do_test multiplex-5.6.2.$jmode {
   552         -      execsql {
   553         -        CREATE TABLE t1(a, b);
   554         -        INSERT INTO t1 VALUES(1, randomblob(15000));
   555         -        INSERT INTO t1 VALUES(2, randomblob(15000));
   556         -        INSERT INTO t1 VALUES(3, randomblob(15000));
   557         -        INSERT INTO t1 VALUES(4, randomblob(15000));
   558         -        INSERT INTO t1 VALUES(5, randomblob(15000));
   559         -      }
   560         -      db close
   561         -      sqlite3_multiplex_initialize "" 1
   562         -      sqlite3 db test.db
   563         -      multiplex_set db main 4096 16
   564         -    } {SQLITE_OK}
   565         -    do_test multiplex-5.6.3.$jmode {
   566         -      catchsql {
   567         -        INSERT INTO t1 VALUES(6, randomblob(15000));
   568         -      }
   569         -    } {1 {disk I/O error}}
   570         -    do_test multiplex-5.6.4.$jmode {
   571         -      db close
   572         -    } {}
   573         -  }
   574         -}
   575         -
   576    532   #-------------------------------------------------------------------------
   577    533   # Test that you can vacuum a multiplex'ed DB.  
   578    534   
   579    535   ifcapable vacuum {
   580    536   
   581    537   sqlite3_multiplex_shutdown
   582    538   do_test multiplex-6.0.0 {
   583    539     multiplex_delete test.db
          540  +  multiplex_delete test.x
   584    541     sqlite3_multiplex_initialize "" 1
   585         -  sqlite3 db test.db
          542  +  sqlite3 db test.x
   586    543     multiplex_set db main 4096 16
   587    544   } {SQLITE_OK}
   588    545   
   589    546   do_test multiplex-6.1.0 {
   590    547     execsql {
   591    548       PRAGMA page_size=1024;
   592    549       PRAGMA journal_mode=DELETE;
................................................................................
   594    551     }
   595    552     execsql {
   596    553       CREATE TABLE t1(a, b);
   597    554       INSERT INTO t1 VALUES(1, randomblob($g_chunk_size));
   598    555       INSERT INTO t1 VALUES(2, randomblob($g_chunk_size));
   599    556     }
   600    557   } {}
   601         -do_test multiplex-6.2.1 { file size [multiplex_name test.db 0] } [list $g_chunk_size]
   602         -do_test multiplex-6.2.2 { file size [multiplex_name test.db 1] } [list $g_chunk_size]
          558  +do_test multiplex-6.2.1 { file size [multiplex_name test.x 0] } [list $g_chunk_size]
          559  +do_test multiplex-6.2.2 { file size [multiplex_name test.x 1] } [list $g_chunk_size]
   603    560   
   604    561   do_test multiplex-6.3.0 {
   605    562     execsql { VACUUM }
   606    563   } {}
   607    564   
   608    565   do_test multiplex-6.99 {
   609    566     db close
   610         -  multiplex_delete test.db
          567  +  multiplex_delete test.x
   611    568     sqlite3_multiplex_shutdown
   612    569   } {SQLITE_OK}
   613    570   
   614    571   }
   615    572   
   616    573   
   617    574   catch { sqlite3_multiplex_shutdown }
   618    575   finish_test

Changes to test/permutations.test.

   771    771     where6.test where7.test where8.test where9.test
   772    772     whereA.test whereB.test wherelimit.test
   773    773     select1.test select2.test select3.test select4.test select5.test
   774    774     select7.test select8.test selectA.test selectC.test
   775    775   } -dbconfig {
   776    776     optimization_control $::dbhandle all 0
   777    777   }
          778  +
          779  +test_suite "prepare" -description {
          780  +  Run tests with the db connection using sqlite3_prepare() instead of _v2().
          781  +} -dbconfig {
          782  +  db_use_legacy_prepare $::dbhandle 1
          783  +  #$::dbhandle cache size 0
          784  +} -files [
          785  +  test_set $allquicktests -exclude *malloc* *ioerr* *fault*
          786  +]
   778    787   
   779    788   # End of tests
   780    789   #############################################################################
   781    790   
   782    791   # run_tests NAME OPTIONS
   783    792   #
   784    793   # where available options are:  

Changes to test/releasetest.tcl.

   151    151       -DSQLITE_ENABLE_OVERSIZE_CELL_CHECK=1
   152    152       -DSQLITE_MAX_ATTACHED=62
   153    153     }
   154    154   }
   155    155   
   156    156   array set ::Platforms {
   157    157     Linux-x86_64 {
          158  +    "Debug-One"               "checksymbols test"
   158    159       "Secure-Delete"           test
   159    160       "Unlock-Notify"           "QUICKTEST_INCLUDE=notify2.test test"
   160    161       "Update-Delete-Limit"     test
   161         -    "Debug-One"               test
   162    162       "Extra-Robustness"        test
   163    163       "Device-Two"              test
   164    164       "Ftrapv"                  test
   165    165       "Default"                 "threadtest test"
   166    166       "Device-One"              fulltest
   167    167     }
   168    168     Linux-i686 {
................................................................................
   172    172       "Default"                 "threadtest fulltest"
   173    173     }
   174    174     Darwin-i386 {
   175    175       "Locking-Style"           test
   176    176       "OS-X"                    "threadtest fulltest"
   177    177     }
   178    178   }
          179  +
   179    180   
   180    181   # End of configuration section.
   181    182   #########################################################################
   182    183   #########################################################################
   183    184   
   184    185   foreach {key value} [array get ::Platforms] {
   185    186     foreach {v t} $value {

Changes to test/selectB.test.

   351    351   
   352    352     do_test selectB-$ii.19 {
   353    353       execsql {
   354    354         SELECT * FROM (
   355    355           SELECT DISTINCT (a/10) FROM t1 UNION ALL SELECT DISTINCT(d%2) FROM t2
   356    356         )
   357    357       }
   358         -  } {0 1 0 1}
          358  +  } {0 1 1 0}
   359    359   
   360    360     do_test selectB-$ii.20 {
   361    361       execsql {
   362    362         SELECT DISTINCT * FROM (
   363    363           SELECT DISTINCT (a/10) FROM t1 UNION ALL SELECT DISTINCT(d%2) FROM t2
   364    364         )
   365    365       }

Changes to test/sqllimits1.test.

   315    315   } {SQLITE_TOOBIG}
   316    316   do_test sqllimits1-5.14.4 {
   317    317     set np1 [expr {$SQLITE_LIMIT_LENGTH + 1}]
   318    318     set ::str1 [string repeat A $np1]
   319    319     catch {sqlite3_bind_text $::STMT 1 $::str1 -1} res
   320    320     set res
   321    321   } {SQLITE_TOOBIG}
   322         -do_test sqllimits1-5.14.5 {
   323         -  catch {sqlite3_bind_text16 $::STMT 1 $::str1 -1} res
   324         -  set res
   325         -} {SQLITE_TOOBIG}
          322  +ifcapable utf16 {
          323  +  do_test sqllimits1-5.14.5 {
          324  +    catch {sqlite3_bind_text16 $::STMT 1 $::str1 -1} res
          325  +    set res
          326  +  } {SQLITE_TOOBIG}
          327  +}
   326    328   do_test sqllimits1-5.14.6 {
   327    329     catch {sqlite3_bind_text $::STMT 1 $::str1 $np1} res
   328    330     set res
   329    331   } {SQLITE_TOOBIG}
   330         -do_test sqllimits1-5.14.7 {
   331         -  catch {sqlite3_bind_text16 $::STMT 1 $::str1 $np1} res
   332         -  set res
   333         -} {SQLITE_TOOBIG}
          332  +ifcapable utf16 {
          333  +  do_test sqllimits1-5.14.7 {
          334  +    catch {sqlite3_bind_text16 $::STMT 1 $::str1 $np1} res
          335  +    set res
          336  +  } {SQLITE_TOOBIG}
          337  +}
   334    338   do_test sqllimits1-5.14.8 {
   335    339     set n [expr {$np1-1}]
   336    340     catch {sqlite3_bind_text $::STMT 1 $::str1 $n} res
   337    341     set res
   338    342   } {}
   339    343   do_test sqllimits1-5.14.9 {
   340    344     catch {sqlite3_bind_text16 $::STMT 1 $::str1 $n} res

Changes to test/temptable.test.

   288    288   # but the permanent index should still be accessible and should still
   289    289   # be updated when its corresponding table changes.
   290    290   #
   291    291   do_test temptable-5.1 {
   292    292     execsql {
   293    293       CREATE TEMP TABLE mask(a,b,c)
   294    294     } db2
          295  +  if {[permutation]=="prepare"} { db2 cache flush }
   295    296     execsql {
   296    297       CREATE INDEX mask ON t2(x);
   297    298       SELECT * FROM t2;
   298    299     }
   299    300   } {3 4}
   300    301   #do_test temptable-5.2 {
   301    302   #  catchsql {

Changes to test/tester.tcl.

    16     16   #-------------------------------------------------------------------------
    17     17   # The commands provided by the code in this file to help with creating 
    18     18   # test cases are as follows:
    19     19   #
    20     20   # Commands to manipulate the db and the file-system at a high level:
    21     21   #
    22     22   #      copy_file              FROM TO
    23         -#      drop_all_table         ?DB?
           23  +#      drop_all_tables        ?DB?
    24     24   #      forcedelete            FILENAME
    25     25   #
    26     26   # Test the capability of the SQLite version built into the interpreter to
    27     27   # determine if a specific test can be run:
    28     28   #
    29     29   #      ifcapable              EXPR
    30     30   #

Added test/tkt-54844eea3f.test.

            1  +# 2011 July 8
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this file is testing that bug [54844eea3f] has been fixed.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +
           18  +set ::testprefix tkt-54844eea3f
           19  +
           20  +do_test 1.0 {
           21  +  execsql {
           22  +    CREATE TABLE t1(a INTEGER PRIMARY KEY);
           23  +    INSERT INTO t1 VALUES(1);
           24  +    INSERT INTO t1 VALUES(4);
           25  +
           26  +    CREATE TABLE t2(b INTEGER PRIMARY KEY);
           27  +    INSERT INTO t2 VALUES(1);
           28  +    INSERT INTO t2 VALUES(2);
           29  +    INSERT INTO t2 SELECT b+2 FROM t2;
           30  +    INSERT INTO t2 SELECT b+4 FROM t2;
           31  +    INSERT INTO t2 SELECT b+8 FROM t2;
           32  +    INSERT INTO t2 SELECT b+16 FROM t2;
           33  +
           34  +    CREATE TABLE t3(c INTEGER PRIMARY KEY);
           35  +    INSERT INTO t3 VALUES(1);
           36  +    INSERT INTO t3 VALUES(2);
           37  +    INSERT INTO t3 VALUES(3);
           38  +  }
           39  +} {}
           40  +
           41  +do_test 1.1 {
           42  +  execsql {
           43  +    SELECT 'test-2', t3.c, (
           44  +          SELECT count(*) 
           45  +          FROM t1 JOIN (SELECT DISTINCT t3.c AS p FROM t2) AS x ON t1.a=x.p
           46  +    )
           47  +    FROM t3;
           48  +  }
           49  +} {test-2 1 1 test-2 2 0 test-2 3 0}
           50  +
           51  +do_test 1.2 {
           52  +  execsql {
           53  +    CREATE TABLE t4(a, b, c);
           54  +    INSERT INTO t4 VALUES('a', 1, 'one');
           55  +    INSERT INTO t4 VALUES('a', 2, 'two');
           56  +    INSERT INTO t4 VALUES('b', 1, 'three');
           57  +    INSERT INTO t4 VALUES('b', 2, 'four');
           58  +    SELECT ( 
           59  +      SELECT c FROM (
           60  +        SELECT * FROM t4 WHERE a=out.a ORDER BY b LIMIT 10 OFFSET 1
           61  +      ) WHERE b=out.b
           62  +    ) FROM t4 AS out;
           63  +  }
           64  +} {{} two {} four}
           65  +
           66  +
           67  +finish_test

Changes to test/triggerD.test.

    10     10   #***********************************************************************
    11     11   #
    12     12   # Verify that when columns named "rowid", "oid", and "_rowid_" appear
    13     13   # in a table as ordinary columns (not as the INTEGER PRIMARY KEY) then
    14     14   # the use of these columns in triggers will refer to the column and not
    15     15   # to the actual ROWID.  Ticket [34d2ae1c6d08b5271ba5e5592936d4a1d913ffe3]
    16     16   #
           17  +# Also, verify that triggers created like this:
           18  +#
           19  +#    CREATE TRIGGER attached.trig AFTER INSERT ON attached.tab ...
           20  +#
           21  +# can be reparsed as a main database.  Ticket [d6ddba6706353915ceedc56b4e3]
           22  +#
    17     23   
    18     24   set testdir [file dirname $argv0]
    19     25   source $testdir/tester.tcl
    20     26   ifcapable {!trigger} {
    21     27     finish_test
    22     28     return
    23     29   }
................................................................................
   166    172       END;
   167    173       INSERT INTO main.t300 VALUES(3);
   168    174       INSERT INTO temp.t300 VALUES(4);
   169    175       SELECT * FROM t301;
   170    176     }
   171    177   } {10003 20004}
   172    178   
          179  +
          180  +#############################################################################
          181  +# 
          182  +# Ticket [d6ddba6706353915ceedc56b4e3e72ecb4d77ba4]
          183  +#
          184  +# The following syntax really should not be allowed:
          185  +#
          186  +#    CREATE TRIGGER xyz.trig BEFORE UPDATE ON xyz.tab BEGIN ...
          187  +#
          188  +# But a long-standing bug does allow it.  And the "xyz.tab" slips into
          189  +# the sqlite_master table.  We cannot fix the bug simply by disallowing
          190  +# "xyz.tab" since that could break legacy applications.  We have to 
          191  +# fix the system so that the "xyz." on "xyz.tab" is ignored.
          192  +# Verify that this is the case.
          193  +#
          194  +do_test triggerD-4.1 {
          195  +  db close
          196  +  file delete -force test.db test2.db
          197  +  sqlite3 db test.db
          198  +  db eval {
          199  +    CREATE TABLE t1(x);
          200  +    ATTACH 'test2.db' AS db2;
          201  +    CREATE TABLE db2.t2(y);
          202  +    CREATE TABLE db2.log(z);
          203  +    CREATE TRIGGER db2.trig AFTER INSERT ON db2.t2 BEGIN
          204  +      INSERT INTO log(z) VALUES(new.y);
          205  +    END;
          206  +    INSERT INTO t2 VALUES(123);
          207  +    SELECT * FROM log;
          208  +  }
          209  +} {123}
          210  +do_test triggerD-4.2 {
          211  +  sqlite3 db2 test2.db
          212  +  db2 eval {
          213  +    INSERT INTO t2 VALUES(234);
          214  +    SELECT * FROM log;
          215  +  }
          216  +} {123 234}
          217  +db2 close
   173    218   
   174    219   finish_test

Added test/vtabF.test.

            1  +# 2011 Aug 1
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library. 
           12  +# This file checks to make sure IS NOT NULL constraints work on
           13  +# virtual tables.
           14  +#
           15  +
           16  +set testdir [file dirname $argv0]
           17  +source $testdir/tester.tcl
           18  +
           19  +ifcapable !vtab||!schema_pragmas { finish_test ; return }
           20  +
           21  +# Register the echo module
           22  +register_echo_module [sqlite3_connection_pointer db]
           23  +
           24  +do_test vtabE-1.1 {
           25  +  execsql {
           26  +    CREATE TABLE t1(a, b);
           27  +    CREATE INDEX i1 ON t1(a);
           28  +    CREATE INDEX i2 ON t1(b);
           29  +    INSERT INTO t1 VALUES(10,110);
           30  +    INSERT INTO t1 VALUES(11,111);
           31  +    INSERT INTO t1 SELECT a+2, b+2 FROM t1;
           32  +    INSERT INTO t1 SELECT null, b+4 FROM t1;
           33  +    INSERT INTO t1 SELECT null, b+8 FROM t1;
           34  +    INSERT INTO t1 SELECT null, b+16 FROM t1;
           35  +    ANALYZE;
           36  +    CREATE VIRTUAL TABLE tv1 USING echo(t1);
           37  +    SELECT b FROM t1 WHERE a IS NOT NULL;
           38  +  }
           39  +} {110 111 112 113}
           40  +do_test vtabE-1.2 {
           41  +  execsql {SELECT b FROM tv1 WHERE a IS NOT NULL}
           42  +} {110 111 112 113}
           43  +
           44  +
           45  +finish_test

Changes to test/wal3.test.

   721    721   #
   722    722   # This test case verifies that if an exclusive lock cannot be obtained
   723    723   # on any aReadMark[] slot (because there are already several readers),
   724    724   # the client takes a shared-lock on a slot without modifying the value
   725    725   # and continues.
   726    726   #
   727    727   set nConn 50
   728         -if { [string match *BSD $tcl_platform(os)] } { set nConn 35 }
          728  +if { [string match *BSD $tcl_platform(os)] } { set nConn 25 }
   729    729   do_test wal3-9.0 {
   730    730     file delete -force test.db test.db-journal test.db wal
   731    731     sqlite3 db test.db
   732    732     execsql {
   733    733       PRAGMA page_size = 1024;
   734    734       PRAGMA journal_mode = WAL;
   735    735       CREATE TABLE whoami(x);
................................................................................
   798    798   
   799    799     do_test wal3-10.$tn.2 {
   800    800       sql2 {PRAGMA integrity_check}
   801    801     } {ok}
   802    802   }
   803    803   
   804    804   finish_test
   805         -

Changes to test/wal5.test.

   249    249       do_test 2.3.$tn.5 { sql1 { INSERT INTO t2 VALUES(3, 4) } } {}
   250    250       do_test 2.3.$tn.6 { file_page_counts } [
   251    251         list 1 [wal_frames db 4 3] 1 [wal_frames db 4 3]
   252    252       ]
   253    253       do_test 2.3.$tn.7 { code1 { do_wal_checkpoint db -mode full } } [
   254    254         list 1 [wal_frames db 4 3] [wal_frames db 3 2]
   255    255       ]
   256         -    if {$tcl_platform(platform) == "windows"} {
   257         -        # on unix, the size_hint is a no-op if no chunk size is set.
   258         -        # the windows implementation does not have a similar check,
   259         -        # and because of this, the db file size has an extra page.
   260         -        do_test 2.3.$tn.8 { file_page_counts } {2 7 2 7}
   261         -    } {
   262         -        do_test 2.3.$tn.8 { file_page_counts } [
   263         -          list 1 [wal_frames db 4 3] 2 [wal_frames db 4 3]
   264         -        ]
   265         -    }
          256  +    do_test 2.3.$tn.8 { file_page_counts } {2 7 2 7}
   266    257     }
   267    258   
   268    259     # Check that checkpoints block on the correct locks. And respond correctly
   269    260     # if they cannot obtain those locks. There are three locks that a checkpoint
   270    261     # may block on (in the following order):
   271    262     #
   272    263     #   1. The writer lock: FULL and RESTART checkpoints block until any writer
................................................................................
   373    364   
   374    365       do_test 3.$tn.6 { code3 { do_wal_checkpoint db3 } } {0 0 0}
   375    366     }
   376    367   }
   377    368   
   378    369   
   379    370   finish_test
   380         -

Changes to test/wal6.test.

    41     41   	    CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
    42     42   	    INSERT INTO t1 VALUES(1,2);
    43     43   	    SELECT * FROM t1;
    44     44   	  }
    45     45   	} {1 2}
    46     46   
    47     47   # Under Windows, you'll get an error trying to delete
    48         -# a file this is already opened.  For now, make sure 
    49         -# we get that error, then close the first connection
           48  +# a file this is already opened.  Close the first connection
    50     49   # so the other tests work.
    51     50   if {$tcl_platform(platform)=="windows"} {
    52     51     if {$jmode=="persist" || $jmode=="truncate"} {
    53         -	  do_test wal6-1.2.$jmode.win {
    54         -	    sqlite3 db2 test.db
    55         -	    catchsql {
    56         -		    PRAGMA journal_mode=WAL;
    57         -	    } db2
    58         -	  } {1 {disk I/O error}}
    59         -  	db2 close
    60         -	  db close
           52  +    db close
    61     53     }
    62     54   }
    63     55   
    64     56   	do_test wal6-1.2.$jmode {
    65     57   	  sqlite3 db2 test.db
    66     58   	  execsql {
    67     59   		PRAGMA journal_mode=WAL;
................................................................................
    85     77   	db close
    86     78   	db2 close
    87     79     forcedelete test.db
    88     80   
    89     81   }
    90     82   
    91     83   finish_test
    92         -

Added test/walpersist.test.

            1  +# 2011 July 26
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# This file contains tests for using WAL with persistent WAL file mode.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +source $testdir/lock_common.tcl
           18  +set ::testprefix walpersist
           19  +
           20  +do_test walpersist-1.0 {
           21  +  db eval {
           22  +    PRAGMA journal_mode=WAL;
           23  +    CREATE TABLE t1(a);
           24  +    INSERT INTO t1 VALUES(randomblob(5000));
           25  +  }
           26  +  file exists test.db-wal
           27  +} {1}
           28  +do_test walpersist-1.1 {
           29  +  file exists test.db-shm
           30  +} {1}
           31  +do_test walpersist-1.2 {
           32  +  db close
           33  +  list [file exists test.db] [file exists test.db-wal] [file exists test.db-shm]
           34  +} {1 0 0}
           35  +do_test walpersist-1.3 {
           36  +  sqlite3 db test.db
           37  +  db eval {SELECT length(a) FROM t1}
           38  +} {5000}
           39  +do_test walpersist-1.4 {
           40  +  list [file exists test.db] [file exists test.db-wal] [file exists test.db-shm]
           41  +} {1 1 1}
           42  +do_test walpersist-1.5 {
           43  +  file_control_persist_wal db -1
           44  +} {0 0}
           45  +do_test walpersist-1.6 {
           46  +  file_control_persist_wal db 1
           47  +} {0 1}
           48  +do_test walpersist-1.7 {
           49  +  file_control_persist_wal db -1
           50  +} {0 1}
           51  +do_test walpersist-1.8 {
           52  +  file_control_persist_wal db 0
           53  +} {0 0}
           54  +do_test walpersist-1.9 {
           55  +  file_control_persist_wal db -1
           56  +} {0 0}
           57  +do_test walpersist-1.10 {
           58  +  file_control_persist_wal db 1
           59  +} {0 1}
           60  +do_test walpersist-1.11 {
           61  +  db close
           62  +  list [file exists test.db] [file exists test.db-wal] [file exists test.db-shm]
           63  +} {1 1 1}
           64  +
           65  +  
           66  +
           67  +
           68  +finish_test

Added test/win32lock.test.

            1  +# 2011 July 11
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is recovery from transient manditory locks
           13  +# that sometimes appear on database files due to anti-virus software.
           14  +#
           15  +
           16  +if {$tcl_platform(platform)!="windows"} return
           17  +
           18  +set testdir [file dirname $argv0]
           19  +source $testdir/tester.tcl
           20  +
           21  +set testprefix win32lock
           22  +
           23  +db close
           24  +sqlite3_shutdown
           25  +test_sqlite3_log xLog
           26  +proc xLog {error_code msg} {
           27  +  lappend ::log $msg 
           28  +}
           29  +sqlite3 db test.db
           30  +
           31  +do_test win32lock-1.1 {
           32  +  db eval {
           33  +    PRAGMA cache_size=10;
           34  +    CREATE TABLE t1(x,y);
           35  +    INSERT INTO t1 VALUES(1,randomblob(100000));
           36  +    INSERT INTO t1 VALUES(2,randomblob(50000));
           37  +    INSERT INTO t1 VALUES(3,randomblob(25000));
           38  +    INSERT INTO t1 VALUES(4,randomblob(12500));
           39  +    SELECT x, length(y) FROM t1 ORDER BY rowid;
           40  +  }
           41  +} {1 100000 2 50000 3 25000 4 12500}
           42  +
           43  +unset -nocomplain delay1 rc msg
           44  +set delay1 50
           45  +set rc 0
           46  +set old_pending_byte [sqlite3_test_control_pending_byte 0x40000000]
           47  +while {1} {
           48  +  sqlite3_sleep 10
           49  +  lock_win32_file test.db 0 $::delay1
           50  +  set rc [catch {db eval {SELECT x, length(y) FROM t1 ORDER BY rowid}} msg]
           51  +  if {$rc} {
           52  +    do_test win32lock-1.2-$delay1-fin {
           53  +       set ::msg
           54  +    } {disk I/O error}
           55  +    break
           56  +  } else {
           57  +    do_test win32lock-1.2-$delay1 {
           58  +       set ::msg
           59  +    } {1 100000 2 50000 3 25000 4 12500}
           60  +    if {[info exists ::log] && $::log!=""} {
           61  +      do_test win32lock-1.2-$delay1-log1 {
           62  +        regsub {\d+} $::log # x
           63  +        set x
           64  +      } {{delayed #ms for lock/sharing conflict}}
           65  +    }
           66  +    incr delay1 50
           67  +  }
           68  +  set ::log {}
           69  +}
           70  +
           71  +do_test win32lock-2.0 {
           72  +  file_control_win32_av_retry db -1 -1
           73  +} {0 10 25}
           74  +do_test win32lock-2.1 {
           75  +  file_control_win32_av_retry db 1 1
           76  +} {0 1 1}
           77  +
           78  +set delay1 50
           79  +while {1} {
           80  +  sqlite3_sleep 10
           81  +  lock_win32_file test.db 0 $::delay1
           82  +  set rc [catch {db eval {SELECT x, length(y) FROM t1 ORDER BY rowid}} msg]
           83  +  if {$rc} {
           84  +    do_test win32lock-2.2-$delay1-fin {
           85  +       set ::msg
           86  +    } {disk I/O error}
           87  +    break
           88  +  } else {
           89  +    do_test win32lock-2.2-$delay1 {
           90  +       set ::msg
           91  +    } {1 100000 2 50000 3 25000 4 12500}
           92  +    if {$::log!=""} {
           93  +      do_test win32lock-2.2-$delay1-log1 {
           94  +        regsub {\d+} $::log # x
           95  +        set x
           96  +      } {{delayed #ms for lock/sharing conflict}}
           97  +    }
           98  +    incr delay1 50
           99  +  }
          100  +  set ::log {}
          101  +}
          102  +
          103  +file_control_win32_av_retry db 10 25
          104  +sqlite3_test_control_pending_byte $old_pending_byte
          105  +sqlite3_shutdown
          106  +test_sqlite3_log 
          107  +sqlite3_initialize
          108  +finish_test

Changes to tool/symbols.sh.

     8      8   echo '****** Exported symbols from a build including RTREE, FTS4 & ICU ******'
     9      9   gcc -c -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \
    10     10     -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \
    11     11     -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \
    12     12     -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \
    13     13     -DSQLITE_ENABLE_ICU \
    14     14     sqlite3.c
    15         -nm sqlite3.o | grep ' T ' | sort -k 3
           15  +nm sqlite3.o | grep ' [TD] ' | sort -k 3
    16     16   
    17     17   echo '****** Surplus symbols from a build including RTREE, FTS4 & ICU ******'
    18         -nm sqlite3.o | grep ' T ' | grep -v ' sqlite3_'
           18  +nm sqlite3.o | grep ' [TD] ' | grep -v ' .*sqlite3_'
    19     19   
    20     20   echo '****** Dependencies of the core. No extensions. No OS interface *******'
    21     21   gcc -c -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \
    22     22     -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \
    23     23     -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \
    24     24     -DSQLITE_OS_OTHER -DSQLITE_THREADSAFE=0 \
    25     25     sqlite3.c

Changes to tool/warnings.sh.

     4      4   # compiler warnings in SQLite.
     5      5   #
     6      6   make sqlite3.c
     7      7   echo '********** No optimizations.  Includes FTS4 and RTREE *********'
     8      8   gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \
     9      9         -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \
    10     10         sqlite3.c
    11         -echo '********** Optimized -O3.  Includes FTS4 and RTREE *********'
           11  +echo '********** No optimizations. ENABLE_STAT2. THREADSAFE=0 *******'
           12  +gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \
           13  +      -ansi -DSQLITE_ENABLE_STAT2 -DSQLITE_THREADSAFE=0 \
           14  +      sqlite3.c
           15  +echo '********** Optimized -O3.  Includes FTS4 and RTREE ************'
    12     16   gcc -O3 -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \
    13     17         -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \
    14     18         sqlite3.c