/ Check-in [110cfd69]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge the latest trunk changes into the sessions branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | sessions
Files: files | file ages | folders
SHA1: 110cfd6920cf3011aeaf7e586f8db867bfc69fbb
User & Date: drh 2011-07-22 12:49:27
Context
2011-07-26
15:50
Fix a problem causing sqlite3changeset_invert() to effectively drop UPDATE changes. check-in: bb3e65d9 user: dan tags: sessions
2011-07-22
12:49
Merge the latest trunk changes into the sessions branch. check-in: 110cfd69 user: drh tags: sessions
11:23
Also add the sqlite3-all.c target to the MSVC makefile. check-in: 8ce2b74a user: drh tags: trunk
2011-07-18
15:22
Fix the sqlite3session_isempty() method so that it returns, as documented, non-zero when no changes have been recorded by the session object. check-in: d04e0fd8 user: dan tags: sessions
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   513    513   	$(TCLSH_CMD) $(TOP)/tool/vdbe-compress.tcl <tsrc/vdbe.c >vdbe.new
   514    514   	mv vdbe.new tsrc/vdbe.c
   515    515   	touch .target_source
   516    516   
   517    517   sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl
   518    518   	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl
   519    519   
          520  +sqlite3-all.c:	sqlite3.c $(TOP)/tool/split-sqlite3c.tcl
          521  +	$(TCLSH_CMD) $(TOP)/tool/split-sqlite3c.tcl
          522  +
   520    523   # Rule to build the amalgamation
   521    524   #
   522    525   sqlite3.lo:	sqlite3.c
   523    526   	$(LTCOMPILE) $(TEMP_STORE) -c sqlite3.c
   524    527   
   525    528   # Rules to build the LEMON compiler generator
   526    529   #
................................................................................
   781    784   	cp $(TOP)/src/parse.y .
   782    785   	rm -f parse.h
   783    786   	./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y
   784    787   	mv parse.h parse.h.temp
   785    788   	$(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h
   786    789   
   787    790   sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION
   788         -	tclsh $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
          791  +	$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
   789    792   
   790    793   keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
   791    794   	$(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c
   792    795   	./mkkeywordhash$(BEXE) >keywordhash.h
   793    796   
   794    797   
   795    798   

Changes to Makefile.msc.

    39     39   
    40     40   # Define -DNDEBUG to compile without debugging (i.e., for production usage)
    41     41   # Omitting the define will cause extra debugging code to be inserted and
    42     42   # includes extra comments when "EXPLAIN stmt" is used.
    43     43   #
    44     44   TCC = $(TCC) -DNDEBUG
    45     45   
    46         -# The library that programs using TCL must link against.
           46  +# The locations of the Tcl header and library files.  Also, the library that
           47  +# non-stubs enabled programs using Tcl must link against.  These variables
           48  +# (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment
           49  +# prior to running nmake in order to match the actual installed location and
           50  +# version on this machine.
    47     51   #
    48         -LIBTCL = tcl85.lib
           52  +!if "$(TCLINCDIR)" == ""
    49     53   TCLINCDIR = c:\tcl\include
           54  +!endif
           55  +
           56  +!if "$(TCLLIBDIR)" == ""
    50     57   TCLLIBDIR = c:\tcl\lib
           58  +!endif
           59  +
           60  +!if "$(LIBTCL)" == ""
           61  +LIBTCL = tcl85.lib
           62  +!endif
    51     63   
    52     64   # This is the command to use for tclsh - normally just "tclsh", but we may
    53         -# know the specific version we want to use
           65  +# know the specific version we want to use.  This variable (TCLSH_CMD) may be
           66  +# overridden via the environment prior to running nmake in order to select a
           67  +# specific Tcl shell to use.
    54     68   #
           69  +!if "$(TCLSH_CMD)" == ""
    55     70   TCLSH_CMD = tclsh85
           71  +!endif
    56     72   
    57     73   # Compiler options needed for programs that use the readline() library.
    58     74   #
    59     75   READLINE_FLAGS = -DHAVE_READLINE=0
    60     76   
    61     77   # The library that programs using readline() must link against.
    62     78   #
................................................................................
   114    130   # to deduce the binary type based on the object files.
   115    131   !IF "$(PLATFORM)"!=""
   116    132   LTLINKOPTS = /MACHINE:$(PLATFORM)
   117    133   LTLIBOPTS = /MACHINE:$(PLATFORM)
   118    134   !ENDIF
   119    135   
   120    136   # nawk compatible awk.
   121         -NAWK = .\gawk.exe
          137  +NAWK = gawk.exe
   122    138   
   123    139   # You should not have to change anything below this line
   124    140   ###############################################################################
   125    141   
   126    142   # Object files for the SQLite library (non-amalgamation).
   127    143   #
   128    144   LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
................................................................................
   435    451     $(TOP)\ext\icu\sqliteicu.h
   436    452   EXTHDR = $(EXTHDR) \
   437    453     $(TOP)\ext\rtree\sqlite3rtree.h
   438    454   
   439    455   # This is the default Makefile target.  The objects listed here
   440    456   # are what get build when you type just "make" with no arguments.
   441    457   #
   442         -all:	libsqlite3.lib sqlite3.exe libtclsqlite3.lib
          458  +all:	dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib
   443    459   
   444    460   libsqlite3.lib:	$(LIBOBJ)
   445    461   	$(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS)
   446    462   
   447    463   libtclsqlite3.lib:	tclsqlite.lo libsqlite3.lib
   448    464   	$(LTLIB) $(LTLIBOPTS) /LIBPATH:$(TCLLIBDIR) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCL:tcl=tclstub) $(TLIBS)
   449    465   
................................................................................
   459    475   # all that automatic generation.
   460    476   #
   461    477   .target_source:	$(SRC) $(TOP)\tool\vdbe-compress.tcl
   462    478   	-rmdir /S/Q tsrc
   463    479   	-mkdir tsrc
   464    480   	for %i in ($(SRC)) do copy /Y %i tsrc
   465    481   	del /Q tsrc\sqlite.h.in tsrc\parse.y
   466         -	$(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl <tsrc\vdbe.c >vdbe.new
          482  +	$(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl < tsrc\vdbe.c > vdbe.new
   467    483   	move vdbe.new tsrc\vdbe.c
   468    484   	echo > .target_source
   469    485   
   470    486   sqlite3.c:	.target_source $(TOP)\tool\mksqlite3c.tcl
   471    487   	$(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl
          488  +
          489  +sqlite3-all.c:	sqlite3.c $(TOP)/tool/split-sqlite3c.tcl
          490  +	$(TCLSH_CMD) $(TOP)/tool/split-sqlite3c.tcl
   472    491   
   473    492   # Rule to build the amalgamation
   474    493   #
   475    494   sqlite3.lo:	sqlite3.c
   476    495   	$(LTCOMPILE) -c sqlite3.c
   477    496   
   478    497   # Rules to build the LEMON compiler generator
................................................................................
   716    735   tclsqlite3.exe:	tclsqlite-shell.lo libsqlite3.lib
   717    736   	$(LTLINK) tclsqlite-shell.lo \
   718    737   		/link $(LTLINKOPTS) /LIBPATH:$(TCLLIBDIR) libsqlite3.lib $(LIBTCL)
   719    738   
   720    739   # Rules to build opcodes.c and opcodes.h
   721    740   #
   722    741   opcodes.c:	opcodes.h $(TOP)\mkopcodec.awk
   723         -	$(NAWK) "/#define OP_/ { print }" opcodes.h | sort /+45 | $(NAWK) -f $(TOP)\mkopcodec.awk >opcodes.c
          742  +	$(NAWK) "/#define OP_/ { print }" opcodes.h | sort /+45 | $(NAWK) -f $(TOP)\mkopcodec.awk > opcodes.c
   724    743   
   725    744   opcodes.h:	parse.h $(TOP)\src\vdbe.c $(TOP)\mkopcodeh.awk
   726         -	type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk >opcodes.h
          745  +	type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk > opcodes.h
   727    746   
   728    747   # Rules to build parse.c and parse.h - the outputs of lemon.
   729    748   #
   730    749   parse.h:	parse.c
   731    750   
   732    751   parse.c:	$(TOP)\src\parse.y lemon.exe $(TOP)\addopcodes.awk
   733    752   	del /Q parse.y parse.h parse.h.temp
   734    753   	copy $(TOP)\src\parse.y .
   735    754   	.\lemon.exe $(OPT_FEATURE_FLAGS) $(OPTS) parse.y
   736    755   	move parse.h parse.h.temp
   737         -	$(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp >parse.h
          756  +	$(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp > parse.h
   738    757   
   739    758   sqlite3.h:	$(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION
   740         -	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP) >sqlite3.h
          759  +	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP) > sqlite3.h
   741    760   
   742    761   mkkeywordhash.exe:	$(TOP)\tool\mkkeywordhash.c
   743    762   	$(BCC) -Femkkeywordhash.exe $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)\tool\mkkeywordhash.c
   744    763   
   745    764   keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
   746         -	.\mkkeywordhash.exe >keywordhash.h
          765  +	.\mkkeywordhash.exe > keywordhash.h
   747    766   
   748    767   
   749    768   
   750    769   # Rules to build the extension objects.
   751    770   #
   752    771   icu.lo:	$(TOP)\ext\icu\icu.c $(HDR) $(EXTHDR)
   753    772   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\icu\icu.c
................................................................................
   835    854   	.\testfixture.exe $(TOP)\test\all.test -soak=1
   836    855   
   837    856   test:	testfixture.exe sqlite3.exe
   838    857   	.\testfixture.exe $(TOP)\test\veryquick.test
   839    858   
   840    859   spaceanal_tcl.h:	$(TOP)\tool\spaceanal.tcl
   841    860   	$(NAWK) -f $(TOP)/tool/tostr.awk \
   842         -		$(TOP)\tool\spaceanal.tcl >spaceanal_tcl.h
          861  +		$(TOP)\tool\spaceanal.tcl > spaceanal_tcl.h
   843    862   
   844    863   sqlite3_analyzer.exe:	$(TESTFIXTURE_SRC) spaceanal_tcl.h
   845    864   	$(LTLINK) -DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 \
   846    865   		-DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE \
   847    866   		-DBUILD_sqlite -I$(TCLINCDIR) \
   848    867   		$(TESTFIXTURE_SRC) \
   849    868   		/link $(LTLINKOPTS) /LIBPATH:$(TCLLIBDIR) $(LIBTCL) $(TLIBS)
................................................................................
   862    881   
   863    882   #
   864    883   # Windows section
   865    884   #
   866    885   dll: sqlite3.dll
   867    886   
   868    887   sqlite3.def: libsqlite3.lib
   869         -	echo EXPORTS >sqlite3.def
          888  +	echo EXPORTS > sqlite3.def
   870    889   	dumpbin /all libsqlite3.lib \
   871    890   		| $(NAWK) "/ 1 _sqlite3_/ { sub(/^.* _/,\"\");print }" \
   872         -		| sort >>sqlite3.def
          891  +		| sort >> sqlite3.def
   873    892   
   874    893   sqlite3.dll: $(LIBOBJ) sqlite3.def
   875    894   	link $(LTLINKOPTS) /DLL /DEF:sqlite3.def /OUT:$@ $(LIBOBJ)

Changes to VERSION.

     1         -3.7.7
            1  +3.7.8

Changes to configure.

     1      1   #! /bin/sh
     2      2   # Guess values for system-dependent variables and create Makefiles.
     3         -# Generated by GNU Autoconf 2.62 for sqlite 3.7.7.
            3  +# Generated by GNU Autoconf 2.62 for sqlite 3.7.8.
     4      4   #
     5      5   # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
     6      6   # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
     7      7   # This configure script is free software; the Free Software Foundation
     8      8   # gives unlimited permission to copy, distribute and modify it.
     9      9   ## --------------------- ##
    10     10   ## M4sh Initialization.  ##
................................................................................
   739    739   MFLAGS=
   740    740   MAKEFLAGS=
   741    741   SHELL=${CONFIG_SHELL-/bin/sh}
   742    742   
   743    743   # Identity of this package.
   744    744   PACKAGE_NAME='sqlite'
   745    745   PACKAGE_TARNAME='sqlite'
   746         -PACKAGE_VERSION='3.7.7'
   747         -PACKAGE_STRING='sqlite 3.7.7'
          746  +PACKAGE_VERSION='3.7.8'
          747  +PACKAGE_STRING='sqlite 3.7.8'
   748    748   PACKAGE_BUGREPORT=''
   749    749   
   750    750   # Factoring default headers for most tests.
   751    751   ac_includes_default="\
   752    752   #include <stdio.h>
   753    753   #ifdef HAVE_SYS_TYPES_H
   754    754   # include <sys/types.h>
................................................................................
  1481   1481   #
  1482   1482   # Report the --help message.
  1483   1483   #
  1484   1484   if test "$ac_init_help" = "long"; then
  1485   1485     # Omit some internal or obsolete options to make the list less imposing.
  1486   1486     # This message is too long to be a string in the A/UX 3.1 sh.
  1487   1487     cat <<_ACEOF
  1488         -\`configure' configures sqlite 3.7.7 to adapt to many kinds of systems.
         1488  +\`configure' configures sqlite 3.7.8 to adapt to many kinds of systems.
  1489   1489   
  1490   1490   Usage: $0 [OPTION]... [VAR=VALUE]...
  1491   1491   
  1492   1492   To assign environment variables (e.g., CC, CFLAGS...), specify them as
  1493   1493   VAR=VALUE.  See below for descriptions of some of the useful variables.
  1494   1494   
  1495   1495   Defaults for the options are specified in brackets.
................................................................................
  1546   1546     --build=BUILD     configure for building on BUILD [guessed]
  1547   1547     --host=HOST       cross-compile to build programs to run on HOST [BUILD]
  1548   1548   _ACEOF
  1549   1549   fi
  1550   1550   
  1551   1551   if test -n "$ac_init_help"; then
  1552   1552     case $ac_init_help in
  1553         -     short | recursive ) echo "Configuration of sqlite 3.7.7:";;
         1553  +     short | recursive ) echo "Configuration of sqlite 3.7.8:";;
  1554   1554      esac
  1555   1555     cat <<\_ACEOF
  1556   1556   
  1557   1557   Optional Features:
  1558   1558     --disable-option-checking  ignore unrecognized --enable/--with options
  1559   1559     --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  1560   1560     --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
  1662   1662       cd "$ac_pwd" || { ac_status=$?; break; }
  1663   1663     done
  1664   1664   fi
  1665   1665   
  1666   1666   test -n "$ac_init_help" && exit $ac_status
  1667   1667   if $ac_init_version; then
  1668   1668     cat <<\_ACEOF
  1669         -sqlite configure 3.7.7
         1669  +sqlite configure 3.7.8
  1670   1670   generated by GNU Autoconf 2.62
  1671   1671   
  1672   1672   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
  1673   1673   2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
  1674   1674   This configure script is free software; the Free Software Foundation
  1675   1675   gives unlimited permission to copy, distribute and modify it.
  1676   1676   _ACEOF
  1677   1677     exit
  1678   1678   fi
  1679   1679   cat >config.log <<_ACEOF
  1680   1680   This file contains any messages produced by compilers while
  1681   1681   running configure, to aid debugging if configure makes a mistake.
  1682   1682   
  1683         -It was created by sqlite $as_me 3.7.7, which was
         1683  +It was created by sqlite $as_me 3.7.8, which was
  1684   1684   generated by GNU Autoconf 2.62.  Invocation command line was
  1685   1685   
  1686   1686     $ $0 $@
  1687   1687   
  1688   1688   _ACEOF
  1689   1689   exec 5>>config.log
  1690   1690   {
................................................................................
 14026  14026   
 14027  14027   exec 6>&1
 14028  14028   
 14029  14029   # Save the log message, to keep $[0] and so on meaningful, and to
 14030  14030   # report actual input values of CONFIG_FILES etc. instead of their
 14031  14031   # values after options handling.
 14032  14032   ac_log="
 14033         -This file was extended by sqlite $as_me 3.7.7, which was
        14033  +This file was extended by sqlite $as_me 3.7.8, which was
 14034  14034   generated by GNU Autoconf 2.62.  Invocation command line was
 14035  14035   
 14036  14036     CONFIG_FILES    = $CONFIG_FILES
 14037  14037     CONFIG_HEADERS  = $CONFIG_HEADERS
 14038  14038     CONFIG_LINKS    = $CONFIG_LINKS
 14039  14039     CONFIG_COMMANDS = $CONFIG_COMMANDS
 14040  14040     $ $0 $@
................................................................................
 14079  14079   $config_commands
 14080  14080   
 14081  14081   Report bugs to <bug-autoconf@gnu.org>."
 14082  14082   
 14083  14083   _ACEOF
 14084  14084   cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 14085  14085   ac_cs_version="\\
 14086         -sqlite config.status 3.7.7
        14086  +sqlite config.status 3.7.8
 14087  14087   configured by $0, generated by GNU Autoconf 2.62,
 14088  14088     with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\"
 14089  14089   
 14090  14090   Copyright (C) 2008 Free Software Foundation, Inc.
 14091  14091   This config.status script is free software; the Free Software Foundation
 14092  14092   gives unlimited permission to copy, distribute and modify it."
 14093  14093   

Changes to ext/fts3/fts3.c.

   308    308   
   309    309   #include "fts3.h"
   310    310   #ifndef SQLITE_CORE 
   311    311   # include "sqlite3ext.h"
   312    312     SQLITE_EXTENSION_INIT1
   313    313   #endif
   314    314   
          315  +static int fts3EvalNext(Fts3Cursor *pCsr);
          316  +static int fts3EvalStart(Fts3Cursor *pCsr);
          317  +static int fts3TermSegReaderCursor(
          318  +    Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **);
          319  +
   315    320   /* 
   316    321   ** Write a 64-bit variable-length integer to memory starting at p[0].
   317    322   ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes.
   318    323   ** The number of bytes written is returned.
   319    324   */
   320    325   int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){
   321    326     unsigned char *q = (unsigned char *) p;
................................................................................
   816    821     for(i=0; i<p->nColumn; i++){
   817    822       fts3Appendf(pRc, &zRet, ",%s(?)", zFunction);
   818    823     }
   819    824     sqlite3_free(zFree);
   820    825     return zRet;
   821    826   }
   822    827   
          828  +/*
          829  +** This function interprets the string at (*pp) as a non-negative integer
          830  +** value. It reads the integer and sets *pnOut to the value read, then 
          831  +** sets *pp to point to the byte immediately following the last byte of
          832  +** the integer value.
          833  +**
          834  +** Only decimal digits ('0'..'9') may be part of an integer value. 
          835  +**
          836  +** If *pp does not being with a decimal digit SQLITE_ERROR is returned and
          837  +** the output value undefined. Otherwise SQLITE_OK is returned.
          838  +**
          839  +** This function is used when parsing the "prefix=" FTS4 parameter.
          840  +*/
   823    841   static int fts3GobbleInt(const char **pp, int *pnOut){
   824         -  const char *p = *pp;
   825         -  int nInt = 0;
          842  +  const char *p = *pp;            /* Iterator pointer */
          843  +  int nInt = 0;                   /* Output value */
          844  +
   826    845     for(p=*pp; p[0]>='0' && p[0]<='9'; p++){
   827    846       nInt = nInt * 10 + (p[0] - '0');
   828    847     }
   829    848     if( p==*pp ) return SQLITE_ERROR;
   830    849     *pnOut = nInt;
   831    850     *pp = p;
   832    851     return SQLITE_OK;
   833    852   }
   834    853   
   835         -
          854  +/*
          855  +** This function is called to allocate an array of Fts3Index structures
          856  +** representing the indexes maintained by the current FTS table. FTS tables
          857  +** always maintain the main "terms" index, but may also maintain one or
          858  +** more "prefix" indexes, depending on the value of the "prefix=" parameter
          859  +** (if any) specified as part of the CREATE VIRTUAL TABLE statement.
          860  +**
          861  +** Argument zParam is passed the value of the "prefix=" option if one was
          862  +** specified, or NULL otherwise.
          863  +**
          864  +** If no error occurs, SQLITE_OK is returned and *apIndex set to point to
          865  +** the allocated array. *pnIndex is set to the number of elements in the
          866  +** array. If an error does occur, an SQLite error code is returned.
          867  +**
          868  +** Regardless of whether or not an error is returned, it is the responsibility
          869  +** of the caller to call sqlite3_free() on the output array to free it.
          870  +*/
   836    871   static int fts3PrefixParameter(
   837    872     const char *zParam,             /* ABC in prefix=ABC parameter to parse */
   838    873     int *pnIndex,                   /* OUT: size of *apIndex[] array */
   839         -  struct Fts3Index **apIndex,     /* OUT: Array of indexes for this table */
   840         -  struct Fts3Index **apFree       /* OUT: Free this with sqlite3_free() */
          874  +  struct Fts3Index **apIndex      /* OUT: Array of indexes for this table */
   841    875   ){
   842         -  struct Fts3Index *aIndex;
   843         -  int nIndex = 1;
          876  +  struct Fts3Index *aIndex;       /* Allocated array */
          877  +  int nIndex = 1;                 /* Number of entries in array */
   844    878   
   845    879     if( zParam && zParam[0] ){
   846    880       const char *p;
   847    881       nIndex++;
   848    882       for(p=zParam; *p; p++){
   849    883         if( *p==',' ) nIndex++;
   850    884       }
   851    885     }
   852    886   
   853    887     aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex);
   854         -  *apIndex = *apFree = aIndex;
          888  +  *apIndex = aIndex;
   855    889     *pnIndex = nIndex;
   856    890     if( !aIndex ){
   857    891       return SQLITE_NOMEM;
   858    892     }
   859    893   
   860    894     memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex);
   861    895     if( zParam ){
................................................................................
   904    938     int nDb;                        /* Bytes required to hold database name */
   905    939     int nName;                      /* Bytes required to hold table name */
   906    940     int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
   907    941     const char **aCol;              /* Array of column names */
   908    942     sqlite3_tokenizer *pTokenizer = 0;        /* Tokenizer for this table */
   909    943   
   910    944     int nIndex;                     /* Size of aIndex[] array */
   911         -  struct Fts3Index *aIndex;       /* Array of indexes for this table */
   912         -  struct Fts3Index *aFree = 0;    /* Free this before returning */
          945  +  struct Fts3Index *aIndex = 0;   /* Array of indexes for this table */
   913    946   
   914    947     /* The results of parsing supported FTS4 key=value options: */
   915    948     int bNoDocsize = 0;             /* True to omit %_docsize table */
   916    949     int bDescIdx = 0;               /* True to store descending indexes */
   917    950     char *zPrefix = 0;              /* Prefix parameter value (or NULL) */
   918    951     char *zCompress = 0;            /* compress=? parameter (or NULL) */
   919    952     char *zUncompress = 0;          /* uncompress=? parameter (or NULL) */
................................................................................
  1042   1075   
  1043   1076     if( pTokenizer==0 ){
  1044   1077       rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
  1045   1078       if( rc!=SQLITE_OK ) goto fts3_init_out;
  1046   1079     }
  1047   1080     assert( pTokenizer );
  1048   1081   
  1049         -  rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree);
         1082  +  rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex);
  1050   1083     if( rc==SQLITE_ERROR ){
  1051   1084       assert( zPrefix );
  1052   1085       *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix);
  1053   1086     }
  1054   1087     if( rc!=SQLITE_OK ) goto fts3_init_out;
  1055   1088   
  1056   1089     /* Allocate and populate the Fts3Table structure. */
................................................................................
  1129   1162     p->nNodeSize = p->nPgsz-35;
  1130   1163   
  1131   1164     /* Declare the table schema to SQLite. */
  1132   1165     fts3DeclareVtab(&rc, p);
  1133   1166   
  1134   1167   fts3_init_out:
  1135   1168     sqlite3_free(zPrefix);
  1136         -  sqlite3_free(aFree);
         1169  +  sqlite3_free(aIndex);
  1137   1170     sqlite3_free(zCompress);
  1138   1171     sqlite3_free(zUncompress);
  1139   1172     sqlite3_free((void *)aCol);
  1140   1173     if( rc!=SQLITE_OK ){
  1141   1174       if( p ){
  1142   1175         fts3DisconnectMethod((sqlite3_vtab *)p);
  1143   1176       }else if( pTokenizer ){
................................................................................
  1720   1753     *p++ = POS_END;
  1721   1754     *pp = p;
  1722   1755     *pp1 = p1 + 1;
  1723   1756     *pp2 = p2 + 1;
  1724   1757   }
  1725   1758   
  1726   1759   /*
  1727         -** nToken==1 searches for adjacent positions.
  1728         -**
  1729   1760   ** This function is used to merge two position lists into one. When it is
  1730   1761   ** called, *pp1 and *pp2 must both point to position lists. A position-list is
  1731   1762   ** the part of a doclist that follows each document id. For example, if a row
  1732   1763   ** contains:
  1733   1764   **
  1734   1765   **     'a b c'|'x y z'|'a b b a'
  1735   1766   **
................................................................................
  1741   1772   ** byte following the 0x00 terminator of their respective position lists.
  1742   1773   **
  1743   1774   ** If isSaveLeft is 0, an entry is added to the output position list for 
  1744   1775   ** each position in *pp2 for which there exists one or more positions in
  1745   1776   ** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e.
  1746   1777   ** when the *pp1 token appears before the *pp2 token, but not more than nToken
  1747   1778   ** slots before it.
         1779  +**
         1780  +** e.g. nToken==1 searches for adjacent positions.
  1748   1781   */
  1749   1782   static int fts3PoslistPhraseMerge(
  1750   1783     char **pp,                      /* IN/OUT: Preallocated output buffer */
  1751   1784     int nToken,                     /* Maximum difference in token positions */
  1752   1785     int isSaveLeft,                 /* Save the left position */
  1753   1786     int isExact,                    /* If *pp1 is exactly nTokens before *pp2 */
  1754   1787     char **pp1,                     /* IN/OUT: Left input list */
................................................................................
  1907   1940       res = 0;
  1908   1941     }
  1909   1942   
  1910   1943     return res;
  1911   1944   }
  1912   1945   
  1913   1946   /* 
  1914         -** A pointer to an instance of this structure is used as the context 
  1915         -** argument to sqlite3Fts3SegReaderIterate()
         1947  +** An instance of this function is used to merge together the (potentially
         1948  +** large number of) doclists for each term that matches a prefix query.
         1949  +** See function fts3TermSelectMerge() for details.
  1916   1950   */
  1917   1951   typedef struct TermSelect TermSelect;
  1918   1952   struct TermSelect {
  1919         -  int isReqPos;
  1920         -  char *aaOutput[16];             /* Malloc'd output buffer */
  1921         -  int anOutput[16];               /* Size of output in bytes */
         1953  +  char *aaOutput[16];             /* Malloc'd output buffers */
         1954  +  int anOutput[16];               /* Size each output buffer in bytes */
  1922   1955   };
  1923   1956   
  1924         -
         1957  +/*
         1958  +** This function is used to read a single varint from a buffer. Parameter
         1959  +** pEnd points 1 byte past the end of the buffer. When this function is
         1960  +** called, if *pp points to pEnd or greater, then the end of the buffer
         1961  +** has been reached. In this case *pp is set to 0 and the function returns.
         1962  +**
         1963  +** If *pp does not point to or past pEnd, then a single varint is read
         1964  +** from *pp. *pp is then set to point 1 byte past the end of the read varint.
         1965  +**
         1966  +** If bDescIdx is false, the value read is added to *pVal before returning.
         1967  +** If it is true, the value read is subtracted from *pVal before this 
         1968  +** function returns.
         1969  +*/
  1925   1970   static void fts3GetDeltaVarint3(
  1926         -  char **pp, 
  1927         -  char *pEnd, 
  1928         -  int bDescIdx,
  1929         -  sqlite3_int64 *pVal
         1971  +  char **pp,                      /* IN/OUT: Point to read varint from */
         1972  +  char *pEnd,                     /* End of buffer */
         1973  +  int bDescIdx,                   /* True if docids are descending */
         1974  +  sqlite3_int64 *pVal             /* IN/OUT: Integer value */
  1930   1975   ){
  1931   1976     if( *pp>=pEnd ){
  1932   1977       *pp = 0;
  1933   1978     }else{
  1934   1979       sqlite3_int64 iVal;
  1935   1980       *pp += sqlite3Fts3GetVarint(*pp, &iVal);
  1936   1981       if( bDescIdx ){
................................................................................
  1937   1982         *pVal -= iVal;
  1938   1983       }else{
  1939   1984         *pVal += iVal;
  1940   1985       }
  1941   1986     }
  1942   1987   }
  1943   1988   
         1989  +/*
         1990  +** This function is used to write a single varint to a buffer. The varint
         1991  +** is written to *pp. Before returning, *pp is set to point 1 byte past the
         1992  +** end of the value written.
         1993  +**
         1994  +** If *pbFirst is zero when this function is called, the value written to
         1995  +** the buffer is that of parameter iVal. 
         1996  +**
         1997  +** If *pbFirst is non-zero when this function is called, then the value 
         1998  +** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal)
         1999  +** (if bDescIdx is non-zero).
         2000  +**
         2001  +** Before returning, this function always sets *pbFirst to 1 and *piPrev
         2002  +** to the value of parameter iVal.
         2003  +*/
  1944   2004   static void fts3PutDeltaVarint3(
  1945   2005     char **pp,                      /* IN/OUT: Output pointer */
  1946   2006     int bDescIdx,                   /* True for descending docids */
  1947   2007     sqlite3_int64 *piPrev,          /* IN/OUT: Previous value written to list */
  1948   2008     int *pbFirst,                   /* IN/OUT: True after first int written */
  1949   2009     sqlite3_int64 iVal              /* Write this value to the list */
  1950   2010   ){
................................................................................
  1957   2017     assert( *pbFirst || *piPrev==0 );
  1958   2018     assert( *pbFirst==0 || iWrite>0 );
  1959   2019     *pp += sqlite3Fts3PutVarint(*pp, iWrite);
  1960   2020     *piPrev = iVal;
  1961   2021     *pbFirst = 1;
  1962   2022   }
  1963   2023   
  1964         -#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2))
  1965   2024   
         2025  +/*
         2026  +** This macro is used by various functions that merge doclists. The two
         2027  +** arguments are 64-bit docid values. If the value of the stack variable
         2028  +** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). 
         2029  +** Otherwise, (i2-i1).
         2030  +**
         2031  +** Using this makes it easier to write code that can merge doclists that are
         2032  +** sorted in either ascending or descending order.
         2033  +*/
         2034  +#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2))
         2035  +
         2036  +/*
         2037  +** This function does an "OR" merge of two doclists (output contains all
         2038  +** positions contained in either argument doclist). If the docids in the 
         2039  +** input doclists are sorted in ascending order, parameter bDescDoclist
         2040  +** should be false. If they are sorted in ascending order, it should be
         2041  +** passed a non-zero value.
         2042  +**
         2043  +** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer
         2044  +** containing the output doclist and SQLITE_OK is returned. In this case
         2045  +** *pnOut is set to the number of bytes in the output doclist.
         2046  +**
         2047  +** If an error occurs, an SQLite error code is returned. The output values
         2048  +** are undefined in this case.
         2049  +*/
  1966   2050   static int fts3DoclistOrMerge(
  1967         -  int bDescIdx,                   /* True if arguments are desc */
         2051  +  int bDescDoclist,               /* True if arguments are desc */
  1968   2052     char *a1, int n1,               /* First doclist */
  1969   2053     char *a2, int n2,               /* Second doclist */
  1970   2054     char **paOut, int *pnOut        /* OUT: Malloc'd doclist */
  1971   2055   ){
  1972   2056     sqlite3_int64 i1 = 0;
  1973   2057     sqlite3_int64 i2 = 0;
  1974   2058     sqlite3_int64 iPrev = 0;
................................................................................
  1985   2069     aOut = sqlite3_malloc(n1+n2);
  1986   2070     if( !aOut ) return SQLITE_NOMEM;
  1987   2071   
  1988   2072     p = aOut;
  1989   2073     fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
  1990   2074     fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
  1991   2075     while( p1 || p2 ){
  1992         -    sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
         2076  +    sqlite3_int64 iDiff = DOCID_CMP(i1, i2);
  1993   2077   
  1994   2078       if( p2 && p1 && iDiff==0 ){
  1995         -      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
         2079  +      fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
  1996   2080         fts3PoslistMerge(&p, &p1, &p2);
  1997         -      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
  1998         -      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
         2081  +      fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
         2082  +      fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
  1999   2083       }else if( !p2 || (p1 && iDiff<0) ){
  2000         -      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
         2084  +      fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
  2001   2085         fts3PoslistCopy(&p, &p1);
  2002         -      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
         2086  +      fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
  2003   2087       }else{
  2004         -      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2);
         2088  +      fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2);
  2005   2089         fts3PoslistCopy(&p, &p2);
  2006         -      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
         2090  +      fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
  2007   2091       }
  2008   2092     }
  2009   2093   
  2010   2094     *paOut = aOut;
  2011   2095     *pnOut = (p-aOut);
  2012   2096     return SQLITE_OK;
  2013   2097   }
  2014   2098   
         2099  +/*
         2100  +** This function does a "phrase" merge of two doclists. In a phrase merge,
         2101  +** the output contains a copy of each position from the right-hand input
         2102  +** doclist for which there is a position in the left-hand input doclist
         2103  +** exactly nDist tokens before it.
         2104  +**
         2105  +** If the docids in the input doclists are sorted in ascending order,
         2106  +** parameter bDescDoclist should be false. If they are sorted in ascending 
         2107  +** order, it should be passed a non-zero value.
         2108  +**
         2109  +** The right-hand input doclist is overwritten by this function.
         2110  +*/
  2015   2111   static void fts3DoclistPhraseMerge(
  2016         -  int bDescIdx,                   /* True if arguments are desc */
         2112  +  int bDescDoclist,               /* True if arguments are desc */
  2017   2113     int nDist,                      /* Distance from left to right (1=adjacent) */
  2018   2114     char *aLeft, int nLeft,         /* Left doclist */
  2019   2115     char *aRight, int *pnRight      /* IN/OUT: Right/output doclist */
  2020   2116   ){
  2021   2117     sqlite3_int64 i1 = 0;
  2022   2118     sqlite3_int64 i2 = 0;
  2023   2119     sqlite3_int64 iPrev = 0;
................................................................................
  2032   2128     assert( nDist>0 );
  2033   2129   
  2034   2130     p = aOut;
  2035   2131     fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
  2036   2132     fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
  2037   2133   
  2038   2134     while( p1 && p2 ){
  2039         -    sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
         2135  +    sqlite3_int64 iDiff = DOCID_CMP(i1, i2);
  2040   2136       if( iDiff==0 ){
  2041   2137         char *pSave = p;
  2042   2138         sqlite3_int64 iPrevSave = iPrev;
  2043   2139         int bFirstOutSave = bFirstOut;
  2044   2140   
  2045         -      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
         2141  +      fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
  2046   2142         if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){
  2047   2143           p = pSave;
  2048   2144           iPrev = iPrevSave;
  2049   2145           bFirstOut = bFirstOutSave;
  2050   2146         }
  2051         -      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
  2052         -      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
         2147  +      fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
         2148  +      fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
  2053   2149       }else if( iDiff<0 ){
  2054   2150         fts3PoslistCopy(0, &p1);
  2055         -      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
         2151  +      fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
  2056   2152       }else{
  2057   2153         fts3PoslistCopy(0, &p2);
  2058         -      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
         2154  +      fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
  2059   2155       }
  2060   2156     }
  2061   2157   
  2062   2158     *pnRight = p - aOut;
  2063   2159   }
  2064   2160   
  2065   2161   
................................................................................
  2068   2164   ** doclist stored in TermSelect.aaOutput[0]. If successful, delete all
  2069   2165   ** other doclists (except the aaOutput[0] one) and return SQLITE_OK.
  2070   2166   **
  2071   2167   ** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
  2072   2168   ** the responsibility of the caller to free any doclists left in the
  2073   2169   ** TermSelect.aaOutput[] array.
  2074   2170   */
  2075         -static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){
         2171  +static int fts3TermSelectFinishMerge(Fts3Table *p, TermSelect *pTS){
  2076   2172     char *aOut = 0;
  2077   2173     int nOut = 0;
  2078   2174     int i;
  2079   2175   
  2080   2176     /* Loop through the doclists in the aaOutput[] array. Merge them all
  2081   2177     ** into a single doclist.
  2082   2178     */
................................................................................
  2109   2205   
  2110   2206     pTS->aaOutput[0] = aOut;
  2111   2207     pTS->anOutput[0] = nOut;
  2112   2208     return SQLITE_OK;
  2113   2209   }
  2114   2210   
  2115   2211   /*
  2116         -** This function is used as the sqlite3Fts3SegReaderIterate() callback when
  2117         -** querying the full-text index for a doclist associated with a term or
  2118         -** term-prefix.
         2212  +** Merge the doclist aDoclist/nDoclist into the TermSelect object passed
         2213  +** as the first argument. The merge is an "OR" merge (see function
         2214  +** fts3DoclistOrMerge() for details).
         2215  +**
         2216  +** This function is called with the doclist for each term that matches
         2217  +** a queried prefix. It merges all these doclists into one, the doclist
         2218  +** for the specified prefix. Since there can be a very large number of
         2219  +** doclists to merge, the merging is done pair-wise using the TermSelect
         2220  +** object.
         2221  +**
         2222  +** This function returns SQLITE_OK if the merge is successful, or an
         2223  +** SQLite error code (SQLITE_NOMEM) if an error occurs.
  2119   2224   */
  2120         -static int fts3TermSelectCb(
  2121         -  Fts3Table *p,                   /* Virtual table object */
  2122         -  void *pContext,                 /* Pointer to TermSelect structure */
  2123         -  char *zTerm,
  2124         -  int nTerm,
  2125         -  char *aDoclist,
  2126         -  int nDoclist
         2225  +static int fts3TermSelectMerge(
         2226  +  Fts3Table *p,                   /* FTS table handle */
         2227  +  TermSelect *pTS,                /* TermSelect object to merge into */
         2228  +  char *aDoclist,                 /* Pointer to doclist */
         2229  +  int nDoclist                    /* Size of aDoclist in bytes */
  2127   2230   ){
  2128         -  TermSelect *pTS = (TermSelect *)pContext;
  2129         -
  2130         -  UNUSED_PARAMETER(p);
  2131         -  UNUSED_PARAMETER(zTerm);
  2132         -  UNUSED_PARAMETER(nTerm);
  2133         -
  2134   2231     if( pTS->aaOutput[0]==0 ){
  2135   2232       /* If this is the first term selected, copy the doclist to the output
  2136   2233       ** buffer using memcpy(). */
  2137   2234       pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
  2138   2235       pTS->anOutput[0] = nDoclist;
  2139   2236       if( pTS->aaOutput[0] ){
  2140   2237         memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
................................................................................
  2197   2294       }
  2198   2295       pCsr->apSegment = apNew;
  2199   2296     }
  2200   2297     pCsr->apSegment[pCsr->nSegment++] = pNew;
  2201   2298     return SQLITE_OK;
  2202   2299   }
  2203   2300   
         2301  +/*
         2302  +** Add seg-reader objects to the Fts3MultiSegReader object passed as the
         2303  +** 8th argument.
         2304  +**
         2305  +** This function returns SQLITE_OK if successful, or an SQLite error code
         2306  +** otherwise.
         2307  +*/
  2204   2308   static int fts3SegReaderCursor(
  2205   2309     Fts3Table *p,                   /* FTS3 table handle */
  2206   2310     int iIndex,                     /* Index to search (from 0 to p->nIndex-1) */
  2207   2311     int iLevel,                     /* Level of segments to scan */
  2208   2312     const char *zTerm,              /* Term to query for */
  2209   2313     int nTerm,                      /* Size of zTerm in bytes */
  2210   2314     int isPrefix,                   /* True for a prefix search */
  2211   2315     int isScan,                     /* True to scan from zTerm to EOF */
  2212         -  Fts3MultiSegReader *pCsr       /* Cursor object to populate */
         2316  +  Fts3MultiSegReader *pCsr        /* Cursor object to populate */
  2213   2317   ){
  2214         -  int rc = SQLITE_OK;
  2215         -  int rc2;
  2216         -  sqlite3_stmt *pStmt = 0;
         2318  +  int rc = SQLITE_OK;             /* Error code */
         2319  +  sqlite3_stmt *pStmt = 0;        /* Statement to iterate through segments */
         2320  +  int rc2;                        /* Result of sqlite3_reset() */
  2217   2321   
  2218   2322     /* If iLevel is less than 0 and this is not a scan, include a seg-reader 
  2219   2323     ** for the pending-terms. If this is a scan, then this call must be being
  2220   2324     ** made by an fts4aux module, not an FTS table. In this case calling
  2221   2325     ** Fts3SegReaderPending might segfault, as the data structures used by 
  2222   2326     ** fts4aux are not completely populated. So it's easiest to filter these
  2223   2327     ** calls out here.  */
................................................................................
  2298   2402     memset(pCsr, 0, sizeof(Fts3MultiSegReader));
  2299   2403   
  2300   2404     return fts3SegReaderCursor(
  2301   2405         p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
  2302   2406     );
  2303   2407   }
  2304   2408   
         2409  +/*
         2410  +** In addition to its current configuration, have the Fts3MultiSegReader
         2411  +** passed as the 4th argument also scan the doclist for term zTerm/nTerm.
         2412  +**
         2413  +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
         2414  +*/
  2305   2415   static int fts3SegReaderCursorAddZero(
  2306         -  Fts3Table *p,
  2307         -  const char *zTerm,
  2308         -  int nTerm,
  2309         -  Fts3MultiSegReader *pCsr
         2416  +  Fts3Table *p,                   /* FTS virtual table handle */
         2417  +  const char *zTerm,              /* Term to scan doclist of */
         2418  +  int nTerm,                      /* Number of bytes in zTerm */
         2419  +  Fts3MultiSegReader *pCsr        /* Fts3MultiSegReader to modify */
  2310   2420   ){
  2311   2421     return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr);
  2312   2422   }
  2313   2423   
  2314         -
  2315         -int sqlite3Fts3TermSegReaderCursor(
         2424  +/*
         2425  +** Open an Fts3MultiSegReader to scan the doclist for term zTerm/nTerm. Or,
         2426  +** if isPrefix is true, to scan the doclist for all terms for which 
         2427  +** zTerm/nTerm is a prefix. If successful, return SQLITE_OK and write
         2428  +** a pointer to the new Fts3MultiSegReader to *ppSegcsr. Otherwise, return
         2429  +** an SQLite error code.
         2430  +**
         2431  +** It is the responsibility of the caller to free this object by eventually
         2432  +** passing it to fts3SegReaderCursorFree() 
         2433  +**
         2434  +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
         2435  +** Output parameter *ppSegcsr is set to 0 if an error occurs.
         2436  +*/
         2437  +static int fts3TermSegReaderCursor(
  2316   2438     Fts3Cursor *pCsr,               /* Virtual table cursor handle */
  2317   2439     const char *zTerm,              /* Term to query for */
  2318   2440     int nTerm,                      /* Size of zTerm in bytes */
  2319   2441     int isPrefix,                   /* True for a prefix search */
  2320   2442     Fts3MultiSegReader **ppSegcsr   /* OUT: Allocated seg-reader cursor */
  2321   2443   ){
  2322         -  Fts3MultiSegReader *pSegcsr;   /* Object to allocate and return */
         2444  +  Fts3MultiSegReader *pSegcsr;    /* Object to allocate and return */
  2323   2445     int rc = SQLITE_NOMEM;          /* Return code */
  2324   2446   
  2325   2447     pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader));
  2326   2448     if( pSegcsr ){
  2327   2449       int i;
  2328   2450       int bFound = 0;               /* True once an index has been found */
  2329   2451       Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
................................................................................
  2359   2481       }
  2360   2482     }
  2361   2483   
  2362   2484     *ppSegcsr = pSegcsr;
  2363   2485     return rc;
  2364   2486   }
  2365   2487   
         2488  +/*
         2489  +** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor().
         2490  +*/
  2366   2491   static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){
  2367   2492     sqlite3Fts3SegReaderFinish(pSegcsr);
  2368   2493     sqlite3_free(pSegcsr);
  2369   2494   }
  2370   2495   
  2371   2496   /*
  2372   2497   ** This function retreives the doclist for the specified term (or term
  2373         -** prefix) from the database. 
  2374         -**
  2375         -** The returned doclist may be in one of two formats, depending on the 
  2376         -** value of parameter isReqPos. If isReqPos is zero, then the doclist is
  2377         -** a sorted list of delta-compressed docids (a bare doclist). If isReqPos
  2378         -** is non-zero, then the returned list is in the same format as is stored 
  2379         -** in the database without the found length specifier at the start of on-disk
  2380         -** doclists.
         2498  +** prefix) from the database.
  2381   2499   */
  2382   2500   static int fts3TermSelect(
  2383   2501     Fts3Table *p,                   /* Virtual table handle */
  2384   2502     Fts3PhraseToken *pTok,          /* Token to query for */
  2385   2503     int iColumn,                    /* Column to query (or -ve for all columns) */
  2386         -  int isReqPos,                   /* True to include position lists in output */
  2387   2504     int *pnOut,                     /* OUT: Size of buffer at *ppOut */
  2388   2505     char **ppOut                    /* OUT: Malloced result buffer */
  2389   2506   ){
  2390   2507     int rc;                         /* Return code */
  2391         -  Fts3MultiSegReader *pSegcsr;   /* Seg-reader cursor for this term */
  2392         -  TermSelect tsc;                 /* Context object for fts3TermSelectCb() */
         2508  +  Fts3MultiSegReader *pSegcsr;    /* Seg-reader cursor for this term */
         2509  +  TermSelect tsc;                 /* Object for pair-wise doclist merging */
  2393   2510     Fts3SegFilter filter;           /* Segment term filter configuration */
  2394   2511   
  2395   2512     pSegcsr = pTok->pSegcsr;
  2396   2513     memset(&tsc, 0, sizeof(TermSelect));
  2397         -  tsc.isReqPos = isReqPos;
  2398   2514   
  2399         -  filter.flags = FTS3_SEGMENT_IGNORE_EMPTY 
         2515  +  filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS
  2400   2516           | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0)
  2401         -        | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
  2402   2517           | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
  2403   2518     filter.iCol = iColumn;
  2404   2519     filter.zTerm = pTok->z;
  2405   2520     filter.nTerm = pTok->n;
  2406   2521   
  2407   2522     rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter);
  2408   2523     while( SQLITE_OK==rc
  2409   2524         && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) 
  2410   2525     ){
  2411         -    rc = fts3TermSelectCb(p, (void *)&tsc, 
  2412         -        pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist
  2413         -    );
         2526  +    rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist);
  2414   2527     }
  2415   2528   
  2416   2529     if( rc==SQLITE_OK ){
  2417         -    rc = fts3TermSelectMerge(p, &tsc);
         2530  +    rc = fts3TermSelectFinishMerge(p, &tsc);
  2418   2531     }
  2419   2532     if( rc==SQLITE_OK ){
  2420   2533       *ppOut = tsc.aaOutput[0];
  2421   2534       *pnOut = tsc.anOutput[0];
  2422   2535     }else{
  2423   2536       int i;
  2424   2537       for(i=0; i<SizeofArray(tsc.aaOutput); i++){
................................................................................
  2436   2549   ** in buffer aList[], size nList bytes.
  2437   2550   **
  2438   2551   ** If the isPoslist argument is true, then it is assumed that the doclist
  2439   2552   ** contains a position-list following each docid. Otherwise, it is assumed
  2440   2553   ** that the doclist is simply a list of docids stored as delta encoded 
  2441   2554   ** varints.
  2442   2555   */
  2443         -static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){
         2556  +static int fts3DoclistCountDocids(char *aList, int nList){
  2444   2557     int nDoc = 0;                   /* Return value */
  2445   2558     if( aList ){
  2446   2559       char *aEnd = &aList[nList];   /* Pointer to one byte after EOF */
  2447   2560       char *p = aList;              /* Cursor */
  2448         -    if( !isPoslist ){
  2449         -      /* The number of docids in the list is the same as the number of 
  2450         -      ** varints. In FTS3 a varint consists of a single byte with the 0x80 
  2451         -      ** bit cleared and zero or more bytes with the 0x80 bit set. So to
  2452         -      ** count the varints in the buffer, just count the number of bytes
  2453         -      ** with the 0x80 bit clear.  */
  2454         -      while( p<aEnd ) nDoc += (((*p++)&0x80)==0);
  2455         -    }else{
  2456         -      while( p<aEnd ){
  2457         -        nDoc++;
  2458         -        while( (*p++)&0x80 );     /* Skip docid varint */
  2459         -        fts3PoslistCopy(0, &p);   /* Skip over position list */
  2460         -      }
         2561  +    while( p<aEnd ){
         2562  +      nDoc++;
         2563  +      while( (*p++)&0x80 );     /* Skip docid varint */
         2564  +      fts3PoslistCopy(0, &p);   /* Skip over position list */
  2461   2565       }
  2462   2566     }
  2463   2567   
  2464   2568     return nDoc;
  2465   2569   }
  2466   2570   
  2467   2571   /*
................................................................................
  2483   2587         pCsr->isEof = 1;
  2484   2588         rc = sqlite3_reset(pCsr->pStmt);
  2485   2589       }else{
  2486   2590         pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
  2487   2591         rc = SQLITE_OK;
  2488   2592       }
  2489   2593     }else{
  2490         -    rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor);
         2594  +    rc = fts3EvalNext((Fts3Cursor *)pCursor);
  2491   2595     }
  2492   2596     assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  2493   2597     return rc;
  2494   2598   }
  2495   2599   
  2496   2600   /*
  2497   2601   ** This is the xFilter interface for the virtual table.  See
................................................................................
  2560   2664         }
  2561   2665         return rc;
  2562   2666       }
  2563   2667   
  2564   2668       rc = sqlite3Fts3ReadLock(p);
  2565   2669       if( rc!=SQLITE_OK ) return rc;
  2566   2670   
  2567         -    rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1);
         2671  +    rc = fts3EvalStart(pCsr);
  2568   2672   
  2569   2673       sqlite3Fts3SegmentsClose(p);
  2570   2674       if( rc!=SQLITE_OK ) return rc;
  2571   2675       pCsr->pNextId = pCsr->aDoclist;
  2572   2676       pCsr->iPrevId = 0;
  2573   2677     }
  2574   2678   
................................................................................
  2967   3071     fts3DbExec(&rc, db,
  2968   3072       "ALTER TABLE %Q.'%q_segdir'   RENAME TO '%q_segdir';",
  2969   3073       p->zDb, p->zName, zName
  2970   3074     );
  2971   3075     return rc;
  2972   3076   }
  2973   3077   
         3078  +/*
         3079  +** The xSavepoint() method.
         3080  +**
         3081  +** Flush the contents of the pending-terms table to disk.
         3082  +*/
  2974   3083   static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  2975   3084     UNUSED_PARAMETER(iSavepoint);
  2976   3085     assert( ((Fts3Table *)pVtab)->inTransaction );
  2977   3086     assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
  2978   3087     TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
  2979   3088     return fts3SyncMethod(pVtab);
  2980   3089   }
         3090  +
         3091  +/*
         3092  +** The xRelease() method.
         3093  +**
         3094  +** This is a no-op.
         3095  +*/
  2981   3096   static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
  2982   3097     TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
  2983   3098     UNUSED_PARAMETER(iSavepoint);
  2984   3099     UNUSED_PARAMETER(pVtab);
  2985   3100     assert( p->inTransaction );
  2986   3101     assert( p->mxSavepoint >= iSavepoint );
  2987   3102     TESTONLY( p->mxSavepoint = iSavepoint-1 );
  2988   3103     return SQLITE_OK;
  2989   3104   }
         3105  +
         3106  +/*
         3107  +** The xRollbackTo() method.
         3108  +**
         3109  +** Discard the contents of the pending terms table.
         3110  +*/
  2990   3111   static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
  2991   3112     Fts3Table *p = (Fts3Table*)pVtab;
  2992   3113     UNUSED_PARAMETER(iSavepoint);
  2993   3114     assert( p->inTransaction );
  2994   3115     assert( p->mxSavepoint >= iSavepoint );
  2995   3116     TESTONLY( p->mxSavepoint = iSavepoint );
  2996   3117     sqlite3Fts3PendingTermsClear(p);
................................................................................
  3131   3252     assert( rc!=SQLITE_OK );
  3132   3253     if( pHash ){
  3133   3254       sqlite3Fts3HashClear(pHash);
  3134   3255       sqlite3_free(pHash);
  3135   3256     }
  3136   3257     return rc;
  3137   3258   }
  3138         -
  3139         -#if !SQLITE_CORE
  3140         -int sqlite3_extension_init(
  3141         -  sqlite3 *db, 
  3142         -  char **pzErrMsg,
  3143         -  const sqlite3_api_routines *pApi
  3144         -){
  3145         -  SQLITE_EXTENSION_INIT2(pApi)
  3146         -  return sqlite3Fts3Init(db);
  3147         -}
  3148         -#endif
  3149         -
  3150   3259   
  3151   3260   /*
  3152   3261   ** Allocate an Fts3MultiSegReader for each token in the expression headed
  3153   3262   ** by pExpr. 
  3154   3263   **
  3155   3264   ** An Fts3SegReader object is a cursor that can seek or scan a range of
  3156   3265   ** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple
................................................................................
  3160   3269   ** If the allocated Fts3MultiSegReader just seeks to a single entry in a
  3161   3270   ** segment b-tree (if the term is not a prefix or it is a prefix for which
  3162   3271   ** there exists prefix b-tree of the right length) then it may be traversed
  3163   3272   ** and merged incrementally. Otherwise, it has to be merged into an in-memory 
  3164   3273   ** doclist and then traversed.
  3165   3274   */
  3166   3275   static void fts3EvalAllocateReaders(
  3167         -  Fts3Cursor *pCsr, 
  3168         -  Fts3Expr *pExpr, 
         3276  +  Fts3Cursor *pCsr,               /* FTS cursor handle */
         3277  +  Fts3Expr *pExpr,                /* Allocate readers for this expression */
  3169   3278     int *pnToken,                   /* OUT: Total number of tokens in phrase. */
  3170   3279     int *pnOr,                      /* OUT: Total number of OR nodes in expr. */
  3171         -  int *pRc
         3280  +  int *pRc                        /* IN/OUT: Error code */
  3172   3281   ){
  3173   3282     if( pExpr && SQLITE_OK==*pRc ){
  3174   3283       if( pExpr->eType==FTSQUERY_PHRASE ){
  3175   3284         int i;
  3176   3285         int nToken = pExpr->pPhrase->nToken;
  3177   3286         *pnToken += nToken;
  3178   3287         for(i=0; i<nToken; i++){
  3179   3288           Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
  3180         -        int rc = sqlite3Fts3TermSegReaderCursor(pCsr, 
         3289  +        int rc = fts3TermSegReaderCursor(pCsr, 
  3181   3290               pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr
  3182   3291           );
  3183   3292           if( rc!=SQLITE_OK ){
  3184   3293             *pRc = rc;
  3185   3294             return;
  3186   3295           }
  3187   3296         }
................................................................................
  3191   3300         *pnOr += (pExpr->eType==FTSQUERY_OR);
  3192   3301         fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc);
  3193   3302         fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc);
  3194   3303       }
  3195   3304     }
  3196   3305   }
  3197   3306   
         3307  +/*
         3308  +** Arguments pList/nList contain the doclist for token iToken of phrase p.
         3309  +** It is merged into the main doclist stored in p->doclist.aAll/nAll.
         3310  +**
         3311  +** This function assumes that pList points to a buffer allocated using
         3312  +** sqlite3_malloc(). This function takes responsibility for eventually
         3313  +** freeing the buffer.
         3314  +*/
  3198   3315   static void fts3EvalPhraseMergeToken(
  3199         -  Fts3Table *pTab,
  3200         -  Fts3Phrase *p,
  3201         -  int iToken,
  3202         -  char *pList,
  3203         -  int nList
         3316  +  Fts3Table *pTab,                /* FTS Table pointer */
         3317  +  Fts3Phrase *p,                  /* Phrase to merge pList/nList into */
         3318  +  int iToken,                     /* Token pList/nList corresponds to */
         3319  +  char *pList,                    /* Pointer to doclist */
         3320  +  int nList                       /* Number of bytes in pList */
  3204   3321   ){
  3205   3322     assert( iToken!=p->iDoclistToken );
  3206   3323   
  3207   3324     if( pList==0 ){
  3208   3325       sqlite3_free(p->doclist.aAll);
  3209   3326       p->doclist.aAll = 0;
  3210   3327       p->doclist.nAll = 0;
................................................................................
  3245   3362       p->doclist.aAll = pRight;
  3246   3363       p->doclist.nAll = nRight;
  3247   3364     }
  3248   3365   
  3249   3366     if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken;
  3250   3367   }
  3251   3368   
         3369  +/*
         3370  +** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist
         3371  +** does not take deferred tokens into account.
         3372  +**
         3373  +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
         3374  +*/
  3252   3375   static int fts3EvalPhraseLoad(
  3253         -  Fts3Cursor *pCsr, 
  3254         -  Fts3Phrase *p
         3376  +  Fts3Cursor *pCsr,               /* FTS Cursor handle */
         3377  +  Fts3Phrase *p                   /* Phrase object */
  3255   3378   ){
  3256   3379     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  3257   3380     int iToken;
  3258   3381     int rc = SQLITE_OK;
  3259   3382   
  3260   3383     for(iToken=0; rc==SQLITE_OK && iToken<p->nToken; iToken++){
  3261   3384       Fts3PhraseToken *pToken = &p->aToken[iToken];
  3262   3385       assert( pToken->pDeferred==0 || pToken->pSegcsr==0 );
  3263   3386   
  3264   3387       if( pToken->pSegcsr ){
  3265   3388         int nThis = 0;
  3266   3389         char *pThis = 0;
  3267         -      rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis);
         3390  +      rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis);
  3268   3391         if( rc==SQLITE_OK ){
  3269   3392           fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis);
  3270   3393         }
  3271   3394       }
  3272   3395       assert( pToken->pSegcsr==0 );
  3273   3396     }
  3274   3397   
  3275   3398     return rc;
  3276   3399   }
  3277   3400   
         3401  +/*
         3402  +** This function is called on each phrase after the position lists for
         3403  +** any deferred tokens have been loaded into memory. It updates the phrases
         3404  +** current position list to include only those positions that are really
         3405  +** instances of the phrase (after considering deferred tokens). If this
         3406  +** means that the phrase does not appear in the current row, doclist.pList
         3407  +** and doclist.nList are both zeroed.
         3408  +**
         3409  +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
         3410  +*/
  3278   3411   static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
  3279         -  int iToken;
  3280         -  int rc = SQLITE_OK;
  3281         -
  3282         -  int nMaxUndeferred = pPhrase->iDoclistToken;
  3283         -  char *aPoslist = 0;
  3284         -  int nPoslist = 0;
  3285         -  int iPrev = -1;
         3412  +  int iToken;                     /* Used to iterate through phrase tokens */
         3413  +  int rc = SQLITE_OK;             /* Return code */
         3414  +  char *aPoslist = 0;             /* Position list for deferred tokens */
         3415  +  int nPoslist = 0;               /* Number of bytes in aPoslist */
         3416  +  int iPrev = -1;                 /* Token number of previous deferred token */
  3286   3417   
  3287   3418     assert( pPhrase->doclist.bFreeList==0 );
  3288   3419   
  3289   3420     for(iToken=0; rc==SQLITE_OK && iToken<pPhrase->nToken; iToken++){
  3290   3421       Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
  3291   3422       Fts3DeferredToken *pDeferred = pToken->pDeferred;
  3292   3423   
................................................................................
  3324   3455           }
  3325   3456         }
  3326   3457         iPrev = iToken;
  3327   3458       }
  3328   3459     }
  3329   3460   
  3330   3461     if( iPrev>=0 ){
         3462  +    int nMaxUndeferred = pPhrase->iDoclistToken;
  3331   3463       if( nMaxUndeferred<0 ){
  3332   3464         pPhrase->doclist.pList = aPoslist;
  3333   3465         pPhrase->doclist.nList = nPoslist;
  3334   3466         pPhrase->doclist.iDocid = pCsr->iPrevId;
  3335   3467         pPhrase->doclist.bFreeList = 1;
  3336   3468       }else{
  3337   3469         int nDistance;
................................................................................
  3372   3504   }
  3373   3505   
  3374   3506   /*
  3375   3507   ** This function is called for each Fts3Phrase in a full-text query 
  3376   3508   ** expression to initialize the mechanism for returning rows. Once this
  3377   3509   ** function has been called successfully on an Fts3Phrase, it may be
  3378   3510   ** used with fts3EvalPhraseNext() to iterate through the matching docids.
         3511  +**
         3512  +** If parameter bOptOk is true, then the phrase may (or may not) use the
         3513  +** incremental loading strategy. Otherwise, the entire doclist is loaded into
         3514  +** memory within this call.
         3515  +**
         3516  +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
  3379   3517   */
  3380   3518   static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
  3381         -  int rc;
         3519  +  int rc;                         /* Error code */
  3382   3520     Fts3PhraseToken *pFirst = &p->aToken[0];
  3383   3521     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  3384   3522   
  3385   3523     if( pCsr->bDesc==pTab->bDescIdx 
  3386   3524      && bOptOk==1 
  3387   3525      && p->nToken==1 
  3388   3526      && pFirst->pSegcsr 
................................................................................
  3402   3540   
  3403   3541     assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr );
  3404   3542     return rc;
  3405   3543   }
  3406   3544   
  3407   3545   /*
  3408   3546   ** This function is used to iterate backwards (from the end to start) 
  3409         -** through doclists.
         3547  +** through doclists. It is used by this module to iterate through phrase
         3548  +** doclists in reverse and by the fts3_write.c module to iterate through
         3549  +** pending-terms lists when writing to databases with "order=desc".
         3550  +**
         3551  +** The doclist may be sorted in ascending (parameter bDescIdx==0) or 
         3552  +** descending (parameter bDescIdx==1) order of docid. Regardless, this
         3553  +** function iterates from the end of the doclist to the beginning.
  3410   3554   */
  3411   3555   void sqlite3Fts3DoclistPrev(
  3412   3556     int bDescIdx,                   /* True if the doclist is desc */
  3413   3557     char *aDoclist,                 /* Pointer to entire doclist */
  3414   3558     int nDoclist,                   /* Length of aDoclist in bytes */
  3415   3559     char **ppIter,                  /* IN/OUT: Iterator pointer */
  3416   3560     sqlite3_int64 *piDocid,         /* IN/OUT: Docid pointer */
................................................................................
  3467   3611   ** SQLITE_OK.
  3468   3612   **
  3469   3613   ** If there is no "next" entry and no error occurs, then *pbEof is set to
  3470   3614   ** 1 before returning. Otherwise, if no error occurs and the iterator is
  3471   3615   ** successfully advanced, *pbEof is set to 0.
  3472   3616   */
  3473   3617   static int fts3EvalPhraseNext(
  3474         -  Fts3Cursor *pCsr, 
  3475         -  Fts3Phrase *p, 
  3476         -  u8 *pbEof
         3618  +  Fts3Cursor *pCsr,               /* FTS Cursor handle */
         3619  +  Fts3Phrase *p,                  /* Phrase object to advance to next docid */
         3620  +  u8 *pbEof                       /* OUT: Set to 1 if EOF */
  3477   3621   ){
  3478   3622     int rc = SQLITE_OK;
  3479   3623     Fts3Doclist *pDL = &p->doclist;
  3480   3624     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  3481   3625   
  3482   3626     if( p->bIncr ){
  3483   3627       assert( p->nToken==1 );
................................................................................
  3515   3659         }
  3516   3660         pDL->pList = pIter;
  3517   3661         fts3PoslistCopy(0, &pIter);
  3518   3662         pDL->nList = (pIter - pDL->pList);
  3519   3663   
  3520   3664         /* pIter now points just past the 0x00 that terminates the position-
  3521   3665         ** list for document pDL->iDocid. However, if this position-list was
  3522         -      ** edited in place by fts3EvalNearTrim2(), then pIter may not actually
         3666  +      ** edited in place by fts3EvalNearTrim(), then pIter may not actually
  3523   3667         ** point to the start of the next docid value. The following line deals
  3524   3668         ** with this case by advancing pIter past the zero-padding added by
  3525         -      ** fts3EvalNearTrim2().  */
         3669  +      ** fts3EvalNearTrim().  */
  3526   3670         while( pIter<pEnd && *pIter==0 ) pIter++;
  3527   3671   
  3528   3672         pDL->pNextDocid = pIter;
  3529   3673         assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter );
  3530   3674         *pbEof = 0;
  3531   3675       }
  3532   3676     }
  3533   3677   
  3534   3678     return rc;
  3535   3679   }
  3536   3680   
         3681  +/*
         3682  +**
         3683  +** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
         3684  +** Otherwise, fts3EvalPhraseStart() is called on all phrases within the
         3685  +** expression. Also the Fts3Expr.bDeferred variable is set to true for any
         3686  +** expressions for which all descendent tokens are deferred.
         3687  +**
         3688  +** If parameter bOptOk is zero, then it is guaranteed that the
         3689  +** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for
         3690  +** each phrase in the expression (subject to deferred token processing).
         3691  +** Or, if bOptOk is non-zero, then one or more tokens within the expression
         3692  +** may be loaded incrementally, meaning doclist.aAll/nAll is not available.
         3693  +**
         3694  +** If an error occurs within this function, *pRc is set to an SQLite error
         3695  +** code before returning.
         3696  +*/
  3537   3697   static void fts3EvalStartReaders(
  3538         -  Fts3Cursor *pCsr, 
  3539         -  Fts3Expr *pExpr, 
  3540         -  int bOptOk,
  3541         -  int *pRc
         3698  +  Fts3Cursor *pCsr,               /* FTS Cursor handle */
         3699  +  Fts3Expr *pExpr,                /* Expression to initialize phrases in */
         3700  +  int bOptOk,                     /* True to enable incremental loading */
         3701  +  int *pRc                        /* IN/OUT: Error code */
  3542   3702   ){
  3543   3703     if( pExpr && SQLITE_OK==*pRc ){
  3544   3704       if( pExpr->eType==FTSQUERY_PHRASE ){
  3545   3705         int i;
  3546   3706         int nToken = pExpr->pPhrase->nToken;
  3547   3707         for(i=0; i<nToken; i++){
  3548   3708           if( pExpr->pPhrase->aToken[i].pDeferred==0 ) break;
................................................................................
  3553   3713         fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc);
  3554   3714         fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc);
  3555   3715         pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred);
  3556   3716       }
  3557   3717     }
  3558   3718   }
  3559   3719   
         3720  +/*
         3721  +** An array of the following structures is assembled as part of the process
         3722  +** of selecting tokens to defer before the query starts executing (as part
         3723  +** of the xFilter() method). There is one element in the array for each
         3724  +** token in the FTS expression.
         3725  +**
         3726  +** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong
         3727  +** to phrases that are connected only by AND and NEAR operators (not OR or
         3728  +** NOT). When determining tokens to defer, each AND/NEAR cluster is considered
         3729  +** separately. The root of a tokens AND/NEAR cluster is stored in 
         3730  +** Fts3TokenAndCost.pRoot.
         3731  +*/
  3560   3732   typedef struct Fts3TokenAndCost Fts3TokenAndCost;
  3561   3733   struct Fts3TokenAndCost {
  3562   3734     Fts3Phrase *pPhrase;            /* The phrase the token belongs to */
  3563   3735     int iToken;                     /* Position of token in phrase */
  3564   3736     Fts3PhraseToken *pToken;        /* The token itself */
  3565         -  Fts3Expr *pRoot; 
  3566         -  int nOvfl;
         3737  +  Fts3Expr *pRoot;                /* Root of NEAR/AND cluster */
         3738  +  int nOvfl;                      /* Number of overflow pages to load doclist */
  3567   3739     int iCol;                       /* The column the token must match */
  3568   3740   };
  3569   3741   
         3742  +/*
         3743  +** This function is used to populate an allocated Fts3TokenAndCost array.
         3744  +**
         3745  +** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
         3746  +** Otherwise, if an error occurs during execution, *pRc is set to an
         3747  +** SQLite error code.
         3748  +*/
  3570   3749   static void fts3EvalTokenCosts(
  3571         -  Fts3Cursor *pCsr, 
  3572         -  Fts3Expr *pRoot, 
  3573         -  Fts3Expr *pExpr, 
  3574         -  Fts3TokenAndCost **ppTC,
  3575         -  Fts3Expr ***ppOr,
  3576         -  int *pRc
         3750  +  Fts3Cursor *pCsr,               /* FTS Cursor handle */
         3751  +  Fts3Expr *pRoot,                /* Root of current AND/NEAR cluster */
         3752  +  Fts3Expr *pExpr,                /* Expression to consider */
         3753  +  Fts3TokenAndCost **ppTC,        /* Write new entries to *(*ppTC)++ */
         3754  +  Fts3Expr ***ppOr,               /* Write new OR root to *(*ppOr)++ */
         3755  +  int *pRc                        /* IN/OUT: Error code */
  3577   3756   ){
  3578   3757     if( *pRc==SQLITE_OK && pExpr ){
  3579   3758       if( pExpr->eType==FTSQUERY_PHRASE ){
  3580   3759         Fts3Phrase *pPhrase = pExpr->pPhrase;
  3581   3760         int i;
  3582   3761         for(i=0; *pRc==SQLITE_OK && i<pPhrase->nToken; i++){
  3583   3762           Fts3TokenAndCost *pTC = (*ppTC)++;
................................................................................
  3601   3780           (*ppOr)++;
  3602   3781         }
  3603   3782         fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
  3604   3783       }
  3605   3784     }
  3606   3785   }
  3607   3786   
         3787  +/*
         3788  +** Determine the average document (row) size in pages. If successful,
         3789  +** write this value to *pnPage and return SQLITE_OK. Otherwise, return
         3790  +** an SQLite error code.
         3791  +**
         3792  +** The average document size in pages is calculated by first calculating 
         3793  +** determining the average size in bytes, B. If B is less than the amount
         3794  +** of data that will fit on a single leaf page of an intkey table in
         3795  +** this database, then the average docsize is 1. Otherwise, it is 1 plus
         3796  +** the number of overflow pages consumed by a record B bytes in size.
         3797  +*/
  3608   3798   static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
  3609   3799     if( pCsr->nRowAvg==0 ){
  3610   3800       /* The average document size, which is required to calculate the cost
  3611         -     ** of each doclist, has not yet been determined. Read the required 
  3612         -     ** data from the %_stat table to calculate it.
  3613         -     **
  3614         -     ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 
  3615         -     ** varints, where nCol is the number of columns in the FTS3 table.
  3616         -     ** The first varint is the number of documents currently stored in
  3617         -     ** the table. The following nCol varints contain the total amount of
  3618         -     ** data stored in all rows of each column of the table, from left
  3619         -     ** to right.
  3620         -     */
         3801  +    ** of each doclist, has not yet been determined. Read the required 
         3802  +    ** data from the %_stat table to calculate it.
         3803  +    **
         3804  +    ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 
         3805  +    ** varints, where nCol is the number of columns in the FTS3 table.
         3806  +    ** The first varint is the number of documents currently stored in
         3807  +    ** the table. The following nCol varints contain the total amount of
         3808  +    ** data stored in all rows of each column of the table, from left
         3809  +    ** to right.
         3810  +    */
  3621   3811       int rc;
  3622   3812       Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
  3623   3813       sqlite3_stmt *pStmt;
  3624   3814       sqlite3_int64 nDoc = 0;
  3625   3815       sqlite3_int64 nByte = 0;
  3626   3816       const char *pEnd;
  3627   3817       const char *a;
................................................................................
  3648   3838       if( rc!=SQLITE_OK ) return rc;
  3649   3839     }
  3650   3840   
  3651   3841     *pnPage = pCsr->nRowAvg;
  3652   3842     return SQLITE_OK;
  3653   3843   }
  3654   3844   
         3845  +/*
         3846  +** This function is called to select the tokens (if any) that will be 
         3847  +** deferred. The array aTC[] has already been populated when this is
         3848  +** called.
         3849  +**
         3850  +** This function is called once for each AND/NEAR cluster in the 
         3851  +** expression. Each invocation determines which tokens to defer within
         3852  +** the cluster with root node pRoot. See comments above the definition
         3853  +** of struct Fts3TokenAndCost for more details.
         3854  +**
         3855  +** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken()
         3856  +** called on each token to defer. Otherwise, an SQLite error code is
         3857  +** returned.
         3858  +*/
  3655   3859   static int fts3EvalSelectDeferred(
  3656         -  Fts3Cursor *pCsr,
  3657         -  Fts3Expr *pRoot,
  3658         -  Fts3TokenAndCost *aTC,
  3659         -  int nTC
         3860  +  Fts3Cursor *pCsr,               /* FTS Cursor handle */
         3861  +  Fts3Expr *pRoot,                /* Consider tokens with this root node */
         3862  +  Fts3TokenAndCost *aTC,          /* Array of expression tokens and costs */
         3863  +  int nTC                         /* Number of entries in aTC[] */
  3660   3864   ){
  3661         -  int nDocSize = 0;
  3662         -  int nDocEst = 0;
  3663         -  int rc = SQLITE_OK;
  3664   3865     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  3665         -  int ii;
         3866  +  int nDocSize = 0;               /* Number of pages per doc loaded */
         3867  +  int rc = SQLITE_OK;             /* Return code */
         3868  +  int ii;                         /* Iterator variable for various purposes */
         3869  +  int nOvfl = 0;                  /* Total overflow pages used by doclists */
         3870  +  int nToken = 0;                 /* Total number of tokens in cluster */
  3666   3871   
  3667         -  int nOvfl = 0;
  3668         -  int nTerm = 0;
         3872  +  int nMinEst = 0;                /* The minimum count for any phrase so far. */
         3873  +  int nLoad4 = 1;                 /* (Phrases that will be loaded)^4. */
  3669   3874   
         3875  +  /* Count the tokens in this AND/NEAR cluster. If none of the doclists
         3876  +  ** associated with the tokens spill onto overflow pages, or if there is
         3877  +  ** only 1 token, exit early. No tokens to defer in this case. */
  3670   3878     for(ii=0; ii<nTC; ii++){
  3671   3879       if( aTC[ii].pRoot==pRoot ){
  3672   3880         nOvfl += aTC[ii].nOvfl;
  3673         -      nTerm++;
         3881  +      nToken++;
  3674   3882       }
  3675   3883     }
  3676         -  if( nOvfl==0 || nTerm<2 ) return SQLITE_OK;
         3884  +  if( nOvfl==0 || nToken<2 ) return SQLITE_OK;
  3677   3885   
         3886  +  /* Obtain the average docsize (in pages). */
  3678   3887     rc = fts3EvalAverageDocsize(pCsr, &nDocSize);
         3888  +  assert( rc!=SQLITE_OK || nDocSize>0 );
  3679   3889   
  3680         -  for(ii=0; ii<nTerm && rc==SQLITE_OK; ii++){
  3681         -    int jj;
  3682         -    Fts3TokenAndCost *pTC = 0;
  3683   3890   
  3684         -    for(jj=0; jj<nTC; jj++){
  3685         -      if( aTC[jj].pToken && aTC[jj].pRoot==pRoot 
  3686         -       && (!pTC || aTC[jj].nOvfl<pTC->nOvfl) 
         3891  +  /* Iterate through all tokens in this AND/NEAR cluster, in ascending order 
         3892  +  ** of the number of overflow pages that will be loaded by the pager layer 
         3893  +  ** to retrieve the entire doclist for the token from the full-text index.
         3894  +  ** Load the doclists for tokens that are either:
         3895  +  **
         3896  +  **   a. The cheapest token in the entire query (i.e. the one visited by the
         3897  +  **      first iteration of this loop), or
         3898  +  **
         3899  +  **   b. Part of a multi-token phrase.
         3900  +  **
         3901  +  ** After each token doclist is loaded, merge it with the others from the
         3902  +  ** same phrase and count the number of documents that the merged doclist
         3903  +  ** contains. Set variable "nMinEst" to the smallest number of documents in 
         3904  +  ** any phrase doclist for which 1 or more token doclists have been loaded.
         3905  +  ** Let nOther be the number of other phrases for which it is certain that
         3906  +  ** one or more tokens will not be deferred.
         3907  +  **
         3908  +  ** Then, for each token, defer it if loading the doclist would result in
         3909  +  ** loading N or more overflow pages into memory, where N is computed as:
         3910  +  **
         3911  +  **    (nMinEst + 4^nOther - 1) / (4^nOther)
         3912  +  */
         3913  +  for(ii=0; ii<nToken && rc==SQLITE_OK; ii++){
         3914  +    int iTC;                      /* Used to iterate through aTC[] array. */
         3915  +    Fts3TokenAndCost *pTC = 0;    /* Set to cheapest remaining token. */
         3916  +
         3917  +    /* Set pTC to point to the cheapest remaining token. */
         3918  +    for(iTC=0; iTC<nTC; iTC++){
         3919  +      if( aTC[iTC].pToken && aTC[iTC].pRoot==pRoot 
         3920  +       && (!pTC || aTC[iTC].nOvfl<pTC->nOvfl) 
  3687   3921         ){
  3688         -        pTC = &aTC[jj];
         3922  +        pTC = &aTC[iTC];
  3689   3923         }
  3690   3924       }
  3691   3925       assert( pTC );
  3692   3926   
  3693         -    /* At this point pTC points to the cheapest remaining token. */
  3694         -    if( ii==0 ){
  3695         -      if( pTC->nOvfl ){
  3696         -        nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10;
  3697         -      }else{
         3927  +    if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){
         3928  +      /* The number of overflow pages to load for this (and therefore all
         3929  +      ** subsequent) tokens is greater than the estimated number of pages 
         3930  +      ** that will be loaded if all subsequent tokens are deferred.
         3931  +      */
         3932  +      Fts3PhraseToken *pToken = pTC->pToken;
         3933  +      rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol);
         3934  +      fts3SegReaderCursorFree(pToken->pSegcsr);
         3935  +      pToken->pSegcsr = 0;
         3936  +    }else{
         3937  +      nLoad4 = nLoad4*4;
         3938  +      if( ii==0 || pTC->pPhrase->nToken>1 ){
         3939  +        /* Either this is the cheapest token in the entire query, or it is
         3940  +        ** part of a multi-token phrase. Either way, the entire doclist will
         3941  +        ** (eventually) be loaded into memory. It may as well be now. */
  3698   3942           Fts3PhraseToken *pToken = pTC->pToken;
  3699   3943           int nList = 0;
  3700   3944           char *pList = 0;
  3701         -        rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList);
         3945  +        rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList);
  3702   3946           assert( rc==SQLITE_OK || pList==0 );
  3703         -
  3704   3947           if( rc==SQLITE_OK ){
  3705         -          nDocEst = fts3DoclistCountDocids(1, pList, nList);
         3948  +          int nCount;
  3706   3949             fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList);
         3950  +          nCount = fts3DoclistCountDocids(
         3951  +              pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll
         3952  +          );
         3953  +          if( ii==0 || nCount<nMinEst ) nMinEst = nCount;
  3707   3954           }
  3708   3955         }
  3709         -    }else{
  3710         -      if( pTC->nOvfl>=(nDocEst*nDocSize) ){
  3711         -        Fts3PhraseToken *pToken = pTC->pToken;
  3712         -        rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol);
  3713         -        fts3SegReaderCursorFree(pToken->pSegcsr);
  3714         -        pToken->pSegcsr = 0;
  3715         -      }
  3716         -      nDocEst = 1 + (nDocEst/4);
  3717   3956       }
  3718   3957       pTC->pToken = 0;
  3719   3958     }
  3720   3959   
  3721   3960     return rc;
  3722   3961   }
  3723   3962   
  3724         -int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){
         3963  +/*
         3964  +** This function is called from within the xFilter method. It initializes
         3965  +** the full-text query currently stored in pCsr->pExpr. To iterate through
         3966  +** the results of a query, the caller does:
         3967  +**
         3968  +**    fts3EvalStart(pCsr);
         3969  +**    while( 1 ){
         3970  +**      fts3EvalNext(pCsr);
         3971  +**      if( pCsr->bEof ) break;
         3972  +**      ... return row pCsr->iPrevId to the caller ...
         3973  +**    }
         3974  +*/
         3975  +static int fts3EvalStart(Fts3Cursor *pCsr){
  3725   3976     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  3726   3977     int rc = SQLITE_OK;
  3727   3978     int nToken = 0;
  3728   3979     int nOr = 0;
  3729   3980   
  3730   3981     /* Allocate a MultiSegReader for each token in the expression. */
  3731         -  fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc);
         3982  +  fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc);
  3732   3983   
  3733         -  /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO:
  3734         -  ** This call will eventually also be responsible for determining which
  3735         -  ** tokens are 'deferred' until the document text is loaded into memory.
  3736         -  **
  3737         -  ** Each token in each phrase is dealt with using one of the following
  3738         -  ** three strategies:
  3739         -  **
  3740         -  **   1. Entire doclist loaded into memory as part of the
  3741         -  **      fts3EvalStartReaders() call.
  3742         -  **
  3743         -  **   2. Doclist loaded into memory incrementally, as part of each
  3744         -  **      sqlite3Fts3EvalNext() call.
  3745         -  **
  3746         -  **   3. Token doclist is never loaded. Instead, documents are loaded into
  3747         -  **      memory and scanned for the token as part of the sqlite3Fts3EvalNext()
  3748         -  **      call. This is known as a "deferred" token.
  3749         -  */
  3750         -
  3751         -  /* If bOptOk is true, check if there are any tokens that should be deferred.
  3752         -  */
  3753         -  if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){
         3984  +  /* Determine which, if any, tokens in the expression should be deferred. */
         3985  +  if( rc==SQLITE_OK && nToken>1 && pTab->bHasStat ){
  3754   3986       Fts3TokenAndCost *aTC;
  3755   3987       Fts3Expr **apOr;
  3756   3988       aTC = (Fts3TokenAndCost *)sqlite3_malloc(
  3757   3989           sizeof(Fts3TokenAndCost) * nToken
  3758   3990         + sizeof(Fts3Expr *) * nOr * 2
  3759   3991       );
  3760   3992       apOr = (Fts3Expr **)&aTC[nToken];
................................................................................
  3762   3994       if( !aTC ){
  3763   3995         rc = SQLITE_NOMEM;
  3764   3996       }else{
  3765   3997         int ii;
  3766   3998         Fts3TokenAndCost *pTC = aTC;
  3767   3999         Fts3Expr **ppOr = apOr;
  3768   4000   
  3769         -      fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc);
         4001  +      fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc);
  3770   4002         nToken = pTC-aTC;
  3771   4003         nOr = ppOr-apOr;
  3772   4004   
  3773   4005         if( rc==SQLITE_OK ){
  3774   4006           rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken);
  3775   4007           for(ii=0; rc==SQLITE_OK && ii<nOr; ii++){
  3776   4008             rc = fts3EvalSelectDeferred(pCsr, apOr[ii], aTC, nToken);
................................................................................
  3777   4009           }
  3778   4010         }
  3779   4011   
  3780   4012         sqlite3_free(aTC);
  3781   4013       }
  3782   4014     }
  3783   4015   
  3784         -  fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc);
         4016  +  fts3EvalStartReaders(pCsr, pCsr->pExpr, 1, &rc);
  3785   4017     return rc;
  3786   4018   }
  3787   4019   
  3788         -static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){
         4020  +/*
         4021  +** Invalidate the current position list for phrase pPhrase.
         4022  +*/
         4023  +static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){
  3789   4024     if( pPhrase->doclist.bFreeList ){
  3790   4025       sqlite3_free(pPhrase->doclist.pList);
  3791   4026     }
  3792   4027     pPhrase->doclist.pList = 0;
  3793   4028     pPhrase->doclist.nList = 0;
  3794   4029     pPhrase->doclist.bFreeList = 0;
  3795   4030   }
  3796   4031   
  3797         -static int fts3EvalNearTrim2(
  3798         -  int nNear,
         4032  +/*
         4033  +** This function is called to edit the position list associated with
         4034  +** the phrase object passed as the fifth argument according to a NEAR
         4035  +** condition. For example:
         4036  +**
         4037  +**     abc NEAR/5 "def ghi"
         4038  +**
         4039  +** Parameter nNear is passed the NEAR distance of the expression (5 in
         4040  +** the example above). When this function is called, *paPoslist points to
         4041  +** the position list, and *pnToken is the number of phrase tokens in, the
         4042  +** phrase on the other side of the NEAR operator to pPhrase. For example,
         4043  +** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to
         4044  +** the position list associated with phrase "abc".
         4045  +**
         4046  +** All positions in the pPhrase position list that are not sufficiently
         4047  +** close to a position in the *paPoslist position list are removed. If this
         4048  +** leaves 0 positions, zero is returned. Otherwise, non-zero.
         4049  +**
         4050  +** Before returning, *paPoslist is set to point to the position lsit 
         4051  +** associated with pPhrase. And *pnToken is set to the number of tokens in
         4052  +** pPhrase.
         4053  +*/
         4054  +static int fts3EvalNearTrim(
         4055  +  int nNear,                      /* NEAR distance. As in "NEAR/nNear". */
  3799   4056     char *aTmp,                     /* Temporary space to use */
  3800   4057     char **paPoslist,               /* IN/OUT: Position list */
  3801   4058     int *pnToken,                   /* IN/OUT: Tokens in phrase of *paPoslist */
  3802   4059     Fts3Phrase *pPhrase             /* The phrase object to trim the doclist of */
  3803   4060   ){
  3804   4061     int nParam1 = nNear + pPhrase->nToken;
  3805   4062     int nParam2 = nNear + *pnToken;
................................................................................
  3823   4080       *paPoslist = pPhrase->doclist.pList;
  3824   4081       *pnToken = pPhrase->nToken;
  3825   4082     }
  3826   4083   
  3827   4084     return res;
  3828   4085   }
  3829   4086   
         4087  +/*
         4088  +** This function is a no-op if *pRc is other than SQLITE_OK when it is called.
         4089  +** Otherwise, it advances the expression passed as the second argument to
         4090  +** point to the next matching row in the database. Expressions iterate through
         4091  +** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero,
         4092  +** or descending if it is non-zero.
         4093  +**
         4094  +** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if
         4095  +** successful, the following variables in pExpr are set:
         4096  +**
         4097  +**   Fts3Expr.bEof                (non-zero if EOF - there is no next row)
         4098  +**   Fts3Expr.iDocid              (valid if bEof==0. The docid of the next row)
         4099  +**
         4100  +** If the expression is of type FTSQUERY_PHRASE, and the expression is not
         4101  +** at EOF, then the following variables are populated with the position list
         4102  +** for the phrase for the visited row:
         4103  +**
         4104  +**   FTs3Expr.pPhrase->doclist.nList        (length of pList in bytes)
         4105  +**   FTs3Expr.pPhrase->doclist.pList        (pointer to position list)
         4106  +**
         4107  +** It says above that this function advances the expression to the next
         4108  +** matching row. This is usually true, but there are the following exceptions:
         4109  +**
         4110  +**   1. Deferred tokens are not taken into account. If a phrase consists
         4111  +**      entirely of deferred tokens, it is assumed to match every row in
         4112  +**      the db. In this case the position-list is not populated at all. 
         4113  +**
         4114  +**      Or, if a phrase contains one or more deferred tokens and one or
         4115  +**      more non-deferred tokens, then the expression is advanced to the 
         4116  +**      next possible match, considering only non-deferred tokens. In other
         4117  +**      words, if the phrase is "A B C", and "B" is deferred, the expression
         4118  +**      is advanced to the next row that contains an instance of "A * C", 
         4119  +**      where "*" may match any single token. The position list in this case
         4120  +**      is populated as for "A * C" before returning.
         4121  +**
         4122  +**   2. NEAR is treated as AND. If the expression is "x NEAR y", it is 
         4123  +**      advanced to point to the next row that matches "x AND y".
         4124  +** 
         4125  +** See fts3EvalTestDeferredAndNear() for details on testing if a row is
         4126  +** really a match, taking into account deferred tokens and NEAR operators.
         4127  +*/
         4128  +static void fts3EvalNextRow(
         4129  +  Fts3Cursor *pCsr,               /* FTS Cursor handle */
         4130  +  Fts3Expr *pExpr,                /* Expr. to advance to next matching row */
         4131  +  int *pRc                        /* IN/OUT: Error code */
         4132  +){
         4133  +  if( *pRc==SQLITE_OK ){
         4134  +    int bDescDoclist = pCsr->bDesc;         /* Used by DOCID_CMP() macro */
         4135  +    assert( pExpr->bEof==0 );
         4136  +    pExpr->bStart = 1;
         4137  +
         4138  +    switch( pExpr->eType ){
         4139  +      case FTSQUERY_NEAR:
         4140  +      case FTSQUERY_AND: {
         4141  +        Fts3Expr *pLeft = pExpr->pLeft;
         4142  +        Fts3Expr *pRight = pExpr->pRight;
         4143  +        assert( !pLeft->bDeferred || !pRight->bDeferred );
         4144  +
         4145  +        if( pLeft->bDeferred ){
         4146  +          /* LHS is entirely deferred. So we assume it matches every row.
         4147  +          ** Advance the RHS iterator to find the next row visited. */
         4148  +          fts3EvalNextRow(pCsr, pRight, pRc);
         4149  +          pExpr->iDocid = pRight->iDocid;
         4150  +          pExpr->bEof = pRight->bEof;
         4151  +        }else if( pRight->bDeferred ){
         4152  +          /* RHS is entirely deferred. So we assume it matches every row.
         4153  +          ** Advance the LHS iterator to find the next row visited. */
         4154  +          fts3EvalNextRow(pCsr, pLeft, pRc);
         4155  +          pExpr->iDocid = pLeft->iDocid;
         4156  +          pExpr->bEof = pLeft->bEof;
         4157  +        }else{
         4158  +          /* Neither the RHS or LHS are deferred. */
         4159  +          fts3EvalNextRow(pCsr, pLeft, pRc);
         4160  +          fts3EvalNextRow(pCsr, pRight, pRc);
         4161  +          while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
         4162  +            sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
         4163  +            if( iDiff==0 ) break;
         4164  +            if( iDiff<0 ){
         4165  +              fts3EvalNextRow(pCsr, pLeft, pRc);
         4166  +            }else{
         4167  +              fts3EvalNextRow(pCsr, pRight, pRc);
         4168  +            }
         4169  +          }
         4170  +          pExpr->iDocid = pLeft->iDocid;
         4171  +          pExpr->bEof = (pLeft->bEof || pRight->bEof);
         4172  +        }
         4173  +        break;
         4174  +      }
         4175  +  
         4176  +      case FTSQUERY_OR: {
         4177  +        Fts3Expr *pLeft = pExpr->pLeft;
         4178  +        Fts3Expr *pRight = pExpr->pRight;
         4179  +        sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
         4180  +
         4181  +        assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
         4182  +        assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );
         4183  +
         4184  +        if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
         4185  +          fts3EvalNextRow(pCsr, pLeft, pRc);
         4186  +        }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
         4187  +          fts3EvalNextRow(pCsr, pRight, pRc);
         4188  +        }else{
         4189  +          fts3EvalNextRow(pCsr, pLeft, pRc);
         4190  +          fts3EvalNextRow(pCsr, pRight, pRc);
         4191  +        }
         4192  +
         4193  +        pExpr->bEof = (pLeft->bEof && pRight->bEof);
         4194  +        iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
         4195  +        if( pRight->bEof || (pLeft->bEof==0 &&  iCmp<0) ){
         4196  +          pExpr->iDocid = pLeft->iDocid;
         4197  +        }else{
         4198  +          pExpr->iDocid = pRight->iDocid;
         4199  +        }
         4200  +
         4201  +        break;
         4202  +      }
         4203  +
         4204  +      case FTSQUERY_NOT: {
         4205  +        Fts3Expr *pLeft = pExpr->pLeft;
         4206  +        Fts3Expr *pRight = pExpr->pRight;
         4207  +
         4208  +        if( pRight->bStart==0 ){
         4209  +          fts3EvalNextRow(pCsr, pRight, pRc);
         4210  +          assert( *pRc!=SQLITE_OK || pRight->bStart );
         4211  +        }
         4212  +
         4213  +        fts3EvalNextRow(pCsr, pLeft, pRc);
         4214  +        if( pLeft->bEof==0 ){
         4215  +          while( !*pRc 
         4216  +              && !pRight->bEof 
         4217  +              && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 
         4218  +          ){
         4219  +            fts3EvalNextRow(pCsr, pRight, pRc);
         4220  +          }
         4221  +        }
         4222  +        pExpr->iDocid = pLeft->iDocid;
         4223  +        pExpr->bEof = pLeft->bEof;
         4224  +        break;
         4225  +      }
         4226  +
         4227  +      default: {
         4228  +        Fts3Phrase *pPhrase = pExpr->pPhrase;
         4229  +        fts3EvalInvalidatePoslist(pPhrase);
         4230  +        *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
         4231  +        pExpr->iDocid = pPhrase->doclist.iDocid;
         4232  +        break;
         4233  +      }
         4234  +    }
         4235  +  }
         4236  +}
         4237  +
         4238  +/*
         4239  +** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR
         4240  +** cluster, then this function returns 1 immediately.
         4241  +**
         4242  +** Otherwise, it checks if the current row really does match the NEAR 
         4243  +** expression, using the data currently stored in the position lists 
         4244  +** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. 
         4245  +**
         4246  +** If the current row is a match, the position list associated with each
         4247  +** phrase in the NEAR expression is edited in place to contain only those
         4248  +** phrase instances sufficiently close to their peers to satisfy all NEAR
         4249  +** constraints. In this case it returns 1. If the NEAR expression does not 
         4250  +** match the current row, 0 is returned. The position lists may or may not
         4251  +** be edited if 0 is returned.
         4252  +*/
  3830   4253   static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
  3831   4254     int res = 1;
  3832   4255   
  3833   4256     /* The following block runs if pExpr is the root of a NEAR query.
  3834   4257     ** For example, the query:
  3835   4258     **
  3836   4259     **         "w" NEAR "x" NEAR "y" NEAR "z"
................................................................................
  3844   4267     **                     |        |
  3845   4268     **                +--NEAR--+   "y"
  3846   4269     **                |        |
  3847   4270     **               "w"      "x"
  3848   4271     **
  3849   4272     ** The right-hand child of a NEAR node is always a phrase. The 
  3850   4273     ** left-hand child may be either a phrase or a NEAR node. There are
  3851         -  ** no exceptions to this.
         4274  +  ** no exceptions to this - it's the way the parser in fts3_expr.c works.
  3852   4275     */
  3853   4276     if( *pRc==SQLITE_OK 
  3854   4277      && pExpr->eType==FTSQUERY_NEAR 
  3855   4278      && pExpr->bEof==0
  3856   4279      && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
  3857   4280     ){
  3858   4281       Fts3Expr *p; 
................................................................................
  3871   4294       }else{
  3872   4295         char *aPoslist = p->pPhrase->doclist.pList;
  3873   4296         int nToken = p->pPhrase->nToken;
  3874   4297   
  3875   4298         for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
  3876   4299           Fts3Phrase *pPhrase = p->pRight->pPhrase;
  3877   4300           int nNear = p->nNear;
  3878         -        res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
         4301  +        res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
  3879   4302         }
  3880   4303     
  3881   4304         aPoslist = pExpr->pRight->pPhrase->doclist.pList;
  3882   4305         nToken = pExpr->pRight->pPhrase->nToken;
  3883   4306         for(p=pExpr->pLeft; p && res; p=p->pLeft){
  3884   4307           int nNear = p->pParent->nNear;
  3885   4308           Fts3Phrase *pPhrase = (
  3886   4309               p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
  3887   4310           );
  3888         -        res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
         4311  +        res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
  3889   4312         }
  3890   4313       }
  3891   4314   
  3892   4315       sqlite3_free(aTmp);
  3893   4316     }
  3894   4317   
  3895   4318     return res;
  3896   4319   }
  3897   4320   
  3898   4321   /*
  3899         -** This macro is used by the fts3EvalNext() function. The two arguments are
  3900         -** 64-bit docid values. If the current query is "ORDER BY docid ASC", then
  3901         -** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then
  3902         -** it returns (i2 - i1). This allows the same code to be used for merging
  3903         -** doclists in ascending or descending order.
         4322  +** This function is a helper function for fts3EvalTestDeferredAndNear().
         4323  +** Assuming no error occurs or has occurred, It returns non-zero if the
         4324  +** expression passed as the second argument matches the row that pCsr 
         4325  +** currently points to, or zero if it does not.
         4326  +**
         4327  +** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
         4328  +** If an error occurs during execution of this function, *pRc is set to 
         4329  +** the appropriate SQLite error code. In this case the returned value is 
         4330  +** undefined.
  3904   4331   */
  3905         -#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2))
  3906         -
  3907         -static void fts3EvalNext(
  3908         -  Fts3Cursor *pCsr, 
  3909         -  Fts3Expr *pExpr, 
  3910         -  int *pRc
         4332  +static int fts3EvalTestExpr(
         4333  +  Fts3Cursor *pCsr,               /* FTS cursor handle */
         4334  +  Fts3Expr *pExpr,                /* Expr to test. May or may not be root. */
         4335  +  int *pRc                        /* IN/OUT: Error code */
  3911   4336   ){
  3912         -  if( *pRc==SQLITE_OK ){
  3913         -    assert( pExpr->bEof==0 );
  3914         -    pExpr->bStart = 1;
  3915         -
  3916         -    switch( pExpr->eType ){
  3917         -      case FTSQUERY_NEAR:
  3918         -      case FTSQUERY_AND: {
  3919         -        Fts3Expr *pLeft = pExpr->pLeft;
  3920         -        Fts3Expr *pRight = pExpr->pRight;
  3921         -        assert( !pLeft->bDeferred || !pRight->bDeferred );
  3922         -        if( pLeft->bDeferred ){
  3923         -          fts3EvalNext(pCsr, pRight, pRc);
  3924         -          pExpr->iDocid = pRight->iDocid;
  3925         -          pExpr->bEof = pRight->bEof;
  3926         -        }else if( pRight->bDeferred ){
  3927         -          fts3EvalNext(pCsr, pLeft, pRc);
  3928         -          pExpr->iDocid = pLeft->iDocid;
  3929         -          pExpr->bEof = pLeft->bEof;
  3930         -        }else{
  3931         -          fts3EvalNext(pCsr, pLeft, pRc);
  3932         -          fts3EvalNext(pCsr, pRight, pRc);
  3933         -
  3934         -          while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
  3935         -            sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
  3936         -            if( iDiff==0 ) break;
  3937         -            if( iDiff<0 ){
  3938         -              fts3EvalNext(pCsr, pLeft, pRc);
  3939         -            }else{
  3940         -              fts3EvalNext(pCsr, pRight, pRc);
  3941         -            }
  3942         -          }
  3943         -
  3944         -          pExpr->iDocid = pLeft->iDocid;
  3945         -          pExpr->bEof = (pLeft->bEof || pRight->bEof);
  3946         -        }
  3947         -        break;
  3948         -      }
  3949         -  
  3950         -      case FTSQUERY_OR: {
  3951         -        Fts3Expr *pLeft = pExpr->pLeft;
  3952         -        Fts3Expr *pRight = pExpr->pRight;
  3953         -        sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
  3954         -
  3955         -        assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
  3956         -        assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );
  3957         -
  3958         -        if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
  3959         -          fts3EvalNext(pCsr, pLeft, pRc);
  3960         -        }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
  3961         -          fts3EvalNext(pCsr, pRight, pRc);
  3962         -        }else{
  3963         -          fts3EvalNext(pCsr, pLeft, pRc);
  3964         -          fts3EvalNext(pCsr, pRight, pRc);
  3965         -        }
  3966         -
  3967         -        pExpr->bEof = (pLeft->bEof && pRight->bEof);
  3968         -        iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
  3969         -        if( pRight->bEof || (pLeft->bEof==0 &&  iCmp<0) ){
  3970         -          pExpr->iDocid = pLeft->iDocid;
  3971         -        }else{
  3972         -          pExpr->iDocid = pRight->iDocid;
  3973         -        }
  3974         -
  3975         -        break;
  3976         -      }
  3977         -
  3978         -      case FTSQUERY_NOT: {
  3979         -        Fts3Expr *pLeft = pExpr->pLeft;
  3980         -        Fts3Expr *pRight = pExpr->pRight;
  3981         -
  3982         -        if( pRight->bStart==0 ){
  3983         -          fts3EvalNext(pCsr, pRight, pRc);
  3984         -          assert( *pRc!=SQLITE_OK || pRight->bStart );
  3985         -        }
  3986         -
  3987         -        fts3EvalNext(pCsr, pLeft, pRc);
  3988         -        if( pLeft->bEof==0 ){
  3989         -          while( !*pRc 
  3990         -              && !pRight->bEof 
  3991         -              && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 
  3992         -          ){
  3993         -            fts3EvalNext(pCsr, pRight, pRc);
  3994         -          }
  3995         -        }
  3996         -        pExpr->iDocid = pLeft->iDocid;
  3997         -        pExpr->bEof = pLeft->bEof;
  3998         -        break;
  3999         -      }
  4000         -
  4001         -      default: {
  4002         -        Fts3Phrase *pPhrase = pExpr->pPhrase;
  4003         -        fts3EvalZeroPoslist(pPhrase);
  4004         -        *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
  4005         -        pExpr->iDocid = pPhrase->doclist.iDocid;
  4006         -        break;
  4007         -      }
  4008         -    }
  4009         -  }
  4010         -}
  4011         -
  4012         -static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){
  4013         -  int bHit = 1;
         4337  +  int bHit = 1;                   /* Return value */
  4014   4338     if( *pRc==SQLITE_OK ){
  4015   4339       switch( pExpr->eType ){
  4016   4340         case FTSQUERY_NEAR:
  4017   4341         case FTSQUERY_AND:
  4018   4342           bHit = (
  4019         -            fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
  4020         -         && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
         4343  +            fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc)
         4344  +         && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc)
  4021   4345            && fts3EvalNearTest(pExpr, pRc)
  4022   4346           );
  4023   4347   
  4024   4348           /* If the NEAR expression does not match any rows, zero the doclist for 
  4025   4349           ** all phrases involved in the NEAR. This is because the snippet(),
  4026   4350           ** offsets() and matchinfo() functions are not supposed to recognize 
  4027   4351           ** any instances of phrases that are part of unmatched NEAR queries. 
................................................................................
  4039   4363           if( bHit==0 
  4040   4364            && pExpr->eType==FTSQUERY_NEAR 
  4041   4365            && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
  4042   4366           ){
  4043   4367             Fts3Expr *p;
  4044   4368             for(p=pExpr; p->pPhrase==0; p=p->pLeft){
  4045   4369               if( p->pRight->iDocid==pCsr->iPrevId ){
  4046         -              fts3EvalZeroPoslist(p->pRight->pPhrase);
         4370  +              fts3EvalInvalidatePoslist(p->pRight->pPhrase);
  4047   4371               }
  4048   4372             }
  4049   4373             if( p->iDocid==pCsr->iPrevId ){
  4050         -            fts3EvalZeroPoslist(p->pPhrase);
         4374  +            fts3EvalInvalidatePoslist(p->pPhrase);
  4051   4375             }
  4052   4376           }
  4053   4377   
  4054   4378           break;
  4055   4379   
  4056   4380         case FTSQUERY_OR: {
  4057         -        int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc);
  4058         -        int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc);
         4381  +        int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc);
         4382  +        int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc);
  4059   4383           bHit = bHit1 || bHit2;
  4060   4384           break;
  4061   4385         }
  4062   4386   
  4063   4387         case FTSQUERY_NOT:
  4064   4388           bHit = (
  4065         -            fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
  4066         -         && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
         4389  +            fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc)
         4390  +         && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc)
  4067   4391           );
  4068   4392           break;
  4069   4393   
  4070   4394         default: {
  4071   4395           if( pCsr->pDeferred 
  4072   4396            && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred)
  4073   4397           ){
  4074   4398             Fts3Phrase *pPhrase = pExpr->pPhrase;
  4075   4399             assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 );
  4076   4400             if( pExpr->bDeferred ){
  4077         -            fts3EvalZeroPoslist(pPhrase);
         4401  +            fts3EvalInvalidatePoslist(pPhrase);
  4078   4402             }
  4079   4403             *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase);
  4080   4404             bHit = (pPhrase->doclist.pList!=0);
  4081   4405             pExpr->iDocid = pCsr->iPrevId;
  4082   4406           }else{
  4083   4407             bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId);
  4084   4408           }
................................................................................
  4086   4410         }
  4087   4411       }
  4088   4412     }
  4089   4413     return bHit;
  4090   4414   }
  4091   4415   
  4092   4416   /*
  4093         -** Return 1 if both of the following are true:
         4417  +** This function is called as the second part of each xNext operation when
         4418  +** iterating through the results of a full-text query. At this point the
         4419  +** cursor points to a row that matches the query expression, with the
         4420  +** following caveats:
         4421  +**
         4422  +**   * Up until this point, "NEAR" operators in the expression have been
         4423  +**     treated as "AND".
         4424  +**
         4425  +**   * Deferred tokens have not yet been considered.
         4426  +**
         4427  +** If *pRc is not SQLITE_OK when this function is called, it immediately
         4428  +** returns 0. Otherwise, it tests whether or not after considering NEAR
         4429  +** operators and deferred tokens the current row is still a match for the
         4430  +** expression. It returns 1 if both of the following are true:
  4094   4431   **
  4095   4432   **   1. *pRc is SQLITE_OK when this function returns, and
  4096   4433   **
  4097   4434   **   2. After scanning the current FTS table row for the deferred tokens,
  4098         -**      it is determined that the row does not match the query.
         4435  +**      it is determined that the row does *not* match the query.
  4099   4436   **
  4100   4437   ** Or, if no error occurs and it seems the current row does match the FTS
  4101   4438   ** query, return 0.
  4102   4439   */
  4103         -static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){
         4440  +static int fts3EvalTestDeferredAndNear(Fts3Cursor *pCsr, int *pRc){
  4104   4441     int rc = *pRc;
  4105   4442     int bMiss = 0;
  4106   4443     if( rc==SQLITE_OK ){
         4444  +
         4445  +    /* If there are one or more deferred tokens, load the current row into
         4446  +    ** memory and scan it to determine the position list for each deferred
         4447  +    ** token. Then, see if this row is really a match, considering deferred
         4448  +    ** tokens and NEAR operators (neither of which were taken into account
         4449  +    ** earlier, by fts3EvalNextRow()). 
         4450  +    */
  4107   4451       if( pCsr->pDeferred ){
  4108   4452         rc = fts3CursorSeek(0, pCsr);
  4109   4453         if( rc==SQLITE_OK ){
  4110   4454           rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
  4111   4455         }
  4112   4456       }
  4113         -    bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc));
         4457  +    bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc));
         4458  +
         4459  +    /* Free the position-lists accumulated for each deferred token above. */
  4114   4460       sqlite3Fts3FreeDeferredDoclists(pCsr);
  4115   4461       *pRc = rc;
  4116   4462     }
  4117   4463     return (rc==SQLITE_OK && bMiss);
  4118   4464   }
  4119   4465   
  4120   4466   /*
  4121   4467   ** Advance to the next document that matches the FTS expression in
  4122   4468   ** Fts3Cursor.pExpr.
  4123   4469   */
  4124         -int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){
         4470  +static int fts3EvalNext(Fts3Cursor *pCsr){
  4125   4471     int rc = SQLITE_OK;             /* Return Code */
  4126   4472     Fts3Expr *pExpr = pCsr->pExpr;
  4127   4473     assert( pCsr->isEof==0 );
  4128   4474     if( pExpr==0 ){
  4129   4475       pCsr->isEof = 1;
  4130   4476     }else{
  4131   4477       do {
  4132   4478         if( pCsr->isRequireSeek==0 ){
  4133   4479           sqlite3_reset(pCsr->pStmt);
  4134   4480         }
  4135   4481         assert( sqlite3_data_count(pCsr->pStmt)==0 );
  4136         -      fts3EvalNext(pCsr, pExpr, &rc);
         4482  +      fts3EvalNextRow(pCsr, pExpr, &rc);
  4137   4483         pCsr->isEof = pExpr->bEof;
  4138   4484         pCsr->isRequireSeek = 1;
  4139   4485         pCsr->isMatchinfoNeeded = 1;
  4140   4486         pCsr->iPrevId = pExpr->iDocid;
  4141         -    }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) );
         4487  +    }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) );
  4142   4488     }
  4143   4489     return rc;
  4144   4490   }
  4145   4491   
  4146   4492   /*
  4147   4493   ** Restart interation for expression pExpr so that the next call to
  4148         -** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental 
         4494  +** fts3EvalNext() visits the first row. Do not allow incremental 
  4149   4495   ** loading or merging of phrase doclists for this iteration.
  4150   4496   **
  4151   4497   ** If *pRc is other than SQLITE_OK when this function is called, it is
  4152   4498   ** a no-op. If an error occurs within this function, *pRc is set to an
  4153   4499   ** SQLite error code before returning.
  4154   4500   */
  4155   4501   static void fts3EvalRestart(
................................................................................
  4157   4503     Fts3Expr *pExpr,
  4158   4504     int *pRc
  4159   4505   ){
  4160   4506     if( pExpr && *pRc==SQLITE_OK ){
  4161   4507       Fts3Phrase *pPhrase = pExpr->pPhrase;
  4162   4508   
  4163   4509       if( pPhrase ){
  4164         -      fts3EvalZeroPoslist(pPhrase);
         4510  +      fts3EvalInvalidatePoslist(pPhrase);
  4165   4511         if( pPhrase->bIncr ){
  4166   4512           assert( pPhrase->nToken==1 );
  4167   4513           assert( pPhrase->aToken[0].pSegcsr );
  4168   4514           sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr);
  4169   4515           *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase);
  4170   4516         }
  4171   4517   
................................................................................
  4273   4619   
  4274   4620         do {
  4275   4621           /* Ensure the %_content statement is reset. */
  4276   4622           if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt);
  4277   4623           assert( sqlite3_data_count(pCsr->pStmt)==0 );
  4278   4624   
  4279   4625           /* Advance to the next document */
  4280         -        fts3EvalNext(pCsr, pRoot, &rc);
         4626  +        fts3EvalNextRow(pCsr, pRoot, &rc);
  4281   4627           pCsr->isEof = pRoot->bEof;
  4282   4628           pCsr->isRequireSeek = 1;
  4283   4629           pCsr->isMatchinfoNeeded = 1;
  4284   4630           pCsr->iPrevId = pRoot->iDocid;
  4285   4631         }while( pCsr->isEof==0 
  4286   4632              && pRoot->eType==FTSQUERY_NEAR 
  4287         -           && fts3EvalLoadDeferred(pCsr, &rc) 
         4633  +           && fts3EvalTestDeferredAndNear(pCsr, &rc) 
  4288   4634         );
  4289   4635   
  4290   4636         if( rc==SQLITE_OK && pCsr->isEof==0 ){
  4291   4637           fts3EvalUpdateCounts(pRoot);
  4292   4638         }
  4293   4639       }
  4294   4640   
................................................................................
  4302   4648         ** order. For this reason, even though it seems more defensive, the 
  4303   4649         ** do loop can not be written:
  4304   4650         **
  4305   4651         **   do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK );
  4306   4652         */
  4307   4653         fts3EvalRestart(pCsr, pRoot, &rc);
  4308   4654         do {
  4309         -        fts3EvalNext(pCsr, pRoot, &rc);
         4655  +        fts3EvalNextRow(pCsr, pRoot, &rc);
  4310   4656           assert( pRoot->bEof==0 );
  4311   4657         }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
  4312         -      fts3EvalLoadDeferred(pCsr, &rc);
         4658  +      fts3EvalTestDeferredAndNear(pCsr, &rc);
  4313   4659       }
  4314   4660     }
  4315   4661     return rc;
  4316   4662   }
  4317   4663   
  4318   4664   /*
  4319   4665   ** This function is used by the matchinfo() module to query a phrase 
................................................................................
  4436   4782   **   * the contents of pPhrase->doclist, and
  4437   4783   **   * any Fts3MultiSegReader objects held by phrase tokens.
  4438   4784   */
  4439   4785   void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){
  4440   4786     if( pPhrase ){
  4441   4787       int i;
  4442   4788       sqlite3_free(pPhrase->doclist.aAll);
  4443         -    fts3EvalZeroPoslist(pPhrase);
         4789  +    fts3EvalInvalidatePoslist(pPhrase);
  4444   4790       memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
  4445   4791       for(i=0; i<pPhrase->nToken; i++){
  4446   4792         fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr);
  4447   4793         pPhrase->aToken[i].pSegcsr = 0;
  4448   4794       }
  4449   4795     }
  4450   4796   }
         4797  +
         4798  +#if !SQLITE_CORE
         4799  +/*
         4800  +** Initialize API pointer table, if required.
         4801  +*/
         4802  +int sqlite3_extension_init(
         4803  +  sqlite3 *db, 
         4804  +  char **pzErrMsg,
         4805  +  const sqlite3_api_routines *pApi
         4806  +){
         4807  +  SQLITE_EXTENSION_INIT2(pApi)
         4808  +  return sqlite3Fts3Init(db);
         4809  +}
         4810  +#endif
  4451   4811   
  4452   4812   #endif

Changes to ext/fts3/fts3Int.h.

    23     23   ** SQLITE_ENABLE_FTS3 macro.  But to avoid confusion we also all
    24     24   ** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3.
    25     25   */
    26     26   #if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3)
    27     27   # define SQLITE_ENABLE_FTS3
    28     28   #endif
    29     29   
    30         -#ifdef SQLITE_ENABLE_FTS3
           30  +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
           31  +
           32  +/* If not building as part of the core, include sqlite3ext.h. */
           33  +#ifndef SQLITE_CORE
           34  +# include "sqlite3ext.h" 
           35  +extern const sqlite3_api_routines *sqlite3_api;
           36  +#endif
           37  +
    31     38   #include "sqlite3.h"
    32     39   #include "fts3_tokenizer.h"
    33     40   #include "fts3_hash.h"
    34     41   
    35     42   /*
    36     43   ** This constant controls how often segments are merged. Once there are
    37     44   ** FTS3_MERGE_COUNT segments of level N, they are merged into a single
................................................................................
   286    293     int nAll;                      /* Size of a[] in bytes */
   287    294     char *pNextDocid;              /* Pointer to next docid */
   288    295   
   289    296     sqlite3_int64 iDocid;          /* Current docid (if pList!=0) */
   290    297     int bFreeList;                 /* True if pList should be sqlite3_free()d */
   291    298     char *pList;                   /* Pointer to position list following iDocid */
   292    299     int nList;                     /* Length of position list */
   293         -} doclist;
          300  +};
   294    301   
   295    302   /*
   296    303   ** A "phrase" is a sequence of one or more tokens that must match in
   297    304   ** sequence.  A single token is the base case and the most common case.
   298    305   ** For a sequence of tokens contained in double-quotes (i.e. "one two three")
   299    306   ** nToken will be the number of tokens in the string.
   300    307   */
................................................................................
   486    493   int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
   487    494   int sqlite3Fts3InitTerm(sqlite3 *db);
   488    495   #endif
   489    496   
   490    497   /* fts3_aux.c */
   491    498   int sqlite3Fts3InitAux(sqlite3 *db);
   492    499   
   493         -int sqlite3Fts3TermSegReaderCursor(
   494         -  Fts3Cursor *pCsr,               /* Virtual table cursor handle */
   495         -  const char *zTerm,              /* Term to query for */
   496         -  int nTerm,                      /* Size of zTerm in bytes */
   497         -  int isPrefix,                   /* True for a prefix search */
   498         -  Fts3MultiSegReader **ppSegcsr   /* OUT: Allocated seg-reader cursor */
   499         -);
   500         -
   501    500   void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
   502    501   
   503         -int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int);
   504         -int sqlite3Fts3EvalNext(Fts3Cursor *pCsr);
   505         -
   506    502   int sqlite3Fts3MsrIncrStart(
   507    503       Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
   508    504   int sqlite3Fts3MsrIncrNext(
   509    505       Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
   510    506   char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); 
   511    507   int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
   512    508   int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
   513    509   
   514    510   int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);
   515    511   
   516         -#endif /* SQLITE_ENABLE_FTS3 */
          512  +#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
   517    513   #endif /* _FTSINT_H */

Changes to ext/fts3/fts3_hash.c.

    26     26   #include "fts3Int.h"
    27     27   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    28     28   
    29     29   #include <assert.h>
    30     30   #include <stdlib.h>
    31     31   #include <string.h>
    32     32   
    33         -#include "sqlite3.h"
    34     33   #include "fts3_hash.h"
    35     34   
    36     35   /*
    37     36   ** Malloc and Free functions
    38     37   */
    39     38   static void *fts3HashMalloc(int n){
    40     39     void *p = sqlite3_malloc(n);

Changes to ext/fts3/fts3_test.c.

    14     14   ** testing. It contains a Tcl command that can be used to test if a document
    15     15   ** matches an FTS NEAR expression.
    16     16   */
    17     17   
    18     18   #include <tcl.h>
    19     19   #include <string.h>
    20     20   #include <assert.h>
           21  +
           22  +#ifdef SQLITE_TEST
    21     23   
    22     24   /* Required so that the "ifdef SQLITE_ENABLE_FTS3" below works */
    23     25   #include "fts3Int.h"
    24     26   
    25     27   #define NM_MAX_TOKEN 12
    26     28   
    27     29   typedef struct NearPhrase NearPhrase;
................................................................................
   315    317   int Sqlitetestfts3_Init(Tcl_Interp *interp){
   316    318     Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);
   317    319     Tcl_CreateObjCommand(interp, 
   318    320         "fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0
   319    321     );
   320    322     return TCL_OK;
   321    323   }
          324  +#endif                  /* ifdef SQLITE_TEST */

Changes to ext/fts3/fts3_tokenizer.c.

    19     19   **
    20     20   **     * The FTS3 module is being built as an extension
    21     21   **       (in which case SQLITE_CORE is not defined), or
    22     22   **
    23     23   **     * The FTS3 module is being built into the core of
    24     24   **       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
    25     25   */
    26         -#include "sqlite3ext.h"
    27         -#ifndef SQLITE_CORE
    28         -  SQLITE_EXTENSION_INIT1
    29         -#endif
    30     26   #include "fts3Int.h"
    31         -
    32     27   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    33     28   
    34     29   #include <assert.h>
    35     30   #include <string.h>
    36     31   
    37     32   /*
    38     33   ** Implementation of the SQL scalar function for accessing the underlying 

Changes to main.mk.

   381    381   
   382    382   sqlite3.c:	target_source $(TOP)/tool/mksqlite3c.tcl
   383    383   	tclsh $(TOP)/tool/mksqlite3c.tcl
   384    384   	echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c
   385    385   	cat sqlite3.c >>tclsqlite3.c
   386    386   	echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c
   387    387   	cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c
          388  +
          389  +sqlite3-all.c:	sqlite3.c $(TOP)/tool/split-sqlite3c.tcl
          390  +	tclsh $(TOP)/tool/split-sqlite3c.tcl
   388    391   
   389    392   fts2amal.c:	target_source $(TOP)/ext/fts2/mkfts2amal.tcl
   390    393   	tclsh $(TOP)/ext/fts2/mkfts2amal.tcl
   391    394   
   392    395   fts3amal.c:	target_source $(TOP)/ext/fts3/mkfts3amal.tcl
   393    396   	tclsh $(TOP)/ext/fts3/mkfts3amal.tcl
   394    397   
................................................................................
   563    566   
   564    567   TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO)
   565    568   $(TEST_EXTENSION): $(TOP)/src/test_loadext.c
   566    569   	$(MKSHLIB) $(TOP)/src/test_loadext.c -o $(TEST_EXTENSION)
   567    570   
   568    571   extensiontest: testfixture$(EXE) $(TEST_EXTENSION)
   569    572   	./testfixture$(EXE) $(TOP)/test/loadext.test
          573  +
          574  +# This target will fail if the SQLite amalgamation contains any exported
          575  +# symbols that do not begin with "sqlite3_". It is run as part of the
          576  +# releasetest.tcl script.
          577  +#
          578  +checksymbols: sqlite3.o
          579  +	nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0
   570    580   
   571    581   
   572    582   # Standard install and cleanup targets
   573    583   #
   574    584   install:	sqlite3 libsqlite3.a sqlite3.h
   575    585   	mv sqlite3 /usr/bin
   576    586   	mv libsqlite3.a /usr/lib

Changes to src/build.c.

  2787   2787       ** the zStmt variable
  2788   2788       */
  2789   2789       if( pStart ){
  2790   2790         assert( pEnd!=0 );
  2791   2791         /* A named index with an explicit CREATE INDEX statement */
  2792   2792         zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s",
  2793   2793           onError==OE_None ? "" : " UNIQUE",
  2794         -        pEnd->z - pName->z + 1,
         2794  +        (int)(pEnd->z - pName->z) + 1,
  2795   2795           pName->z);
  2796   2796       }else{
  2797   2797         /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */
  2798   2798         /* zStmt = sqlite3MPrintf(""); */
  2799   2799         zStmt = 0;
  2800   2800       }
  2801   2801   

Changes to src/date.c.

   423    423   ** routine will always fail.
   424    424   */
   425    425   static int osLocaltime(time_t *t, struct tm *pTm){
   426    426     int rc;
   427    427   #if (!defined(HAVE_LOCALTIME_R) || !HAVE_LOCALTIME_R) \
   428    428         && (!defined(HAVE_LOCALTIME_S) || !HAVE_LOCALTIME_S)
   429    429     struct tm *pX;
          430  +#if SQLITE_THREADSAFE>0
   430    431     sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER);
          432  +#endif
   431    433     sqlite3_mutex_enter(mutex);
   432    434     pX = localtime(t);
   433    435   #ifndef SQLITE_OMIT_BUILTIN_TEST
   434    436     if( sqlite3GlobalConfig.bLocaltimeFault ) pX = 0;
   435    437   #endif
   436    438     if( pX ) *pTm = *pX;
   437    439     sqlite3_mutex_leave(mutex);

Changes to src/delete.c.

   374    374       int iRowSet = ++pParse->nMem;   /* Register for rowset of rows to delete */
   375    375       int iRowid = ++pParse->nMem;    /* Used for storing rowid values. */
   376    376       int regRowid;                   /* Actual register containing rowids */
   377    377   
   378    378       /* Collect rowids of every row to be deleted.
   379    379       */
   380    380       sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet);
   381         -    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,WHERE_DUPLICATES_OK);
          381  +    pWInfo = sqlite3WhereBegin(
          382  +        pParse, pTabList, pWhere, 0, 0, WHERE_DUPLICATES_OK
          383  +    );
   382    384       if( pWInfo==0 ) goto delete_from_cleanup;
   383    385       regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid);
   384    386       sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid);
   385    387       if( db->flags & SQLITE_CountRows ){
   386    388         sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1);
   387    389       }
   388    390       sqlite3WhereEnd(pWInfo);

Changes to src/expr.c.

   898    898       Table *pTab;
   899    899       pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase);
   900    900       pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName);
   901    901       pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias);
   902    902       pNewItem->jointype = pOldItem->jointype;
   903    903       pNewItem->iCursor = pOldItem->iCursor;
   904    904       pNewItem->isPopulated = pOldItem->isPopulated;
          905  +    pNewItem->isCorrelated = pOldItem->isCorrelated;
   905    906       pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex);
   906    907       pNewItem->notIndexed = pOldItem->notIndexed;
   907    908       pNewItem->pIndex = pOldItem->pIndex;
   908    909       pTab = pNewItem->pTab = pOldItem->pTab;
   909    910       if( pTab ){
   910    911         pTab->nRef++;
   911    912       }

Changes to src/fkey.c.

   556    556     sNameContext.pParse = pParse;
   557    557     sqlite3ResolveExprNames(&sNameContext, pWhere);
   558    558   
   559    559     /* Create VDBE to loop through the entries in pSrc that match the WHERE
   560    560     ** clause. If the constraint is not deferred, throw an exception for
   561    561     ** each row found. Otherwise, for deferred constraints, increment the
   562    562     ** deferred constraint counter by nIncr for each row selected.  */
   563         -  pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0);
          563  +  pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0);
   564    564     if( nIncr>0 && pFKey->isDeferred==0 ){
   565    565       sqlite3ParseToplevel(pParse)->mayAbort = 1;
   566    566     }
   567    567     sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr);
   568    568     if( pWInfo ){
   569    569       sqlite3WhereEnd(pWInfo);
   570    570     }

Changes to src/loadext.c.

    80     80   # define sqlite3_progress_handler 0
    81     81   #endif
    82     82   
    83     83   #ifdef SQLITE_OMIT_VIRTUALTABLE
    84     84   # define sqlite3_create_module 0
    85     85   # define sqlite3_create_module_v2 0
    86     86   # define sqlite3_declare_vtab 0
           87  +# define sqlite3_vtab_config 0
           88  +# define sqlite3_vtab_on_conflict 0
    87     89   #endif
    88     90   
    89     91   #ifdef SQLITE_OMIT_SHARED_CACHE
    90     92   # define sqlite3_enable_shared_cache 0
    91     93   #endif
    92     94   
    93     95   #ifdef SQLITE_OMIT_TRACE
................................................................................
   103    105   #ifdef SQLITE_OMIT_INCRBLOB
   104    106   #define sqlite3_bind_zeroblob  0
   105    107   #define sqlite3_blob_bytes     0
   106    108   #define sqlite3_blob_close     0
   107    109   #define sqlite3_blob_open      0
   108    110   #define sqlite3_blob_read      0
   109    111   #define sqlite3_blob_write     0
          112  +#define sqlite3_blob_reopen    0
   110    113   #endif
   111    114   
   112    115   /*
   113    116   ** The following structure contains pointers to all SQLite API routines.
   114    117   ** A pointer to this structure is passed into extensions when they are
   115    118   ** loaded so that the extension can make calls back into the SQLite
   116    119   ** library.
................................................................................
   368    371     sqlite3_wal_checkpoint,
   369    372     sqlite3_wal_hook,
   370    373   #else
   371    374     0,
   372    375     0,
   373    376     0,
   374    377   #endif
          378  +  sqlite3_blob_reopen,
          379  +  sqlite3_vtab_config,
          380  +  sqlite3_vtab_on_conflict,
   375    381   };
   376    382   
   377    383   /*
   378    384   ** Attempt to load an SQLite extension library contained in the file
   379    385   ** zFile.  The entry point is zProc.  zProc may be 0 in which case a
   380    386   ** default entry point name (sqlite3_extension_init) is used.  Use
   381    387   ** of the default name is recommended.

Changes to src/mem3.c.

   429    429   
   430    430   /*
   431    431   ** Free an outstanding memory allocation.
   432    432   **
   433    433   ** This function assumes that the necessary mutexes, if any, are
   434    434   ** already held by the caller. Hence "Unsafe".
   435    435   */
   436         -void memsys3FreeUnsafe(void *pOld){
          436  +static void memsys3FreeUnsafe(void *pOld){
   437    437     Mem3Block *p = (Mem3Block*)pOld;
   438    438     int i;
   439    439     u32 size, x;
   440    440     assert( sqlite3_mutex_held(mem3.mutex) );
   441    441     assert( p>mem3.aPool && p<&mem3.aPool[mem3.nPool] );
   442    442     i = p - mem3.aPool;
   443    443     assert( (mem3.aPool[i-1].u.hdr.size4x&1)==1 );
................................................................................
   504    504     memsys3Leave();
   505    505     return (void*)p; 
   506    506   }
   507    507   
   508    508   /*
   509    509   ** Free memory.
   510    510   */
   511         -void memsys3Free(void *pPrior){
          511  +static void memsys3Free(void *pPrior){
   512    512     assert( pPrior );
   513    513     memsys3Enter();
   514    514     memsys3FreeUnsafe(pPrior);
   515    515     memsys3Leave();
   516    516   }
   517    517   
   518    518   /*
   519    519   ** Change the size of an existing memory allocation
   520    520   */
   521         -void *memsys3Realloc(void *pPrior, int nBytes){
          521  +static void *memsys3Realloc(void *pPrior, int nBytes){
   522    522     int nOld;
   523    523     void *p;
   524    524     if( pPrior==0 ){
   525    525       return sqlite3_malloc(nBytes);
   526    526     }
   527    527     if( nBytes<=0 ){
   528    528       sqlite3_free(pPrior);

Changes to src/os.c.

   132    132   ){
   133    133     int rc;
   134    134     DO_OS_MALLOC_TEST(0);
   135    135     /* 0x87f3f is a mask of SQLITE_OPEN_ flags that are valid to be passed
   136    136     ** down into the VFS layer.  Some SQLITE_OPEN_ flags (for example,
   137    137     ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before
   138    138     ** reaching the VFS. */
   139         -  rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x87f3f, pFlagsOut);
          139  +  rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x87f7f, pFlagsOut);
   140    140     assert( rc==SQLITE_OK || pFile->pMethods==0 );
   141    141     return rc;
   142    142   }
   143    143   int sqlite3OsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
   144    144     return pVfs->xDelete(pVfs, zPath, dirSync);
   145    145   }
   146    146   int sqlite3OsAccess(

Changes to src/os_unix.c.

   673    673     case EIO:
   674    674     case EBADF:
   675    675     case EINVAL:
   676    676     case ENOTCONN:
   677    677     case ENODEV:
   678    678     case ENXIO:
   679    679     case ENOENT:
          680  +#ifdef ESTALE                     /* ESTALE is not defined on Interix systems */
   680    681     case ESTALE:
          682  +#endif
   681    683     case ENOSYS:
   682    684       /* these should force the client to close the file and reconnect */
   683    685       
   684    686     default: 
   685    687       return sqliteIOErr;
   686    688     }
   687    689   }
................................................................................
  3668   3670   */
  3669   3671   static void unixShmPurge(unixFile *pFd){
  3670   3672     unixShmNode *p = pFd->pInode->pShmNode;
  3671   3673     assert( unixMutexHeld() );
  3672   3674     if( p && p->nRef==0 ){
  3673   3675       int i;
  3674   3676       assert( p->pInode==pFd->pInode );
  3675         -    if( p->mutex ) sqlite3_mutex_free(p->mutex);
         3677  +    sqlite3_mutex_free(p->mutex);
  3676   3678       for(i=0; i<p->nRegion; i++){
  3677   3679         if( p->h>=0 ){
  3678   3680           munmap(p->apRegion[i], p->szRegion);
  3679   3681         }else{
  3680   3682           sqlite3_free(p->apRegion[i]);
  3681   3683         }
  3682   3684       }

Changes to src/os_win.c.

   397    397     sqlite3_log(errcode,
   398    398         "os_win.c:%d: (%d) %s(%s) - %s",
   399    399         iLine, iErrno, zFunc, zPath, zMsg
   400    400     );
   401    401   
   402    402     return errcode;
   403    403   }
          404  +
          405  +/*
          406  +** The number of times that a ReadFile(), WriteFile(), and DeleteFile()
          407  +** will be retried following a locking error - probably caused by 
          408  +** antivirus software.  Also the initial delay before the first retry.
          409  +** The delay increases linearly with each retry.
          410  +*/
          411  +#ifndef SQLITE_WIN32_IOERR_RETRY
          412  +# define SQLITE_WIN32_IOERR_RETRY 10
          413  +#endif
          414  +#ifndef SQLITE_WIN32_IOERR_RETRY_DELAY
          415  +# define SQLITE_WIN32_IOERR_RETRY_DELAY 25
          416  +#endif
          417  +static int win32IoerrRetry = SQLITE_WIN32_IOERR_RETRY;
          418  +static int win32IoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY;
          419  +
          420  +/*
          421  +** If a ReadFile() or WriteFile() error occurs, invoke this routine
          422  +** to see if it should be retried.  Return TRUE to retry.  Return FALSE
          423  +** to give up with an error.
          424  +*/
          425  +static int retryIoerr(int *pnRetry){
          426  +  DWORD e;
          427  +  if( *pnRetry>=win32IoerrRetry ){
          428  +    return 0;
          429  +  }
          430  +  e = GetLastError();
          431  +  if( e==ERROR_ACCESS_DENIED ||
          432  +      e==ERROR_LOCK_VIOLATION ||
          433  +      e==ERROR_SHARING_VIOLATION ){
          434  +    Sleep(win32IoerrRetryDelay*(1+*pnRetry));
          435  +    ++*pnRetry;
          436  +    return 1;
          437  +  }
          438  +  return 0;
          439  +}
          440  +
          441  +/*
          442  +** Log a I/O error retry episode.
          443  +*/
          444  +static void logIoerr(int nRetry){
          445  +  if( nRetry ){
          446  +    sqlite3_log(SQLITE_IOERR, 
          447  +      "delayed %dms for lock/sharing conflict",
          448  +      win32IoerrRetryDelay*nRetry*(nRetry+1)/2
          449  +    );
          450  +  }
          451  +}
   404    452   
   405    453   #if SQLITE_OS_WINCE
   406    454   /*************************************************************************
   407    455   ** This section contains code for WinCE only.
   408    456   */
   409    457   /*
   410    458   ** WindowsCE does not have a localtime() function.  So create a
................................................................................
   816    864     sqlite3_file *id,          /* File to read from */
   817    865     void *pBuf,                /* Write content into this buffer */
   818    866     int amt,                   /* Number of bytes to read */
   819    867     sqlite3_int64 offset       /* Begin reading at this offset */
   820    868   ){
   821    869     winFile *pFile = (winFile*)id;  /* file handle */
   822    870     DWORD nRead;                    /* Number of bytes actually read from file */
          871  +  int nRetry = 0;                 /* Number of retrys */
   823    872   
   824    873     assert( id!=0 );
   825    874     SimulateIOError(return SQLITE_IOERR_READ);
   826    875     OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype));
   827    876   
   828    877     if( seekWinFile(pFile, offset) ){
   829    878       return SQLITE_FULL;
   830    879     }
   831         -  if( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){
          880  +  while( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){
          881  +    if( retryIoerr(&nRetry) ) continue;
   832    882       pFile->lastErrno = GetLastError();
   833    883       return winLogError(SQLITE_IOERR_READ, "winRead", pFile->zPath);
   834    884     }
          885  +  logIoerr(nRetry);
   835    886     if( nRead<(DWORD)amt ){
   836    887       /* Unread parts of the buffer must be zero-filled */
   837    888       memset(&((char*)pBuf)[nRead], 0, amt-nRead);
   838    889       return SQLITE_IOERR_SHORT_READ;
   839    890     }
   840    891   
   841    892     return SQLITE_OK;
................................................................................
   849    900     sqlite3_file *id,               /* File to write into */
   850    901     const void *pBuf,               /* The bytes to be written */
   851    902     int amt,                        /* Number of bytes to write */
   852    903     sqlite3_int64 offset            /* Offset into the file to begin writing at */
   853    904   ){
   854    905     int rc;                         /* True if error has occured, else false */
   855    906     winFile *pFile = (winFile*)id;  /* File handle */
          907  +  int nRetry = 0;                 /* Number of retries */
   856    908   
   857    909     assert( amt>0 );
   858    910     assert( pFile );
   859    911     SimulateIOError(return SQLITE_IOERR_WRITE);
   860    912     SimulateDiskfullError(return SQLITE_FULL);
   861    913   
   862    914     OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype));
................................................................................
   863    915   
   864    916     rc = seekWinFile(pFile, offset);
   865    917     if( rc==0 ){
   866    918       u8 *aRem = (u8 *)pBuf;        /* Data yet to be written */
   867    919       int nRem = amt;               /* Number of bytes yet to be written */
   868    920       DWORD nWrite;                 /* Bytes written by each WriteFile() call */
   869    921   
   870         -    while( nRem>0 && WriteFile(pFile->h, aRem, nRem, &nWrite, 0) && nWrite>0 ){
          922  +    while( nRem>0 ){
          923  +      if( !WriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){
          924  +        if( retryIoerr(&nRetry) ) continue;
          925  +        break;
          926  +      }
          927  +      if( nWrite<=0 ) break;
   871    928         aRem += nWrite;
   872    929         nRem -= nWrite;
   873    930       }
   874    931       if( nRem>0 ){
   875    932         pFile->lastErrno = GetLastError();
   876    933         rc = 1;
   877    934       }
................................................................................
   879    936   
   880    937     if( rc ){
   881    938       if(   ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL )
   882    939          || ( pFile->lastErrno==ERROR_DISK_FULL )){
   883    940         return SQLITE_FULL;
   884    941       }
   885    942       return winLogError(SQLITE_IOERR_WRITE, "winWrite", pFile->zPath);
          943  +  }else{
          944  +    logIoerr(nRetry);
   886    945     }
   887    946     return SQLITE_OK;
   888    947   }
   889    948   
   890    949   /*
   891    950   ** Truncate an open file to a specified size
   892    951   */
................................................................................
  1294   1353         SimulateIOErrorBenign(1);
  1295   1354         winTruncate(id, sz);
  1296   1355         SimulateIOErrorBenign(0);
  1297   1356         return SQLITE_OK;
  1298   1357       }
  1299   1358       case SQLITE_FCNTL_SYNC_OMITTED: {
  1300   1359         return SQLITE_OK;
         1360  +    }
         1361  +    case SQLITE_FCNTL_WIN32_AV_RETRY: {
         1362  +      int *a = (int*)pArg;
         1363  +      if( a[0]>0 ){
         1364  +        win32IoerrRetry = a[0];
         1365  +      }else{
         1366  +        a[0] = win32IoerrRetry;
         1367  +      }
         1368  +      if( a[1]>0 ){
         1369  +        win32IoerrRetryDelay = a[1];
         1370  +      }else{
         1371  +        a[1] = win32IoerrRetryDelay;
         1372  +      }
         1373  +      return SQLITE_OK;
  1301   1374       }
  1302   1375     }
  1303   1376     return SQLITE_NOTFOUND;
  1304   1377   }
  1305   1378   
  1306   1379   /*
  1307   1380   ** Return the sector size in bytes of the underlying block device for
................................................................................
  2312   2385   ** will open a journal file shortly after it is created in order to do
  2313   2386   ** whatever it does.  While this other process is holding the
  2314   2387   ** file open, we will be unable to delete it.  To work around this
  2315   2388   ** problem, we delay 100 milliseconds and try to delete again.  Up
  2316   2389   ** to MX_DELETION_ATTEMPTs deletion attempts are run before giving
  2317   2390   ** up and returning an error.
  2318   2391   */
  2319         -#define MX_DELETION_ATTEMPTS 5
  2320   2392   static int winDelete(
  2321   2393     sqlite3_vfs *pVfs,          /* Not used on win32 */
  2322   2394     const char *zFilename,      /* Name of file to delete */
  2323   2395     int syncDir                 /* Not used on win32 */
  2324   2396   ){
  2325   2397     int cnt = 0;
  2326         -  DWORD rc;
  2327         -  DWORD error = 0;
         2398  +  int rc;
  2328   2399     void *zConverted;
  2329   2400     UNUSED_PARAMETER(pVfs);
  2330   2401     UNUSED_PARAMETER(syncDir);
  2331   2402   
  2332   2403     SimulateIOError(return SQLITE_IOERR_DELETE);
  2333   2404     zConverted = convertUtf8Filename(zFilename);
  2334   2405     if( zConverted==0 ){
  2335   2406       return SQLITE_NOMEM;
  2336   2407     }
  2337   2408     if( isNT() ){
  2338         -    do{
  2339         -      DeleteFileW(zConverted);
  2340         -    }while(   (   ((rc = GetFileAttributesW(zConverted)) != INVALID_FILE_ATTRIBUTES)
  2341         -               || ((error = GetLastError()) == ERROR_ACCESS_DENIED))
  2342         -           && (++cnt < MX_DELETION_ATTEMPTS)
  2343         -           && (Sleep(100), 1) );
         2409  +    rc = 1;
         2410  +    while( GetFileAttributesW(zConverted)!=INVALID_FILE_ATTRIBUTES &&
         2411  +           (rc = DeleteFileW(zConverted))==0 && retryIoerr(&cnt) ){}
         2412  +    rc = rc ? SQLITE_OK : SQLITE_ERROR;
  2344   2413   /* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. 
  2345   2414   ** Since the ASCII version of these Windows API do not exist for WINCE,
  2346   2415   ** it's important to not reference them for WINCE builds.
  2347   2416   */
  2348   2417   #if SQLITE_OS_WINCE==0
  2349   2418     }else{
  2350         -    do{
  2351         -      DeleteFileA(zConverted);
  2352         -    }while(   (   ((rc = GetFileAttributesA(zConverted)) != INVALID_FILE_ATTRIBUTES)
  2353         -               || ((error = GetLastError()) == ERROR_ACCESS_DENIED))
  2354         -           && (++cnt < MX_DELETION_ATTEMPTS)
  2355         -           && (Sleep(100), 1) );
         2419  +    rc = 1;
         2420  +    while( GetFileAttributesA(zConverted)!=INVALID_FILE_ATTRIBUTES &&
         2421  +           (rc = DeleteFileA(zConverted))==0 && retryIoerr(&cnt) ){}
         2422  +    rc = rc ? SQLITE_OK : SQLITE_ERROR;
  2356   2423   #endif
  2357   2424     }
         2425  +  if( rc ){
         2426  +    rc = winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename);
         2427  +  }else{
         2428  +    logIoerr(cnt);
         2429  +  }
  2358   2430     free(zConverted);
  2359         -  OSTRACE(("DELETE \"%s\" %s\n", zFilename,
  2360         -       ( (rc==INVALID_FILE_ATTRIBUTES) && (error==ERROR_FILE_NOT_FOUND)) ?
  2361         -         "ok" : "failed" ));
  2362         - 
  2363         -  return (   (rc == INVALID_FILE_ATTRIBUTES) 
  2364         -          && (error == ERROR_FILE_NOT_FOUND)) ? SQLITE_OK :
  2365         -                 winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename);
         2431  +  OSTRACE(("DELETE \"%s\" %s\n", zFilename, (rc ? "failed" : "ok" )));
         2432  +  return rc;
  2366   2433   }
  2367   2434   
  2368   2435   /*
  2369   2436   ** Check the existance and status of a file.
  2370   2437   */
  2371   2438   static int winAccess(
  2372   2439     sqlite3_vfs *pVfs,         /* Not used on win32 */

Changes to src/resolve.c.

   992    992       }
   993    993     
   994    994       /* Recursively resolve names in all subqueries
   995    995       */
   996    996       for(i=0; i<p->pSrc->nSrc; i++){
   997    997         struct SrcList_item *pItem = &p->pSrc->a[i];
   998    998         if( pItem->pSelect ){
          999  +        NameContext *pNC;         /* Used to iterate name contexts */
         1000  +        int nRef = 0;             /* Refcount for pOuterNC and outer contexts */
   999   1001           const char *zSavedContext = pParse->zAuthContext;
         1002  +
         1003  +        /* Count the total number of references to pOuterNC and all of its
         1004  +        ** parent contexts. After resolving references to expressions in
         1005  +        ** pItem->pSelect, check if this value has changed. If so, then
         1006  +        ** SELECT statement pItem->pSelect must be correlated. Set the
         1007  +        ** pItem->isCorrelated flag if this is the case. */
         1008  +        for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef += pNC->nRef;
         1009  +
  1000   1010           if( pItem->zName ) pParse->zAuthContext = pItem->zName;
  1001   1011           sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC);
  1002   1012           pParse->zAuthContext = zSavedContext;
  1003   1013           if( pParse->nErr || db->mallocFailed ) return WRC_Abort;
         1014  +
         1015  +        for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef -= pNC->nRef;
         1016  +        assert( pItem->isCorrelated==0 && nRef<=0 );
         1017  +        pItem->isCorrelated = (nRef!=0);
  1004   1018         }
  1005   1019       }
  1006   1020     
  1007   1021       /* If there are no aggregate functions in the result-set, and no GROUP BY 
  1008   1022       ** expression, do not allow aggregates in any of the other expressions.
  1009   1023       */
  1010   1024       assert( (p->selFlags & SF_Aggregate)==0 );

Changes to src/select.c.

  3717   3717     ExprList *pOrderBy;    /* The ORDER BY clause.  May be NULL */
  3718   3718     ExprList *pGroupBy;    /* The GROUP BY clause.  May be NULL */
  3719   3719     Expr *pHaving;         /* The HAVING clause.  May be NULL */
  3720   3720     int isDistinct;        /* True if the DISTINCT keyword is present */
  3721   3721     int distinct;          /* Table to use for the distinct set */
  3722   3722     int rc = 1;            /* Value to return from this function */
  3723   3723     int addrSortIndex;     /* Address of an OP_OpenEphemeral instruction */
         3724  +  int addrDistinctIndex; /* Address of an OP_OpenEphemeral instruction */
  3724   3725     AggInfo sAggInfo;      /* Information used by aggregate queries */
  3725   3726     int iEnd;              /* Address of the end of the query */
  3726   3727     sqlite3 *db;           /* The database connection */
  3727   3728   
  3728   3729   #ifndef SQLITE_OMIT_EXPLAIN
  3729   3730     int iRestoreSelectId = pParse->iSelectId;
  3730   3731     pParse->iSelectId = pParse->iNextSelectId++;
................................................................................
  3843   3844       }
  3844   3845       rc = multiSelect(pParse, p, pDest);
  3845   3846       explainSetInteger(pParse->iSelectId, iRestoreSelectId);
  3846   3847       return rc;
  3847   3848     }
  3848   3849   #endif
  3849   3850   
  3850         -  /* If possible, rewrite the query to use GROUP BY instead of DISTINCT.
  3851         -  ** GROUP BY might use an index, DISTINCT never does.
  3852         -  */
  3853         -  assert( p->pGroupBy==0 || (p->selFlags & SF_Aggregate)!=0 );
  3854         -  if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ){
  3855         -    p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0);
  3856         -    pGroupBy = p->pGroupBy;
  3857         -    p->selFlags &= ~SF_Distinct;
  3858         -  }
  3859         -
  3860   3851     /* If there is both a GROUP BY and an ORDER BY clause and they are
  3861   3852     ** identical, then disable the ORDER BY clause since the GROUP BY
  3862   3853     ** will cause elements to come out in the correct order.  This is
  3863   3854     ** an optimization - the correct answer should result regardless.
  3864   3855     ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER
  3865   3856     ** to disable this optimization for testing purposes.
  3866   3857     */
  3867   3858     if( sqlite3ExprListCompare(p->pGroupBy, pOrderBy)==0
  3868   3859            && (db->flags & SQLITE_GroupByOrder)==0 ){
  3869   3860       pOrderBy = 0;
  3870   3861     }
         3862  +
         3863  +  /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and 
         3864  +  ** if the select-list is the same as the ORDER BY list, then this query
         3865  +  ** can be rewritten as a GROUP BY. In other words, this:
         3866  +  **
         3867  +  **     SELECT DISTINCT xyz FROM ... ORDER BY xyz
         3868  +  **
         3869  +  ** is transformed to:
         3870  +  **
         3871  +  **     SELECT xyz FROM ... GROUP BY xyz
         3872  +  **
         3873  +  ** The second form is preferred as a single index (or temp-table) may be 
         3874  +  ** used for both the ORDER BY and DISTINCT processing. As originally 
         3875  +  ** written the query must use a temp-table for at least one of the ORDER 
         3876  +  ** BY and DISTINCT, and an index or separate temp-table for the other.
         3877  +  */
         3878  +  if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct 
         3879  +   && sqlite3ExprListCompare(pOrderBy, p->pEList)==0
         3880  +  ){
         3881  +    p->selFlags &= ~SF_Distinct;
         3882  +    p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0);
         3883  +    pGroupBy = p->pGroupBy;
         3884  +    pOrderBy = 0;
         3885  +  }
  3871   3886   
  3872   3887     /* If there is an ORDER BY clause, then this sorting
  3873   3888     ** index might end up being unused if the data can be 
  3874   3889     ** extracted in pre-sorted order.  If that is the case, then the
  3875   3890     ** OP_OpenEphemeral instruction will be changed to an OP_Noop once
  3876   3891     ** we figure out that the sorting index is not needed.  The addrSortIndex
  3877   3892     ** variable is used to facilitate that change.
................................................................................
  3900   3915     p->nSelectRow = (double)LARGEST_INT64;
  3901   3916     computeLimitRegisters(pParse, p, iEnd);
  3902   3917   
  3903   3918     /* Open a virtual index to use for the distinct set.
  3904   3919     */
  3905   3920     if( p->selFlags & SF_Distinct ){
  3906   3921       KeyInfo *pKeyInfo;
  3907         -    assert( isAgg || pGroupBy );
  3908   3922       distinct = pParse->nTab++;
  3909   3923       pKeyInfo = keyInfoFromExprList(pParse, p->pEList);
  3910         -    sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0,
  3911         -                        (char*)pKeyInfo, P4_KEYINFO_HANDOFF);
         3924  +    addrDistinctIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0,
         3925  +        (char*)pKeyInfo, P4_KEYINFO_HANDOFF);
  3912   3926       sqlite3VdbeChangeP5(v, BTREE_UNORDERED);
  3913   3927     }else{
  3914         -    distinct = -1;
         3928  +    distinct = addrDistinctIndex = -1;
  3915   3929     }
  3916   3930   
  3917   3931     /* Aggregate and non-aggregate queries are handled differently */
  3918   3932     if( !isAgg && pGroupBy==0 ){
  3919         -    /* This case is for non-aggregate queries
  3920         -    ** Begin the database scan
  3921         -    */
  3922         -    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0);
         3933  +    ExprList *pDist = (isDistinct ? p->pEList : 0);
         3934  +
         3935  +    /* Begin the database scan. */
         3936  +    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, pDist, 0);
  3923   3937       if( pWInfo==0 ) goto select_end;
  3924   3938       if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut;
  3925   3939   
  3926   3940       /* If sorting index that was created by a prior OP_OpenEphemeral 
  3927   3941       ** instruction ended up not being needed, then change the OP_OpenEphemeral
  3928   3942       ** into an OP_Noop.
  3929   3943       */
  3930   3944       if( addrSortIndex>=0 && pOrderBy==0 ){
  3931   3945         sqlite3VdbeChangeToNoop(v, addrSortIndex, 1);
  3932   3946         p->addrOpenEphm[2] = -1;
  3933   3947       }
  3934   3948   
  3935         -    /* Use the standard inner loop
  3936         -    */
  3937         -    assert(!isDistinct);
  3938         -    selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, -1, pDest,
         3949  +    if( pWInfo->eDistinct ){
         3950  +      VdbeOp *pOp;                /* No longer required OpenEphemeral instr. */
         3951  +     
         3952  +      assert( addrDistinctIndex>0 );
         3953  +      pOp = sqlite3VdbeGetOp(v, addrDistinctIndex);
         3954  +
         3955  +      assert( isDistinct );
         3956  +      assert( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED 
         3957  +           || pWInfo->eDistinct==WHERE_DISTINCT_UNIQUE 
         3958  +      );
         3959  +      distinct = -1;
         3960  +      if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED ){
         3961  +        int iJump;
         3962  +        int iExpr;
         3963  +        int iFlag = ++pParse->nMem;
         3964  +        int iBase = pParse->nMem+1;
         3965  +        int iBase2 = iBase + pEList->nExpr;
         3966  +        pParse->nMem += (pEList->nExpr*2);
         3967  +
         3968  +        /* Change the OP_OpenEphemeral coded earlier to an OP_Integer. The
         3969  +        ** OP_Integer initializes the "first row" flag.  */
         3970  +        pOp->opcode = OP_Integer;
         3971  +        pOp->p1 = 1;
         3972  +        pOp->p2 = iFlag;
         3973  +
         3974  +        sqlite3ExprCodeExprList(pParse, pEList, iBase, 1);
         3975  +        iJump = sqlite3VdbeCurrentAddr(v) + 1 + pEList->nExpr + 1 + 1;
         3976  +        sqlite3VdbeAddOp2(v, OP_If, iFlag, iJump-1);
         3977  +        for(iExpr=0; iExpr<pEList->nExpr; iExpr++){
         3978  +          CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[iExpr].pExpr);
         3979  +          sqlite3VdbeAddOp3(v, OP_Ne, iBase+iExpr, iJump, iBase2+iExpr);
         3980  +          sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ);
         3981  +          sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
         3982  +        }
         3983  +        sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iContinue);
         3984  +
         3985  +        sqlite3VdbeAddOp2(v, OP_Integer, 0, iFlag);
         3986  +        assert( sqlite3VdbeCurrentAddr(v)==iJump );
         3987  +        sqlite3VdbeAddOp3(v, OP_Move, iBase, iBase2, pEList->nExpr);
         3988  +      }else{
         3989  +        pOp->opcode = OP_Noop;
         3990  +      }
         3991  +    }
         3992  +
         3993  +    /* Use the standard inner loop. */
         3994  +    selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, distinct, pDest,
  3939   3995                       pWInfo->iContinue, pWInfo->iBreak);
  3940   3996   
  3941   3997       /* End the database scan loop.
  3942   3998       */
  3943   3999       sqlite3WhereEnd(pWInfo);
  3944   4000     }else{
  3945   4001       /* This is the processing for aggregate queries */
................................................................................
  4041   4097   
  4042   4098         /* Begin a loop that will extract all source rows in GROUP BY order.
  4043   4099         ** This might involve two separate loops with an OP_Sort in between, or
  4044   4100         ** it might be a single loop that uses an index to extract information
  4045   4101         ** in the right order to begin with.
  4046   4102         */
  4047   4103         sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset);
  4048         -      pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0);
         4104  +      pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0, 0);
  4049   4105         if( pWInfo==0 ) goto select_end;
  4050   4106         if( pGroupBy==0 ){
  4051   4107           /* The optimizer is able to deliver rows in group by order so
  4052   4108           ** we do not have to sort.  The OP_OpenEphemeral table will be
  4053   4109           ** cancelled later because we still need to use the pKeyInfo
  4054   4110           */
  4055   4111           pGroupBy = p->pGroupBy;
................................................................................
  4303   4359           }
  4304   4360     
  4305   4361           /* This case runs if the aggregate has no GROUP BY clause.  The
  4306   4362           ** processing is much simpler since there is only a single row
  4307   4363           ** of output.
  4308   4364           */
  4309   4365           resetAccumulator(pParse, &sAggInfo);
  4310         -        pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, flag);
         4366  +        pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, 0, flag);
  4311   4367           if( pWInfo==0 ){
  4312   4368             sqlite3ExprListDelete(db, pDel);
  4313   4369             goto select_end;
  4314   4370           }
  4315   4371           updateAccumulator(pParse, &sAggInfo);
  4316   4372           if( !pMinMax && flag ){
  4317   4373             sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iBreak);

Changes to src/shell.c.

  2628   2628     "   -stats               print memory stats before each finalize\n"
  2629   2629     "   -nullvalue 'text'    set text string for NULL values\n"
  2630   2630     "   -version             show SQLite version\n"
  2631   2631     "   -vfs NAME            use NAME as the default VFS\n"
  2632   2632   #ifdef SQLITE_ENABLE_VFSTRACE
  2633   2633     "   -vfstrace            enable tracing of all VFS calls\n"
  2634   2634   #endif
         2635  +#ifdef SQLITE_ENABLE_MULTIPLEX
         2636  +  "   -multiplex           enable the multiplexor VFS\n"
         2637  +#endif
  2635   2638   ;
  2636   2639   static void usage(int showDetail){
  2637   2640     fprintf(stderr,
  2638   2641         "Usage: %s [OPTIONS] FILENAME [SQL]\n"  
  2639   2642         "FILENAME is the name of an SQLite database. A new database is created\n"
  2640   2643         "if the file does not previously exist.\n", Argv0);
  2641   2644     if( showDetail ){
................................................................................
  2729   2732            const char *zOldVfsName,
  2730   2733            int (*xOut)(const char*,void*),
  2731   2734            void *pOutArg,
  2732   2735            int makeDefault
  2733   2736         );
  2734   2737         vfstrace_register("trace",0,(int(*)(const char*,void*))fputs,stderr,1);
  2735   2738   #endif
         2739  +#ifdef SQLITE_ENABLE_MULTIPLEX
         2740  +    }else if( strcmp(argv[i],"-multiplex")==0 ){
         2741  +      extern int sqlite3_multiple_initialize(const char*,int);
         2742  +      sqlite3_multiplex_initialize(0, 1);
         2743  +#endif
  2736   2744       }else if( strcmp(argv[i],"-vfs")==0 ){
  2737   2745         sqlite3_vfs *pVfs = sqlite3_vfs_find(argv[++i]);
  2738   2746         if( pVfs ){
  2739   2747           sqlite3_vfs_register(pVfs, 1);
  2740   2748         }else{
  2741   2749           fprintf(stderr, "no such VFS: \"%s\"\n", argv[i]);
  2742   2750           exit(1);
................................................................................
  2847   2855         stdin_is_interactive = 1;
  2848   2856       }else if( strcmp(z,"-batch")==0 ){
  2849   2857         stdin_is_interactive = 0;
  2850   2858       }else if( strcmp(z,"-heap")==0 ){
  2851   2859         i++;
  2852   2860       }else if( strcmp(z,"-vfs")==0 ){
  2853   2861         i++;
         2862  +#ifdef SQLITE_ENABLE_VFSTRACE
  2854   2863       }else if( strcmp(z,"-vfstrace")==0 ){
  2855   2864         i++;
         2865  +#endif
         2866  +#ifdef SQLITE_ENABLE_MULTIPLEX
         2867  +    }else if( strcmp(z,"-multiplex")==0 ){
         2868  +      i++;
         2869  +#endif
  2856   2870       }else if( strcmp(z,"-help")==0 || strcmp(z, "--help")==0 ){
  2857   2871         usage(1);
  2858   2872       }else{
  2859   2873         fprintf(stderr,"%s: Error: unknown option: %s\n", Argv0, z);
  2860   2874         fprintf(stderr,"Use -help for a list of options.\n");
  2861   2875         return 1;
  2862   2876       }

Changes to src/sqlite.h.in.

   732    732   ** when the database connection has [PRAGMA synchronous] set to OFF.)^
   733    733   ** Some specialized VFSes need this signal in order to operate correctly
   734    734   ** when [PRAGMA synchronous | PRAGMA synchronous=OFF] is set, but most 
   735    735   ** VFSes do not need this signal and should silently ignore this opcode.
   736    736   ** Applications should not call [sqlite3_file_control()] with this
   737    737   ** opcode as doing so may disrupt the operation of the specialized VFSes
   738    738   ** that do require it.  
          739  +**
          740  +** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic
          741  +** retry counts and intervals for certain disk I/O operations for the
          742  +** windows [VFS] in order to work to provide robustness against
          743  +** anti-virus programs.  By default, the windows VFS will retry file read,
          744  +** file write, and file delete opertions up to 10 times, with a delay
          745  +** of 25 milliseconds before the first retry and with the delay increasing
          746  +** by an additional 25 milliseconds with each subsequent retry.  This
          747  +** opcode allows those to values (10 retries and 25 milliseconds of delay)
          748  +** to be adjusted.  The values are changed for all database connections
          749  +** within the same process.  The argument is a pointer to an array of two
          750  +** integers where the first integer i the new retry count and the second
          751  +** integer is the delay.  If either integer is negative, then the setting
          752  +** is not changed but instead the prior value of that setting is written
          753  +** into the array entry, allowing the current retry settings to be
          754  +** interrogated.  The zDbName parameter is ignored.
          755  +** 
   739    756   */
   740    757   #define SQLITE_FCNTL_LOCKSTATE        1
   741    758   #define SQLITE_GET_LOCKPROXYFILE      2
   742    759   #define SQLITE_SET_LOCKPROXYFILE      3
   743    760   #define SQLITE_LAST_ERRNO             4
   744    761   #define SQLITE_FCNTL_SIZE_HINT        5
   745    762   #define SQLITE_FCNTL_CHUNK_SIZE       6
   746    763   #define SQLITE_FCNTL_FILE_POINTER     7
   747    764   #define SQLITE_FCNTL_SYNC_OMITTED     8
   748         -
          765  +#define SQLITE_FCNTL_WIN32_AV_RETRY   9
   749    766   
   750    767   /*
   751    768   ** CAPI3REF: Mutex Handle
   752    769   **
   753    770   ** The mutex module within SQLite defines [sqlite3_mutex] to be an
   754    771   ** abstract type for a mutex object.  The SQLite core never looks
   755    772   ** at the internal representation of an [sqlite3_mutex].  It only

Changes to src/sqlite3ext.h.

   208    208     const char *(*sourceid)(void);
   209    209     int (*stmt_status)(sqlite3_stmt*,int,int);
   210    210     int (*strnicmp)(const char*,const char*,int);
   211    211     int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*);
   212    212     int (*wal_autocheckpoint)(sqlite3*,int);
   213    213     int (*wal_checkpoint)(sqlite3*,const char*);
   214    214     void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*);
          215  +  int (*blob_reopen)(sqlite3_blob*,sqlite3_int64);
          216  +  int (*vtab_config)(sqlite3*,int op,...);
          217  +  int (*vtab_on_conflict)(sqlite3*);
   215    218   };
   216    219   
   217    220   /*
   218    221   ** The following macros redefine the API routines so that they are
   219    222   ** redirected throught the global sqlite3_api structure.
   220    223   **
   221    224   ** This header file is also used by the loadext.c source file
................................................................................
   408    411   #define sqlite3_sourceid               sqlite3_api->sourceid
   409    412   #define sqlite3_stmt_status            sqlite3_api->stmt_status
   410    413   #define sqlite3_strnicmp               sqlite3_api->strnicmp
   411    414   #define sqlite3_unlock_notify          sqlite3_api->unlock_notify
   412    415   #define sqlite3_wal_autocheckpoint     sqlite3_api->wal_autocheckpoint
   413    416   #define sqlite3_wal_checkpoint         sqlite3_api->wal_checkpoint
   414    417   #define sqlite3_wal_hook               sqlite3_api->wal_hook
          418  +#define sqlite3_blob_reopen            sqlite3_api->blob_reopen
          419  +#define sqlite3_vtab_config            sqlite3_api->vtab_config
          420  +#define sqlite3_vtab_on_conflict       sqlite3_api->vtab_on_conflict
   415    421   #endif /* SQLITE_CORE */
   416    422   
   417    423   #define SQLITE_EXTENSION_INIT1     const sqlite3_api_routines *sqlite3_api = 0;
   418    424   #define SQLITE_EXTENSION_INIT2(v)  sqlite3_api = v;
   419    425   
   420    426   #endif /* _SQLITE3EXT_H_ */

Changes to src/sqliteInt.h.

   960    960   #define SQLITE_ColumnCache    0x02        /* Disable the column cache */
   961    961   #define SQLITE_IndexSort      0x04        /* Disable indexes for sorting */
   962    962   #define SQLITE_IndexSearch    0x08        /* Disable indexes for searching */
   963    963   #define SQLITE_IndexCover     0x10        /* Disable index covering table */
   964    964   #define SQLITE_GroupByOrder   0x20        /* Disable GROUPBY cover of ORDERBY */
   965    965   #define SQLITE_FactorOutConst 0x40        /* Disable factoring out constants */
   966    966   #define SQLITE_IdxRealAsInt   0x80        /* Store REAL as INT in indices */
          967  +#define SQLITE_DistinctOpt    0x80        /* DISTINCT using indexes */
   967    968   #define SQLITE_OptMask        0xff        /* Mask of all disablable opts */
   968    969   
   969    970   /*
   970    971   ** Possible values for the sqlite.magic field.
   971    972   ** The numbers are obtained at random and have no special meaning, other
   972    973   ** than being distinct from one another.
   973    974   */
................................................................................
  1851   1852       char *zName;      /* Name of the table */
  1852   1853       char *zAlias;     /* The "B" part of a "A AS B" phrase.  zName is the "A" */
  1853   1854       Table *pTab;      /* An SQL table corresponding to zName */
  1854   1855       Select *pSelect;  /* A SELECT statement used in place of a table name */
  1855   1856       u8 isPopulated;   /* Temporary table associated with SELECT is populated */
  1856   1857       u8 jointype;      /* Type of join between this able and the previous */
  1857   1858       u8 notIndexed;    /* True if there is a NOT INDEXED clause */
         1859  +    u8 isCorrelated;  /* True if sub-query is correlated */
  1858   1860   #ifndef SQLITE_OMIT_EXPLAIN
  1859   1861       u8 iSelectId;     /* If pSelect!=0, the id of the sub-select in EQP */
  1860   1862   #endif
  1861   1863       int iCursor;      /* The VDBE cursor number used to access this table */
  1862   1864       Expr *pOn;        /* The ON clause of a join */
  1863   1865       IdList *pUsing;   /* The USING clause of a join */
  1864   1866       Bitmask colUsed;  /* Bit N (1<<N) set if column N of pTab is used */
................................................................................
  1970   1972   ** into the second half to give some continuity.
  1971   1973   */
  1972   1974   struct WhereInfo {
  1973   1975     Parse *pParse;       /* Parsing and code generating context */
  1974   1976     u16 wctrlFlags;      /* Flags originally passed to sqlite3WhereBegin() */
  1975   1977     u8 okOnePass;        /* Ok to use one-pass algorithm for UPDATE or DELETE */
  1976   1978     u8 untestedTerms;    /* Not all WHERE terms resolved by outer loop */
         1979  +  u8 eDistinct;
  1977   1980     SrcList *pTabList;             /* List of tables in the join */
  1978   1981     int iTop;                      /* The very beginning of the WHERE loop */
  1979   1982     int iContinue;                 /* Jump here to continue with next record */
  1980   1983     int iBreak;                    /* Jump here to break out of the loop */
  1981   1984     int nLevel;                    /* Number of nested loop */
  1982   1985     struct WhereClause *pWC;       /* Decomposition of the WHERE clause */
  1983   1986     double savedNQueryLoop;        /* pParse->nQueryLoop outside the WHERE loop */
  1984   1987     double nRowOut;                /* Estimated number of output rows */
  1985   1988     WhereLevel a[1];               /* Information about each nest loop in WHERE */
  1986   1989   };
  1987   1990   
         1991  +#define WHERE_DISTINCT_UNIQUE 1
         1992  +#define WHERE_DISTINCT_ORDERED 2
         1993  +
  1988   1994   /*
  1989   1995   ** A NameContext defines a context in which to resolve table and column
  1990   1996   ** names.  The context consists of a list of tables (the pSrcList) field and
  1991   1997   ** a list of named expression (pEList).  The named expression list may
  1992   1998   ** be NULL.  The pSrc corresponds to the FROM clause of a SELECT or
  1993   1999   ** to the table being operated on by INSERT, UPDATE, or DELETE.  The
  1994   2000   ** pEList corresponds to the result set of a SELECT and is NULL for
................................................................................
  2743   2749   int sqlite3IsReadOnly(Parse*, Table*, int);
  2744   2750   void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
  2745   2751   #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY)
  2746   2752   Expr *sqlite3LimitWhere(Parse *, SrcList *, Expr *, ExprList *, Expr *, Expr *, char *);
  2747   2753   #endif
  2748   2754   void sqlite3DeleteFrom(Parse*, SrcList*, Expr*);
  2749   2755   void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int);
  2750         -WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**, u16);
         2756  +WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**,ExprList*,u16);
  2751   2757   void sqlite3WhereEnd(WhereInfo*);
  2752   2758   int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int);
  2753   2759   void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int);
  2754   2760   void sqlite3ExprCodeMove(Parse*, int, int, int);
  2755   2761   void sqlite3ExprCodeCopy(Parse*, int, int, int);
  2756   2762   void sqlite3ExprCacheStore(Parse*, int, int, int);
  2757   2763   void sqlite3ExprCachePush(Parse*);

Changes to src/tclsqlite.c.

   103    103   };
   104    104   
   105    105   typedef struct IncrblobChannel IncrblobChannel;
   106    106   
   107    107   /*
   108    108   ** There is one instance of this structure for each SQLite database
   109    109   ** that has been opened by the SQLite TCL interface.
          110  +**
          111  +** If this module is built with SQLITE_TEST defined (to create the SQLite
          112  +** testfixture executable), then it may be configured to use either
          113  +** sqlite3_prepare_v2() or sqlite3_prepare() to prepare SQL statements.
          114  +** If SqliteDb.bLegacyPrepare is true, sqlite3_prepare() is used.
   110    115   */
   111    116   typedef struct SqliteDb SqliteDb;
   112    117   struct SqliteDb {
   113    118     sqlite3 *db;               /* The "real" database structure. MUST BE FIRST */
   114    119     Tcl_Interp *interp;        /* The interpreter used for this database */
   115    120     char *zBusy;               /* The busy callback routine */
   116    121     char *zCommit;             /* The commit hook callback routine */
................................................................................
   132    137     SqlPreparedStmt *stmtList; /* List of prepared statements*/
   133    138     SqlPreparedStmt *stmtLast; /* Last statement in the list */
   134    139     int maxStmt;               /* The next maximum number of stmtList */
   135    140     int nStmt;                 /* Number of statements in stmtList */
   136    141     IncrblobChannel *pIncrblob;/* Linked list of open incrblob channels */
   137    142     int nStep, nSort, nIndex;  /* Statistics for most recent operation */
   138    143     int nTransaction;          /* Number of nested [transaction] methods */
          144  +#ifdef SQLITE_TEST
          145  +  int bLegacyPrepare;        /* True to use sqlite3_prepare() */
          146  +#endif
   139    147   };
   140    148   
   141    149   struct IncrblobChannel {
   142    150     sqlite3_blob *pBlob;      /* sqlite3 blob handle */
   143    151     SqliteDb *pDb;            /* Associated database connection */
   144    152     int iSeek;                /* Current seek offset */
   145    153     Tcl_Channel channel;      /* Channel identifier */
................................................................................
   425    433     }
   426    434     pNew->interp = pDb->interp;
   427    435     pNew->pScript = 0;
   428    436     pNew->pNext = pDb->pFunc;
   429    437     pDb->pFunc = pNew;
   430    438     return pNew;
   431    439   }
          440  +
          441  +/*
          442  +** Free a single SqlPreparedStmt object.
          443  +*/
          444  +static void dbFreeStmt(SqlPreparedStmt *pStmt){
          445  +#ifdef SQLITE_TEST
          446  +  if( sqlite3_sql(pStmt->pStmt)==0 ){
          447  +    Tcl_Free((char *)pStmt->zSql);
          448  +  }
          449  +#endif
          450  +  sqlite3_finalize(pStmt->pStmt);
          451  +  Tcl_Free((char *)pStmt);
          452  +}
   432    453   
   433    454   /*
   434    455   ** Finalize and free a list of prepared statements
   435    456   */
   436         -static void flushStmtCache( SqliteDb *pDb ){
          457  +static void flushStmtCache(SqliteDb *pDb){
   437    458     SqlPreparedStmt *pPreStmt;
          459  +  SqlPreparedStmt *pNext;
   438    460   
   439         -  while(  pDb->stmtList ){
   440         -    sqlite3_finalize( pDb->stmtList->pStmt );
   441         -    pPreStmt = pDb->stmtList;
   442         -    pDb->stmtList = pDb->stmtList->pNext;
   443         -    Tcl_Free( (char*)pPreStmt );
          461  +  for(pPreStmt = pDb->stmtList; pPreStmt; pPreStmt=pNext){
          462  +    pNext = pPreStmt->pNext;
          463  +    dbFreeStmt(pPreStmt);
   444    464     }
   445    465     pDb->nStmt = 0;
   446    466     pDb->stmtLast = 0;
          467  +  pDb->stmtList = 0;
   447    468   }
   448    469   
   449    470   /*
   450    471   ** TCL calls this procedure when an sqlite3 database command is
   451    472   ** deleted.
   452    473   */
   453    474   static void DbDeleteCmd(void *db){
................................................................................
  1069   1090       }
  1070   1091       sqlite3_exec(pDb->db, "ROLLBACK", 0, 0, 0);
  1071   1092     }
  1072   1093     pDb->disableAuth--;
  1073   1094   
  1074   1095     return rc;
  1075   1096   }
         1097  +
         1098  +/*
         1099  +** Unless SQLITE_TEST is defined, this function is a simple wrapper around
         1100  +** sqlite3_prepare_v2(). If SQLITE_TEST is defined, then it uses either
         1101  +** sqlite3_prepare_v2() or legacy interface sqlite3_prepare(), depending
         1102  +** on whether or not the [db_use_legacy_prepare] command has been used to 
         1103  +** configure the connection.
         1104  +*/
         1105  +static int dbPrepare(
         1106  +  SqliteDb *pDb,                  /* Database object */
         1107  +  const char *zSql,               /* SQL to compile */
         1108  +  sqlite3_stmt **ppStmt,          /* OUT: Prepared statement */
         1109  +  const char **pzOut              /* OUT: Pointer to next SQL statement */
         1110  +){
         1111  +#ifdef SQLITE_TEST
         1112  +  if( pDb->bLegacyPrepare ){
         1113  +    return sqlite3_prepare(pDb->db, zSql, -1, ppStmt, pzOut);
         1114  +  }
         1115  +#endif
         1116  +  return sqlite3_prepare_v2(pDb->db, zSql, -1, ppStmt, pzOut);
         1117  +}
  1076   1118   
  1077   1119   /*
  1078   1120   ** Search the cache for a prepared-statement object that implements the
  1079   1121   ** first SQL statement in the buffer pointed to by parameter zIn. If
  1080   1122   ** no such prepared-statement can be found, allocate and prepare a new
  1081   1123   ** one. In either case, bind the current values of the relevant Tcl
  1082   1124   ** variables to any $var, :var or @var variables in the statement. Before
................................................................................
  1140   1182     }
  1141   1183     
  1142   1184     /* If no prepared statement was found. Compile the SQL text. Also allocate
  1143   1185     ** a new SqlPreparedStmt structure.  */
  1144   1186     if( pPreStmt==0 ){
  1145   1187       int nByte;
  1146   1188   
  1147         -    if( SQLITE_OK!=sqlite3_prepare_v2(pDb->db, zSql, -1, &pStmt, pzOut) ){
         1189  +    if( SQLITE_OK!=dbPrepare(pDb, zSql, &pStmt, pzOut) ){
  1148   1190         Tcl_SetObjResult(interp, dbTextToObj(sqlite3_errmsg(pDb->db)));
  1149   1191         return TCL_ERROR;
  1150   1192       }
  1151   1193       if( pStmt==0 ){
  1152   1194         if( SQLITE_OK!=sqlite3_errcode(pDb->db) ){
  1153   1195           /* A compile-time error in the statement. */
  1154   1196           Tcl_SetObjResult(interp, dbTextToObj(sqlite3_errmsg(pDb->db)));
................................................................................
  1167   1209       pPreStmt = (SqlPreparedStmt*)Tcl_Alloc(nByte);
  1168   1210       memset(pPreStmt, 0, nByte);
  1169   1211   
  1170   1212       pPreStmt->pStmt = pStmt;
  1171   1213       pPreStmt->nSql = (*pzOut - zSql);
  1172   1214       pPreStmt->zSql = sqlite3_sql(pStmt);
  1173   1215       pPreStmt->apParm = (Tcl_Obj **)&pPreStmt[1];
         1216  +#ifdef SQLITE_TEST
         1217  +    if( pPreStmt->zSql==0 ){
         1218  +      char *zCopy = Tcl_Alloc(pPreStmt->nSql + 1);
         1219  +      memcpy(zCopy, zSql, pPreStmt->nSql);
         1220  +      zCopy[pPreStmt->nSql] = '\0';
         1221  +      pPreStmt->zSql = zCopy;
         1222  +    }
         1223  +#endif
  1174   1224     }
  1175   1225     assert( pPreStmt );
  1176   1226     assert( strlen30(pPreStmt->zSql)==pPreStmt->nSql );
  1177   1227     assert( 0==memcmp(pPreStmt->zSql, zSql, pPreStmt->nSql) );
  1178   1228   
  1179   1229     /* Bind values to parameters that begin with $ or : */  
  1180   1230     for(i=1; i<=nVar; i++){
................................................................................
  1220   1270     }
  1221   1271     pPreStmt->nParm = iParm;
  1222   1272     *ppPreStmt = pPreStmt;
  1223   1273   
  1224   1274     return TCL_OK;
  1225   1275   }
  1226   1276   
  1227         -
  1228   1277   /*
  1229   1278   ** Release a statement reference obtained by calling dbPrepareAndBind().
  1230   1279   ** There should be exactly one call to this function for each call to
  1231   1280   ** dbPrepareAndBind().
  1232   1281   **
  1233   1282   ** If the discard parameter is non-zero, then the statement is deleted
  1234   1283   ** immediately. Otherwise it is added to the LRU list and may be returned
................................................................................
  1245   1294     for(i=0; i<pPreStmt->nParm; i++){
  1246   1295       Tcl_DecrRefCount(pPreStmt->apParm[i]);
  1247   1296     }
  1248   1297     pPreStmt->nParm = 0;
  1249   1298   
  1250   1299     if( pDb->maxStmt<=0 || discard ){
  1251   1300       /* If the cache is turned off, deallocated the statement */
  1252         -    sqlite3_finalize(pPreStmt->pStmt);
  1253         -    Tcl_Free((char *)pPreStmt);
         1301  +    dbFreeStmt(pPreStmt);
  1254   1302     }else{
  1255   1303       /* Add the prepared statement to the beginning of the cache list. */
  1256   1304       pPreStmt->pNext = pDb->stmtList;
  1257   1305       pPreStmt->pPrev = 0;
  1258   1306       if( pDb->stmtList ){
  1259   1307        pDb->stmtList->pPrev = pPreStmt;
  1260   1308       }
................................................................................
  1266   1314         assert( pDb->nStmt>0 );
  1267   1315       }
  1268   1316       pDb->nStmt++;
  1269   1317      
  1270   1318       /* If we have too many statement in cache, remove the surplus from 
  1271   1319       ** the end of the cache list.  */
  1272   1320       while( pDb->nStmt>pDb->maxStmt ){
  1273         -      sqlite3_finalize(pDb->stmtLast->pStmt);
  1274         -      pDb->stmtLast = pDb->stmtLast->pPrev;
  1275         -      Tcl_Free((char*)pDb->stmtLast->pNext);
         1321  +      SqlPreparedStmt *pLast = pDb->stmtLast;
         1322  +      pDb->stmtLast = pLast->pPrev;
  1276   1323         pDb->stmtLast->pNext = 0;
  1277   1324         pDb->nStmt--;
         1325  +      dbFreeStmt(pLast);
  1278   1326       }
  1279   1327     }
  1280   1328   }
  1281   1329   
  1282   1330   /*
  1283   1331   ** Structure used with dbEvalXXX() functions:
  1284   1332   **
................................................................................
  1403   1451   ** A return value of TCL_OK means there is a row of data available. The
  1404   1452   ** data may be accessed using dbEvalRowInfo() and dbEvalColumnValue(). This
  1405   1453   ** is analogous to a return of SQLITE_ROW from sqlite3_step(). If TCL_BREAK
  1406   1454   ** is returned, then the SQL script has finished executing and there are
  1407   1455   ** no further rows available. This is similar to SQLITE_DONE.
  1408   1456   */
  1409   1457   static int dbEvalStep(DbEvalContext *p){
         1458  +  const char *zPrevSql = 0;       /* Previous value of p->zSql */
         1459  +
  1410   1460     while( p->zSql[0] || p->pPreStmt ){
  1411   1461       int rc;
  1412   1462       if( p->pPreStmt==0 ){
         1463  +      zPrevSql = (p->zSql==zPrevSql ? 0 : p->zSql);
  1413   1464         rc = dbPrepareAndBind(p->pDb, p->zSql, &p->zSql, &p->pPreStmt);
  1414   1465         if( rc!=TCL_OK ) return rc;
  1415   1466       }else{
  1416   1467         int rcs;
  1417   1468         SqliteDb *pDb = p->pDb;
  1418   1469         SqlPreparedStmt *pPreStmt = p->pPreStmt;
  1419   1470         sqlite3_stmt *pStmt = pPreStmt->pStmt;
................................................................................
  1432   1483         pDb->nIndex = sqlite3_stmt_status(pStmt,SQLITE_STMTSTATUS_AUTOINDEX,1);
  1433   1484         dbReleaseColumnNames(p);
  1434   1485         p->pPreStmt = 0;
  1435   1486   
  1436   1487         if( rcs!=SQLITE_OK ){
  1437   1488           /* If a run-time error occurs, report the error and stop reading
  1438   1489           ** the SQL.  */
  1439         -        Tcl_SetObjResult(pDb->interp, dbTextToObj(sqlite3_errmsg(pDb->db)));
  1440   1490           dbReleaseStmt(pDb, pPreStmt, 1);
         1491  +#if SQLITE_TEST
         1492  +        if( p->pDb->bLegacyPrepare && rcs==SQLITE_SCHEMA && zPrevSql ){
         1493  +          /* If the runtime error was an SQLITE_SCHEMA, and the database
         1494  +          ** handle is configured to use the legacy sqlite3_prepare() 
         1495  +          ** interface, retry prepare()/step() on the same SQL statement.
         1496  +          ** This only happens once. If there is a second SQLITE_SCHEMA
         1497  +          ** error, the error will be returned to the caller. */
         1498  +          p->zSql = zPrevSql;
         1499  +          continue;
         1500  +        }
         1501  +#endif
         1502  +        Tcl_SetObjResult(pDb->interp, dbTextToObj(sqlite3_errmsg(pDb->db)));
  1441   1503           return TCL_ERROR;
  1442   1504         }else{
  1443   1505           dbReleaseStmt(pDb, pPreStmt, 0);
  1444   1506         }
  1445   1507       }
  1446   1508     }
  1447   1509   
................................................................................
  3068   3130         if( Tcl_GetBooleanFromObj(interp, objv[i+1], &b) ) return TCL_ERROR;
  3069   3131         if( b ){
  3070   3132           flags |= SQLITE_OPEN_NOMUTEX;
  3071   3133           flags &= ~SQLITE_OPEN_FULLMUTEX;
  3072   3134         }else{
  3073   3135           flags &= ~SQLITE_OPEN_NOMUTEX;
  3074   3136         }
  3075         -   }else if( strcmp(zArg, "-fullmutex")==0 ){
         3137  +    }else if( strcmp(zArg, "-fullmutex")==0 ){
  3076   3138         int b;
  3077   3139         if( Tcl_GetBooleanFromObj(interp, objv[i+1], &b) ) return TCL_ERROR;
  3078   3140         if( b ){
  3079   3141           flags |= SQLITE_OPEN_FULLMUTEX;
  3080   3142           flags &= ~SQLITE_OPEN_NOMUTEX;
  3081   3143         }else{
  3082   3144           flags &= ~SQLITE_OPEN_FULLMUTEX;
................................................................................
  3669   3731     if( !slave ){
  3670   3732       return TCL_ERROR;
  3671   3733     }
  3672   3734   
  3673   3735     init_all(slave);
  3674   3736     return TCL_OK;
  3675   3737   }
         3738  +
         3739  +/*
         3740  +** Tclcmd: db_use_legacy_prepare DB BOOLEAN
         3741  +**
         3742  +**   The first argument to this command must be a database command created by
         3743  +**   [sqlite3]. If the second argument is true, then the handle is configured
         3744  +**   to use the sqlite3_prepare_v2() function to prepare statements. If it
         3745  +**   is false, sqlite3_prepare().
         3746  +*/
         3747  +static int db_use_legacy_prepare_cmd(
         3748  +  ClientData cd,
         3749  +  Tcl_Interp *interp,
         3750  +  int objc,
         3751  +  Tcl_Obj *CONST objv[]
         3752  +){
         3753  +  Tcl_CmdInfo cmdInfo;
         3754  +  SqliteDb *pDb;
         3755  +  int bPrepare;
         3756  +
         3757  +  if( objc!=3 ){
         3758  +    Tcl_WrongNumArgs(interp, 1, objv, "DB BOOLEAN");
         3759  +    return TCL_ERROR;
         3760  +  }
         3761  +
         3762  +  if( !Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){
         3763  +    Tcl_AppendResult(interp, "no such db: ", Tcl_GetString(objv[1]), (char*)0);
         3764  +    return TCL_ERROR;
         3765  +  }
         3766  +  pDb = (SqliteDb*)cmdInfo.objClientData;
         3767  +  if( Tcl_GetBooleanFromObj(interp, objv[2], &bPrepare) ){
         3768  +    return TCL_ERROR;
         3769  +  }
         3770  +
         3771  +  pDb->bLegacyPrepare = bPrepare;
         3772  +
         3773  +  Tcl_ResetResult(interp);
         3774  +  return TCL_OK;
         3775  +}
  3676   3776   #endif
  3677   3777   
  3678   3778   /*
  3679   3779   ** Configure the interpreter passed as the first argument to have access
  3680   3780   ** to the commands and linked variables that make up:
  3681   3781   **
  3682   3782   **   * the [sqlite3] extension itself, 
................................................................................
  3779   3879   #if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)
  3780   3880       TestSession_Init(interp);
  3781   3881   #endif
  3782   3882   #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
  3783   3883       Sqlitetestfts3_Init(interp);
  3784   3884   #endif
  3785   3885   
  3786         -    Tcl_CreateObjCommand(interp,"load_testfixture_extensions",init_all_cmd,0,0);
         3886  +    Tcl_CreateObjCommand(
         3887  +        interp, "load_testfixture_extensions", init_all_cmd, 0, 0
         3888  +    );
         3889  +    Tcl_CreateObjCommand(
         3890  +        interp, "db_use_legacy_prepare", db_use_legacy_prepare_cmd, 0, 0
         3891  +    );
  3787   3892   
  3788   3893   #ifdef SQLITE_SSE
  3789   3894       Sqlitetestsse_Init(interp);
  3790   3895   #endif
  3791   3896     }
  3792   3897   #endif
  3793   3898   }

Changes to src/test1.c.

  5092   5092         return TCL_ERROR;
  5093   5093       }
  5094   5094     }
  5095   5095   #endif
  5096   5096     return TCL_OK;  
  5097   5097   }
  5098   5098   
         5099  +/*
         5100  +** tclcmd:   file_control_win32_av_retry DB  NRETRY  DELAY
         5101  +**
         5102  +** This TCL command runs the sqlite3_file_control interface with
         5103  +** the SQLITE_FCNTL_WIN32_AV_RETRY opcode.
         5104  +*/
         5105  +static int file_control_win32_av_retry(
         5106  +  ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
         5107  +  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
         5108  +  int objc,              /* Number of arguments */
         5109  +  Tcl_Obj *CONST objv[]  /* Command arguments */
         5110  +){
         5111  +  sqlite3 *db;
         5112  +  int rc;
         5113  +  int a[2];
         5114  +  char z[100];
         5115  +
         5116  +  if( objc!=4 ){
         5117  +    Tcl_AppendResult(interp, "wrong # args: should be \"",
         5118  +        Tcl_GetStringFromObj(objv[0], 0), " DB NRETRY DELAY", 0);
         5119  +    return TCL_ERROR;
         5120  +  }
         5121  +  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
         5122  +    return TCL_ERROR;
         5123  +  }
         5124  +  if( Tcl_GetIntFromObj(interp, objv[2], &a[0]) ) return TCL_ERROR;
         5125  +  if( Tcl_GetIntFromObj(interp, objv[3], &a[1]) ) return TCL_ERROR;
         5126  +  rc = sqlite3_file_control(db, NULL, SQLITE_FCNTL_WIN32_AV_RETRY, (void*)a);
         5127  +  sqlite3_snprintf(sizeof(z), z, "%d %d %d", rc, a[0], a[1]);
         5128  +  Tcl_AppendResult(interp, z, (char*)0);
         5129  +  return TCL_OK;  
         5130  +}
         5131  +
  5099   5132   
  5100   5133   /*
  5101   5134   ** tclcmd:   sqlite3_vfs_list
  5102   5135   **
  5103   5136   **   Return a tcl list containing the names of all registered vfs's.
  5104   5137   */
  5105   5138   static int vfs_list(
................................................................................
  5570   5603       }
  5571   5604     }
  5572   5605   
  5573   5606     Tcl_ResetResult(interp);
  5574   5607     return TCL_OK;
  5575   5608   }
  5576   5609   
         5610  +#if SQLITE_OS_WIN
         5611  +/*
         5612  +** Information passed from the main thread into the windows file locker
         5613  +** background thread.
         5614  +*/
         5615  +struct win32FileLocker {
         5616  +  HANDLE h;           /* Handle of the file to be locked */
         5617  +  int delay1;         /* Delay before locking */
         5618  +  int delay2;         /* Delay before unlocking */
         5619  +  int ok;             /* Finished ok */
         5620  +  int err;            /* True if an error occurs */
         5621  +};
         5622  +#endif
         5623  +
         5624  +
         5625  +#if SQLITE_OS_WIN
         5626  +/*
         5627  +** The background thread that does file locking.
         5628  +*/
         5629  +static void win32_file_locker(void *pAppData){
         5630  +  struct win32FileLocker *p = (struct win32FileLocker*)pAppData;
         5631  +  if( p->delay1 ) Sleep(p->delay1);
         5632  +  if( LockFile(p->h, 0, 0, 100000000, 0) ){
         5633  +    Sleep(p->delay2);
         5634  +    UnlockFile(p->h, 0, 0, 100000000, 0);
         5635  +    p->ok = 1;
         5636  +  }else{
         5637  +    p->err = 1;
         5638  +  }
         5639  +  CloseHandle(p->h);
         5640  +  p->h = 0;
         5641  +  p->delay1 = 0;
         5642  +  p->delay2 = 0;
         5643  +}
         5644  +#endif
         5645  +
         5646  +#if SQLITE_OS_WIN
         5647  +/*
         5648  +**      lock_win32_file FILENAME DELAY1 DELAY2
         5649  +**
         5650  +** Get an exclusive manditory lock on file for DELAY2 milliseconds.
         5651  +** Wait DELAY1 milliseconds before acquiring the lock.
         5652  +*/
         5653  +static int win32_file_lock(
         5654  +  void * clientData,
         5655  +  Tcl_Interp *interp,
         5656  +  int objc,
         5657  +  Tcl_Obj *CONST objv[]
         5658  +){
         5659  +  static struct win32FileLocker x = { 0, 0, 0 };
         5660  +  const char *zFilename;
         5661  +  int retry = 0;
         5662  +  
         5663  +  if( objc!=4 && objc!=1 ){
         5664  +    Tcl_WrongNumArgs(interp, 1, objv, "FILENAME DELAY1 DELAY2");
         5665  +    return TCL_ERROR;
         5666  +  }
         5667  +  if( objc==1 ){
         5668  +    char zBuf[200];
         5669  +    sqlite3_snprintf(sizeof(zBuf), zBuf, "%d %d %d %d %d",
         5670  +                     x.ok, x.err, x.delay1, x.delay2, x.h);
         5671  +    Tcl_AppendResult(interp, zBuf, (char*)0);
         5672  +    return TCL_OK;
         5673  +  }
         5674  +  while( x.h && retry<30 ){
         5675  +    retry++;
         5676  +    Sleep(100);
         5677  +  }
         5678  +  if( x.h ){
         5679  +    Tcl_AppendResult(interp, "busy", (char*)0);
         5680  +    return TCL_ERROR;
         5681  +  }
         5682  +  if( Tcl_GetIntFromObj(interp, objv[2], &x.delay1) ) return TCL_ERROR;
         5683  +  if( Tcl_GetIntFromObj(interp, objv[3], &x.delay2) ) return TCL_ERROR;
         5684  +  zFilename = Tcl_GetString(objv[1]);
         5685  +  x.h = CreateFile(zFilename, GENERIC_READ|GENERIC_WRITE,
         5686  +              FILE_SHARE_READ|FILE_SHARE_WRITE, 0, OPEN_ALWAYS,
         5687  +              FILE_ATTRIBUTE_NORMAL, 0);
         5688  +  if( !x.h ){
         5689  +    Tcl_AppendResult(interp, "cannot open file: ", zFilename, (char*)0);
         5690  +    return TCL_ERROR;
         5691  +  }
         5692  +  _beginthread(win32_file_locker, 0, (void*)&x);
         5693  +  Sleep(0);
         5694  +  return TCL_OK;
         5695  +}
         5696  +#endif
         5697  +
  5577   5698   
  5578   5699   /*
  5579   5700   **      optimization_control DB OPT BOOLEAN
  5580   5701   **
  5581   5702   ** Enable or disable query optimizations using the sqlite3_test_control()
  5582   5703   ** interface.  Disable if BOOLEAN is false and enable if BOOLEAN is true.
  5583   5704   ** OPT is the name of the optimization to be disabled.
................................................................................
  5750   5871        { "sqlite3_extended_result_codes", test_extended_result_codes, 0},
  5751   5872        { "sqlite3_limit",                 test_limit,                 0},
  5752   5873   
  5753   5874        { "save_prng_state",               save_prng_state,    0 },
  5754   5875        { "restore_prng_state",            restore_prng_state, 0 },
  5755   5876        { "reset_prng_state",              reset_prng_state,   0 },
  5756   5877        { "optimization_control",          optimization_control,0},
         5878  +#if SQLITE_OS_WIN
         5879  +     { "lock_win32_file",               win32_file_lock,    0 },
         5880  +#endif
  5757   5881        { "tcl_objproc",                   runAsObjProc,       0 },
  5758   5882   
  5759   5883        /* sqlite3_column_*() API */
  5760   5884        { "sqlite3_column_count",          test_column_count  ,0 },
  5761   5885        { "sqlite3_data_count",            test_data_count    ,0 },
  5762   5886        { "sqlite3_column_type",           test_column_type   ,0 },
  5763   5887        { "sqlite3_column_blob",           test_column_blob   ,0 },
................................................................................
  5798   5922        { "vfs_initfail_test",          vfs_initfail_test,   0   },
  5799   5923        { "vfs_unregister_all",         vfs_unregister_all,  0   },
  5800   5924        { "vfs_reregister_all",         vfs_reregister_all,  0   },
  5801   5925        { "file_control_test",          file_control_test,   0   },
  5802   5926        { "file_control_lasterrno_test", file_control_lasterrno_test,  0   },
  5803   5927        { "file_control_lockproxy_test", file_control_lockproxy_test,  0   },
  5804   5928        { "file_control_chunksize_test", file_control_chunksize_test,  0   },
  5805         -     { "file_control_sizehint_test", file_control_sizehint_test,  0   },
         5929  +     { "file_control_sizehint_test",  file_control_sizehint_test,   0   },
         5930  +     { "file_control_win32_av_retry", file_control_win32_av_retry,  0   },
  5806   5931        { "sqlite3_vfs_list",           vfs_list,     0   },
  5807   5932        { "sqlite3_create_function_v2", test_create_function_v2, 0 },
  5808   5933   
  5809   5934        /* Functions from os.h */
  5810   5935   #ifndef SQLITE_OMIT_UTF16
  5811   5936        { "add_test_collate",        test_collate, 0            },
  5812   5937        { "add_test_collate_needed", test_collate_needed, 0     },

Changes to src/test_config.c.

   579    579     LINKVAR( MAX_LIKE_PATTERN_LENGTH );
   580    580     LINKVAR( MAX_TRIGGER_DEPTH );
   581    581     LINKVAR( DEFAULT_TEMP_CACHE_SIZE );
   582    582     LINKVAR( DEFAULT_CACHE_SIZE );
   583    583     LINKVAR( DEFAULT_PAGE_SIZE );
   584    584     LINKVAR( DEFAULT_FILE_FORMAT );
   585    585     LINKVAR( MAX_ATTACHED );
          586  +  LINKVAR( MAX_DEFAULT_PAGE_SIZE );
   586    587   
   587    588     {
   588    589       static const int cv_TEMP_STORE = SQLITE_TEMP_STORE;
   589    590       Tcl_LinkVar(interp, "TEMP_STORE", (char *)&(cv_TEMP_STORE),
   590    591                   TCL_LINK_INT | TCL_LINK_READ_ONLY);
   591    592     }
   592    593   }

Changes to src/test_multiplex.c.

    41     41   ** The multiplex VFS allows databases up to 32 GiB in size.  But it splits
    42     42   ** the files up into 1 GiB pieces, so that they will work even on filesystems
    43     43   ** that do not support large files.
    44     44   */
    45     45   #include "sqlite3.h"
    46     46   #include <string.h>
    47     47   #include <assert.h>
           48  +#include <stdlib.h>
    48     49   #include "test_multiplex.h"
    49     50   
    50     51   #ifndef SQLITE_CORE
    51     52     #define SQLITE_CORE 1  /* Disable the API redefinition in sqlite3ext.h */
    52     53   #endif
    53     54   #include "sqlite3ext.h"
    54     55   
................................................................................
    74     75   #define sqlite3_mutex_held(X)     ((void)(X),1)
    75     76   #define sqlite3_mutex_notheld(X)  ((void)(X),1)
    76     77   #endif /* SQLITE_THREADSAFE==0 */
    77     78   
    78     79   
    79     80   /************************ Shim Definitions ******************************/
    80     81   
    81         -#define SQLITE_MULTIPLEX_VFS_NAME "multiplex"
           82  +#ifndef SQLITE_MULTIPLEX_VFS_NAME
           83  +# define SQLITE_MULTIPLEX_VFS_NAME "multiplex"
           84  +#endif
    82     85   
    83     86   /* This is the limit on the chunk size.  It may be changed by calling
    84     87   ** the xFileControl() interface.  It will be rounded up to a 
    85         -** multiple of MAX_PAGE_SIZE.  We default it here to 1GB.
           88  +** multiple of MAX_PAGE_SIZE.  We default it here to 2GiB less 64KiB.
    86     89   */
    87         -#define SQLITE_MULTIPLEX_CHUNK_SIZE (MAX_PAGE_SIZE*16384)
           90  +#ifndef SQLITE_MULTIPLEX_CHUNK_SIZE
           91  +# define SQLITE_MULTIPLEX_CHUNK_SIZE 2147418112
           92  +#endif
    88     93   
    89     94   /* Default limit on number of chunks.  Care should be taken
    90     95   ** so that values for chunks numbers fit in the SQLITE_MULTIPLEX_EXT_FMT
    91     96   ** format specifier. It may be changed by calling
    92     97   ** the xFileControl() interface.
    93     98   */
    94         -#define SQLITE_MULTIPLEX_MAX_CHUNKS 32
           99  +#ifndef SQLITE_MULTIPLEX_MAX_CHUNKS
          100  +# define SQLITE_MULTIPLEX_MAX_CHUNKS 32
          101  +#endif
    95    102   
    96    103   /* If SQLITE_MULTIPLEX_EXT_OVWR is defined, the 
    97    104   ** last SQLITE_MULTIPLEX_EXT_SZ characters of the 
    98    105   ** filename will be overwritten, otherwise, the 
    99    106   ** multiplex extension is simply appended to the filename.
   100    107   ** Ex.  (undefined) test.db -> test.db01
   101    108   **      (defined)   test.db -> test.01
................................................................................
   115    122   ** makeup a single SQLite DB file.  This allows the size of the DB
   116    123   ** to exceed the limits imposed by the file system.
   117    124   **
   118    125   ** There is an instance of the following object for each defined multiplex
   119    126   ** group.
   120    127   */
   121    128   struct multiplexGroup {
   122         -  sqlite3_file **pReal;            /* Handles to each chunk */
   123         -  char *bOpen;                     /* array of bools - 0 if chunk not opened */
          129  +  struct multiplexReal {           /* For each chunk */
          130  +    sqlite3_file *p;                  /* Handle for the chunk */
          131  +    char *z;                          /* Name of this chunk */
          132  +  } *aReal;                        /* list of all chunks */
          133  +  int nReal;                       /* Number of chunks */
   124    134     char *zName;                     /* Base filename of this group */
   125    135     int nName;                       /* Length of base filename */
   126    136     int flags;                       /* Flags used for original opening */
   127         -  int nChunkSize;                  /* Chunk size used for this group */
   128         -  int nMaxChunks;                  /* Max number of chunks for this group */
          137  +  unsigned int szChunk;            /* Chunk size used for this group */
   129    138     int bEnabled;                    /* TRUE to use Multiplex VFS for this file */
   130    139     multiplexGroup *pNext, *pPrev;   /* Doubly linked list of all group objects */
   131    140   };
   132    141   
   133    142   /*
   134    143   ** An instance of the following object represents each open connection
   135    144   ** to a file that is multiplex'ed.  This object is a 
................................................................................
   180    189     ** shim, the following mutex must be held.
   181    190     */
   182    191     sqlite3_mutex *pMutex;
   183    192   
   184    193     /* List of multiplexGroup objects.
   185    194     */
   186    195     multiplexGroup *pGroups;
   187         -
   188         -  /* Storage for temp file names.  Allocated during 
   189         -  ** initialization to the max pathname of the underlying VFS.
   190         -  */
   191         -  char *zName;
   192         -
   193    196   } gMultiplex;
   194    197   
   195    198   /************************* Utility Routines *********************************/
   196    199   /*
   197    200   ** Acquire and release the mutex used to serialize access to the
   198    201   ** list of multiplexGroups.
   199    202   */
................................................................................
   261    264       j = multiplexStrlen30(zBuf);
   262    265       if( (j + 8 + 1 + 3 + 1) <= nBuf ){
   263    266         /* Make 3 attempts to generate a unique name. */
   264    267         do {
   265    268           attempts++;
   266    269           sqlite3_randomness(8, &zBuf[j]);
   267    270           for(i=0; i<8; i++){
   268         -          zBuf[j+i] = (char)zChars[ ((unsigned char)zBuf[j+i])%(sizeof(zChars)-1) ];
          271  +          unsigned char uc = (unsigned char)zBuf[j+i];
          272  +          zBuf[j+i] = (char)zChars[uc%(sizeof(zChars)-1)];
   269    273           }
   270    274           memcpy(&zBuf[j+i], ".tmp", 5);
   271    275           rc = pOrigVfs->xAccess(pOrigVfs, zBuf, SQLITE_ACCESS_EXISTS, &exists);
   272    276         } while ( (rc==SQLITE_OK) && exists && (attempts<3) );
   273    277         if( rc==SQLITE_OK && exists ){
   274    278           rc = SQLITE_ERROR;
   275    279         }
   276    280       }
   277    281     }
   278    282   
   279    283     return rc;
   280    284   }
          285  +
          286  +/* Compute the filename for the iChunk-th chunk
          287  +*/
          288  +static int multiplexSubFilename(multiplexGroup *pGroup, int iChunk){
          289  +  if( iChunk>=pGroup->nReal ){
          290  +    struct multiplexReal *p;
          291  +    p = sqlite3_realloc(pGroup->aReal, (iChunk+1)*sizeof(*p));
          292  +    if( p==0 ){
          293  +      return SQLITE_NOMEM;
          294  +    }
          295  +    memset(&p[pGroup->nReal], 0, sizeof(p[0])*(iChunk+1-pGroup->nReal));
          296  +    pGroup->aReal = p;
          297  +    pGroup->nReal = iChunk+1;
          298  +  }
          299  +  if( pGroup->aReal[iChunk].z==0 ){
          300  +    char *z;
          301  +    int n = pGroup->nName;
          302  +    pGroup->aReal[iChunk].z = z = sqlite3_malloc( n+3 );
          303  +    if( z==0 ){
          304  +      return SQLITE_NOMEM;
          305  +    }
          306  +    memcpy(z, pGroup->zName, n+1);
          307  +    if( iChunk>0 ){
          308  +#ifdef SQLITE_ENABLE_8_3_NAMES
          309  +      if( n>3 && z[n-3]=='.' ){
          310  +        n--;
          311  +      }else if( n>4 && z[n-4]=='.' ){
          312  +        n -= 2;
          313  +      }
          314  +#endif
          315  +      sqlite3_snprintf(3,&z[n],"%02d",iChunk);
          316  +    }
          317  +  }
          318  +  return SQLITE_OK;
          319  +}
   281    320   
   282    321   /* Translate an sqlite3_file* that is really a multiplexGroup* into
   283    322   ** the sqlite3_file* for the underlying original VFS.
   284    323   */
   285         -static sqlite3_file *multiplexSubOpen(multiplexConn *pConn, int iChunk, int *rc, int *pOutFlags){
   286         -  multiplexGroup *pGroup = pConn->pGroup;
          324  +static sqlite3_file *multiplexSubOpen(
          325  +  multiplexGroup *pGroup,
          326  +  int iChunk,
          327  +  int *rc,
          328  +  int *pOutFlags
          329  +){
          330  +  sqlite3_file *pSubOpen = 0;
   287    331     sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;        /* Real VFS */
   288         -  if( iChunk<pGroup->nMaxChunks ){
   289         -    sqlite3_file *pSubOpen = pGroup->pReal[iChunk];    /* Real file descriptor */
   290         -    if( !pGroup->bOpen[iChunk] ){
   291         -      memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1);
   292         -      if( iChunk ){
   293         -#ifdef SQLITE_MULTIPLEX_EXT_OVWR
   294         -        sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, SQLITE_MULTIPLEX_EXT_FMT, iChunk);
   295         -#else
   296         -        sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName, SQLITE_MULTIPLEX_EXT_FMT, iChunk);
   297         -#endif
   298         -      }
   299         -      *rc = pOrigVfs->xOpen(pOrigVfs, gMultiplex.zName, pSubOpen, pGroup->flags, pOutFlags);
   300         -      if( *rc==SQLITE_OK ){
   301         -        pGroup->bOpen[iChunk] = -1;
   302         -        return pSubOpen;
   303         -      }
   304         -      return NULL;
          332  +  *rc = multiplexSubFilename(pGroup, iChunk);
          333  +  if( (*rc)==SQLITE_OK && (pSubOpen = pGroup->aReal[iChunk].p)==0 ){
          334  +    pSubOpen = sqlite3_malloc( pOrigVfs->szOsFile );
          335  +    if( pSubOpen==0 ){
          336  +      *rc = SQLITE_NOMEM;
          337  +      return 0;
   305    338       }
   306         -    *rc = SQLITE_OK;
   307         -    return pSubOpen;
          339  +    pGroup->aReal[iChunk].p = pSubOpen;
          340  +    *rc = pOrigVfs->xOpen(pOrigVfs, pGroup->aReal[iChunk].z, pSubOpen,
          341  +                          pGroup->flags, pOutFlags);
          342  +    if( *rc!=SQLITE_OK ){
          343  +      sqlite3_free(pSubOpen);
          344  +      pGroup->aReal[iChunk].p = 0;
          345  +      return 0;
          346  +    }
   308    347     }
   309         -  *rc = SQLITE_FULL;
   310         -  return NULL;
          348  +  return pSubOpen;
   311    349   }
   312    350   
   313    351   /*
   314    352   ** This is the implementation of the multiplex_control() SQL function.
   315    353   */
   316    354   static void multiplexControlFunc(
   317    355     sqlite3_context *context,
................................................................................
   361    399     const sqlite3_api_routines *pApi
   362    400   ){
   363    401     int rc;
   364    402     rc = sqlite3_create_function(db, "multiplex_control", 2, SQLITE_ANY, 
   365    403         0, multiplexControlFunc, 0, 0);
   366    404     return rc;
   367    405   }
          406  +
          407  +/*
          408  +** Close a single sub-file in the connection group.
          409  +*/
          410  +static void multiplexSubClose(
          411  +  multiplexGroup *pGroup,
          412  +  int iChunk,
          413  +  sqlite3_vfs *pOrigVfs
          414  +){
          415  +  sqlite3_file *pSubOpen = pGroup->aReal[iChunk].p;
          416  +  if( pSubOpen ){
          417  +    if( pOrigVfs ) pOrigVfs->xDelete(pOrigVfs, pGroup->aReal[iChunk].z, 0);
          418  +    pSubOpen->pMethods->xClose(pSubOpen);
          419  +    sqlite3_free(pGroup->aReal[iChunk].p);
          420  +  }
          421  +  sqlite3_free(pGroup->aReal[iChunk].z);
          422  +  memset(&pGroup->aReal[iChunk], 0, sizeof(pGroup->aReal[iChunk]));
          423  +}
          424  +
          425  +/*
          426  +** Deallocate memory held by a multiplexGroup
          427  +*/
          428  +static void multiplexFreeComponents(multiplexGroup *pGroup){
          429  +  int i;
          430  +  for(i=0; i<pGroup->nReal; i++){ multiplexSubClose(pGroup, i, 0); }
          431  +  sqlite3_free(pGroup->aReal);
          432  +  pGroup->aReal = 0;
          433  +  pGroup->nReal = 0;
          434  +}
          435  +
   368    436   
   369    437   /************************* VFS Method Wrappers *****************************/
   370    438   
   371    439   /*
   372    440   ** This is the xOpen method used for the "multiplex" VFS.
   373    441   **
   374    442   ** Most of the work is done by the underlying original VFS.  This method
................................................................................
   378    446   static int multiplexOpen(
   379    447     sqlite3_vfs *pVfs,         /* The multiplex VFS */
   380    448     const char *zName,         /* Name of file to be opened */
   381    449     sqlite3_file *pConn,       /* Fill in this file descriptor */
   382    450     int flags,                 /* Flags to control the opening */
   383    451     int *pOutFlags             /* Flags showing results of opening */
   384    452   ){
   385         -  int rc = SQLITE_OK;                            /* Result code */
   386         -  multiplexConn *pMultiplexOpen;                 /* The new multiplex file descriptor */
   387         -  multiplexGroup *pGroup;                        /* Corresponding multiplexGroup object */
   388         -  sqlite3_file *pSubOpen;                        /* Real file descriptor */
          453  +  int rc = SQLITE_OK;                  /* Result code */
          454  +  multiplexConn *pMultiplexOpen;       /* The new multiplex file descriptor */
          455  +  multiplexGroup *pGroup;              /* Corresponding multiplexGroup object */
          456  +  sqlite3_file *pSubOpen = 0;                    /* Real file descriptor */
   389    457     sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;   /* Real VFS */
   390    458     int nName;
   391         -  int i;
   392    459     int sz;
          460  +  char *zToFree = 0;
   393    461   
   394    462     UNUSED_PARAMETER(pVfs);
          463  +  memset(pConn, 0, pVfs->szOsFile);
   395    464   
   396    465     /* We need to create a group structure and manage
   397    466     ** access to this group of files.
   398    467     */
   399    468     multiplexEnter();
   400    469     pMultiplexOpen = (multiplexConn*)pConn;
   401    470   
   402    471     /* If the second argument to this function is NULL, generate a 
   403    472     ** temporary file name to use.  This will be handled by the
   404    473     ** original xOpen method.  We just need to allocate space for
   405    474     ** it.
   406    475     */
   407    476     if( !zName ){
   408         -    rc = multiplexGetTempname(pOrigVfs, pOrigVfs->mxPathname, gMultiplex.zName);
   409         -    zName = gMultiplex.zName;
          477  +    zName = zToFree = sqlite3_malloc( pOrigVfs->mxPathname + 10 );
          478  +    if( zName==0 ){
          479  +      rc = SQLITE_NOMEM;
          480  +    }else{
          481  +      rc = multiplexGetTempname(pOrigVfs, pOrigVfs->mxPathname, zToFree);
          482  +    }
   410    483     }
   411    484   
   412    485     if( rc==SQLITE_OK ){
   413    486       /* allocate space for group */
   414    487       nName = multiplexStrlen30(zName);
   415         -    sz = sizeof(multiplexGroup)                                /* multiplexGroup */
   416         -       + (sizeof(sqlite3_file *)*SQLITE_MULTIPLEX_MAX_CHUNKS)  /* pReal[] */
   417         -       + (pOrigVfs->szOsFile*SQLITE_MULTIPLEX_MAX_CHUNKS)      /* *pReal */
   418         -       + SQLITE_MULTIPLEX_MAX_CHUNKS                           /* bOpen[] */
   419         -       + nName + 1;                                            /* zName */
   420         -#ifndef SQLITE_MULTIPLEX_EXT_OVWR
   421         -    sz += SQLITE_MULTIPLEX_EXT_SZ;
   422         -    assert(nName+SQLITE_MULTIPLEX_EXT_SZ < pOrigVfs->mxPathname);
   423         -#else
   424         -    assert(nName >= SQLITE_MULTIPLEX_EXT_SZ);
   425         -    assert(nName < pOrigVfs->mxPathname);
   426         -#endif
          488  +    sz = sizeof(multiplexGroup)                             /* multiplexGroup */
          489  +       + nName + 1;                                         /* zName */
   427    490       pGroup = sqlite3_malloc( sz );
   428    491       if( pGroup==0 ){
   429         -      rc=SQLITE_NOMEM;
          492  +      rc = SQLITE_NOMEM;
   430    493       }
   431    494     }
   432    495   
   433    496     if( rc==SQLITE_OK ){
   434    497       /* assign pointers to extra space allocated */
   435    498       char *p = (char *)&pGroup[1];
   436    499       pMultiplexOpen->pGroup = pGroup;
   437    500       memset(pGroup, 0, sz);
   438    501       pGroup->bEnabled = -1;
   439         -    pGroup->nChunkSize = SQLITE_MULTIPLEX_CHUNK_SIZE;
   440         -    pGroup->nMaxChunks = SQLITE_MULTIPLEX_MAX_CHUNKS;
   441         -    pGroup->pReal = (sqlite3_file **)p;
   442         -    p += (sizeof(sqlite3_file *)*pGroup->nMaxChunks);
   443         -    for(i=0; i<pGroup->nMaxChunks; i++){
   444         -      pGroup->pReal[i] = (sqlite3_file *)p;
   445         -      p += pOrigVfs->szOsFile;
          502  +    pGroup->szChunk = SQLITE_MULTIPLEX_CHUNK_SIZE;
          503  +    if( flags & SQLITE_OPEN_URI ){
          504  +      const char *zChunkSize;
          505  +      zChunkSize = sqlite3_uri_parameter(zName, "chunksize");
          506  +      if( zChunkSize ){
          507  +        unsigned int n = 0;
          508  +        int i;
          509  +        for(i=0; zChunkSize[i]>='0' && zChunkSize[i]<='9'; i++){
          510  +          n = n*10 + zChunkSize[i] - '0';
          511  +        }
          512  +        if( n>0 ){
          513  +          pGroup->szChunk = (n+0xffff)&~0xffff;
          514  +        }else{
          515  +          /* A zero or negative chunksize disabled the multiplexor */
          516  +          pGroup->bEnabled = 0;
          517  +        }
          518  +      }
   446    519       }
   447         -    /* bOpen[] vals should all be zero from memset above */
   448         -    pGroup->bOpen = p;
   449         -    p += pGroup->nMaxChunks;
   450    520       pGroup->zName = p;
   451    521       /* save off base filename, name length, and original open flags  */
   452    522       memcpy(pGroup->zName, zName, nName+1);
   453    523       pGroup->nName = nName;
   454    524       pGroup->flags = flags;
   455         -    pSubOpen = multiplexSubOpen(pMultiplexOpen, 0, &rc, pOutFlags);
          525  +    rc = multiplexSubFilename(pGroup, 1);
          526  +    if( rc==SQLITE_OK ){
          527  +      pSubOpen = multiplexSubOpen(pGroup, 0, &rc, pOutFlags);
          528  +    }
   456    529       if( pSubOpen ){
   457         -      /* if this file is already larger than chunk size, disable 
   458         -      ** the multiplex feature.
   459         -      */
          530  +      int exists, rc2, rc3;
   460    531         sqlite3_int64 sz;
   461         -      int rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz);
   462         -      if( (rc2==SQLITE_OK) && (sz>pGroup->nChunkSize) ){
   463         -        pGroup->bEnabled = 0;
          532  +
          533  +      rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz);
          534  +      if( rc2==SQLITE_OK ){
          535  +        /* If the first overflow file exists and if the size of the main file
          536  +        ** is different from the chunk size, that means the chunk size is set
          537  +        ** set incorrectly.  So fix it.
          538  +        **
          539  +        ** Or, if the first overflow file does not exist and the main file is
          540  +        ** larger than the chunk size, that means the chunk size is too small.
          541  +        ** But we have no way of determining the intended chunk size, so 
          542  +        ** just disable the multiplexor all togethre.
          543  +        */
          544  +        rc3 = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[1].z,
          545  +            SQLITE_ACCESS_EXISTS, &exists);
          546  +        if( rc3==SQLITE_OK && exists && sz==(sz&0xffff0000) && sz>0
          547  +            && sz!=pGroup->szChunk ){
          548  +          pGroup->szChunk = sz;
          549  +        }else if( rc3==SQLITE_OK && !exists && sz>pGroup->szChunk ){
          550  +          pGroup->bEnabled = 0;
          551  +        }
   464    552         }
          553  +
   465    554         if( pSubOpen->pMethods->iVersion==1 ){
   466    555           pMultiplexOpen->base.pMethods = &gMultiplex.sIoMethodsV1;
   467    556         }else{
   468    557           pMultiplexOpen->base.pMethods = &gMultiplex.sIoMethodsV2;
   469    558         }
   470    559         /* place this group at the head of our list */
   471    560         pGroup->pNext = gMultiplex.pGroups;
   472    561         if( gMultiplex.pGroups ) gMultiplex.pGroups->pPrev = pGroup;
   473    562         gMultiplex.pGroups = pGroup;
   474    563       }else{
          564  +      multiplexFreeComponents(pGroup);
   475    565         sqlite3_free(pGroup);
   476    566       }
   477    567     }
   478    568     multiplexLeave();
          569  +  sqlite3_free(zToFree);
   479    570     return rc;
   480    571   }
   481    572   
   482    573   /*
   483    574   ** This is the xDelete method used for the "multiplex" VFS.
   484         -** It attempts to delete the filename specified, as well
   485         -** as additional files with the SQLITE_MULTIPLEX_EXT_FMT extension.
          575  +** It attempts to delete the filename specified.
   486    576   */
   487    577   static int multiplexDelete(
   488    578     sqlite3_vfs *pVfs,         /* The multiplex VFS */
   489    579     const char *zName,         /* Name of file to delete */
   490    580     int syncDir
   491    581   ){
   492    582     sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;   /* Real VFS */
   493         -  int rc = SQLITE_OK;
   494         -  int nName = multiplexStrlen30(zName);
   495         -  int i;
   496         -
   497         -  UNUSED_PARAMETER(pVfs);
   498         -
   499         -  multiplexEnter();
   500         -  memcpy(gMultiplex.zName, zName, nName+1);
   501         -  for(i=0; i<SQLITE_MULTIPLEX_MAX_CHUNKS; i++){
   502         -    int rc2;
   503         -    int exists = 0;
   504         -    if( i ){
   505         -#ifdef SQLITE_MULTIPLEX_EXT_OVWR
   506         -        sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, 
   507         -            gMultiplex.zName+nName-SQLITE_MULTIPLEX_EXT_SZ, 
   508         -            SQLITE_MULTIPLEX_EXT_FMT, i);
   509         -#else
   510         -        sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, 
   511         -            gMultiplex.zName+nName, 
   512         -            SQLITE_MULTIPLEX_EXT_FMT, i);
   513         -#endif
   514         -    }
   515         -    rc2 = pOrigVfs->xAccess(pOrigVfs, gMultiplex.zName, 
   516         -        SQLITE_ACCESS_EXISTS, &exists);
   517         -    if( rc2==SQLITE_OK && exists ){
   518         -      /* if it exists, delete it */
   519         -      rc2 = pOrigVfs->xDelete(pOrigVfs, gMultiplex.zName, syncDir);
   520         -      if( rc2!=SQLITE_OK ) rc = rc2;
   521         -    }else{
   522         -      /* stop at first "gap" */
   523         -      break;
   524         -    }
   525         -  }
   526         -  multiplexLeave();
   527         -  return rc;
          583  +  return pOrigVfs->xDelete(pOrigVfs, zName, syncDir);
   528    584   }
   529    585   
   530    586   static int multiplexAccess(sqlite3_vfs *a, const char *b, int c, int *d){
   531    587     return gMultiplex.pOrigVfs->xAccess(gMultiplex.pOrigVfs, b, c, d);
   532    588   }
   533    589   static int multiplexFullPathname(sqlite3_vfs *a, const char *b, int c, char *d){
   534    590     return gMultiplex.pOrigVfs->xFullPathname(gMultiplex.pOrigVfs, b, c, d);
................................................................................
   568    624   ** The group structure for this file is unlinked from 
   569    625   ** our list of groups and freed.
   570    626   */
   571    627   static int multiplexClose(sqlite3_file *pConn){
   572    628     multiplexConn *p = (multiplexConn*)pConn;
   573    629     multiplexGroup *pGroup = p->pGroup;
   574    630     int rc = SQLITE_OK;
   575         -  int i;
   576    631     multiplexEnter();
   577         -  /* close any open handles */
   578         -  for(i=0; i<pGroup->nMaxChunks; i++){
   579         -    if( pGroup->bOpen[i] ){
   580         -      sqlite3_file *pSubOpen = pGroup->pReal[i];
   581         -      int rc2 = pSubOpen->pMethods->xClose(pSubOpen);
   582         -      if( rc2!=SQLITE_OK ) rc = rc2;
   583         -      pGroup->bOpen[i] = 0;
   584         -    }
   585         -  }
          632  +  multiplexFreeComponents(pGroup);
   586    633     /* remove from linked list */
   587    634     if( pGroup->pNext ) pGroup->pNext->pPrev = pGroup->pPrev;
   588    635     if( pGroup->pPrev ){
   589    636       pGroup->pPrev->pNext = pGroup->pNext;
   590    637     }else{
   591    638       gMultiplex.pGroups = pGroup->pNext;
   592    639     }
................................................................................
   606    653     sqlite3_int64 iOfst
   607    654   ){
   608    655     multiplexConn *p = (multiplexConn*)pConn;
   609    656     multiplexGroup *pGroup = p->pGroup;
   610    657     int rc = SQLITE_OK;
   611    658     multiplexEnter();
   612    659     if( !pGroup->bEnabled ){
   613         -    sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
   614         -    rc = ( !pSubOpen ) ? SQLITE_IOERR_READ : pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst);
          660  +    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
          661  +    if( pSubOpen==0 ){
          662  +      rc = SQLITE_IOERR_READ;
          663  +    }else{
          664  +      rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst);
          665  +    }
   615    666     }else{
   616    667       while( iAmt > 0 ){
   617         -      int i = (int)(iOfst / pGroup->nChunkSize);
   618         -      sqlite3_file *pSubOpen = multiplexSubOpen(p, i, &rc, NULL);
          668  +      int i = (int)(iOfst / pGroup->szChunk);
          669  +      sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL);
   619    670         if( pSubOpen ){
   620         -        int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - pGroup->nChunkSize;
          671  +        int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) - pGroup->szChunk;
   621    672           if( extra<0 ) extra = 0;
   622    673           iAmt -= extra;
   623         -        rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst % pGroup->nChunkSize);
          674  +        rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt,
          675  +                                       iOfst % pGroup->szChunk);
   624    676           if( rc!=SQLITE_OK ) break;
   625    677           pBuf = (char *)pBuf + iAmt;
   626    678           iOfst += iAmt;
   627    679           iAmt = extra;
   628    680         }else{
   629    681           rc = SQLITE_IOERR_READ;
   630    682           break;
................................................................................
   646    698     sqlite3_int64 iOfst
   647    699   ){
   648    700     multiplexConn *p = (multiplexConn*)pConn;
   649    701     multiplexGroup *pGroup = p->pGroup;
   650    702     int rc = SQLITE_OK;
   651    703     multiplexEnter();
   652    704     if( !pGroup->bEnabled ){
   653         -    sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
   654         -    rc = ( !pSubOpen ) ? SQLITE_IOERR_WRITE : pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst);
          705  +    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
          706  +    if( pSubOpen==0 ){
          707  +      rc = SQLITE_IOERR_WRITE;
          708  +    }else{
          709  +      rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst);
          710  +    }
   655    711     }else{
   656    712       while( iAmt > 0 ){
   657         -      int i = (int)(iOfst / pGroup->nChunkSize);
   658         -      sqlite3_file *pSubOpen = multiplexSubOpen(p, i, &rc, NULL);
          713  +      int i = (int)(iOfst / pGroup->szChunk);
          714  +      sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL);
   659    715         if( pSubOpen ){
   660         -        int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - pGroup->nChunkSize;
          716  +        int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) -
          717  +                    pGroup->szChunk;
   661    718           if( extra<0 ) extra = 0;
   662    719           iAmt -= extra;
   663         -        rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst % pGroup->nChunkSize);
          720  +        rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt,
          721  +                                        iOfst % pGroup->szChunk);
   664    722           if( rc!=SQLITE_OK ) break;
   665    723           pBuf = (char *)pBuf + iAmt;
   666    724           iOfst += iAmt;
   667    725           iAmt = extra;
   668    726         }else{
   669    727           rc = SQLITE_IOERR_WRITE;
   670    728           break;
................................................................................
   681    739   */
   682    740   static int multiplexTruncate(sqlite3_file *pConn, sqlite3_int64 size){
   683    741     multiplexConn *p = (multiplexConn*)pConn;
   684    742     multiplexGroup *pGroup = p->pGroup;
   685    743     int rc = SQLITE_OK;
   686    744     multiplexEnter();
   687    745     if( !pGroup->bEnabled ){
   688         -    sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
   689         -    rc = ( !pSubOpen ) ? SQLITE_IOERR_TRUNCATE : pSubOpen->pMethods->xTruncate(pSubOpen, size);
          746  +    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
          747  +    if( pSubOpen==0 ){
          748  +      rc = SQLITE_IOERR_TRUNCATE;
          749  +    }else{
          750  +      rc = pSubOpen->pMethods->xTruncate(pSubOpen, size);
          751  +    }
   690    752     }else{
   691    753       int rc2;
   692    754       int i;
   693    755       sqlite3_file *pSubOpen;
   694    756       sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;   /* Real VFS */
   695         -    memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1);
   696    757       /* delete the chunks above the truncate limit */
   697         -    for(i=(int)(size / pGroup->nChunkSize)+1; i<pGroup->nMaxChunks; i++){
   698         -      /* close any open chunks before deleting them */
   699         -      if( pGroup->bOpen[i] ){
   700         -        pSubOpen = pGroup->pReal[i];
   701         -        rc2 = pSubOpen->pMethods->xClose(pSubOpen);
   702         -        if( rc2!=SQLITE_OK ) rc = SQLITE_IOERR_TRUNCATE;
   703         -        pGroup->bOpen[i] = 0;
   704         -      }
   705         -#ifdef SQLITE_MULTIPLEX_EXT_OVWR
   706         -      sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, 
   707         -          gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, 
   708         -          SQLITE_MULTIPLEX_EXT_FMT, i);
   709         -#else
   710         -      sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, 
   711         -          gMultiplex.zName+pGroup->nName, 
   712         -          SQLITE_MULTIPLEX_EXT_FMT, i);
   713         -#endif
   714         -      rc2 = pOrigVfs->xDelete(pOrigVfs, gMultiplex.zName, 0);
   715         -      if( rc2!=SQLITE_OK ) rc = SQLITE_IOERR_TRUNCATE;
          758  +    for(i=(int)(size / pGroup->szChunk)+1; i<pGroup->nReal; i++){
          759  +      multiplexSubClose(pGroup, i, pOrigVfs);
   716    760       }
   717         -    pSubOpen = multiplexSubOpen(p, (int)(size / pGroup->nChunkSize), &rc2, NULL);
          761  +    pSubOpen = multiplexSubOpen(pGroup, (int)(size/pGroup->szChunk), &rc2,0);
   718    762       if( pSubOpen ){
   719         -      rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->nChunkSize);
          763  +      rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->szChunk);
   720    764         if( rc2!=SQLITE_OK ) rc = rc2;
   721    765       }else{
   722    766         rc = SQLITE_IOERR_TRUNCATE;
   723    767       }
   724    768     }
   725    769     multiplexLeave();
   726    770     return rc;
................................................................................
   730    774   */
   731    775   static int multiplexSync(sqlite3_file *pConn, int flags){
   732    776     multiplexConn *p = (multiplexConn*)pConn;
   733    777     multiplexGroup *pGroup = p->pGroup;
   734    778     int rc = SQLITE_OK;
   735    779     int i;
   736    780     multiplexEnter();
   737         -  for(i=0; i<pGroup->nMaxChunks; i++){
   738         -    /* if we don't have it open, we don't need to sync it */
   739         -    if( pGroup->bOpen[i] ){
   740         -      sqlite3_file *pSubOpen = pGroup->pReal[i];
          781  +  for(i=0; i<pGroup->nReal; i++){
          782  +    sqlite3_file *pSubOpen = pGroup->aReal[i].p;
          783  +    if( pSubOpen ){
   741    784         int rc2 = pSubOpen->pMethods->xSync(pSubOpen, flags);
   742    785         if( rc2!=SQLITE_OK ) rc = rc2;
   743    786       }
   744    787     }
   745    788     multiplexLeave();
   746    789     return rc;
   747    790   }
................................................................................
   753    796     multiplexConn *p = (multiplexConn*)pConn;
   754    797     multiplexGroup *pGroup = p->pGroup;
   755    798     int rc = SQLITE_OK;
   756    799     int rc2;
   757    800     int i;
   758    801     multiplexEnter();
   759    802     if( !pGroup->bEnabled ){
   760         -    sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
   761         -    rc = ( !pSubOpen ) ? SQLITE_IOERR_FSTAT : pSubOpen->pMethods->xFileSize(pSubOpen, pSize);
          803  +    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
          804  +    if( pSubOpen==0 ){
          805  +      rc = SQLITE_IOERR_FSTAT;
          806  +    }else{
          807  +      rc = pSubOpen->pMethods->xFileSize(pSubOpen, pSize);
          808  +    }
   762    809     }else{
          810  +    sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;
   763    811       *pSize = 0;
   764         -    for(i=0; i<pGroup->nMaxChunks; i++){
   765         -      sqlite3_file *pSubOpen = NULL;
   766         -      /* if not opened already, check to see if the chunk exists */
   767         -      if( pGroup->bOpen[i] ){
   768         -        pSubOpen = pGroup->pReal[i];
          812  +    for(i=0; 1; i++){
          813  +      sqlite3_file *pSubOpen = 0;
          814  +      int exists = 0;
          815  +      rc = multiplexSubFilename(pGroup, i);
          816  +      if( rc ) break;
          817  +      rc2 = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[i].z,
          818  +          SQLITE_ACCESS_EXISTS, &exists);
          819  +      if( rc2==SQLITE_OK && exists){
          820  +        /* if it exists, open it */
          821  +        pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL);
   769    822         }else{
   770         -        sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;   /* Real VFS */
   771         -        int exists = 0;
   772         -        memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1);
   773         -        if( i ){
   774         -#ifdef SQLITE_MULTIPLEX_EXT_OVWR
   775         -          sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, 
   776         -              gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, 
   777         -              SQLITE_MULTIPLEX_EXT_FMT, i);
   778         -#else
   779         -          sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, 
   780         -              gMultiplex.zName+pGroup->nName, 
   781         -              SQLITE_MULTIPLEX_EXT_FMT, i);
   782         -#endif
   783         -        }
   784         -        rc2 = pOrigVfs->xAccess(pOrigVfs, gMultiplex.zName, 
   785         -            SQLITE_ACCESS_EXISTS, &exists);
   786         -        if( rc2==SQLITE_OK && exists){
   787         -          /* if it exists, open it */
   788         -          pSubOpen = multiplexSubOpen(p, i, &rc, NULL);
   789         -        }else{
   790         -          /* stop at first "gap" */
   791         -          break;
   792         -        }
          823  +        /* stop at first "gap" */
          824  +        break;
   793    825         }
   794    826         if( pSubOpen ){
   795    827           sqlite3_int64 sz;
   796    828           rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz);
   797    829           if( rc2!=SQLITE_OK ){
   798    830             rc = rc2;
   799    831           }else{
   800         -          if( sz>pGroup->nChunkSize ){
          832  +          if( sz>pGroup->szChunk ){
   801    833               rc = SQLITE_IOERR_FSTAT;
   802    834             }
   803    835             *pSize += sz;
   804    836           }
   805    837         }else{
   806    838           break;
   807    839         }
................................................................................
   812    844   }
   813    845   
   814    846   /* Pass xLock requests through to the original VFS unchanged.
   815    847   */
   816    848   static int multiplexLock(sqlite3_file *pConn, int lock){
   817    849     multiplexConn *p = (multiplexConn*)pConn;
   818    850     int rc;
   819         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          851  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   820    852     if( pSubOpen ){
   821    853       return pSubOpen->pMethods->xLock(pSubOpen, lock);
   822    854     }
   823    855     return SQLITE_BUSY;
   824    856   }
   825    857   
   826    858   /* Pass xUnlock requests through to the original VFS unchanged.
   827    859   */
   828    860   static int multiplexUnlock(sqlite3_file *pConn, int lock){
   829    861     multiplexConn *p = (multiplexConn*)pConn;
   830    862     int rc;
   831         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          863  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   832    864     if( pSubOpen ){
   833    865       return pSubOpen->pMethods->xUnlock(pSubOpen, lock);
   834    866     }
   835    867     return SQLITE_IOERR_UNLOCK;
   836    868   }
   837    869   
   838    870   /* Pass xCheckReservedLock requests through to the original VFS unchanged.
   839    871   */
   840    872   static int multiplexCheckReservedLock(sqlite3_file *pConn, int *pResOut){
   841    873     multiplexConn *p = (multiplexConn*)pConn;
   842    874     int rc;
   843         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          875  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   844    876     if( pSubOpen ){
   845    877       return pSubOpen->pMethods->xCheckReservedLock(pSubOpen, pResOut);
   846    878     }
   847    879     return SQLITE_IOERR_CHECKRESERVEDLOCK;
   848    880   }
   849    881   
   850    882   /* Pass xFileControl requests through to the original VFS unchanged,
................................................................................
   863    895           int bEnabled = *(int *)pArg;
   864    896           pGroup->bEnabled = bEnabled;
   865    897           rc = SQLITE_OK;
   866    898         }
   867    899         break;
   868    900       case MULTIPLEX_CTRL_SET_CHUNK_SIZE:
   869    901         if( pArg ) {
   870         -        int nChunkSize = *(int *)pArg;
   871         -        if( nChunkSize<1 ){
          902  +        unsigned int szChunk = *(unsigned*)pArg;
          903  +        if( szChunk<1 ){
   872    904             rc = SQLITE_MISUSE;
   873    905           }else{
   874    906             /* Round up to nearest multiple of MAX_PAGE_SIZE. */
   875         -          nChunkSize = (nChunkSize + (MAX_PAGE_SIZE-1));
   876         -          nChunkSize &= ~(MAX_PAGE_SIZE-1);
   877         -          pGroup->nChunkSize = nChunkSize;
          907  +          szChunk = (szChunk + (MAX_PAGE_SIZE-1));
          908  +          szChunk &= ~(MAX_PAGE_SIZE-1);
          909  +          pGroup->szChunk = szChunk;
   878    910             rc = SQLITE_OK;
   879    911           }
   880    912         }
   881    913         break;
   882    914       case MULTIPLEX_CTRL_SET_MAX_CHUNKS:
   883         -      if( pArg ) {
   884         -        int nMaxChunks = *(int *)pArg;
   885         -        if(( nMaxChunks<1 ) || ( nMaxChunks>SQLITE_MULTIPLEX_MAX_CHUNKS )){
   886         -          rc = SQLITE_MISUSE;
   887         -        }else{
   888         -          pGroup->nMaxChunks = nMaxChunks;
   889         -          rc = SQLITE_OK;
   890         -        }
   891         -      }
          915  +      rc = SQLITE_OK;
   892    916         break;
   893    917       case SQLITE_FCNTL_SIZE_HINT:
   894    918       case SQLITE_FCNTL_CHUNK_SIZE:
   895    919         /* no-op these */
   896    920         rc = SQLITE_OK;
   897    921         break;
   898    922       default:
   899         -      pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          923  +      pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
   900    924         if( pSubOpen ){
   901    925           rc = pSubOpen->pMethods->xFileControl(pSubOpen, op, pArg);
   902    926         }
   903    927         break;
   904    928     }
   905    929     return rc;
   906    930   }
   907    931   
   908    932   /* Pass xSectorSize requests through to the original VFS unchanged.
   909    933   */
   910    934   static int multiplexSectorSize(sqlite3_file *pConn){
   911    935     multiplexConn *p = (multiplexConn*)pConn;
   912    936     int rc;
   913         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          937  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   914    938     if( pSubOpen ){
   915    939       return pSubOpen->pMethods->xSectorSize(pSubOpen);
   916    940     }
   917    941     return DEFAULT_SECTOR_SIZE;
   918    942   }
   919    943   
   920    944   /* Pass xDeviceCharacteristics requests through to the original VFS unchanged.
   921    945   */
   922    946   static int multiplexDeviceCharacteristics(sqlite3_file *pConn){
   923    947     multiplexConn *p = (multiplexConn*)pConn;
   924    948     int rc;
   925         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          949  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   926    950     if( pSubOpen ){
   927    951       return pSubOpen->pMethods->xDeviceCharacteristics(pSubOpen);
   928    952     }
   929    953     return 0;
   930    954   }
   931    955   
   932    956   /* Pass xShmMap requests through to the original VFS unchanged.
................................................................................
   936    960     int iRegion,                    /* Region to retrieve */
   937    961     int szRegion,                   /* Size of regions */
   938    962     int bExtend,                    /* True to extend file if necessary */
   939    963     void volatile **pp              /* OUT: Mapped memory */
   940    964   ){
   941    965     multiplexConn *p = (multiplexConn*)pConn;
   942    966     int rc;
   943         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          967  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   944    968     if( pSubOpen ){
   945         -    return pSubOpen->pMethods->xShmMap(pSubOpen, iRegion, szRegion, bExtend, pp);
          969  +    return pSubOpen->pMethods->xShmMap(pSubOpen, iRegion, szRegion, bExtend,pp);
   946    970     }
   947    971     return SQLITE_IOERR;
   948    972   }
   949    973   
   950    974   /* Pass xShmLock requests through to the original VFS unchanged.
   951    975   */
   952    976   static int multiplexShmLock(
................................................................................
   953    977     sqlite3_file *pConn,       /* Database file holding the shared memory */
   954    978     int ofst,                  /* First lock to acquire or release */
   955    979     int n,                     /* Number of locks to acquire or release */
   956    980     int flags                  /* What to do with the lock */
   957    981   ){
   958    982     multiplexConn *p = (multiplexConn*)pConn;
   959    983     int rc;
   960         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          984  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   961    985     if( pSubOpen ){
   962    986       return pSubOpen->pMethods->xShmLock(pSubOpen, ofst, n, flags);
   963    987     }
   964    988     return SQLITE_BUSY;
   965    989   }
   966    990   
   967    991   /* Pass xShmBarrier requests through to the original VFS unchanged.
   968    992   */
   969    993   static void multiplexShmBarrier(sqlite3_file *pConn){
   970    994     multiplexConn *p = (multiplexConn*)pConn;
   971    995     int rc;
   972         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
          996  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   973    997     if( pSubOpen ){
   974    998       pSubOpen->pMethods->xShmBarrier(pSubOpen);
   975    999     }
   976   1000   }
   977   1001   
   978   1002   /* Pass xShmUnmap requests through to the original VFS unchanged.
   979   1003   */
   980   1004   static int multiplexShmUnmap(sqlite3_file *pConn, int deleteFlag){
   981   1005     multiplexConn *p = (multiplexConn*)pConn;
   982   1006     int rc;
   983         -  sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL);
         1007  +  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
   984   1008     if( pSubOpen ){
   985   1009       return pSubOpen->pMethods->xShmUnmap(pSubOpen, deleteFlag);
   986   1010     }
   987   1011     return SQLITE_OK;
   988   1012   }
   989   1013   
   990   1014   /************************** Public Interfaces *****************************/
................................................................................
  1006   1030     pOrigVfs = sqlite3_vfs_find(zOrigVfsName);
  1007   1031     if( pOrigVfs==0 ) return SQLITE_ERROR;
  1008   1032     assert( pOrigVfs!=&gMultiplex.sThisVfs );
  1009   1033     gMultiplex.pMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST);
  1010   1034     if( !gMultiplex.pMutex ){
  1011   1035       return SQLITE_NOMEM;
  1012   1036     }
  1013         -  gMultiplex.zName = sqlite3_malloc(pOrigVfs->mxPathname);
  1014         -  if( !gMultiplex.zName ){
  1015         -    sqlite3_mutex_free(gMultiplex.pMutex);
  1016         -    return SQLITE_NOMEM;
  1017         -  }
  1018   1037     gMultiplex.pGroups = NULL;
  1019   1038     gMultiplex.isInitialized = 1;
  1020   1039     gMultiplex.pOrigVfs = pOrigVfs;
  1021   1040     gMultiplex.sThisVfs = *pOrigVfs;
  1022   1041     gMultiplex.sThisVfs.szOsFile += sizeof(multiplexConn);
  1023   1042     gMultiplex.sThisVfs.zName = SQLITE_MULTIPLEX_VFS_NAME;
  1024   1043     gMultiplex.sThisVfs.xOpen = multiplexOpen;
................................................................................
  1043   1062     gMultiplex.sIoMethodsV1.xSync = multiplexSync;
  1044   1063     gMultiplex.sIoMethodsV1.xFileSize = multiplexFileSize;
  1045   1064     gMultiplex.sIoMethodsV1.xLock = multiplexLock;
  1046   1065     gMultiplex.sIoMethodsV1.xUnlock = multiplexUnlock;
  1047   1066     gMultiplex.sIoMethodsV1.xCheckReservedLock = multiplexCheckReservedLock;
  1048   1067     gMultiplex.sIoMethodsV1.xFileControl = multiplexFileControl;
  1049   1068     gMultiplex.sIoMethodsV1.xSectorSize = multiplexSectorSize;
  1050         -  gMultiplex.sIoMethodsV1.xDeviceCharacteristics = multiplexDeviceCharacteristics;
         1069  +  gMultiplex.sIoMethodsV1.xDeviceCharacteristics =
         1070  +                                            multiplexDeviceCharacteristics;
  1051   1071     gMultiplex.sIoMethodsV2 = gMultiplex.sIoMethodsV1;
  1052   1072     gMultiplex.sIoMethodsV2.iVersion = 2;
  1053   1073     gMultiplex.sIoMethodsV2.xShmMap = multiplexShmMap;
  1054   1074     gMultiplex.sIoMethodsV2.xShmLock = multiplexShmLock;
  1055   1075     gMultiplex.sIoMethodsV2.xShmBarrier = multiplexShmBarrier;
  1056   1076     gMultiplex.sIoMethodsV2.xShmUnmap = multiplexShmUnmap;
  1057   1077     sqlite3_vfs_register(&gMultiplex.sThisVfs, makeDefault);
................................................................................
  1070   1090   ** THIS ROUTINE IS NOT THREADSAFE.  Call this routine exactly once while
  1071   1091   ** shutting down in order to free all remaining multiplex groups.
  1072   1092   */
  1073   1093   int sqlite3_multiplex_shutdown(void){
  1074   1094     if( gMultiplex.isInitialized==0 ) return SQLITE_MISUSE;
  1075   1095     if( gMultiplex.pGroups ) return SQLITE_MISUSE;
  1076   1096     gMultiplex.isInitialized = 0;
  1077         -  sqlite3_free(gMultiplex.zName);
  1078   1097     sqlite3_mutex_free(gMultiplex.pMutex);
  1079   1098     sqlite3_vfs_unregister(&gMultiplex.sThisVfs);
  1080   1099     memset(&gMultiplex, 0, sizeof(gMultiplex));
  1081   1100     return SQLITE_OK;
  1082   1101   }
  1083   1102   
  1084   1103   /***************************** Test Code ***********************************/
................................................................................
  1172   1191             Tcl_NewStringObj(pGroup->zName, -1));
  1173   1192       Tcl_ListObjAppendElement(interp, pGroupTerm,
  1174   1193             Tcl_NewIntObj(pGroup->nName));
  1175   1194       Tcl_ListObjAppendElement(interp, pGroupTerm,
  1176   1195             Tcl_NewIntObj(pGroup->flags));
  1177   1196   
  1178   1197       /* count number of chunks with open handles */
  1179         -    for(i=0; i<pGroup->nMaxChunks; i++){
  1180         -      if( pGroup->bOpen[i] ) nChunks++;
         1198  +    for(i=0; i<pGroup->nReal; i++){
         1199  +      if( pGroup->aReal[i].p!=0 ) nChunks++;
  1181   1200       }
  1182   1201       Tcl_ListObjAppendElement(interp, pGroupTerm,
  1183   1202             Tcl_NewIntObj(nChunks));
  1184   1203   
  1185   1204       Tcl_ListObjAppendElement(interp, pGroupTerm,
  1186         -          Tcl_NewIntObj(pGroup->nChunkSize));
         1205  +          Tcl_NewIntObj(pGroup->szChunk));
  1187   1206       Tcl_ListObjAppendElement(interp, pGroupTerm,
  1188         -          Tcl_NewIntObj(pGroup->nMaxChunks));
         1207  +          Tcl_NewIntObj(pGroup->nReal));
  1189   1208   
  1190   1209       Tcl_ListObjAppendElement(interp, pResult, pGroupTerm);
  1191   1210     }
  1192   1211     multiplexLeave();
  1193   1212     Tcl_SetObjResult(interp, pResult);
  1194   1213     return TCL_OK;
  1195   1214   }

Changes to src/trigger.c.

   113    113     }else{
   114    114       /* Figure out the db that the the trigger will be created in */
   115    115       iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName);
   116    116       if( iDb<0 ){
   117    117         goto trigger_cleanup;
   118    118       }
   119    119     }
          120  +  if( !pTableName || db->mallocFailed ){
          121  +    goto trigger_cleanup;
          122  +  }
          123  +
          124  +  /* A long-standing parser bug is that this syntax was allowed:
          125  +  **
          126  +  **    CREATE TRIGGER attached.demo AFTER INSERT ON attached.tab ....
          127  +  **                                                 ^^^^^^^^
          128  +  **
          129  +  ** To maintain backwards compatibility, ignore the database
          130  +  ** name on pTableName if we are reparsing our of SQLITE_MASTER.
          131  +  */
          132  +  if( db->init.busy && iDb!=1 ){
          133  +    sqlite3DbFree(db, pTableName->a[0].zDatabase);
          134  +    pTableName->a[0].zDatabase = 0;
          135  +  }
   120    136   
   121    137     /* If the trigger name was unqualified, and the table is a temp table,
   122    138     ** then set iDb to 1 to create the trigger in the temporary database.
   123    139     ** If sqlite3SrcListLookup() returns 0, indicating the table does not
   124    140     ** exist, the error is caught by the block below.
   125    141     */
   126         -  if( !pTableName || db->mallocFailed ){
   127         -    goto trigger_cleanup;
   128         -  }
   129    142     pTab = sqlite3SrcListLookup(pParse, pTableName);
   130    143     if( db->init.busy==0 && pName2->n==0 && pTab
   131    144           && pTab->pSchema==db->aDb[1].pSchema ){
   132    145       iDb = 1;
   133    146     }
   134    147   
   135    148     /* Ensure the table name matches database name and that the table exists */

Changes to src/update.c.

   307    307     if( sqlite3ResolveExprNames(&sNC, pWhere) ){
   308    308       goto update_cleanup;
   309    309     }
   310    310   
   311    311     /* Begin the database scan
   312    312     */
   313    313     sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid);
   314         -  pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0, WHERE_ONEPASS_DESIRED);
          314  +  pWInfo = sqlite3WhereBegin(
          315  +      pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED
          316  +  );
   315    317     if( pWInfo==0 ) goto update_cleanup;
   316    318     okOnePass = pWInfo->okOnePass;
   317    319   
   318    320     /* Remember the rowid of every item to be updated.
   319    321     */
   320    322     sqlite3VdbeAddOp2(v, OP_Rowid, iCur, regOldRowid);
   321    323     if( !okOnePass ){

Changes to src/util.c.

  1145   1145     if( x>=0 ) return x;
  1146   1146     if( x==(int)0x80000000 ) return 0x7fffffff;
  1147   1147     return -x;
  1148   1148   }
  1149   1149   
  1150   1150   #ifdef SQLITE_ENABLE_8_3_NAMES
  1151   1151   /*
  1152         -** If SQLITE_ENABLE_8_3_NAME is set at compile-time and if the database
         1152  +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database
  1153   1153   ** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and
  1154   1154   ** if filename in z[] has a suffix (a.k.a. "extension") that is longer than
  1155   1155   ** three characters, then shorten the suffix on z[] to be the last three
  1156   1156   ** characters of the original suffix.
         1157  +**
         1158  +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always
         1159  +** do the suffix shortening regardless of URI parameter.
  1157   1160   **
  1158   1161   ** Examples:
  1159   1162   **
  1160   1163   **     test.db-journal    =>   test.nal
  1161   1164   **     test.db-wal        =>   test.wal
  1162   1165   **     test.db-shm        =>   test.shm
  1163   1166   */
  1164   1167   void sqlite3FileSuffix3(const char *zBaseFilename, char *z){
         1168  +#if SQLITE_ENABLE_8_3_NAMES<2
  1165   1169     const char *zOk;
  1166   1170     zOk = sqlite3_uri_parameter(zBaseFilename, "8_3_names");
  1167         -  if( zOk && sqlite3GetBoolean(zOk) ){
         1171  +  if( zOk && sqlite3GetBoolean(zOk) )
         1172  +#endif
         1173  +  {
  1168   1174       int i, sz;
  1169   1175       sz = sqlite3Strlen30(z);
  1170   1176       for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){}
  1171   1177       if( z[i]=='.' && ALWAYS(sz>i+4) ) memcpy(&z[i+1], &z[sz-3], 4);
  1172   1178     }
  1173   1179   }
  1174   1180   #endif

Changes to src/vdbe.c.

  3178   3178       ** sqlite3BtreeCreateTable() with the BTREE_BLOBKEY flag before
  3179   3179       ** opening it. If a transient table is required, just use the
  3180   3180       ** automatically created table with root-page 1 (an BLOB_INTKEY table).
  3181   3181       */
  3182   3182       if( pOp->p4.pKeyInfo ){
  3183   3183         int pgno;
  3184   3184         assert( pOp->p4type==P4_KEYINFO );
  3185         -      rc = sqlite3BtreeCreateTable(pCx->pBt, &pgno, BTREE_BLOBKEY); 
         3185  +      rc = sqlite3BtreeCreateTable(pCx->pBt, &pgno, BTREE_BLOBKEY | pOp->p5); 
  3186   3186         if( rc==SQLITE_OK ){
  3187   3187           assert( pgno==MASTER_ROOT+1 );
  3188   3188           rc = sqlite3BtreeCursor(pCx->pBt, pgno, 1, 
  3189   3189                                   (KeyInfo*)pOp->p4.z, pCx->pCursor);
  3190   3190           pCx->pKeyInfo = pOp->p4.pKeyInfo;
  3191   3191           pCx->pKeyInfo->enc = ENC(p->db);
  3192   3192         }

Changes to src/vdbeapi.c.

   484    484     }
   485    485     db = v->db;
   486    486     sqlite3_mutex_enter(db->mutex);
   487    487     while( (rc = sqlite3Step(v))==SQLITE_SCHEMA
   488    488            && cnt++ < SQLITE_MAX_SCHEMA_RETRY
   489    489            && (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){
   490    490       sqlite3_reset(pStmt);
   491         -    v->expired = 0;
          491  +    assert( v->expired==0 );
   492    492     }
   493    493     if( rc2!=SQLITE_OK && ALWAYS(v->isPrepareV2) && ALWAYS(db->pErr) ){
   494    494       /* This case occurs after failing to recompile an sql statement. 
   495    495       ** The error message from the SQL compiler has already been loaded 
   496    496       ** into the database handle. This block copies the error message 
   497    497       ** from the database handle into the statement and sets the statement
   498    498       ** program counter to 0 to ensure that when the statement is 

Changes to src/vdbeaux.c.

  1502   1502     p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort);
  1503   1503     if( pParse->explain && nMem<10 ){
  1504   1504       nMem = 10;
  1505   1505     }
  1506   1506     memset(zCsr, 0, zEnd-zCsr);
  1507   1507     zCsr += (zCsr - (u8*)0)&7;
  1508   1508     assert( EIGHT_BYTE_ALIGNMENT(zCsr) );
         1509  +  p->expired = 0;
  1509   1510   
  1510   1511     /* Memory for registers, parameters, cursor, etc, is allocated in two
  1511   1512     ** passes.  On the first pass, we try to reuse unused space at the 
  1512   1513     ** end of the opcode array.  If we are unable to satisfy all memory
  1513   1514     ** requirements by reusing the opcode array tail, then the second
  1514   1515     ** pass will fill in the rest using a fresh allocation.  
  1515   1516     **

Changes to src/where.c.

   249    249   #define WHERE_IDX_ONLY     0x00800000  /* Use index only - omit table */
   250    250   #define WHERE_ORDERBY      0x01000000  /* Output will appear in correct order */
   251    251   #define WHERE_REVERSE      0x02000000  /* Scan in reverse order */
   252    252   #define WHERE_UNIQUE       0x04000000  /* Selects no more than one row */
   253    253   #define WHERE_VIRTUALTABLE 0x08000000  /* Use virtual-table processing */
   254    254   #define WHERE_MULTI_OR     0x10000000  /* OR using multiple indices */
   255    255   #define WHERE_TEMP_INDEX   0x20000000  /* Uses an ephemeral index */
          256  +#define WHERE_DISTINCT     0x40000000  /* Correct order for DISTINCT */
   256    257   
   257    258   /*
   258    259   ** Initialize a preallocated WhereClause structure.
   259    260   */
   260    261   static void whereClauseInit(
   261    262     WhereClause *pWC,        /* The WhereClause to be initialized */
   262    263     Parse *pParse,           /* The parsing context */
................................................................................
  1393   1394       if( (exprTableUsage(pMaskSet, pList->a[iFirst++].pExpr)&allowed)!=0 ){
  1394   1395         return 1;
  1395   1396       }
  1396   1397     }
  1397   1398     return 0;
  1398   1399   }
  1399   1400   
         1401  +/*
         1402  +** This function searches the expression list passed as the second argument
         1403  +** for an expression of type TK_COLUMN that refers to the same column and
         1404  +** uses the same collation sequence as the iCol'th column of index pIdx.
         1405  +** Argument iBase is the cursor number used for the table that pIdx refers
         1406  +** to.
         1407  +**
         1408  +** If such an expression is found, its index in pList->a[] is returned. If
         1409  +** no expression is found, -1 is returned.
         1410  +*/
         1411  +static int findIndexCol(
         1412  +  Parse *pParse,                  /* Parse context */
         1413  +  ExprList *pList,                /* Expression list to search */
         1414  +  int iBase,                      /* Cursor for table associated with pIdx */
         1415  +  Index *pIdx,                    /* Index to match column of */
         1416  +  int iCol                        /* Column of index to match */
         1417  +){
         1418  +  int i;
         1419  +  const char *zColl = pIdx->azColl[iCol];
         1420  +
         1421  +  for(i=0; i<pList->nExpr; i++){
         1422  +    Expr *p = pList->a[i].pExpr;
         1423  +    if( p->op==TK_COLUMN
         1424  +     && p->iColumn==pIdx->aiColumn[iCol]
         1425  +     && p->iTable==iBase
         1426  +    ){
         1427  +      CollSeq *pColl = sqlite3ExprCollSeq(pParse, p);
         1428  +      if( ALWAYS(pColl) && 0==sqlite3StrICmp(pColl->zName, zColl) ){
         1429  +        return i;
         1430  +      }
         1431  +    }
         1432  +  }
         1433  +
         1434  +  return -1;
         1435  +}
         1436  +
         1437  +/*
         1438  +** This routine determines if pIdx can be used to assist in processing a
         1439  +** DISTINCT qualifier. In other words, it tests whether or not using this
         1440  +** index for the outer loop guarantees that rows with equal values for
         1441  +** all expressions in the pDistinct list are delivered grouped together.
         1442  +**
         1443  +** For example, the query 
         1444  +**
         1445  +**   SELECT DISTINCT a, b, c FROM tbl WHERE a = ?
         1446  +**
         1447  +** can benefit from any index on columns "b" and "c".
         1448  +*/
         1449  +static int isDistinctIndex(
         1450  +  Parse *pParse,                  /* Parsing context */
         1451  +  WhereClause *pWC,               /* The WHERE clause */
         1452  +  Index *pIdx,                    /* The index being considered */
         1453  +  int base,                       /* Cursor number for the table pIdx is on */
         1454  +  ExprList *pDistinct,            /* The DISTINCT expressions */
         1455  +  int nEqCol                      /* Number of index columns with == */
         1456  +){
         1457  +  Bitmask mask = 0;               /* Mask of unaccounted for pDistinct exprs */
         1458  +  int i;                          /* Iterator variable */
         1459  +
         1460  +  if( pIdx->zName==0 || pDistinct==0 || pDistinct->nExpr>=BMS ) return 0;
         1461  +  testcase( pDistinct->nExpr==BMS-1 );
         1462  +
         1463  +  /* Loop through all the expressions in the distinct list. If any of them
         1464  +  ** are not simple column references, return early. Otherwise, test if the
         1465  +  ** WHERE clause contains a "col=X" clause. If it does, the expression
         1466  +  ** can be ignored. If it does not, and the column does not belong to the
         1467  +  ** same table as index pIdx, return early. Finally, if there is no
         1468  +  ** matching "col=X" expression and the column is on the same table as pIdx,
         1469  +  ** set the corresponding bit in variable mask.
         1470  +  */
         1471  +  for(i=0; i<pDistinct->nExpr; i++){
         1472  +    WhereTerm *pTerm;
         1473  +    Expr *p = pDistinct->a[i].pExpr;
         1474  +    if( p->op!=TK_COLUMN ) return 0;
         1475  +    pTerm = findTerm(pWC, p->iTable, p->iColumn, ~(Bitmask)0, WO_EQ, 0);
         1476  +    if( pTerm ){
         1477  +      Expr *pX = pTerm->pExpr;
         1478  +      CollSeq *p1 = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
         1479  +      CollSeq *p2 = sqlite3ExprCollSeq(pParse, p);
         1480  +      if( p1==p2 ) continue;
         1481  +    }
         1482  +    if( p->iTable!=base ) return 0;
         1483  +    mask |= (((Bitmask)1) << i);
         1484  +  }
         1485  +
         1486  +  for(i=nEqCol; mask && i<pIdx->nColumn; i++){
         1487  +    int iExpr = findIndexCol(pParse, pDistinct, base, pIdx, i);
         1488  +    if( iExpr<0 ) break;
         1489  +    mask &= ~(((Bitmask)1) << iExpr);
         1490  +  }
         1491  +
         1492  +  return (mask==0);
         1493  +}
         1494  +
         1495  +
         1496  +/*
         1497  +** Return true if the DISTINCT expression-list passed as the third argument
         1498  +** is redundant. A DISTINCT list is redundant if the database contains a
         1499  +** UNIQUE index that guarantees that the result of the query will be distinct
         1500  +** anyway.
         1501  +*/
         1502  +static int isDistinctRedundant(
         1503  +  Parse *pParse,
         1504  +  SrcList *pTabList,
         1505  +  WhereClause *pWC,
         1506  +  ExprList *pDistinct
         1507  +){
         1508  +  Table *pTab;
         1509  +  Index *pIdx;
         1510  +  int i;                          
         1511  +  int iBase;
         1512  +
         1513  +  /* If there is more than one table or sub-select in the FROM clause of
         1514  +  ** this query, then it will not be possible to show that the DISTINCT 
         1515  +  ** clause is redundant. */
         1516  +  if( pTabList->nSrc!=1 ) return 0;
         1517  +  iBase = pTabList->a[0].iCursor;
         1518  +  pTab = pTabList->a[0].pTab;
         1519  +
         1520  +  /* If any of the expressions is an IPK column on table iBase, then return 
         1521  +  ** true. Note: The (p->iTable==iBase) part of this test may be false if the
         1522  +  ** current SELECT is a correlated sub-query.
         1523  +  */
         1524  +  for(i=0; i<pDistinct->nExpr; i++){
         1525  +    Expr *p = pDistinct->a[i].pExpr;
         1526  +    if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1;
         1527  +  }
         1528  +
         1529  +  /* Loop through all indices on the table, checking each to see if it makes
         1530  +  ** the DISTINCT qualifier redundant. It does so if:
         1531  +  **
         1532  +  **   1. The index is itself UNIQUE, and
         1533  +  **
         1534  +  **   2. All of the columns in the index are either part of the pDistinct
         1535  +  **      list, or else the WHERE clause contains a term of the form "col=X",
         1536  +  **      where X is a constant value. The collation sequences of the
         1537  +  **      comparison and select-list expressions must match those of the index.
         1538  +  */
         1539  +  for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
         1540  +    if( pIdx->onError==OE_None ) continue;
         1541  +    for(i=0; i<pIdx->nColumn; i++){
         1542  +      int iCol = pIdx->aiColumn[i];
         1543  +      if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) 
         1544  +       && 0>findIndexCol(pParse, pDistinct, iBase, pIdx, i)
         1545  +      ){
         1546  +        break;
         1547  +      }
         1548  +    }
         1549  +    if( i==pIdx->nColumn ){
         1550  +      /* This index implies that the DISTINCT qualifier is redundant. */
         1551  +      return 1;
         1552  +    }
         1553  +  }
         1554  +
         1555  +  return 0;
         1556  +}
  1400   1557   
  1401   1558   /*
  1402   1559   ** This routine decides if pIdx can be used to satisfy the ORDER BY
  1403   1560   ** clause.  If it can, it returns 1.  If pIdx cannot satisfy the
  1404   1561   ** ORDER BY clause, this routine returns 0.
  1405   1562   **
  1406   1563   ** pOrderBy is an ORDER BY clause from a SELECT statement.  pTab is the
................................................................................
  1429   1586   ){
  1430   1587     int i, j;                       /* Loop counters */
  1431   1588     int sortOrder = 0;              /* XOR of index and ORDER BY sort direction */
  1432   1589     int nTerm;                      /* Number of ORDER BY terms */
  1433   1590     struct ExprList_item *pTerm;    /* A term of the ORDER BY clause */
  1434   1591     sqlite3 *db = pParse->db;
  1435   1592   
  1436         -  assert( pOrderBy!=0 );
         1593  +  if( !pOrderBy ) return 0;
         1594  +  if( wsFlags & WHERE_COLUMN_IN ) return 0;
         1595  +  if( pIdx->bUnordered ) return 0;
         1596  +
  1437   1597     nTerm = pOrderBy->nExpr;
  1438   1598     assert( nTerm>0 );
  1439   1599   
  1440   1600     /* Argument pIdx must either point to a 'real' named index structure, 
  1441   1601     ** or an index structure allocated on the stack by bestBtreeIndex() to
  1442   1602     ** represent the rowid index that is part of every table.  */
  1443   1603     assert( pIdx->zName || (pIdx->nColumn==1 && pIdx->aiColumn[0]==-1) );
................................................................................
  1742   1902     double nTableRow;           /* Rows in the input table */
  1743   1903     double logN;                /* log(nTableRow) */
  1744   1904     double costTempIdx;         /* per-query cost of the transient index */
  1745   1905     WhereTerm *pTerm;           /* A single term of the WHERE clause */
  1746   1906     WhereTerm *pWCEnd;          /* End of pWC->a[] */
  1747   1907     Table *pTable;              /* Table tht might be indexed */
  1748   1908   
         1909  +  if( pParse->nQueryLoop<=(double)1 ){
         1910  +    /* There is no point in building an automatic index for a single scan */
         1911  +    return;
         1912  +  }
  1749   1913     if( (pParse->db->flags & SQLITE_AutoIndex)==0 ){
  1750   1914       /* Automatic indices are disabled at run-time */
  1751   1915       return;
  1752   1916     }
  1753   1917     if( (pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 ){
  1754   1918       /* We already have some kind of index in use for this query. */
  1755   1919       return;
  1756   1920     }
  1757   1921     if( pSrc->notIndexed ){
  1758   1922       /* The NOT INDEXED clause appears in the SQL. */
  1759   1923       return;
         1924  +  }
         1925  +  if( pSrc->isCorrelated ){
         1926  +    /* The source is a correlated sub-query. No point in indexing it. */
         1927  +    return;
  1760   1928     }
  1761   1929   
  1762   1930     assert( pParse->nQueryLoop >= (double)1 );
  1763   1931     pTable = pSrc->pTab;
  1764   1932     nTableRow = pTable->nRowEst;
  1765   1933     logN = estLog(nTableRow);
  1766   1934     costTempIdx = 2*logN*(nTableRow/pParse->nQueryLoop + 1);
................................................................................
  2685   2853   static void bestBtreeIndex(
  2686   2854     Parse *pParse,              /* The parsing context */
  2687   2855     WhereClause *pWC,           /* The WHERE clause */
  2688   2856     struct SrcList_item *pSrc,  /* The FROM clause term to search */
  2689   2857     Bitmask notReady,           /* Mask of cursors not available for indexing */
  2690   2858     Bitmask notValid,           /* Cursors not available for any purpose */
  2691   2859     ExprList *pOrderBy,         /* The ORDER BY clause */
         2860  +  ExprList *pDistinct,        /* The select-list if query is DISTINCT */
  2692   2861     WhereCost *pCost            /* Lowest cost query plan */
  2693   2862   ){
  2694   2863     int iCur = pSrc->iCursor;   /* The cursor of the table to be accessed */
  2695   2864     Index *pProbe;              /* An index we are evaluating */
  2696   2865     Index *pIdx;                /* Copy of pProbe, or zero for IPK index */
  2697   2866     int eqTermMask;             /* Current mask of valid equality operators */
  2698   2867     int idxEqTermMask;          /* Index mask of valid equality operators */
................................................................................
  2825   2994       **             SELECT a, b, c FROM tbl WHERE a = 1;
  2826   2995       */
  2827   2996       int nEq;                      /* Number of == or IN terms matching index */
  2828   2997       int bInEst = 0;               /* True if "x IN (SELECT...)" seen */
  2829   2998       int nInMul = 1;               /* Number of distinct equalities to lookup */
  2830   2999       int estBound = 100;           /* Estimated reduction in search space */
  2831   3000       int nBound = 0;               /* Number of range constraints seen */
  2832         -    int bSort = 0;                /* True if external sort required */
         3001  +    int bSort = !!pOrderBy;       /* True if external sort required */
         3002  +    int bDist = !!pDistinct;      /* True if index cannot help with DISTINCT */
  2833   3003       int bLookup = 0;              /* True if not a covering index */
  2834   3004       WhereTerm *pTerm;             /* A single term of the WHERE clause */
  2835   3005   #ifdef SQLITE_ENABLE_STAT2
  2836   3006       WhereTerm *pFirstTerm = 0;    /* First term matching the index */
  2837   3007   #endif
  2838   3008   
  2839   3009       /* Determine the values of nEq and nInMul */
................................................................................
  2889   3059         }
  2890   3060       }
  2891   3061   
  2892   3062       /* If there is an ORDER BY clause and the index being considered will
  2893   3063       ** naturally scan rows in the required order, set the appropriate flags
  2894   3064       ** in wsFlags. Otherwise, if there is an ORDER BY clause but the index
  2895   3065       ** will scan rows in a different order, set the bSort variable.  */
  2896         -    if( pOrderBy ){
  2897         -      if( (wsFlags & WHERE_COLUMN_IN)==0
  2898         -        && pProbe->bUnordered==0
  2899         -        && isSortingIndex(pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy,
  2900         -                          nEq, wsFlags, &rev)
  2901         -      ){
  2902         -        wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY;
  2903         -        wsFlags |= (rev ? WHERE_REVERSE : 0);
  2904         -      }else{
  2905         -        bSort = 1;
  2906         -      }
         3066  +    if( isSortingIndex(
         3067  +          pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, nEq, wsFlags, &rev)
         3068  +    ){
         3069  +      bSort = 0;
         3070  +      wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY;
         3071  +      wsFlags |= (rev ? WHERE_REVERSE : 0);
         3072  +    }
         3073  +
         3074  +    /* If there is a DISTINCT qualifier and this index will scan rows in
         3075  +    ** order of the DISTINCT expressions, clear bDist and set the appropriate
         3076  +    ** flags in wsFlags. */
         3077  +    if( isDistinctIndex(pParse, pWC, pProbe, iCur, pDistinct, nEq) ){
         3078  +      bDist = 0;
         3079  +      wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT;
  2907   3080       }
  2908   3081   
  2909   3082       /* If currently calculating the cost of using an index (not the IPK
  2910   3083       ** index), determine if all required column data may be obtained without 
  2911   3084       ** using the main table (i.e. if the index is a covering
  2912   3085       ** index for this query). If it is, set the WHERE_IDX_ONLY flag in
  2913   3086       ** wsFlags. Otherwise, set the bLookup variable to true.  */
................................................................................
  2934   3107       nRow = (double)(aiRowEst[nEq] * nInMul);
  2935   3108       if( bInEst && nRow*2>aiRowEst[0] ){
  2936   3109         nRow = aiRowEst[0]/2;
  2937   3110         nInMul = (int)(nRow / aiRowEst[nEq]);
  2938   3111       }
  2939   3112   
  2940   3113   #ifdef SQLITE_ENABLE_STAT2
  2941         -    /* If the constraint is of the form x=VALUE and histogram
         3114  +    /* If the constraint is of the form x=VALUE or x IN (E1,E2,...)
         3115  +    ** and we do not think that values of x are unique and if histogram
  2942   3116       ** data is available for column x, then it might be possible
  2943   3117       ** to get a better estimate on the number of rows based on
  2944   3118       ** VALUE and how common that value is according to the histogram.
  2945   3119       */
  2946         -    if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 ){
         3120  +    if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 && aiRowEst[1]>1 ){
  2947   3121         if( pFirstTerm->eOperator & (WO_EQ|WO_ISNULL) ){
  2948   3122           testcase( pFirstTerm->eOperator==WO_EQ );
  2949   3123           testcase( pFirstTerm->eOperator==WO_ISNULL );
  2950   3124           whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, &nRow);
  2951   3125         }else if( pFirstTerm->eOperator==WO_IN && bInEst==0 ){
  2952   3126           whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, &nRow);
  2953   3127         }
................................................................................
  3016   3190       ** adds C*N*log10(N) to the cost, where N is the number of rows to be 
  3017   3191       ** sorted and C is a factor between 1.95 and 4.3.  We will split the
  3018   3192       ** difference and select C of 3.0.
  3019   3193       */
  3020   3194       if( bSort ){
  3021   3195         cost += nRow*estLog(nRow)*3;
  3022   3196       }
         3197  +    if( bDist ){
         3198  +      cost += nRow*estLog(nRow)*3;
         3199  +    }
  3023   3200   
  3024   3201       /**** Cost of using this index has now been computed ****/
  3025   3202   
  3026   3203       /* If there are additional constraints on this table that cannot
  3027   3204       ** be used with the current index, but which might lower the number
  3028   3205       ** of output rows, adjust the nRow value accordingly.  This only 
  3029   3206       ** matters if the current index is the least costly, so do not bother
................................................................................
  3161   3338       if( p->needToFreeIdxStr ){
  3162   3339         sqlite3_free(p->idxStr);
  3163   3340       }
  3164   3341       sqlite3DbFree(pParse->db, p);
  3165   3342     }else
  3166   3343   #endif
  3167   3344     {
  3168         -    bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost);
         3345  +    bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, 0, pCost);
  3169   3346     }
  3170   3347   }
  3171   3348   
  3172   3349   /*
  3173   3350   ** Disable a term in the WHERE clause.  Except, do not disable the term
  3174   3351   ** if it controls a LEFT OUTER JOIN and it did not originate in the ON
  3175   3352   ** or USING clause of that join.
................................................................................
  4123   4300       iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn);
  4124   4301   
  4125   4302       for(ii=0; ii<pOrWc->nTerm; ii++){
  4126   4303         WhereTerm *pOrTerm = &pOrWc->a[ii];
  4127   4304         if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){
  4128   4305           WhereInfo *pSubWInfo;          /* Info for single OR-term scan */
  4129   4306           /* Loop through table entries that match term pOrTerm. */
  4130         -        pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0,
         4307  +        pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, 0,
  4131   4308                           WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE |
  4132   4309                           WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY);
  4133   4310           if( pSubWInfo ){
  4134   4311             explainOneScan(
  4135   4312                 pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0
  4136   4313             );
  4137   4314             if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){
................................................................................
  4364   4541   ** output order, then the *ppOrderBy is unchanged.
  4365   4542   */
  4366   4543   WhereInfo *sqlite3WhereBegin(
  4367   4544     Parse *pParse,        /* The parser context */
  4368   4545     SrcList *pTabList,    /* A list of all tables to be scanned */
  4369   4546     Expr *pWhere,         /* The WHERE clause */
  4370   4547     ExprList **ppOrderBy, /* An ORDER BY clause, or NULL */
         4548  +  ExprList *pDistinct,  /* The select-list for DISTINCT queries - or NULL */
  4371   4549     u16 wctrlFlags        /* One of the WHERE_* flags defined in sqliteInt.h */
  4372   4550   ){
  4373   4551     int i;                     /* Loop counter */
  4374   4552     int nByteWInfo;            /* Num. bytes allocated for WhereInfo struct */
  4375   4553     int nTabList;              /* Number of elements in pTabList */
  4376   4554     WhereInfo *pWInfo;         /* Will become the return value of this function */
  4377   4555     Vdbe *v = pParse->pVdbe;   /* The virtual database engine */
................................................................................
  4423   4601     pWInfo->pParse = pParse;
  4424   4602     pWInfo->pTabList = pTabList;
  4425   4603     pWInfo->iBreak = sqlite3VdbeMakeLabel(v);
  4426   4604     pWInfo->pWC = pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo];
  4427   4605     pWInfo->wctrlFlags = wctrlFlags;
  4428   4606     pWInfo->savedNQueryLoop = pParse->nQueryLoop;
  4429   4607     pMaskSet = (WhereMaskSet*)&pWC[1];
         4608  +
         4609  +  /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via
         4610  +  ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */
         4611  +  if( db->flags & SQLITE_DistinctOpt ) pDistinct = 0;
  4430   4612   
  4431   4613     /* Split the WHERE clause into separate subexpressions where each
  4432   4614     ** subexpression is separated by an AND operator.
  4433   4615     */
  4434   4616     initMaskSet(pMaskSet);
  4435   4617     whereClauseInit(pWC, pParse, pMaskSet);
  4436   4618     sqlite3ExprCodeConstants(pParse, pWhere);
................................................................................
  4490   4672     ** want to analyze these virtual terms, so start analyzing at the end
  4491   4673     ** and work forward so that the added virtual terms are never processed.
  4492   4674     */
  4493   4675     exprAnalyzeAll(pTabList, pWC);
  4494   4676     if( db->mallocFailed ){
  4495   4677       goto whereBeginError;
  4496   4678     }
         4679  +
         4680  +  /* Check if the DISTINCT qualifier, if there is one, is redundant. 
         4681  +  ** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to
         4682  +  ** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT.
         4683  +  */
         4684  +  if( pDistinct && isDistinctRedundant(pParse, pTabList, pWC, pDistinct) ){
         4685  +    pDistinct = 0;
         4686  +    pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
         4687  +  }
  4497   4688   
  4498   4689     /* Chose the best index to use for each table in the FROM clause.
  4499   4690     **
  4500   4691     ** This loop fills in the following fields:
  4501   4692     **
  4502   4693     **   pWInfo->a[].pIdx      The index to use for this level of the loop.
  4503   4694     **   pWInfo->a[].wsFlags   WHERE_xxx flags associated with pIdx
................................................................................
  4574   4765       notIndexed = 0;
  4575   4766       for(isOptimal=(iFrom<nTabList-1); isOptimal>=0 && bestJ<0; isOptimal--){
  4576   4767         Bitmask mask;             /* Mask of tables not yet ready */
  4577   4768         for(j=iFrom, pTabItem=&pTabList->a[j]; j<nTabList; j++, pTabItem++){
  4578   4769           int doNotReorder;    /* True if this table should not be reordered */
  4579   4770           WhereCost sCost;     /* Cost information from best[Virtual]Index() */
  4580   4771           ExprList *pOrderBy;  /* ORDER BY clause for index to optimize */
         4772  +        ExprList *pDist;     /* DISTINCT clause for index to optimize */
  4581   4773     
  4582   4774           doNotReorder =  (pTabItem->jointype & (JT_LEFT|JT_CROSS))!=0;
  4583   4775           if( j!=iFrom && doNotReorder ) break;
  4584   4776           m = getMask(pMaskSet, pTabItem->iCursor);
  4585   4777           if( (m & notReady)==0 ){
  4586   4778             if( j==iFrom ) iFrom++;
  4587   4779             continue;
  4588   4780           }
  4589   4781           mask = (isOptimal ? m : notReady);
  4590   4782           pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0);
         4783  +        pDist = (i==0 ? pDistinct : 0);
  4591   4784           if( pTabItem->pIndex==0 ) nUnconstrained++;
  4592   4785     
  4593   4786           WHERETRACE(("=== trying table %d with isOptimal=%d ===\n",
  4594   4787                       j, isOptimal));
  4595   4788           assert( pTabItem->pTab );
  4596   4789   #ifndef SQLITE_OMIT_VIRTUALTABLE
  4597   4790           if( IsVirtual(pTabItem->pTab) ){
................................................................................
  4598   4791             sqlite3_index_info **pp = &pWInfo->a[j].pIdxInfo;
  4599   4792             bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy,
  4600   4793                              &sCost, pp);
  4601   4794           }else 
  4602   4795   #endif
  4603   4796           {
  4604   4797             bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy,
  4605         -                         &sCost);
         4798  +              pDist, &sCost);
  4606   4799           }
  4607   4800           assert( isOptimal || (sCost.used&notReady)==0 );
  4608   4801   
  4609   4802           /* If an INDEXED BY clause is present, then the plan must use that
  4610   4803           ** index if it uses any index at all */
  4611   4804           assert( pTabItem->pIndex==0 
  4612   4805                     || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0
................................................................................
  4658   4851       assert( bestJ>=0 );
  4659   4852       assert( notReady & getMask(pMaskSet, pTabList->a[bestJ].iCursor) );
  4660   4853       WHERETRACE(("*** Optimizer selects table %d for loop %d"
  4661   4854                   " with cost=%g and nRow=%g\n",
  4662   4855                   bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow));
  4663   4856       if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){
  4664   4857         *ppOrderBy = 0;
         4858  +    }
         4859  +    if( (bestPlan.plan.wsFlags & WHERE_DISTINCT)!=0 ){
         4860  +      assert( pWInfo->eDistinct==0 );
         4861  +      pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
  4665   4862       }
  4666   4863       andFlags &= bestPlan.plan.wsFlags;
  4667   4864       pLevel->plan = bestPlan.plan;
  4668   4865       testcase( bestPlan.plan.wsFlags & WHERE_INDEXED );
  4669   4866       testcase( bestPlan.plan.wsFlags & WHERE_TEMP_INDEX );
  4670   4867       if( bestPlan.plan.wsFlags & (WHERE_INDEXED|WHERE_TEMP_INDEX) ){
  4671   4868         pLevel->iIdxCur = pParse->nTab++;

Changes to test/all.test.

    34     34   run_test_suite journaltest 
    35     35   run_test_suite inmemory_journal
    36     36   run_test_suite pcache0 
    37     37   run_test_suite pcache10 
    38     38   run_test_suite pcache50 
    39     39   run_test_suite pcache90 
    40     40   run_test_suite pcache100
           41  +run_test_suite prepare
    41     42   
    42     43   if {$::tcl_platform(platform)=="unix"} {
    43     44     ifcapable !default_autovacuum {
    44     45       run_test_suite autovacuum_crash
    45     46     }
    46     47   }
    47     48   
    48     49   finish_test
    49     50   
    50     51   

Changes to test/alter2.test.

   133    133     execsql {
   134    134       SELECT sum(a), c FROM abc GROUP BY c;
   135    135     }
   136    136   } {8 {} 1 10}
   137    137   do_test alter2-1.9 {
   138    138     # ALTER TABLE abc ADD COLUMN d;
   139    139     alter_table abc {CREATE TABLE abc(a, b, c, d);}
          140  +  if {[permutation] == "prepare"} { db cache flush }
   140    141     execsql { SELECT * FROM abc; }
   141    142     execsql {
   142    143       UPDATE abc SET d = 11 WHERE c IS NULL AND a<4;
   143    144       SELECT * FROM abc;
   144    145     }
   145    146   } {1 2 10 {} 3 4 {} 11 5 6 {} {}}
   146    147   do_test alter2-1.10 {

Changes to test/autoindex1.test.

   243    243     1 0 0 {SCAN TABLE sheep AS s (~1000000 rows)} 
   244    244     1 1 1 {SEARCH TABLE flock_owner AS prev USING INDEX sqlite_autoindex_flock_owner_1 (flock_no=? AND owner_change_date<?) (~2 rows)} 
   245    245     1 0 0 {EXECUTE CORRELATED SCALAR SUBQUERY 2} 
   246    246     2 0 0 {SEARCH TABLE flock_owner AS later USING COVERING INDEX sqlite_autoindex_flock_owner_1 (flock_no=? AND owner_change_date>? AND owner_change_date<?) (~1 rows)} 
   247    247     0 0 0 {SCAN TABLE sheep AS x USING INDEX sheep_reg_flock_index (~1000000 rows)} 
   248    248     0 1 1 {SEARCH SUBQUERY 1 AS y USING AUTOMATIC COVERING INDEX (sheep_no=?) (~8 rows)}
   249    249   }
          250  +
          251  +
          252  +do_execsql_test autoindex1-700 {
          253  +  CREATE TABLE t5(a, b, c);
          254  +  EXPLAIN QUERY PLAN SELECT a FROM t5 WHERE b=10 ORDER BY c;
          255  +} {
          256  +  0 0 0 {SCAN TABLE t5 (~100000 rows)} 
          257  +  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
          258  +}
          259  +
   250    260   
   251    261   finish_test

Changes to test/collate5.test.

    53     53       INSERT INTO collate5t1 VALUES('N', NULL);
    54     54     } 
    55     55   } {}
    56     56   do_test collate5-1.1 {
    57     57     execsql {
    58     58       SELECT DISTINCT a FROM collate5t1;
    59     59     }
    60         -} {A B N}
           60  +} {a b n}
    61     61   do_test collate5-1.2 {
    62     62     execsql {
    63     63       SELECT DISTINCT b FROM collate5t1;
    64     64     }
    65         -} {{} Apple apple banana}
           65  +} {apple Apple banana {}}
    66     66   do_test collate5-1.3 {
    67     67     execsql {
    68     68       SELECT DISTINCT a, b FROM collate5t1;
    69     69     }
    70         -} {A Apple a apple B banana N {}}
           70  +} {a apple A Apple b banana n {}}
    71     71   
    72     72   # Ticket #3376
    73     73   #
    74     74   do_test collate5-1.11 {
    75     75     execsql {
    76     76       CREATE TABLE tkt3376(a COLLATE nocase PRIMARY KEY);
    77     77       INSERT INTO tkt3376 VALUES('abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz');

Added test/distinct.test.

            1  +# 2011 July 1
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is the DISTINCT modifier.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +
           18  +set testprefix distinct
           19  +
           20  +
           21  +proc is_distinct_noop {sql} {
           22  +  set sql1 $sql
           23  +  set sql2 [string map {DISTINCT ""} $sql]
           24  +
           25  +  set program1 [list]
           26  +  set program2 [list]
           27  +  db eval "EXPLAIN $sql1" {
           28  +    if {$opcode != "Noop"} { lappend program1 $opcode }
           29  +  }
           30  +  db eval "EXPLAIN $sql2" {
           31  +    if {$opcode != "Noop"} { lappend program2 $opcode }
           32  +  }
           33  +
           34  +  return [expr {$program1==$program2}]
           35  +}
           36  +
           37  +proc do_distinct_noop_test {tn sql} {
           38  +  uplevel [list do_test $tn [list is_distinct_noop $sql] 1]
           39  +}
           40  +proc do_distinct_not_noop_test {tn sql} {
           41  +  uplevel [list do_test $tn [list is_distinct_noop $sql] 0]
           42  +}
           43  +
           44  +proc do_temptables_test {tn sql temptables} {
           45  +  uplevel [list do_test $tn [subst -novar {
           46  +    set ret ""
           47  +    db eval "EXPLAIN [set sql]" {
           48  +      if {$opcode == "OpenEphemeral"} { 
           49  +        if {$p5 != "10" && $p5!="00"} { error "p5 = $p5" }
           50  +        if {$p5 == "10"} {
           51  +          lappend ret hash
           52  +        } else {
           53  +          lappend ret btree
           54  +        }
           55  +      }
           56  +    }
           57  +    set ret
           58  +  }] $temptables]
           59  +}
           60  +
           61  +
           62  +#-------------------------------------------------------------------------
           63  +# The following tests - distinct-1.* - check that the planner correctly 
           64  +# detects cases where a UNIQUE index means that a DISTINCT clause is 
           65  +# redundant. Currently the planner only detects such cases when there
           66  +# is a single table in the FROM clause.
           67  +#
           68  +do_execsql_test 1.0 {
           69  +  CREATE TABLE t1(a, b, c, d);
           70  +  CREATE UNIQUE INDEX i1 ON t1(b, c);
           71  +  CREATE UNIQUE INDEX i2 ON t1(d COLLATE nocase);
           72  +
           73  +  CREATE TABLE t2(x INTEGER PRIMARY KEY, y);
           74  +
           75  +  CREATE TABLE t3(c1 PRIMARY KEY, c2);
           76  +  CREATE INDEX i3 ON t3(c2);
           77  +}
           78  +foreach {tn noop sql} {
           79  +
           80  +  1   1   "SELECT DISTINCT b, c FROM t1"
           81  +  2   1   "SELECT DISTINCT c FROM t1 WHERE b = ?"
           82  +  3   1   "SELECT DISTINCT rowid FROM t1"
           83  +  4   1   "SELECT DISTINCT rowid, a FROM t1"
           84  +  5   1   "SELECT DISTINCT x FROM t2"
           85  +  6   1   "SELECT DISTINCT * FROM t2"
           86  +  7   1   "SELECT DISTINCT * FROM (SELECT * FROM t2)"
           87  +
           88  +  8   1   "SELECT DISTINCT * FROM t1"
           89  +
           90  +  8   0   "SELECT DISTINCT a, b FROM t1"
           91  +
           92  +  9   0   "SELECT DISTINCT c FROM t1 WHERE b IN (1,2)"
           93  +  10  0   "SELECT DISTINCT c FROM t1"
           94  +  11  0   "SELECT DISTINCT b FROM t1"
           95  +
           96  +  12  0   "SELECT DISTINCT a, d FROM t1"
           97  +  13  0   "SELECT DISTINCT a, b, c COLLATE nocase FROM t1"
           98  +  14  1   "SELECT DISTINCT a, d COLLATE nocase FROM t1"
           99  +  15  0   "SELECT DISTINCT a, d COLLATE binary FROM t1"
          100  +  16  1   "SELECT DISTINCT a, b, c COLLATE binary FROM t1"
          101  +
          102  +  16  0   "SELECT DISTINCT t1.rowid FROM t1, t2"
          103  +  17  0   { /* Technically, it would be possible to detect that DISTINCT
          104  +            ** is a no-op in cases like the following. But SQLite does not
          105  +            ** do so. */
          106  +            SELECT DISTINCT t1.rowid FROM t1, t2 WHERE t1.rowid=t2.rowid }
          107  +
          108  +  18  1   "SELECT DISTINCT c1, c2 FROM t3"
          109  +  19  1   "SELECT DISTINCT c1 FROM t3"
          110  +  20  1   "SELECT DISTINCT * FROM t3"
          111  +  21  0   "SELECT DISTINCT c2 FROM t3"
          112  +
          113  +  22  0   "SELECT DISTINCT * FROM (SELECT 1, 2, 3 UNION SELECT 4, 5, 6)"
          114  +  23  1   "SELECT DISTINCT rowid FROM (SELECT 1, 2, 3 UNION SELECT 4, 5, 6)"
          115  +
          116  +  24  0   "SELECT DISTINCT rowid/2 FROM t1"
          117  +  25  1   "SELECT DISTINCT rowid/2, rowid FROM t1"
          118  +  26  1   "SELECT DISTINCT rowid/2, b FROM t1 WHERE c = ?"
          119  +} {
          120  +  if {$noop} {
          121  +    do_distinct_noop_test 1.$tn $sql
          122  +  } else {
          123  +    do_distinct_not_noop_test 1.$tn $sql
          124  +  }
          125  +}
          126  +
          127  +#-------------------------------------------------------------------------
          128  +# The following tests - distinct-2.* - test cases where an index is
          129  +# used to deliver results in order of the DISTINCT expressions. 
          130  +#
          131  +drop_all_tables
          132  +do_execsql_test 2.0 {
          133  +  CREATE TABLE t1(a, b, c);
          134  +
          135  +  CREATE INDEX i1 ON t1(a, b);
          136  +  CREATE INDEX i2 ON t1(b COLLATE nocase, c COLLATE nocase);
          137  +
          138  +  INSERT INTO t1 VALUES('a', 'b', 'c');
          139  +  INSERT INTO t1 VALUES('A', 'B', 'C');
          140  +  INSERT INTO t1 VALUES('a', 'b', 'c');
          141  +  INSERT INTO t1 VALUES('A', 'B', 'C');
          142  +}
          143  +
          144  +foreach {tn sql temptables res} {
          145  +  1   "a, b FROM t1"                                       {}      {A B a b}
          146  +  2   "b, a FROM t1"                                       {}      {B A b a}
          147  +  3   "a, b, c FROM t1"                                    {hash}  {a b c A B C}
          148  +  4   "a, b, c FROM t1 ORDER BY a, b, c"                   {btree} {A B C a b c}
          149  +  5   "b FROM t1 WHERE a = 'a'"                            {}      {b}
          150  +  6   "b FROM t1"                                          {hash}  {b B}
          151  +  7   "a FROM t1"                                          {}      {A a}
          152  +  8   "b COLLATE nocase FROM t1"                           {}      {b}
          153  +  9   "b COLLATE nocase FROM t1 ORDER BY b COLLATE nocase" {}      {B}
          154  +} {
          155  +  do_execsql_test    2.$tn.1 "SELECT DISTINCT $sql" $res
          156  +  do_temptables_test 2.$tn.2 "SELECT DISTINCT $sql" $temptables
          157  +}
          158  +
          159  +do_execsql_test 2.A {
          160  +  SELECT (SELECT DISTINCT o.a FROM t1 AS i) FROM t1 AS o;
          161  +} {a A a A}
          162  +
          163  +
          164  +
          165  +
          166  +finish_test

Changes to test/e_select.test.

  1234   1234   do_select_tests e_select-5 {
  1235   1235     3.1 "SELECT ALL x FROM h2" {One Two Three Four one two three four}
  1236   1236     3.2 "SELECT ALL x FROM h1, h2 ON (x=b)" {One one Four four}
  1237   1237   
  1238   1238     3.1 "SELECT x FROM h2" {One Two Three Four one two three four}
  1239   1239     3.2 "SELECT x FROM h1, h2 ON (x=b)" {One one Four four}
  1240   1240   
  1241         -  4.1 "SELECT DISTINCT x FROM h2" {four one three two}
  1242         -  4.2 "SELECT DISTINCT x FROM h1, h2 ON (x=b)" {four one}
         1241  +  4.1 "SELECT DISTINCT x FROM h2" {One Two Three Four}
         1242  +  4.2 "SELECT DISTINCT x FROM h1, h2 ON (x=b)" {One Four}
  1243   1243   } 
  1244   1244   
  1245   1245   # EVIDENCE-OF: R-02054-15343 For the purposes of detecting duplicate
  1246   1246   # rows, two NULL values are considered to be equal.
  1247   1247   #
  1248   1248   do_select_tests e_select-5.5 {
  1249   1249     1  "SELECT DISTINCT d FROM h3" {{} 2 2,3 2,4 3}
  1250   1250   }
  1251   1251   
  1252   1252   # EVIDENCE-OF: R-58359-52112 The normal rules for selecting a collation
  1253   1253   # sequence to compare text values with apply.
  1254   1254   #
  1255   1255   do_select_tests e_select-5.6 {
  1256         -  1  "SELECT DISTINCT b FROM h1"                  {I IV four i iv one}
  1257         -  2  "SELECT DISTINCT b COLLATE nocase FROM h1"   {four i iv one}
  1258         -  3  "SELECT DISTINCT x FROM h2"                  {four one three two}
         1256  +  1  "SELECT DISTINCT b FROM h1"                  {one I i four IV iv}
         1257  +  2  "SELECT DISTINCT b COLLATE nocase FROM h1"   {one I four IV}
         1258  +  3  "SELECT DISTINCT x FROM h2"                  {One Two Three Four}
  1259   1259     4  "SELECT DISTINCT x COLLATE binary FROM h2"   {
  1260         -    Four One Three Two four one three two
         1260  +    One Two Three Four one two three four
  1261   1261     }
  1262   1262   }
  1263   1263   
  1264   1264   #-------------------------------------------------------------------------
  1265   1265   # The following tests - e_select-7.* - test that statements made to do
  1266   1266   # with compound SELECT statements are correct.
  1267   1267   #

Changes to test/exists.test.

   155    155         sql1 { DROP INDEX IF EXISTS aux.i2 }
   156    156         sql2 { SELECT name FROM aux.sqlite_master WHERE type = 'index' }
   157    157       } {}
   158    158       do_test 3.$tn.2.2 {
   159    159         sql1 { DROP INDEX IF EXISTS i2 }
   160    160         sql2 { CREATE INDEX aux.i2 ON t2(x) }
   161    161         sql1 { DROP INDEX IF EXISTS i2 }
   162         -      sql2 { SELECT name FROM aux.sqlite_master WHERE type = 'index' }
          162  +      sql2 { SELECT * FROM aux.sqlite_master WHERE type = 'index' }
   163    163       } {}
   164    164   
   165    165       # VIEW objects.
   166    166       #
   167    167       do_test 3.$tn.3.1 {
   168    168         sql1 { DROP VIEW IF EXISTS aux.v1 }
   169    169         sql2 { CREATE VIEW aux.v1 AS SELECT * FROM t2 }

Changes to test/fts3auto.test.

   103    103   
   104    104     do_execsql_test $tn$title.4 "
   105    105       SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl 
   106    106       WHERE $tbl MATCH '$match' ORDER BY docid ASC
   107    107     " $matchinfo_asc
   108    108   }
   109    109   
   110         -#    fts3_make_deferrable TABLE TOKEN
          110  +#    fts3_make_deferrable TABLE TOKEN ?NROW?
   111    111   #
   112         -proc fts3_make_deferrable {tbl token} {
          112  +proc fts3_make_deferrable {tbl token {nRow 0}} {
   113    113   
   114    114     set stmt [sqlite3_prepare db "SELECT * FROM $tbl" -1 dummy]
   115    115     set name [sqlite3_column_name $stmt 0]
   116    116     sqlite3_finalize $stmt
   117    117   
   118         -  set nRow [db one "SELECT count(*) FROM $tbl"]
          118  +  if {$nRow==0} {
          119  +    set nRow [db one "SELECT count(*) FROM $tbl"]
          120  +  }
   119    121     set pgsz [db one "PRAGMA page_size"]
   120    122     execsql BEGIN
   121    123     for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} {
   122    124       set doc [string repeat "$token " 100]
   123    125       execsql "INSERT INTO $tbl ($name) VALUES(\$doc)"
   124    126     }
   125    127     execsql "INSERT INTO $tbl ($name) VALUES('aaaaaaa ${token}aaaaa')"
................................................................................
   648    650     do_fts3query_test 6.$tn.2 t1 {b:G AND c:I}
   649    651     do_fts3query_test 6.$tn.3 t1 {b:G NEAR c:I}
   650    652     do_fts3query_test 6.$tn.4 t1 {a:C OR b:G OR c:K OR d:C}
   651    653     do_fts3query_test 6.$tn.5 t1 {a:G OR b:G}
   652    654   
   653    655     catchsql { COMMIT }
   654    656   }
          657  +
          658  +foreach {tn create} {
          659  +  1    "fts4(x)"
          660  +  2    "fts4(x, order=DESC)"
          661  +} {
          662  +  execsql [subst {
          663  +    DROP TABLE IF EXISTS t1;
          664  +    CREATE VIRTUAL TABLE t1 USING $create;
          665  +  }]
          666  +
          667  +  foreach {x} {
          668  +    "F E N O T K X V A X I E X A P G Q V H U"
          669  +    "R V A E T C V Q N I E L O N U G J K L U"
          670  +    "U Y I G W M V F J L X I D C H F P J Q B"
          671  +    "S G D Z X R P G S S Y B K A S G A I L L"
          672  +    "L S I C H T Z S R Q P R N K J X L F M J"
          673  +    "C C C D P X B Z C M A D A C X S B T X V"
          674  +    "W Y J M D R G V R K B X S A W R I T N C"
          675  +    "P K L W T M S P O Y Y V V O E H Q A I R"
          676  +    "C D Y I C Z F H J C O Y A Q F L S B D K"
          677  +    "P G S C Y C Y V I M B D S Z D D Y W I E"
          678  +    "Z K Z U E E S F Y X T U A L W O U J C Q"
          679  +    "P A T Z S W L P L Q V Y Y I P W U X S S"
          680  +    "I U I H U O F Z F R H R F T N D X A G M"
          681  +    "N A B M S H K X S O Y D T X S B R Y H Z"
          682  +    "L U D A S K I L S V Z J P U B E B Y H M"
          683  +  } {
          684  +    execsql { INSERT INTO t1 VALUES($x) }
          685  +  }
          686  +
          687  +  # Add extra documents to the database such that token "B" will be considered
          688  +  # deferrable if considering the other tokens means that 2 or fewer documents
          689  +  # will be loaded into memory.
          690  +  #
          691  +  fts3_make_deferrable t1 B 2
          692  +
          693  +  # B is not deferred in either of the first two tests below, since filtering
          694  +  # on "M" or "D" returns 10 documents or so. But filtering on "M * D" only
          695  +  # returns 2, so B is deferred in this case.
          696  +  #
          697  +  do_fts3query_test 7.$tn.1             t1 {"M B"}
          698  +  do_fts3query_test 7.$tn.2             t1 {"B D"}
          699  +  do_fts3query_test 7.$tn.3 -deferred B t1 {"M B D"}
          700  +}
   655    701   
   656    702   set sqlite_fts3_enable_parentheses $sfep
   657    703   finish_test
   658    704   

Changes to test/func3.test.

    13     13   # with functions created using sqlite3_create_function_v2() is 
    14     14   # correctly invoked.
    15     15   #
    16     16   set testdir [file dirname $argv0]
    17     17   source $testdir/tester.tcl
    18     18   
    19     19   
    20         -do_test func3-1.1 {
    21         -  set destroyed 0
    22         -  proc destroy {} { set ::destroyed 1 }
    23         -  sqlite3_create_function_v2 db f2 -1 any -func f2 -destroy destroy
    24         -  set destroyed
    25         -} 0
    26         -do_test func3-1.2 {
    27         -  sqlite3_create_function_v2 db f2 -1 utf8 -func f2
    28         -  set destroyed
    29         -} 0
    30         -do_test func3-1.3 {
    31         -  sqlite3_create_function_v2 db f2 -1 utf16le -func f2
    32         -  set destroyed
    33         -} 0
    34         -do_test func3-1.4 {
    35         -  sqlite3_create_function_v2 db f2 -1 utf16be -func f2
    36         -  set destroyed
    37         -} 1
           20  +ifcapable utf16 {
           21  +  do_test func3-1.1 {
           22  +    set destroyed 0
           23  +    proc destroy {} { set ::destroyed 1 }
           24  +    sqlite3_create_function_v2 db f2 -1 any -func f2 -destroy destroy
           25  +    set destroyed
           26  +  } 0
           27  +  do_test func3-1.2 {
           28  +    sqlite3_create_function_v2 db f2 -1 utf8 -func f2
           29  +    set destroyed
           30  +  } 0
           31  +  do_test func3-1.3 {
           32  +    sqlite3_create_function_v2 db f2 -1 utf16le -func f2
           33  +    set destroyed
           34  +  } 0
           35  +  do_test func3-1.4 {
           36  +    sqlite3_create_function_v2 db f2 -1 utf16be -func f2
           37  +    set destroyed
           38  +  } 1
           39  +}
    38     40   
    39     41   do_test func3-2.1 {
    40     42     set destroyed 0
    41     43     proc destroy {} { set ::destroyed 1 }
    42     44     sqlite3_create_function_v2 db f3 -1 utf8 -func f3 -destroy destroy
    43     45     set destroyed
    44     46   } 0

Changes to test/fuzzer1.test.

  1372   1372   do_test fuzzer1-2.3 {
  1373   1373     execsql {
  1374   1374       SELECT DISTINCT streetname.n FROM f2, streetname
  1375   1375        WHERE f2.word MATCH 'tayle'
  1376   1376          AND f2.distance<=200
  1377   1377          AND streetname.n>=f2.word AND streetname.n<=(f2.word || x'F7BFBFBF')
  1378   1378     }
  1379         -} {steelewood tallia tallu talwyn taymouth thelema trailer {tyler finley}}
         1379  +} {{tyler finley} trailer taymouth steelewood tallia tallu talwyn thelema}
  1380   1380   
  1381   1381   
  1382   1382   finish_test

Changes to test/incrblob_err.test.

    31     31     set bytes [file size [info script]]
    32     32     execsql {
    33     33       CREATE TABLE blobs(k, v BLOB);
    34     34       INSERT INTO blobs VALUES(1, zeroblob($::bytes));
    35     35     }
    36     36   } -tclbody {
    37     37     set ::blob [db incrblob blobs v 1]
           38  +  fconfigure $::blob -translation binary
    38     39     set rc [catch {puts -nonewline $::blob $::data}]
    39     40     if {$rc} { error "out of memory" }
    40     41   } 
    41     42   
    42     43   do_malloc_test 2 -tclprep {
    43     44     execsql {
    44     45       CREATE TABLE blobs(k, v BLOB);
................................................................................
    67     68     } elseif {$::r ne $::data} {
    68     69       error "Bad data read..."
    69     70     }
    70     71     set rc [catch {close $::blob}]
    71     72     if {$rc} { 
    72     73       error "out of memory" 
    73     74     }
    74         -} 
    75         -
           75  +}
    76     76   
    77     77   do_ioerr_test incrblob_err-4 -cksum 1 -sqlprep {
    78     78     CREATE TABLE blobs(k, v BLOB);
    79     79     INSERT INTO blobs VALUES(1, $::data);
    80     80   } -tclbody {
    81     81     set ::blob [db incrblob blobs v 1]
    82     82     read $::blob
................................................................................
    83     83   }
    84     84   
    85     85   do_ioerr_test incrblob_err-5 -cksum 1 -sqlprep {
    86     86     CREATE TABLE blobs(k, v BLOB);
    87     87     INSERT INTO blobs VALUES(1, zeroblob(length(CAST($::data AS BLOB))));
    88     88   } -tclbody {
    89     89     set ::blob [db incrblob blobs v 1]
           90  +  fconfigure $::blob -translation binary
    90     91     puts -nonewline $::blob $::data
    91     92     close $::blob
    92     93   }
    93     94   
    94     95   do_ioerr_test incrblob_err-6 -cksum 1 -sqlprep {
    95     96     CREATE TABLE blobs(k, v BLOB);
    96     97     INSERT INTO blobs VALUES(1, $::data || $::data || $::data);
    97     98   } -tclbody {
    98     99     set ::blob [db incrblob blobs v 1]
          100  +  fconfigure $::blob -translation binary
    99    101     seek $::blob -20 end
   100    102     puts -nonewline $::blob "12345678900987654321"
   101    103     close $::blob
   102    104   }
   103    105   
   104    106   do_ioerr_test incrblob_err-7 -cksum 1 -sqlprep {
   105    107     PRAGMA auto_vacuum = 1;

Changes to test/insert4.test.

   108    108   #
   109    109   do_test insert4-2.4.1 {
   110    110     execsql {
   111    111       DELETE FROM t3;
   112    112       INSERT INTO t3 SELECT DISTINCT * FROM t2;
   113    113       SELECT * FROM t3;
   114    114     }
   115         -} {1 9 9 1}
          115  +} {9 1 1 9}
   116    116   xferopt_test insert4-2.4.2 0
   117    117   do_test insert4-2.4.3 {
   118    118     catchsql {
   119    119       DELETE FROM t1;
   120    120       INSERT INTO t1 SELECT DISTINCT * FROM t2;
   121    121     }
   122    122   } {1 {constraint failed}}

Changes to test/like.test.

    66     66   } {ABC abc}
    67     67   do_test like-1.4 {
    68     68     execsql {
    69     69       SELECT x FROM t1 WHERE x LIKE 'aBc' ORDER BY 1;
    70     70     }
    71     71   } {ABC abc}
    72     72   do_test like-1.5.1 {
           73  +  # Use sqlite3_exec() to verify fix for ticket [25ee81271091] 2011-06-26
           74  +  sqlite3_exec db {PRAGMA case_sensitive_like=on}
           75  +} {0 {}}
           76  +do_test like-1.5.2 {
    73     77     execsql {
    74         -    PRAGMA case_sensitive_like=on;
    75     78       SELECT x FROM t1 WHERE x LIKE 'abc' ORDER BY 1;
    76     79     }
    77     80   } {abc}
    78         -do_test like-1.5.2 {
           81  +do_test like-1.5.3 {
    79     82     execsql {
    80     83       PRAGMA case_sensitive_like; -- no argument; does not change setting
    81     84       SELECT x FROM t1 WHERE x LIKE 'abc' ORDER BY 1;
    82     85     }
    83     86   } {abc}
    84     87   do_test like-1.6 {
    85     88     execsql {

Changes to test/misc5.test.

   501    501             )    
   502    502             WHERE artist <> '' 
   503    503           )  
   504    504          )       
   505    505         )  
   506    506         ORDER BY LOWER(artist) ASC;
   507    507       }
   508         -  } {one}
          508  +  } {two}
   509    509   }
   510    510   
   511    511   # Ticket #1370.  Do not overwrite small files (less than 1024 bytes)
   512    512   # when trying to open them as a database.
   513    513   #
   514    514   if {[permutation] == ""} {
   515    515     do_test misc5-4.1 {

Changes to test/multiplex.test.

    45     45     list $msg
    46     46   }
    47     47   
    48     48   # This attempts to delete the base file and 
    49     49   # and files with the chunk extension.
    50     50   proc multiplex_delete {name} {
    51     51     global g_max_chunks
           52  +  forcedelete $name
    52     53     for {set i 0} {$i<$g_max_chunks} {incr i} {
    53     54       forcedelete [multiplex_name $name $i]
    54     55       forcedelete [multiplex_name $name-journal $i]
    55     56       forcedelete [multiplex_name $name-wal $i]
    56     57     }
    57     58   }
    58     59   
................................................................................
    74     75   do_test multiplex-1.7 { sqlite3_multiplex_initialize "" 1 }        {SQLITE_OK}
    75     76   do_test multiplex-1.8 { sqlite3_multiplex_shutdown }               {SQLITE_OK}
    76     77   
    77     78   
    78     79   do_test multiplex-1.9.1  { sqlite3_multiplex_initialize "" 1 }     {SQLITE_OK}
    79     80   do_test multiplex-1.9.2  { sqlite3 db test.db }                    {}
    80     81   do_test multiplex-1.9.3  { multiplex_set db main 32768 16 }        {SQLITE_OK}
    81         -do_test multiplex-1.9.4  { multiplex_set db main 32768 -1 }        {SQLITE_MISUSE}
    82         -do_test multiplex-1.9.5  { multiplex_set db main -1 16 }           {SQLITE_MISUSE}
           82  +do_test multiplex-1.9.4  { multiplex_set db main 32768 -1 }        {SQLITE_OK}
    83     83   do_test multiplex-1.9.6  { multiplex_set db main 31 16 }           {SQLITE_OK}
    84         -do_test multiplex-1.9.7  { multiplex_set db main 32768 100 }       {SQLITE_MISUSE}
           84  +do_test multiplex-1.9.7  { multiplex_set db main 32768 100 }       {SQLITE_OK}
    85     85   do_test multiplex-1.9.8  { multiplex_set db main 1073741824 1 }    {SQLITE_OK}
    86     86   do_test multiplex-1.9.9  { db close }                              {}
    87     87   do_test multiplex-1.9.10 { sqlite3_multiplex_shutdown }            {SQLITE_OK}
    88     88   
    89     89   do_test multiplex-1.10.1  { sqlite3_multiplex_initialize "" 1 }                                  {SQLITE_OK}
    90     90   do_test multiplex-1.10.2  { sqlite3 db test.db }                                                 {}
    91     91   do_test multiplex-1.10.3  { lindex [ catchsql { SELECT multiplex_control(2, 32768); } ] 0 }      {0}
    92         -do_test multiplex-1.10.4  { lindex [ catchsql { SELECT multiplex_control(3, -1); } ] 0 }         {1}
    93         -do_test multiplex-1.10.5  { lindex [ catchsql { SELECT multiplex_control(2, -1); } ] 0 }         {1}
           92  +do_test multiplex-1.10.4  { lindex [ catchsql { SELECT multiplex_control(3, -1); } ] 0 }         {0}
    94     93   do_test multiplex-1.10.6  { lindex [ catchsql { SELECT multiplex_control(2, 31); } ] 0 }         {0}
    95         -do_test multiplex-1.10.7  { lindex [ catchsql { SELECT multiplex_control(3, 100); } ] 0 }        {1}
           94  +do_test multiplex-1.10.7  { lindex [ catchsql { SELECT multiplex_control(3, 100); } ] 0 }        {0}
    96     95   do_test multiplex-1.10.8  { lindex [ catchsql { SELECT multiplex_control(2, 1073741824); } ] 0 } {0}
    97     96   do_test multiplex-1.10.9  { db close }                                                           {}
    98     97   do_test multiplex-1.10.10 { sqlite3_multiplex_shutdown }                                         {SQLITE_OK}
    99     98   
   100     99   do_test multiplex-1.11.1  { sqlite3_multiplex_initialize "" 1 }               {SQLITE_OK}
   101    100   do_test multiplex-1.11.2  { sqlite3 db test.db }                              {}
   102    101   do_test multiplex-1.11.3  { sqlite3_multiplex_control db main enable 0  }     {SQLITE_OK}
................................................................................
   142    141   #
   143    142   #   multiplex-2.7.*: Disable/enable tests.
   144    143   #
   145    144   
   146    145   sqlite3_multiplex_initialize "" 1
   147    146   multiplex_set db main 32768 16
   148    147   
          148  +file delete -force test.x
   149    149   do_test multiplex-2.1.2 {
   150         -  sqlite3 db test.db
          150  +  sqlite3 db test.x
   151    151     execsql {
   152    152       PRAGMA page_size=1024;
   153    153       PRAGMA auto_vacuum=OFF;
   154    154       PRAGMA journal_mode=DELETE;
   155    155     }
   156    156     execsql {
   157    157       CREATE TABLE t1(a, b);
   158    158       INSERT INTO t1 VALUES(1, randomblob(1100));
   159    159       INSERT INTO t1 VALUES(2, randomblob(1100));
   160    160     }
   161    161   } {}
   162         -do_test multiplex-2.1.3 { file size [multiplex_name test.db 0] } {4096}
          162  +do_test multiplex-2.1.3 { file size [multiplex_name test.x 0] } {4096}
   163    163   do_test multiplex-2.1.4 {
   164    164     execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) }
   165    165   } {}
   166    166   
   167    167   do_test multiplex-2.2.1 {
   168    168     execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) }
   169    169   } {}
   170         -do_test multiplex-2.2.3 { file size [multiplex_name test.db 0] } {6144}
          170  +do_test multiplex-2.2.3 { file size [multiplex_name test.x 0] } {6144}
   171    171   
   172    172   do_test multiplex-2.3.1 {
   173         -  sqlite3 db2 test2.db
          173  +  sqlite3 db2 test2.x
   174    174     db2 close
   175    175   } {}
   176    176   
   177    177   
   178    178   do_test multiplex-2.4.1 {
   179    179     sqlite3_multiplex_shutdown
   180    180   } {SQLITE_MISUSE}
   181    181   do_test multiplex-2.4.2 {
   182    182     execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) }
   183    183   } {}
   184         -do_test multiplex-2.4.4 { file size [multiplex_name test.db 0] } {7168}
          184  +do_test multiplex-2.4.4 { file size [multiplex_name test.x 0] } {7168}
   185    185   do_test multiplex-2.4.99 {
   186    186     db close
   187    187     sqlite3_multiplex_shutdown
   188    188   } {SQLITE_OK}
   189    189   
   190    190   
   191    191   do_test multiplex-2.5.1 {
   192         -  multiplex_delete test.db
          192  +  multiplex_delete test.x
   193    193     sqlite3_multiplex_initialize "" 1
   194         -  sqlite3 db test.db
          194  +  sqlite3 db test.x
   195    195     multiplex_set db main 4096 16
   196    196   } {SQLITE_OK}
   197    197   
   198    198   do_test multiplex-2.5.2 {
   199    199     execsql {
   200    200       PRAGMA page_size = 1024;
   201    201       PRAGMA journal_mode = delete;
................................................................................
   232    232     db eval {SELECT a,length(b) FROM t1 WHERE a=2}
   233    233   } {2 4000}
   234    234   
   235    235   do_test multiplex-2.5.8 {
   236    236     db eval {SELECT a,length(b) FROM t1 WHERE a=4}
   237    237   } {4 4000}
   238    238   
   239         -do_test multiplex-2.5.9 { file size [multiplex_name test.db 0] } [list $g_chunk_size]
   240         -do_test multiplex-2.5.10 { file size [multiplex_name test.db 1] } [list $g_chunk_size]
          239  +do_test multiplex-2.5.9 { file size [multiplex_name test.x 0] } [list $g_chunk_size]
          240  +do_test multiplex-2.5.10 { file size [multiplex_name test.x 1] } [list $g_chunk_size]
   241    241   
   242    242   do_test multiplex-2.5.99 {
   243    243     db close
   244    244     sqlite3_multiplex_shutdown
   245    245   } {SQLITE_OK}
   246    246   
   247    247   
................................................................................
   519    519   do_faultsim_test multiplex-5.5 -prep {
   520    520     catch { sqlite3_multiplex_shutdown }
   521    521   } -body {
   522    522     sqlite3_multiplex_initialize "" 1
   523    523     multiplex_set db main 32768 16
   524    524   }
   525    525   
   526         -# test that mismatch filesize is detected
   527         -#
   528         -# Do not run this test if $::G(perm:presql) is set. If it is set, then the
   529         -# expected IO error will occur within the Tcl [sqlite3] wrapper, not within
   530         -# the first SQL statement executed below. This breaks the test case.
   531         -#
   532         -if {0==[info exists ::G(perm:presql)] || $::G(perm:presql) == ""} {
   533         -  set all_journal_modes {delete persist truncate memory off}
   534         -  foreach jmode $all_journal_modes {
   535         -    do_test multiplex-5.6.1.$jmode {
   536         -      sqlite3_multiplex_shutdown
   537         -      multiplex_delete test.db
   538         -      sqlite3 db test.db
   539         -      db eval {
   540         -        PRAGMA page_size = 1024;
   541         -        PRAGMA auto_vacuum = off;
   542         -      }
   543         -      db eval "PRAGMA journal_mode = $jmode;"
   544         -    } $jmode
   545         -    do_test multiplex-5.6.2.$jmode {
   546         -      execsql {
   547         -        CREATE TABLE t1(a, b);
   548         -        INSERT INTO t1 VALUES(1, randomblob(15000));
   549         -        INSERT INTO t1 VALUES(2, randomblob(15000));
   550         -        INSERT INTO t1 VALUES(3, randomblob(15000));
   551         -        INSERT INTO t1 VALUES(4, randomblob(15000));
   552         -        INSERT INTO t1 VALUES(5, randomblob(15000));
   553         -      }
   554         -      db close
   555         -      sqlite3_multiplex_initialize "" 1
   556         -      sqlite3 db test.db
   557         -      multiplex_set db main 4096 16
   558         -    } {SQLITE_OK}
   559         -    do_test multiplex-5.6.3.$jmode {
   560         -      catchsql {
   561         -        INSERT INTO t1 VALUES(6, randomblob(15000));
   562         -      }
   563         -    } {1 {disk I/O error}}
   564         -    do_test multiplex-5.6.4.$jmode {
   565         -      db close
   566         -    } {}
   567         -  }
   568         -}
   569         -
   570    526   #-------------------------------------------------------------------------
   571    527   # Test that you can vacuum a multiplex'ed DB.  
   572    528   
   573    529   ifcapable vacuum {
   574    530   
   575    531   sqlite3_multiplex_shutdown
   576    532   do_test multiplex-6.0.0 {
   577    533     multiplex_delete test.db
          534  +  multiplex_delete test.x
   578    535     sqlite3_multiplex_initialize "" 1
   579         -  sqlite3 db test.db
          536  +  sqlite3 db test.x
   580    537     multiplex_set db main 4096 16
   581    538   } {SQLITE_OK}
   582    539   
   583    540   do_test multiplex-6.1.0 {
   584    541     execsql {
   585    542       PRAGMA page_size=1024;
   586    543       PRAGMA journal_mode=DELETE;
................................................................................
   588    545     }
   589    546     execsql {
   590    547       CREATE TABLE t1(a, b);
   591    548       INSERT INTO t1 VALUES(1, randomblob($g_chunk_size));
   592    549       INSERT INTO t1 VALUES(2, randomblob($g_chunk_size));
   593    550     }
   594    551   } {}
   595         -do_test multiplex-6.2.1 { file size [multiplex_name test.db 0] } [list $g_chunk_size]
   596         -do_test multiplex-6.2.2 { file size [multiplex_name test.db 1] } [list $g_chunk_size]
          552  +do_test multiplex-6.2.1 { file size [multiplex_name test.x 0] } [list $g_chunk_size]
          553  +do_test multiplex-6.2.2 { file size [multiplex_name test.x 1] } [list $g_chunk_size]
   597    554   
   598    555   do_test multiplex-6.3.0 {
   599    556     execsql { VACUUM }
   600    557   } {}
   601    558   
   602    559   do_test multiplex-6.99 {
   603    560     db close
   604         -  multiplex_delete test.db
          561  +  multiplex_delete test.x
   605    562     sqlite3_multiplex_shutdown
   606    563   } {SQLITE_OK}
   607    564   
   608    565   }
   609    566   
   610    567   
   611    568   catch { sqlite3_multiplex_shutdown }
   612    569   finish_test

Changes to test/permutations.test.

   778    778     where6.test where7.test where8.test where9.test
   779    779     whereA.test whereB.test wherelimit.test
   780    780     select1.test select2.test select3.test select4.test select5.test
   781    781     select7.test select8.test selectA.test selectC.test
   782    782   } -dbconfig {
   783    783     optimization_control $::dbhandle all 0
   784    784   }
          785  +
          786  +test_suite "prepare" -description {
          787  +  Run tests with the db connection using sqlite3_prepare() instead of _v2().
          788  +} -dbconfig {
          789  +  db_use_legacy_prepare $::dbhandle 1
          790  +  #$::dbhandle cache size 0
          791  +} -files [
          792  +  test_set $allquicktests -exclude *malloc* *ioerr* *fault*
          793  +]
   785    794   
   786    795   # End of tests
   787    796   #############################################################################
   788    797   
   789    798   # run_tests NAME OPTIONS
   790    799   #
   791    800   # where available options are:  

Changes to test/releasetest.tcl.

   151    151       -DSQLITE_ENABLE_OVERSIZE_CELL_CHECK=1
   152    152       -DSQLITE_MAX_ATTACHED=62
   153    153     }
   154    154   }
   155    155   
   156    156   array set ::Platforms {
   157    157     Linux-x86_64 {
          158  +    "Debug-One"               "checksymbols test"
   158    159       "Secure-Delete"           test
   159    160       "Unlock-Notify"           "QUICKTEST_INCLUDE=notify2.test test"
   160    161       "Update-Delete-Limit"     test
   161         -    "Debug-One"               test
   162    162       "Extra-Robustness"        test
   163    163       "Device-Two"              test
   164    164       "Ftrapv"                  test
   165    165       "Default"                 "threadtest test"
   166    166       "Device-One"              fulltest
   167    167     }
   168    168     Linux-i686 {
................................................................................
   172    172       "Default"                 "threadtest fulltest"
   173    173     }
   174    174     Darwin-i386 {
   175    175       "Locking-Style"           test
   176    176       "OS-X"                    "threadtest fulltest"
   177    177     }
   178    178   }
          179  +
   179    180   
   180    181   # End of configuration section.
   181    182   #########################################################################
   182    183   #########################################################################
   183    184   
   184    185   foreach {key value} [array get ::Platforms] {
   185    186     foreach {v t} $value {

Changes to test/selectB.test.

   351    351   
   352    352     do_test selectB-$ii.19 {
   353    353       execsql {
   354    354         SELECT * FROM (
   355    355           SELECT DISTINCT (a/10) FROM t1 UNION ALL SELECT DISTINCT(d%2) FROM t2
   356    356         )
   357    357       }
   358         -  } {0 1 0 1}
          358  +  } {0 1 1 0}
   359    359   
   360    360     do_test selectB-$ii.20 {
   361    361       execsql {
   362    362         SELECT DISTINCT * FROM (
   363    363           SELECT DISTINCT (a/10) FROM t1 UNION ALL SELECT DISTINCT(d%2) FROM t2
   364    364         )
   365    365       }

Changes to test/sqllimits1.test.

   315    315   } {SQLITE_TOOBIG}
   316    316   do_test sqllimits1-5.14.4 {
   317    317     set np1 [expr {$SQLITE_LIMIT_LENGTH + 1}]
   318    318     set ::str1 [string repeat A $np1]
   319    319     catch {sqlite3_bind_text $::STMT 1 $::str1 -1} res
   320    320     set res
   321    321   } {SQLITE_TOOBIG}
   322         -do_test sqllimits1-5.14.5 {
   323         -  catch {sqlite3_bind_text16 $::STMT 1 $::str1 -1} res
   324         -  set res
   325         -} {SQLITE_TOOBIG}
          322  +ifcapable utf16 {
          323  +  do_test sqllimits1-5.14.5 {
          324  +    catch {sqlite3_bind_text16 $::STMT 1 $::str1 -1} res
          325  +    set res
          326  +  } {SQLITE_TOOBIG}
          327  +}
   326    328   do_test sqllimits1-5.14.6 {
   327    329     catch {sqlite3_bind_text $::STMT 1 $::str1 $np1} res
   328    330     set res
   329    331   } {SQLITE_TOOBIG}
   330         -do_test sqllimits1-5.14.7 {
   331         -  catch {sqlite3_bind_text16 $::STMT 1 $::str1 $np1} res
   332         -  set res
   333         -} {SQLITE_TOOBIG}
          332  +ifcapable utf16 {
          333  +  do_test sqllimits1-5.14.7 {
          334  +    catch {sqlite3_bind_text16 $::STMT 1 $::str1 $np1} res
          335  +    set res
          336  +  } {SQLITE_TOOBIG}
          337  +}
   334    338   do_test sqllimits1-5.14.8 {
   335    339     set n [expr {$np1-1}]
   336    340     catch {sqlite3_bind_text $::STMT 1 $::str1 $n} res
   337    341     set res
   338    342   } {}
   339    343   do_test sqllimits1-5.14.9 {
   340    344     catch {sqlite3_bind_text16 $::STMT 1 $::str1 $n} res

Changes to test/temptable.test.

   288    288   # but the permanent index should still be accessible and should still
   289    289   # be updated when its corresponding table changes.
   290    290   #
   291    291   do_test temptable-5.1 {
   292    292     execsql {
   293    293       CREATE TEMP TABLE mask(a,b,c)
   294    294     } db2
          295  +  if {[permutation]=="prepare"} { db2 cache flush }
   295    296     execsql {
   296    297       CREATE INDEX mask ON t2(x);
   297    298       SELECT * FROM t2;
   298    299     }
   299    300   } {3 4}
   300    301   #do_test temptable-5.2 {
   301    302   #  catchsql {

Added test/tkt-54844eea3f.test.

            1  +# 2011 July 8
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this file is testing that bug [54844eea3f] has been fixed.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +
           18  +set ::testprefix tkt-54844eea3f
           19  +
           20  +do_test 1.0 {
           21  +  execsql {
           22  +    CREATE TABLE t1(a INTEGER PRIMARY KEY);
           23  +    INSERT INTO t1 VALUES(1);
           24  +    INSERT INTO t1 VALUES(4);
           25  +
           26  +    CREATE TABLE t2(b INTEGER PRIMARY KEY);
           27  +    INSERT INTO t2 VALUES(1);
           28  +    INSERT INTO t2 VALUES(2);
           29  +    INSERT INTO t2 SELECT b+2 FROM t2;
           30  +    INSERT INTO t2 SELECT b+4 FROM t2;
           31  +    INSERT INTO t2 SELECT b+8 FROM t2;
           32  +    INSERT INTO t2 SELECT b+16 FROM t2;
           33  +
           34  +    CREATE TABLE t3(c INTEGER PRIMARY KEY);
           35  +    INSERT INTO t3 VALUES(1);
           36  +    INSERT INTO t3 VALUES(2);
           37  +    INSERT INTO t3 VALUES(3);
           38  +  }
           39  +} {}
           40  +
           41  +do_test 1.1 {
           42  +  execsql {
           43  +    SELECT 'test-2', t3.c, (
           44  +          SELECT count(*) 
           45  +          FROM t1 JOIN (SELECT DISTINCT t3.c AS p FROM t2) AS x ON t1.a=x.p
           46  +    )
           47  +    FROM t3;
           48  +  }
           49  +} {test-2 1 1 test-2 2 0 test-2 3 0}
           50  +
           51  +do_test 1.2 {
           52  +  execsql {
           53  +    CREATE TABLE t4(a, b, c);
           54  +    INSERT INTO t4 VALUES('a', 1, 'one');
           55  +    INSERT INTO t4 VALUES('a', 2, 'two');
           56  +    INSERT INTO t4 VALUES('b', 1, 'three');
           57  +    INSERT INTO t4 VALUES('b', 2, 'four');
           58  +    SELECT ( 
           59  +      SELECT c FROM (
           60  +        SELECT * FROM t4 WHERE a=out.a ORDER BY b LIMIT 10 OFFSET 1
           61  +      ) WHERE b=out.b
           62  +    ) FROM t4 AS out;
           63  +  }
           64  +} {{} two {} four}
           65  +
           66  +
           67  +finish_test

Changes to test/triggerD.test.

    10     10   #***********************************************************************
    11     11   #
    12     12   # Verify that when columns named "rowid", "oid", and "_rowid_" appear
    13     13   # in a table as ordinary columns (not as the INTEGER PRIMARY KEY) then
    14     14   # the use of these columns in triggers will refer to the column and not
    15     15   # to the actual ROWID.  Ticket [34d2ae1c6d08b5271ba5e5592936d4a1d913ffe3]
    16     16   #
           17  +# Also, verify that triggers created like this:
           18  +#
           19  +#    CREATE TRIGGER attached.trig AFTER INSERT ON attached.tab ...
           20  +#
           21  +# can be reparsed as a main database.  Ticket [d6ddba6706353915ceedc56b4e3]
           22  +#
    17     23   
    18     24   set testdir [file dirname $argv0]
    19     25   source $testdir/tester.tcl
    20     26   ifcapable {!trigger} {
    21     27     finish_test
    22     28     return
    23     29   }
................................................................................
   166    172       END;
   167    173       INSERT INTO main.t300 VALUES(3);
   168    174       INSERT INTO temp.t300 VALUES(4);
   169    175       SELECT * FROM t301;
   170    176     }
   171    177   } {10003 20004}
   172    178   
          179  +
          180  +#############################################################################
          181  +# 
          182  +# Ticket [d6ddba6706353915ceedc56b4e3e72ecb4d77ba4]
          183  +#
          184  +# The following syntax really should not be allowed:
          185  +#
          186  +#    CREATE TRIGGER xyz.trig BEFORE UPDATE ON xyz.tab BEGIN ...
          187  +#
          188  +# But a long-standing bug does allow it.  And the "xyz.tab" slips into
          189  +# the sqlite_master table.  We cannot fix the bug simply by disallowing
          190  +# "xyz.tab" since that could break legacy applications.  We have to 
          191  +# fix the system so that the "xyz." on "xyz.tab" is ignored.
          192  +# Verify that this is the case.
          193  +#
          194  +do_test triggerD-4.1 {
          195  +  db close
          196  +  file delete -force test.db test2.db
          197  +  sqlite3 db test.db
          198  +  db eval {
          199  +    CREATE TABLE t1(x);
          200  +    ATTACH 'test2.db' AS db2;
          201  +    CREATE TABLE db2.t2(y);
          202  +    CREATE TABLE db2.log(z);
          203  +    CREATE TRIGGER db2.trig AFTER INSERT ON db2.t2 BEGIN
          204  +      INSERT INTO log(z) VALUES(new.y);
          205  +    END;
          206  +    INSERT INTO t2 VALUES(123);
          207  +    SELECT * FROM log;
          208  +  }
          209  +} {123}
          210  +do_test triggerD-4.2 {
          211  +  sqlite3 db2 test2.db
          212  +  db2 eval {
          213  +    INSERT INTO t2 VALUES(234);
          214  +    SELECT * FROM log;
          215  +  }
          216  +} {123 234}
          217  +db2 close
   173    218   
   174    219   finish_test

Changes to test/wal3.test.

   703    703   #
   704    704   # This test case verifies that if an exclusive lock cannot be obtained
   705    705   # on any aReadMark[] slot (because there are already several readers),
   706    706   # the client takes a shared-lock on a slot without modifying the value
   707    707   # and continues.
   708    708   #
   709    709   set nConn 50
   710         -if { [string match *BSD $tcl_platform(os)] } { set nConn 35 }
          710  +if { [string match *BSD $tcl_platform(os)] } { set nConn 25 }
   711    711   do_test wal3-9.0 {
   712    712     file delete -force test.db test.db-journal test.db wal
   713    713     sqlite3 db test.db
   714    714     execsql {
   715    715       PRAGMA page_size = 1024;
   716    716       PRAGMA journal_mode = WAL;
   717    717       CREATE TABLE whoami(x);
................................................................................
   780    780   
   781    781     do_test wal3-10.$tn.2 {
   782    782       sql2 {PRAGMA integrity_check}
   783    783     } {ok}
   784    784   }
   785    785   
   786    786   finish_test
   787         -

Changes to test/wal6.test.

    39     39   	    CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
    40     40   	    INSERT INTO t1 VALUES(1,2);
    41     41   	    SELECT * FROM t1;
    42     42   	  }
    43     43   	} {1 2}
    44     44   
    45     45   # Under Windows, you'll get an error trying to delete
    46         -# a file this is already opened.  For now, make sure 
    47         -# we get that error, then close the first connection
           46  +# a file this is already opened.  Close the first connection
    48     47   # so the other tests work.
    49     48   if {$tcl_platform(platform)=="windows"} {
    50     49     if {$jmode=="persist" || $jmode=="truncate"} {
    51         -	  do_test wal6-1.2.$jmode.win {
    52         -	    sqlite3 db2 test.db
    53         -	    catchsql {
    54         -		    PRAGMA journal_mode=WAL;
    55         -	    } db2
    56         -	  } {1 {disk I/O error}}
    57         -  	db2 close
    58         -	  db close
           50  +    db close
    59     51     }
    60     52   }
    61     53   
    62     54   	do_test wal6-1.2.$jmode {
    63     55   	  sqlite3 db2 test.db
    64     56   	  execsql {
    65     57   		PRAGMA journal_mode=WAL;
................................................................................
    83     75   	db close
    84     76   	db2 close
    85     77     forcedelete test.db
    86     78   
    87     79   }
    88     80   
    89     81   finish_test
    90         -

Added test/win32lock.test.

            1  +# 2011 July 11
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is recovery from transient manditory locks
           13  +# that sometimes appear on database files due to anti-virus software.
           14  +#
           15  +
           16  +if {$tcl_platform(platform)!="windows"} return
           17  +
           18  +set testdir [file dirname $argv0]
           19  +source $testdir/tester.tcl
           20  +
           21  +set testprefix win32lock
           22  +
           23  +db close
           24  +sqlite3_shutdown
           25  +test_sqlite3_log xLog
           26  +proc xLog {error_code msg} {
           27  +  lappend ::log $msg 
           28  +}
           29  +sqlite3 db test.db
           30  +
           31  +do_test win32lock-1.1 {
           32  +  db eval {
           33  +    PRAGMA cache_size=10;
           34  +    CREATE TABLE t1(x,y);
           35  +    INSERT INTO t1 VALUES(1,randomblob(100000));
           36  +    INSERT INTO t1 VALUES(2,randomblob(50000));
           37  +    INSERT INTO t1 VALUES(3,randomblob(25000));
           38  +    INSERT INTO t1 VALUES(4,randomblob(12500));
           39  +    SELECT x, length(y) FROM t1 ORDER BY rowid;
           40  +  }
           41  +} {1 100000 2 50000 3 25000 4 12500}
           42  +
           43  +unset -nocomplain delay1 rc msg
           44  +set delay1 50
           45  +set rc 0
           46  +set old_pending_byte [sqlite3_test_control_pending_byte 0x40000000]
           47  +while {1} {
           48  +  sqlite3_sleep 10
           49  +  lock_win32_file test.db 0 $::delay1
           50  +  set rc [catch {db eval {SELECT x, length(y) FROM t1 ORDER BY rowid}} msg]
           51  +  if {$rc} {
           52  +    do_test win32lock-1.2-$delay1-fin {
           53  +       set ::msg
           54  +    } {disk I/O error}
           55  +    break
           56  +  } else {
           57  +    do_test win32lock-1.2-$delay1 {
           58  +       set ::msg
           59  +    } {1 100000 2 50000 3 25000 4 12500}
           60  +    if {$::log!=""} {
           61  +      do_test win32lock-1.2-$delay1-log1 {
           62  +        regsub {\d+} $::log # x
           63  +        set x
           64  +      } {{delayed #ms for lock/sharing conflict}}
           65  +    }
           66  +    incr delay1 50
           67  +  }
           68  +  set ::log {}
           69  +}
           70  +
           71  +do_test win32lock-2.0 {
           72  +  file_control_win32_av_retry db -1 -1
           73  +} {0 10 25}
           74  +do_test win32lock-2.1 {
           75  +  file_control_win32_av_retry db 1 1
           76  +} {0 1 1}
           77  +
           78  +set delay1 50
           79  +while {1} {
           80  +  sqlite3_sleep 10
           81  +  lock_win32_file test.db 0 $::delay1
           82  +  set rc [catch {db eval {SELECT x, length(y) FROM t1 ORDER BY rowid}} msg]
           83  +  if {$rc} {
           84  +    do_test win32lock-2.2-$delay1-fin {
           85  +       set ::msg
           86  +    } {disk I/O error}
           87  +    break
           88  +  } else {
           89  +    do_test win32lock-2.2-$delay1 {
           90  +       set ::msg
           91  +    } {1 100000 2 50000 3 25000 4 12500}
           92  +    if {$::log!=""} {
           93  +      do_test win32lock-2.2-$delay1-log1 {
           94  +        regsub {\d+} $::log # x
           95  +        set x
           96  +      } {{delayed #ms for lock/sharing conflict}}
           97  +    }
           98  +    incr delay1 50
           99  +  }
          100  +  set ::log {}
          101  +}
          102  +
          103  +file_control_win32_av_retry db 10 25
          104  +sqlite3_test_control_pending_byte $old_pending_byte
          105  +sqlite3_shutdown
          106  +test_sqlite3_log 
          107  +sqlite3_initialize
          108  +finish_test

Changes to tool/symbols.sh.

     8      8   echo '****** Exported symbols from a build including RTREE, FTS4 & ICU ******'
     9      9   gcc -c -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \
    10     10     -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \
    11     11     -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \
    12     12     -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \
    13     13     -DSQLITE_ENABLE_ICU \
    14     14     sqlite3.c
    15         -nm sqlite3.o | grep ' T ' | sort -k 3
           15  +nm sqlite3.o | grep ' [TD] ' | sort -k 3
    16     16   
    17     17   echo '****** Surplus symbols from a build including RTREE, FTS4 & ICU ******'
    18         -nm sqlite3.o | grep ' T ' | grep -v ' sqlite3_'
           18  +nm sqlite3.o | grep ' [TD] ' | grep -v ' .*sqlite3_'
    19     19   
    20     20   echo '****** Dependencies of the core. No extensions. No OS interface *******'
    21     21   gcc -c -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \
    22     22     -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \
    23     23     -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \
    24     24     -DSQLITE_OS_OTHER -DSQLITE_THREADSAFE=0 \
    25     25     sqlite3.c

Changes to tool/warnings.sh.

     4      4   # compiler warnings in SQLite.
     5      5   #
     6      6   make sqlite3.c
     7      7   echo '********** No optimizations.  Includes FTS4 and RTREE *********'
     8      8   gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \
     9      9         -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \
    10     10         sqlite3.c
    11         -echo '********** Optimized -O3.  Includes FTS4 and RTREE *********'
           11  +echo '********** No optimizations. ENABLE_STAT2. THREADSAFE=0 *******'
           12  +gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \
           13  +      -ansi -DSQLITE_ENABLE_STAT2 -DSQLITE_THREADSAFE=0 \
           14  +      sqlite3.c
           15  +echo '********** Optimized -O3.  Includes FTS4 and RTREE ************'
    12     16   gcc -O3 -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \
    13     17         -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \
    14     18         sqlite3.c