Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -515,10 +515,13 @@ touch .target_source sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl +sqlite3-all.c: sqlite3.c $(TOP)/tool/split-sqlite3c.tcl + $(TCLSH_CMD) $(TOP)/tool/split-sqlite3c.tcl + # Rule to build the amalgamation # sqlite3.lo: sqlite3.c $(LTCOMPILE) $(TEMP_STORE) -c sqlite3.c @@ -783,11 +786,11 @@ ./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y mv parse.h parse.h.temp $(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION - tclsh $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h + $(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h keywordhash.h: $(TOP)/tool/mkkeywordhash.c $(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c ./mkkeywordhash$(BEXE) >keywordhash.h Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -41,20 +41,36 @@ # Omitting the define will cause extra debugging code to be inserted and # includes extra comments when "EXPLAIN stmt" is used. # TCC = $(TCC) -DNDEBUG -# The library that programs using TCL must link against. +# The locations of the Tcl header and library files. Also, the library that +# non-stubs enabled programs using Tcl must link against. These variables +# (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment +# prior to running nmake in order to match the actual installed location and +# version on this machine. # -LIBTCL = tcl85.lib +!if "$(TCLINCDIR)" == "" TCLINCDIR = c:\tcl\include +!endif + +!if "$(TCLLIBDIR)" == "" TCLLIBDIR = c:\tcl\lib +!endif + +!if "$(LIBTCL)" == "" +LIBTCL = tcl85.lib +!endif # This is the command to use for tclsh - normally just "tclsh", but we may -# know the specific version we want to use +# know the specific version we want to use. This variable (TCLSH_CMD) may be +# overridden via the environment prior to running nmake in order to select a +# specific Tcl shell to use. # +!if "$(TCLSH_CMD)" == "" TCLSH_CMD = tclsh85 +!endif # Compiler options needed for programs that use the readline() library. # READLINE_FLAGS = -DHAVE_READLINE=0 @@ -116,11 +132,11 @@ LTLINKOPTS = /MACHINE:$(PLATFORM) LTLIBOPTS = /MACHINE:$(PLATFORM) !ENDIF # nawk compatible awk. -NAWK = .\gawk.exe +NAWK = gawk.exe # You should not have to change anything below this line ############################################################################### # Object files for the SQLite library (non-amalgamation). @@ -437,11 +453,11 @@ $(TOP)\ext\rtree\sqlite3rtree.h # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # -all: libsqlite3.lib sqlite3.exe libtclsqlite3.lib +all: dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib libsqlite3.lib: $(LIBOBJ) $(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS) libtclsqlite3.lib: tclsqlite.lo libsqlite3.lib @@ -461,16 +477,19 @@ .target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl -rmdir /S/Q tsrc -mkdir tsrc for %i in ($(SRC)) do copy /Y %i tsrc del /Q tsrc\sqlite.h.in tsrc\parse.y - $(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl vdbe.new + $(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl < tsrc\vdbe.c > vdbe.new move vdbe.new tsrc\vdbe.c echo > .target_source sqlite3.c: .target_source $(TOP)\tool\mksqlite3c.tcl $(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl + +sqlite3-all.c: sqlite3.c $(TOP)/tool/split-sqlite3c.tcl + $(TCLSH_CMD) $(TOP)/tool/split-sqlite3c.tcl # Rule to build the amalgamation # sqlite3.lo: sqlite3.c $(LTCOMPILE) -c sqlite3.c @@ -718,14 +737,14 @@ /link $(LTLINKOPTS) /LIBPATH:$(TCLLIBDIR) libsqlite3.lib $(LIBTCL) # Rules to build opcodes.c and opcodes.h # opcodes.c: opcodes.h $(TOP)\mkopcodec.awk - $(NAWK) "/#define OP_/ { print }" opcodes.h | sort /+45 | $(NAWK) -f $(TOP)\mkopcodec.awk >opcodes.c + $(NAWK) "/#define OP_/ { print }" opcodes.h | sort /+45 | $(NAWK) -f $(TOP)\mkopcodec.awk > opcodes.c opcodes.h: parse.h $(TOP)\src\vdbe.c $(TOP)\mkopcodeh.awk - type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk >opcodes.h + type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk > opcodes.h # Rules to build parse.c and parse.h - the outputs of lemon. # parse.h: parse.c @@ -732,20 +751,20 @@ parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\addopcodes.awk del /Q parse.y parse.h parse.h.temp copy $(TOP)\src\parse.y . .\lemon.exe $(OPT_FEATURE_FLAGS) $(OPTS) parse.y move parse.h parse.h.temp - $(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp >parse.h + $(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp > parse.h sqlite3.h: $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION - $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP) >sqlite3.h + $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP) > sqlite3.h mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c $(BCC) -Femkkeywordhash.exe $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)\tool\mkkeywordhash.c keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe - .\mkkeywordhash.exe >keywordhash.h + .\mkkeywordhash.exe > keywordhash.h # Rules to build the extension objects. # @@ -837,11 +856,11 @@ test: testfixture.exe sqlite3.exe .\testfixture.exe $(TOP)\test\veryquick.test spaceanal_tcl.h: $(TOP)\tool\spaceanal.tcl $(NAWK) -f $(TOP)/tool/tostr.awk \ - $(TOP)\tool\spaceanal.tcl >spaceanal_tcl.h + $(TOP)\tool\spaceanal.tcl > spaceanal_tcl.h sqlite3_analyzer.exe: $(TESTFIXTURE_SRC) spaceanal_tcl.h $(LTLINK) -DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 \ -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE \ -DBUILD_sqlite -I$(TCLINCDIR) \ @@ -864,12 +883,12 @@ # Windows section # dll: sqlite3.dll sqlite3.def: libsqlite3.lib - echo EXPORTS >sqlite3.def + echo EXPORTS > sqlite3.def dumpbin /all libsqlite3.lib \ | $(NAWK) "/ 1 _sqlite3_/ { sub(/^.* _/,\"\");print }" \ - | sort >>sqlite3.def + | sort >> sqlite3.def sqlite3.dll: $(LIBOBJ) sqlite3.def link $(LTLINKOPTS) /DLL /DEF:sqlite3.def /OUT:$@ $(LIBOBJ) Index: VERSION ================================================================== --- VERSION +++ VERSION @@ -1,1 +1,1 @@ -3.7.7 +3.7.8 Index: configure ================================================================== --- configure +++ configure @@ -1,8 +1,8 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.7.7. +# Generated by GNU Autoconf 2.62 for sqlite 3.7.8. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. # This configure script is free software; the Free Software Foundation # gives unlimited permission to copy, distribute and modify it. @@ -741,12 +741,12 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.7.7' -PACKAGE_STRING='sqlite 3.7.7' +PACKAGE_VERSION='3.7.8' +PACKAGE_STRING='sqlite 3.7.8' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. ac_includes_default="\ #include @@ -1483,11 +1483,11 @@ # if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.7.7 to adapt to many kinds of systems. +\`configure' configures sqlite 3.7.8 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... To assign environment variables (e.g., CC, CFLAGS...), specify them as VAR=VALUE. See below for descriptions of some of the useful variables. @@ -1548,11 +1548,11 @@ _ACEOF fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.7.7:";; + short | recursive ) echo "Configuration of sqlite 3.7.8:";; esac cat <<\_ACEOF Optional Features: --disable-option-checking ignore unrecognized --enable/--with options @@ -1664,11 +1664,11 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.7.7 +sqlite configure 3.7.8 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation @@ -1678,11 +1678,11 @@ fi cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.7.7, which was +It was created by sqlite $as_me 3.7.8, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ _ACEOF @@ -14028,11 +14028,11 @@ # Save the log message, to keep $[0] and so on meaningful, and to # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.7.7, which was +This file was extended by sqlite $as_me 3.7.8, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS CONFIG_LINKS = $CONFIG_LINKS @@ -14081,11 +14081,11 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.7.7 +sqlite config.status 3.7.8 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" Copyright (C) 2008 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -310,10 +310,15 @@ #ifndef SQLITE_CORE # include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 #endif +static int fts3EvalNext(Fts3Cursor *pCsr); +static int fts3EvalStart(Fts3Cursor *pCsr); +static int fts3TermSegReaderCursor( + Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); + /* ** Write a 64-bit variable-length integer to memory starting at p[0]. ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. ** The number of bytes written is returned. */ @@ -818,31 +823,60 @@ } sqlite3_free(zFree); return zRet; } +/* +** This function interprets the string at (*pp) as a non-negative integer +** value. It reads the integer and sets *pnOut to the value read, then +** sets *pp to point to the byte immediately following the last byte of +** the integer value. +** +** Only decimal digits ('0'..'9') may be part of an integer value. +** +** If *pp does not being with a decimal digit SQLITE_ERROR is returned and +** the output value undefined. Otherwise SQLITE_OK is returned. +** +** This function is used when parsing the "prefix=" FTS4 parameter. +*/ static int fts3GobbleInt(const char **pp, int *pnOut){ - const char *p = *pp; - int nInt = 0; + const char *p = *pp; /* Iterator pointer */ + int nInt = 0; /* Output value */ + for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ nInt = nInt * 10 + (p[0] - '0'); } if( p==*pp ) return SQLITE_ERROR; *pnOut = nInt; *pp = p; return SQLITE_OK; } - +/* +** This function is called to allocate an array of Fts3Index structures +** representing the indexes maintained by the current FTS table. FTS tables +** always maintain the main "terms" index, but may also maintain one or +** more "prefix" indexes, depending on the value of the "prefix=" parameter +** (if any) specified as part of the CREATE VIRTUAL TABLE statement. +** +** Argument zParam is passed the value of the "prefix=" option if one was +** specified, or NULL otherwise. +** +** If no error occurs, SQLITE_OK is returned and *apIndex set to point to +** the allocated array. *pnIndex is set to the number of elements in the +** array. If an error does occur, an SQLite error code is returned. +** +** Regardless of whether or not an error is returned, it is the responsibility +** of the caller to call sqlite3_free() on the output array to free it. +*/ static int fts3PrefixParameter( const char *zParam, /* ABC in prefix=ABC parameter to parse */ int *pnIndex, /* OUT: size of *apIndex[] array */ - struct Fts3Index **apIndex, /* OUT: Array of indexes for this table */ - struct Fts3Index **apFree /* OUT: Free this with sqlite3_free() */ + struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ ){ - struct Fts3Index *aIndex; - int nIndex = 1; + struct Fts3Index *aIndex; /* Allocated array */ + int nIndex = 1; /* Number of entries in array */ if( zParam && zParam[0] ){ const char *p; nIndex++; for(p=zParam; *p; p++){ @@ -849,11 +883,11 @@ if( *p==',' ) nIndex++; } } aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); - *apIndex = *apFree = aIndex; + *apIndex = aIndex; *pnIndex = nIndex; if( !aIndex ){ return SQLITE_NOMEM; } @@ -906,12 +940,11 @@ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ int nIndex; /* Size of aIndex[] array */ - struct Fts3Index *aIndex; /* Array of indexes for this table */ - struct Fts3Index *aFree = 0; /* Free this before returning */ + struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ /* The results of parsing supported FTS4 key=value options: */ int bNoDocsize = 0; /* True to omit %_docsize table */ int bDescIdx = 0; /* True to store descending indexes */ char *zPrefix = 0; /* Prefix parameter value (or NULL) */ @@ -1044,11 +1077,11 @@ rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr); if( rc!=SQLITE_OK ) goto fts3_init_out; } assert( pTokenizer ); - rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree); + rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex); if( rc==SQLITE_ERROR ){ assert( zPrefix ); *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix); } if( rc!=SQLITE_OK ) goto fts3_init_out; @@ -1131,11 +1164,11 @@ /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); fts3_init_out: sqlite3_free(zPrefix); - sqlite3_free(aFree); + sqlite3_free(aIndex); sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ @@ -1722,12 +1755,10 @@ *pp1 = p1 + 1; *pp2 = p2 + 1; } /* -** nToken==1 searches for adjacent positions. -** ** This function is used to merge two position lists into one. When it is ** called, *pp1 and *pp2 must both point to position lists. A position-list is ** the part of a doclist that follows each document id. For example, if a row ** contains: ** @@ -1743,10 +1774,12 @@ ** If isSaveLeft is 0, an entry is added to the output position list for ** each position in *pp2 for which there exists one or more positions in ** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. ** when the *pp1 token appears before the *pp2 token, but not more than nToken ** slots before it. +** +** e.g. nToken==1 searches for adjacent positions. */ static int fts3PoslistPhraseMerge( char **pp, /* IN/OUT: Preallocated output buffer */ int nToken, /* Maximum difference in token positions */ int isSaveLeft, /* Save the left position */ @@ -1909,26 +1942,38 @@ return res; } /* -** A pointer to an instance of this structure is used as the context -** argument to sqlite3Fts3SegReaderIterate() +** An instance of this function is used to merge together the (potentially +** large number of) doclists for each term that matches a prefix query. +** See function fts3TermSelectMerge() for details. */ typedef struct TermSelect TermSelect; struct TermSelect { - int isReqPos; - char *aaOutput[16]; /* Malloc'd output buffer */ - int anOutput[16]; /* Size of output in bytes */ + char *aaOutput[16]; /* Malloc'd output buffers */ + int anOutput[16]; /* Size each output buffer in bytes */ }; - +/* +** This function is used to read a single varint from a buffer. Parameter +** pEnd points 1 byte past the end of the buffer. When this function is +** called, if *pp points to pEnd or greater, then the end of the buffer +** has been reached. In this case *pp is set to 0 and the function returns. +** +** If *pp does not point to or past pEnd, then a single varint is read +** from *pp. *pp is then set to point 1 byte past the end of the read varint. +** +** If bDescIdx is false, the value read is added to *pVal before returning. +** If it is true, the value read is subtracted from *pVal before this +** function returns. +*/ static void fts3GetDeltaVarint3( - char **pp, - char *pEnd, - int bDescIdx, - sqlite3_int64 *pVal + char **pp, /* IN/OUT: Point to read varint from */ + char *pEnd, /* End of buffer */ + int bDescIdx, /* True if docids are descending */ + sqlite3_int64 *pVal /* IN/OUT: Integer value */ ){ if( *pp>=pEnd ){ *pp = 0; }else{ sqlite3_int64 iVal; @@ -1939,10 +1984,25 @@ *pVal += iVal; } } } +/* +** This function is used to write a single varint to a buffer. The varint +** is written to *pp. Before returning, *pp is set to point 1 byte past the +** end of the value written. +** +** If *pbFirst is zero when this function is called, the value written to +** the buffer is that of parameter iVal. +** +** If *pbFirst is non-zero when this function is called, then the value +** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) +** (if bDescIdx is non-zero). +** +** Before returning, this function always sets *pbFirst to 1 and *piPrev +** to the value of parameter iVal. +*/ static void fts3PutDeltaVarint3( char **pp, /* IN/OUT: Output pointer */ int bDescIdx, /* True for descending docids */ sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ int *pbFirst, /* IN/OUT: True after first int written */ @@ -1959,14 +2019,38 @@ *pp += sqlite3Fts3PutVarint(*pp, iWrite); *piPrev = iVal; *pbFirst = 1; } -#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2)) +/* +** This macro is used by various functions that merge doclists. The two +** arguments are 64-bit docid values. If the value of the stack variable +** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). +** Otherwise, (i2-i1). +** +** Using this makes it easier to write code that can merge doclists that are +** sorted in either ascending or descending order. +*/ +#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2)) + +/* +** This function does an "OR" merge of two doclists (output contains all +** positions contained in either argument doclist). If the docids in the +** input doclists are sorted in ascending order, parameter bDescDoclist +** should be false. If they are sorted in ascending order, it should be +** passed a non-zero value. +** +** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer +** containing the output doclist and SQLITE_OK is returned. In this case +** *pnOut is set to the number of bytes in the output doclist. +** +** If an error occurs, an SQLite error code is returned. The output values +** are undefined in this case. +*/ static int fts3DoclistOrMerge( - int bDescIdx, /* True if arguments are desc */ + int bDescDoclist, /* True if arguments are desc */ char *a1, int n1, /* First doclist */ char *a2, int n2, /* Second doclist */ char **paOut, int *pnOut /* OUT: Malloc'd doclist */ ){ sqlite3_int64 i1 = 0; @@ -1987,35 +2071,47 @@ p = aOut; fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); while( p1 || p2 ){ - sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); if( p2 && p1 && iDiff==0 ){ - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); fts3PoslistMerge(&p, &p1, &p2); - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); }else if( !p2 || (p1 && iDiff<0) ){ - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); fts3PoslistCopy(&p, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); }else{ - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); fts3PoslistCopy(&p, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); } } *paOut = aOut; *pnOut = (p-aOut); return SQLITE_OK; } +/* +** This function does a "phrase" merge of two doclists. In a phrase merge, +** the output contains a copy of each position from the right-hand input +** doclist for which there is a position in the left-hand input doclist +** exactly nDist tokens before it. +** +** If the docids in the input doclists are sorted in ascending order, +** parameter bDescDoclist should be false. If they are sorted in ascending +** order, it should be passed a non-zero value. +** +** The right-hand input doclist is overwritten by this function. +*/ static void fts3DoclistPhraseMerge( - int bDescIdx, /* True if arguments are desc */ + int bDescDoclist, /* True if arguments are desc */ int nDist, /* Distance from left to right (1=adjacent) */ char *aLeft, int nLeft, /* Left doclist */ char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ ){ sqlite3_int64 i1 = 0; @@ -2034,30 +2130,30 @@ p = aOut; fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); while( p1 && p2 ){ - sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); if( iDiff==0 ){ char *pSave = p; sqlite3_int64 iPrevSave = iPrev; int bFirstOutSave = bFirstOut; - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ p = pSave; iPrev = iPrevSave; bFirstOut = bFirstOutSave; } - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); }else if( iDiff<0 ){ fts3PoslistCopy(0, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); }else{ fts3PoslistCopy(0, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); } } *pnRight = p - aOut; } @@ -2070,11 +2166,11 @@ ** ** If an OOM error occurs, return SQLITE_NOMEM. In this case it is ** the responsibility of the caller to free any doclists left in the ** TermSelect.aaOutput[] array. */ -static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){ +static int fts3TermSelectFinishMerge(Fts3Table *p, TermSelect *pTS){ char *aOut = 0; int nOut = 0; int i; /* Loop through the doclists in the aaOutput[] array. Merge them all @@ -2111,28 +2207,29 @@ pTS->anOutput[0] = nOut; return SQLITE_OK; } /* -** This function is used as the sqlite3Fts3SegReaderIterate() callback when -** querying the full-text index for a doclist associated with a term or -** term-prefix. +** Merge the doclist aDoclist/nDoclist into the TermSelect object passed +** as the first argument. The merge is an "OR" merge (see function +** fts3DoclistOrMerge() for details). +** +** This function is called with the doclist for each term that matches +** a queried prefix. It merges all these doclists into one, the doclist +** for the specified prefix. Since there can be a very large number of +** doclists to merge, the merging is done pair-wise using the TermSelect +** object. +** +** This function returns SQLITE_OK if the merge is successful, or an +** SQLite error code (SQLITE_NOMEM) if an error occurs. */ -static int fts3TermSelectCb( - Fts3Table *p, /* Virtual table object */ - void *pContext, /* Pointer to TermSelect structure */ - char *zTerm, - int nTerm, - char *aDoclist, - int nDoclist +static int fts3TermSelectMerge( + Fts3Table *p, /* FTS table handle */ + TermSelect *pTS, /* TermSelect object to merge into */ + char *aDoclist, /* Pointer to doclist */ + int nDoclist /* Size of aDoclist in bytes */ ){ - TermSelect *pTS = (TermSelect *)pContext; - - UNUSED_PARAMETER(p); - UNUSED_PARAMETER(zTerm); - UNUSED_PARAMETER(nTerm); - if( pTS->aaOutput[0]==0 ){ /* If this is the first term selected, copy the doclist to the output ** buffer using memcpy(). */ pTS->aaOutput[0] = sqlite3_malloc(nDoclist); pTS->anOutput[0] = nDoclist; @@ -2199,23 +2296,30 @@ } pCsr->apSegment[pCsr->nSegment++] = pNew; return SQLITE_OK; } +/* +** Add seg-reader objects to the Fts3MultiSegReader object passed as the +** 8th argument. +** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ static int fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ int isScan, /* True to scan from zTerm to EOF */ - Fts3MultiSegReader *pCsr /* Cursor object to populate */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ - int rc = SQLITE_OK; - int rc2; - sqlite3_stmt *pStmt = 0; + int rc = SQLITE_OK; /* Error code */ + sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ + int rc2; /* Result of sqlite3_reset() */ /* If iLevel is less than 0 and this is not a scan, include a seg-reader ** for the pending-terms. If this is a scan, then this call must be being ** made by an fts4aux module, not an FTS table. In this case calling ** Fts3SegReaderPending might segfault, as the data structures used by @@ -2300,28 +2404,46 @@ return fts3SegReaderCursor( p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr ); } +/* +** In addition to its current configuration, have the Fts3MultiSegReader +** passed as the 4th argument also scan the doclist for term zTerm/nTerm. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ static int fts3SegReaderCursorAddZero( - Fts3Table *p, - const char *zTerm, - int nTerm, - Fts3MultiSegReader *pCsr + Fts3Table *p, /* FTS virtual table handle */ + const char *zTerm, /* Term to scan doclist of */ + int nTerm, /* Number of bytes in zTerm */ + Fts3MultiSegReader *pCsr /* Fts3MultiSegReader to modify */ ){ return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr); } - -int sqlite3Fts3TermSegReaderCursor( +/* +** Open an Fts3MultiSegReader to scan the doclist for term zTerm/nTerm. Or, +** if isPrefix is true, to scan the doclist for all terms for which +** zTerm/nTerm is a prefix. If successful, return SQLITE_OK and write +** a pointer to the new Fts3MultiSegReader to *ppSegcsr. Otherwise, return +** an SQLite error code. +** +** It is the responsibility of the caller to free this object by eventually +** passing it to fts3SegReaderCursorFree() +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +** Output parameter *ppSegcsr is set to 0 if an error occurs. +*/ +static int fts3TermSegReaderCursor( Fts3Cursor *pCsr, /* Virtual table cursor handle */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ ){ - Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ + Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ int rc = SQLITE_NOMEM; /* Return code */ pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); if( pSegcsr ){ int i; @@ -2361,62 +2483,53 @@ *ppSegcsr = pSegcsr; return rc; } +/* +** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). +*/ static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ sqlite3Fts3SegReaderFinish(pSegcsr); sqlite3_free(pSegcsr); } /* ** This function retreives the doclist for the specified term (or term -** prefix) from the database. -** -** The returned doclist may be in one of two formats, depending on the -** value of parameter isReqPos. If isReqPos is zero, then the doclist is -** a sorted list of delta-compressed docids (a bare doclist). If isReqPos -** is non-zero, then the returned list is in the same format as is stored -** in the database without the found length specifier at the start of on-disk -** doclists. +** prefix) from the database. */ static int fts3TermSelect( Fts3Table *p, /* Virtual table handle */ Fts3PhraseToken *pTok, /* Token to query for */ int iColumn, /* Column to query (or -ve for all columns) */ - int isReqPos, /* True to include position lists in output */ int *pnOut, /* OUT: Size of buffer at *ppOut */ char **ppOut /* OUT: Malloced result buffer */ ){ int rc; /* Return code */ - Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ - TermSelect tsc; /* Context object for fts3TermSelectCb() */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ + TermSelect tsc; /* Object for pair-wise doclist merging */ Fts3SegFilter filter; /* Segment term filter configuration */ pSegcsr = pTok->pSegcsr; memset(&tsc, 0, sizeof(TermSelect)); - tsc.isReqPos = isReqPos; - filter.flags = FTS3_SEGMENT_IGNORE_EMPTY + filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) - | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0) | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); filter.iCol = iColumn; filter.zTerm = pTok->z; filter.nTerm = pTok->n; rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); while( SQLITE_OK==rc && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) ){ - rc = fts3TermSelectCb(p, (void *)&tsc, - pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist - ); + rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); } if( rc==SQLITE_OK ){ - rc = fts3TermSelectMerge(p, &tsc); + rc = fts3TermSelectFinishMerge(p, &tsc); } if( rc==SQLITE_OK ){ *ppOut = tsc.aaOutput[0]; *pnOut = tsc.anOutput[0]; }else{ @@ -2438,28 +2551,19 @@ ** If the isPoslist argument is true, then it is assumed that the doclist ** contains a position-list following each docid. Otherwise, it is assumed ** that the doclist is simply a list of docids stored as delta encoded ** varints. */ -static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ +static int fts3DoclistCountDocids(char *aList, int nList){ int nDoc = 0; /* Return value */ if( aList ){ char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ char *p = aList; /* Cursor */ - if( !isPoslist ){ - /* The number of docids in the list is the same as the number of - ** varints. In FTS3 a varint consists of a single byte with the 0x80 - ** bit cleared and zero or more bytes with the 0x80 bit set. So to - ** count the varints in the buffer, just count the number of bytes - ** with the 0x80 bit clear. */ - while( piPrevId = sqlite3_column_int64(pCsr->pStmt, 0); rc = SQLITE_OK; } }else{ - rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor); + rc = fts3EvalNext((Fts3Cursor *)pCursor); } assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } @@ -2562,11 +2666,11 @@ } rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; - rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1); + rc = fts3EvalStart(pCsr); sqlite3Fts3SegmentsClose(p); if( rc!=SQLITE_OK ) return rc; pCsr->pNextId = pCsr->aDoclist; pCsr->iPrevId = 0; @@ -2969,26 +3073,43 @@ p->zDb, p->zName, zName ); return rc; } +/* +** The xSavepoint() method. +** +** Flush the contents of the pending-terms table to disk. +*/ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ UNUSED_PARAMETER(iSavepoint); assert( ((Fts3Table *)pVtab)->inTransaction ); assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); return fts3SyncMethod(pVtab); } + +/* +** The xRelease() method. +** +** This is a no-op. +*/ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); UNUSED_PARAMETER(iSavepoint); UNUSED_PARAMETER(pVtab); assert( p->inTransaction ); assert( p->mxSavepoint >= iSavepoint ); TESTONLY( p->mxSavepoint = iSavepoint-1 ); return SQLITE_OK; } + +/* +** The xRollbackTo() method. +** +** Discard the contents of the pending terms table. +*/ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts3Table *p = (Fts3Table*)pVtab; UNUSED_PARAMETER(iSavepoint); assert( p->inTransaction ); assert( p->mxSavepoint >= iSavepoint ); @@ -3133,22 +3254,10 @@ sqlite3Fts3HashClear(pHash); sqlite3_free(pHash); } return rc; } - -#if !SQLITE_CORE -int sqlite3_extension_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi) - return sqlite3Fts3Init(db); -} -#endif - /* ** Allocate an Fts3MultiSegReader for each token in the expression headed ** by pExpr. ** @@ -3162,24 +3271,24 @@ ** there exists prefix b-tree of the right length) then it may be traversed ** and merged incrementally. Otherwise, it has to be merged into an in-memory ** doclist and then traversed. */ static void fts3EvalAllocateReaders( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Allocate readers for this expression */ int *pnToken, /* OUT: Total number of tokens in phrase. */ int *pnOr, /* OUT: Total number of OR nodes in expr. */ - int *pRc + int *pRc /* IN/OUT: Error code */ ){ if( pExpr && SQLITE_OK==*pRc ){ if( pExpr->eType==FTSQUERY_PHRASE ){ int i; int nToken = pExpr->pPhrase->nToken; *pnToken += nToken; for(i=0; ipPhrase->aToken[i]; - int rc = sqlite3Fts3TermSegReaderCursor(pCsr, + int rc = fts3TermSegReaderCursor(pCsr, pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr ); if( rc!=SQLITE_OK ){ *pRc = rc; return; @@ -3193,16 +3302,24 @@ fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); } } } +/* +** Arguments pList/nList contain the doclist for token iToken of phrase p. +** It is merged into the main doclist stored in p->doclist.aAll/nAll. +** +** This function assumes that pList points to a buffer allocated using +** sqlite3_malloc(). This function takes responsibility for eventually +** freeing the buffer. +*/ static void fts3EvalPhraseMergeToken( - Fts3Table *pTab, - Fts3Phrase *p, - int iToken, - char *pList, - int nList + Fts3Table *pTab, /* FTS Table pointer */ + Fts3Phrase *p, /* Phrase to merge pList/nList into */ + int iToken, /* Token pList/nList corresponds to */ + char *pList, /* Pointer to doclist */ + int nList /* Number of bytes in pList */ ){ assert( iToken!=p->iDoclistToken ); if( pList==0 ){ sqlite3_free(p->doclist.aAll); @@ -3247,13 +3364,19 @@ } if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; } +/* +** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist +** does not take deferred tokens into account. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ static int fts3EvalPhraseLoad( - Fts3Cursor *pCsr, - Fts3Phrase *p + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p /* Phrase object */ ){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int iToken; int rc = SQLITE_OK; @@ -3262,11 +3385,11 @@ assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); if( pToken->pSegcsr ){ int nThis = 0; char *pThis = 0; - rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis); + rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); if( rc==SQLITE_OK ){ fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); } } assert( pToken->pSegcsr==0 ); @@ -3273,18 +3396,26 @@ } return rc; } +/* +** This function is called on each phrase after the position lists for +** any deferred tokens have been loaded into memory. It updates the phrases +** current position list to include only those positions that are really +** instances of the phrase (after considering deferred tokens). If this +** means that the phrase does not appear in the current row, doclist.pList +** and doclist.nList are both zeroed. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ - int iToken; - int rc = SQLITE_OK; - - int nMaxUndeferred = pPhrase->iDoclistToken; - char *aPoslist = 0; - int nPoslist = 0; - int iPrev = -1; + int iToken; /* Used to iterate through phrase tokens */ + int rc = SQLITE_OK; /* Return code */ + char *aPoslist = 0; /* Position list for deferred tokens */ + int nPoslist = 0; /* Number of bytes in aPoslist */ + int iPrev = -1; /* Token number of previous deferred token */ assert( pPhrase->doclist.bFreeList==0 ); for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; @@ -3326,10 +3457,11 @@ iPrev = iToken; } } if( iPrev>=0 ){ + int nMaxUndeferred = pPhrase->iDoclistToken; if( nMaxUndeferred<0 ){ pPhrase->doclist.pList = aPoslist; pPhrase->doclist.nList = nPoslist; pPhrase->doclist.iDocid = pCsr->iPrevId; pPhrase->doclist.bFreeList = 1; @@ -3374,13 +3506,19 @@ /* ** This function is called for each Fts3Phrase in a full-text query ** expression to initialize the mechanism for returning rows. Once this ** function has been called successfully on an Fts3Phrase, it may be ** used with fts3EvalPhraseNext() to iterate through the matching docids. +** +** If parameter bOptOk is true, then the phrase may (or may not) use the +** incremental loading strategy. Otherwise, the entire doclist is loaded into +** memory within this call. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. */ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ - int rc; + int rc; /* Error code */ Fts3PhraseToken *pFirst = &p->aToken[0]; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; if( pCsr->bDesc==pTab->bDescIdx && bOptOk==1 @@ -3404,11 +3542,17 @@ return rc; } /* ** This function is used to iterate backwards (from the end to start) -** through doclists. +** through doclists. It is used by this module to iterate through phrase +** doclists in reverse and by the fts3_write.c module to iterate through +** pending-terms lists when writing to databases with "order=desc". +** +** The doclist may be sorted in ascending (parameter bDescIdx==0) or +** descending (parameter bDescIdx==1) order of docid. Regardless, this +** function iterates from the end of the doclist to the beginning. */ void sqlite3Fts3DoclistPrev( int bDescIdx, /* True if the doclist is desc */ char *aDoclist, /* Pointer to entire doclist */ int nDoclist, /* Length of aDoclist in bytes */ @@ -3469,13 +3613,13 @@ ** If there is no "next" entry and no error occurs, then *pbEof is set to ** 1 before returning. Otherwise, if no error occurs and the iterator is ** successfully advanced, *pbEof is set to 0. */ static int fts3EvalPhraseNext( - Fts3Cursor *pCsr, - Fts3Phrase *p, - u8 *pbEof + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ ){ int rc = SQLITE_OK; Fts3Doclist *pDL = &p->doclist; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; @@ -3517,14 +3661,14 @@ fts3PoslistCopy(0, &pIter); pDL->nList = (pIter - pDL->pList); /* pIter now points just past the 0x00 that terminates the position- ** list for document pDL->iDocid. However, if this position-list was - ** edited in place by fts3EvalNearTrim2(), then pIter may not actually + ** edited in place by fts3EvalNearTrim(), then pIter may not actually ** point to the start of the next docid value. The following line deals ** with this case by advancing pIter past the zero-padding added by - ** fts3EvalNearTrim2(). */ + ** fts3EvalNearTrim(). */ while( pIterpNextDocid = pIter; assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); *pbEof = 0; @@ -3532,15 +3676,31 @@ } return rc; } +/* +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, fts3EvalPhraseStart() is called on all phrases within the +** expression. Also the Fts3Expr.bDeferred variable is set to true for any +** expressions for which all descendent tokens are deferred. +** +** If parameter bOptOk is zero, then it is guaranteed that the +** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for +** each phrase in the expression (subject to deferred token processing). +** Or, if bOptOk is non-zero, then one or more tokens within the expression +** may be loaded incrementally, meaning doclist.aAll/nAll is not available. +** +** If an error occurs within this function, *pRc is set to an SQLite error +** code before returning. +*/ static void fts3EvalStartReaders( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - int bOptOk, - int *pRc + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expression to initialize phrases in */ + int bOptOk, /* True to enable incremental loading */ + int *pRc /* IN/OUT: Error code */ ){ if( pExpr && SQLITE_OK==*pRc ){ if( pExpr->eType==FTSQUERY_PHRASE ){ int i; int nToken = pExpr->pPhrase->nToken; @@ -3555,27 +3715,46 @@ pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); } } } +/* +** An array of the following structures is assembled as part of the process +** of selecting tokens to defer before the query starts executing (as part +** of the xFilter() method). There is one element in the array for each +** token in the FTS expression. +** +** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong +** to phrases that are connected only by AND and NEAR operators (not OR or +** NOT). When determining tokens to defer, each AND/NEAR cluster is considered +** separately. The root of a tokens AND/NEAR cluster is stored in +** Fts3TokenAndCost.pRoot. +*/ typedef struct Fts3TokenAndCost Fts3TokenAndCost; struct Fts3TokenAndCost { Fts3Phrase *pPhrase; /* The phrase the token belongs to */ int iToken; /* Position of token in phrase */ Fts3PhraseToken *pToken; /* The token itself */ - Fts3Expr *pRoot; - int nOvfl; + Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ + int nOvfl; /* Number of overflow pages to load doclist */ int iCol; /* The column the token must match */ }; +/* +** This function is used to populate an allocated Fts3TokenAndCost array. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if an error occurs during execution, *pRc is set to an +** SQLite error code. +*/ static void fts3EvalTokenCosts( - Fts3Cursor *pCsr, - Fts3Expr *pRoot, - Fts3Expr *pExpr, - Fts3TokenAndCost **ppTC, - Fts3Expr ***ppOr, - int *pRc + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ + Fts3Expr *pExpr, /* Expression to consider */ + Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ + Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ + int *pRc /* IN/OUT: Error code */ ){ if( *pRc==SQLITE_OK && pExpr ){ if( pExpr->eType==FTSQUERY_PHRASE ){ Fts3Phrase *pPhrase = pExpr->pPhrase; int i; @@ -3603,23 +3782,34 @@ fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); } } } +/* +** Determine the average document (row) size in pages. If successful, +** write this value to *pnPage and return SQLITE_OK. Otherwise, return +** an SQLite error code. +** +** The average document size in pages is calculated by first calculating +** determining the average size in bytes, B. If B is less than the amount +** of data that will fit on a single leaf page of an intkey table in +** this database, then the average docsize is 1. Otherwise, it is 1 plus +** the number of overflow pages consumed by a record B bytes in size. +*/ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ if( pCsr->nRowAvg==0 ){ /* The average document size, which is required to calculate the cost - ** of each doclist, has not yet been determined. Read the required - ** data from the %_stat table to calculate it. - ** - ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 - ** varints, where nCol is the number of columns in the FTS3 table. - ** The first varint is the number of documents currently stored in - ** the table. The following nCol varints contain the total amount of - ** data stored in all rows of each column of the table, from left - ** to right. - */ + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ int rc; Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; sqlite3_stmt *pStmt; sqlite3_int64 nDoc = 0; sqlite3_int64 nByte = 0; @@ -3650,109 +3840,151 @@ *pnPage = pCsr->nRowAvg; return SQLITE_OK; } +/* +** This function is called to select the tokens (if any) that will be +** deferred. The array aTC[] has already been populated when this is +** called. +** +** This function is called once for each AND/NEAR cluster in the +** expression. Each invocation determines which tokens to defer within +** the cluster with root node pRoot. See comments above the definition +** of struct Fts3TokenAndCost for more details. +** +** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() +** called on each token to defer. Otherwise, an SQLite error code is +** returned. +*/ static int fts3EvalSelectDeferred( - Fts3Cursor *pCsr, - Fts3Expr *pRoot, - Fts3TokenAndCost *aTC, - int nTC + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Consider tokens with this root node */ + Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ + int nTC /* Number of entries in aTC[] */ ){ - int nDocSize = 0; - int nDocEst = 0; - int rc = SQLITE_OK; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int ii; + int nDocSize = 0; /* Number of pages per doc loaded */ + int rc = SQLITE_OK; /* Return code */ + int ii; /* Iterator variable for various purposes */ + int nOvfl = 0; /* Total overflow pages used by doclists */ + int nToken = 0; /* Total number of tokens in cluster */ - int nOvfl = 0; - int nTerm = 0; + int nMinEst = 0; /* The minimum count for any phrase so far. */ + int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ + /* Count the tokens in this AND/NEAR cluster. If none of the doclists + ** associated with the tokens spill onto overflow pages, or if there is + ** only 1 token, exit early. No tokens to defer in this case. */ for(ii=0; ii0 ); - for(ii=0; iinOvfl) + /* Iterate through all tokens in this AND/NEAR cluster, in ascending order + ** of the number of overflow pages that will be loaded by the pager layer + ** to retrieve the entire doclist for the token from the full-text index. + ** Load the doclists for tokens that are either: + ** + ** a. The cheapest token in the entire query (i.e. the one visited by the + ** first iteration of this loop), or + ** + ** b. Part of a multi-token phrase. + ** + ** After each token doclist is loaded, merge it with the others from the + ** same phrase and count the number of documents that the merged doclist + ** contains. Set variable "nMinEst" to the smallest number of documents in + ** any phrase doclist for which 1 or more token doclists have been loaded. + ** Let nOther be the number of other phrases for which it is certain that + ** one or more tokens will not be deferred. + ** + ** Then, for each token, defer it if loading the doclist would result in + ** loading N or more overflow pages into memory, where N is computed as: + ** + ** (nMinEst + 4^nOther - 1) / (4^nOther) + */ + for(ii=0; iinOvfl) ){ - pTC = &aTC[jj]; + pTC = &aTC[iTC]; } } assert( pTC ); - /* At this point pTC points to the cheapest remaining token. */ - if( ii==0 ){ - if( pTC->nOvfl ){ - nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; - }else{ + if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ + /* The number of overflow pages to load for this (and therefore all + ** subsequent) tokens is greater than the estimated number of pages + ** that will be loaded if all subsequent tokens are deferred. + */ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + }else{ + nLoad4 = nLoad4*4; + if( ii==0 || pTC->pPhrase->nToken>1 ){ + /* Either this is the cheapest token in the entire query, or it is + ** part of a multi-token phrase. Either way, the entire doclist will + ** (eventually) be loaded into memory. It may as well be now. */ Fts3PhraseToken *pToken = pTC->pToken; int nList = 0; char *pList = 0; - rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList); + rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); assert( rc==SQLITE_OK || pList==0 ); - if( rc==SQLITE_OK ){ - nDocEst = fts3DoclistCountDocids(1, pList, nList); + int nCount; fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); + nCount = fts3DoclistCountDocids( + pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll + ); + if( ii==0 || nCountnOvfl>=(nDocEst*nDocSize) ){ - Fts3PhraseToken *pToken = pTC->pToken; - rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); - fts3SegReaderCursorFree(pToken->pSegcsr); - pToken->pSegcsr = 0; - } - nDocEst = 1 + (nDocEst/4); } pTC->pToken = 0; } return rc; } -int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ +/* +** This function is called from within the xFilter method. It initializes +** the full-text query currently stored in pCsr->pExpr. To iterate through +** the results of a query, the caller does: +** +** fts3EvalStart(pCsr); +** while( 1 ){ +** fts3EvalNext(pCsr); +** if( pCsr->bEof ) break; +** ... return row pCsr->iPrevId to the caller ... +** } +*/ +static int fts3EvalStart(Fts3Cursor *pCsr){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int rc = SQLITE_OK; int nToken = 0; int nOr = 0; /* Allocate a MultiSegReader for each token in the expression. */ - fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc); + fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); - /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO: - ** This call will eventually also be responsible for determining which - ** tokens are 'deferred' until the document text is loaded into memory. - ** - ** Each token in each phrase is dealt with using one of the following - ** three strategies: - ** - ** 1. Entire doclist loaded into memory as part of the - ** fts3EvalStartReaders() call. - ** - ** 2. Doclist loaded into memory incrementally, as part of each - ** sqlite3Fts3EvalNext() call. - ** - ** 3. Token doclist is never loaded. Instead, documents are loaded into - ** memory and scanned for the token as part of the sqlite3Fts3EvalNext() - ** call. This is known as a "deferred" token. - */ - - /* If bOptOk is true, check if there are any tokens that should be deferred. - */ - if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){ + /* Determine which, if any, tokens in the expression should be deferred. */ + if( rc==SQLITE_OK && nToken>1 && pTab->bHasStat ){ Fts3TokenAndCost *aTC; Fts3Expr **apOr; aTC = (Fts3TokenAndCost *)sqlite3_malloc( sizeof(Fts3TokenAndCost) * nToken + sizeof(Fts3Expr *) * nOr * 2 @@ -3764,11 +3996,11 @@ }else{ int ii; Fts3TokenAndCost *pTC = aTC; Fts3Expr **ppOr = apOr; - fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc); + fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); nToken = pTC-aTC; nOr = ppOr-apOr; if( rc==SQLITE_OK ){ rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); @@ -3779,25 +4011,50 @@ sqlite3_free(aTC); } } - fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc); + fts3EvalStartReaders(pCsr, pCsr->pExpr, 1, &rc); return rc; } -static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){ +/* +** Invalidate the current position list for phrase pPhrase. +*/ +static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ if( pPhrase->doclist.bFreeList ){ sqlite3_free(pPhrase->doclist.pList); } pPhrase->doclist.pList = 0; pPhrase->doclist.nList = 0; pPhrase->doclist.bFreeList = 0; } -static int fts3EvalNearTrim2( - int nNear, +/* +** This function is called to edit the position list associated with +** the phrase object passed as the fifth argument according to a NEAR +** condition. For example: +** +** abc NEAR/5 "def ghi" +** +** Parameter nNear is passed the NEAR distance of the expression (5 in +** the example above). When this function is called, *paPoslist points to +** the position list, and *pnToken is the number of phrase tokens in, the +** phrase on the other side of the NEAR operator to pPhrase. For example, +** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to +** the position list associated with phrase "abc". +** +** All positions in the pPhrase position list that are not sufficiently +** close to a position in the *paPoslist position list are removed. If this +** leaves 0 positions, zero is returned. Otherwise, non-zero. +** +** Before returning, *paPoslist is set to point to the position lsit +** associated with pPhrase. And *pnToken is set to the number of tokens in +** pPhrase. +*/ +static int fts3EvalNearTrim( + int nNear, /* NEAR distance. As in "NEAR/nNear". */ char *aTmp, /* Temporary space to use */ char **paPoslist, /* IN/OUT: Position list */ int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ ){ @@ -3825,10 +4082,176 @@ } return res; } +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is called. +** Otherwise, it advances the expression passed as the second argument to +** point to the next matching row in the database. Expressions iterate through +** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, +** or descending if it is non-zero. +** +** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if +** successful, the following variables in pExpr are set: +** +** Fts3Expr.bEof (non-zero if EOF - there is no next row) +** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) +** +** If the expression is of type FTSQUERY_PHRASE, and the expression is not +** at EOF, then the following variables are populated with the position list +** for the phrase for the visited row: +** +** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) +** FTs3Expr.pPhrase->doclist.pList (pointer to position list) +** +** It says above that this function advances the expression to the next +** matching row. This is usually true, but there are the following exceptions: +** +** 1. Deferred tokens are not taken into account. If a phrase consists +** entirely of deferred tokens, it is assumed to match every row in +** the db. In this case the position-list is not populated at all. +** +** Or, if a phrase contains one or more deferred tokens and one or +** more non-deferred tokens, then the expression is advanced to the +** next possible match, considering only non-deferred tokens. In other +** words, if the phrase is "A B C", and "B" is deferred, the expression +** is advanced to the next row that contains an instance of "A * C", +** where "*" may match any single token. The position list in this case +** is populated as for "A * C" before returning. +** +** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is +** advanced to point to the next row that matches "x AND y". +** +** See fts3EvalTestDeferredAndNear() for details on testing if a row is +** really a match, taking into account deferred tokens and NEAR operators. +*/ +static void fts3EvalNextRow( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expr. to advance to next matching row */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ + assert( pExpr->bEof==0 ); + pExpr->bStart = 1; + + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + assert( !pLeft->bDeferred || !pRight->bDeferred ); + + if( pLeft->bDeferred ){ + /* LHS is entirely deferred. So we assume it matches every row. + ** Advance the RHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + /* RHS is entirely deferred. So we assume it matches every row. + ** Advance the LHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + /* Neither the RHS or LHS are deferred. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else{ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ + fts3EvalNextRow(pCsr, pRight, pRc); + }else{ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + } + + pExpr->bEof = (pLeft->bEof && pRight->bEof); + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } + + break; + } + + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + if( pRight->bStart==0 ){ + fts3EvalNextRow(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } + + fts3EvalNextRow(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; + } + + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalInvalidatePoslist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; + break; + } + } + } +} + +/* +** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR +** cluster, then this function returns 1 immediately. +** +** Otherwise, it checks if the current row really does match the NEAR +** expression, using the data currently stored in the position lists +** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. +** +** If the current row is a match, the position list associated with each +** phrase in the NEAR expression is edited in place to contain only those +** phrase instances sufficiently close to their peers to satisfy all NEAR +** constraints. In this case it returns 1. If the NEAR expression does not +** match the current row, 0 is returned. The position lists may or may not +** be edited if 0 is returned. +*/ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ int res = 1; /* The following block runs if pExpr is the root of a NEAR query. ** For example, the query: @@ -3846,11 +4269,11 @@ ** | | ** "w" "x" ** ** The right-hand child of a NEAR node is always a phrase. The ** left-hand child may be either a phrase or a NEAR node. There are - ** no exceptions to this. + ** no exceptions to this - it's the way the parser in fts3_expr.c works. */ if( *pRc==SQLITE_OK && pExpr->eType==FTSQUERY_NEAR && pExpr->bEof==0 && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) @@ -3873,21 +4296,21 @@ int nToken = p->pPhrase->nToken; for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ Fts3Phrase *pPhrase = p->pRight->pPhrase; int nNear = p->nNear; - res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); } aPoslist = pExpr->pRight->pPhrase->doclist.pList; nToken = pExpr->pRight->pPhrase->nToken; for(p=pExpr->pLeft; p && res; p=p->pLeft){ int nNear = p->pParent->nNear; Fts3Phrase *pPhrase = ( p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase ); - res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); } } sqlite3_free(aTmp); } @@ -3894,132 +4317,33 @@ return res; } /* -** This macro is used by the fts3EvalNext() function. The two arguments are -** 64-bit docid values. If the current query is "ORDER BY docid ASC", then -** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then -** it returns (i2 - i1). This allows the same code to be used for merging -** doclists in ascending or descending order. +** This function is a helper function for fts3EvalTestDeferredAndNear(). +** Assuming no error occurs or has occurred, It returns non-zero if the +** expression passed as the second argument matches the row that pCsr +** currently points to, or zero if it does not. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** If an error occurs during execution of this function, *pRc is set to +** the appropriate SQLite error code. In this case the returned value is +** undefined. */ -#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2)) - -static void fts3EvalNext( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - int *pRc +static int fts3EvalTestExpr( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ + int *pRc /* IN/OUT: Error code */ ){ - if( *pRc==SQLITE_OK ){ - assert( pExpr->bEof==0 ); - pExpr->bStart = 1; - - switch( pExpr->eType ){ - case FTSQUERY_NEAR: - case FTSQUERY_AND: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - assert( !pLeft->bDeferred || !pRight->bDeferred ); - if( pLeft->bDeferred ){ - fts3EvalNext(pCsr, pRight, pRc); - pExpr->iDocid = pRight->iDocid; - pExpr->bEof = pRight->bEof; - }else if( pRight->bDeferred ){ - fts3EvalNext(pCsr, pLeft, pRc); - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = pLeft->bEof; - }else{ - fts3EvalNext(pCsr, pLeft, pRc); - fts3EvalNext(pCsr, pRight, pRc); - - while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ - sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - if( iDiff==0 ) break; - if( iDiff<0 ){ - fts3EvalNext(pCsr, pLeft, pRc); - }else{ - fts3EvalNext(pCsr, pRight, pRc); - } - } - - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = (pLeft->bEof || pRight->bEof); - } - break; - } - - case FTSQUERY_OR: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - - assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); - assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); - - if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - fts3EvalNext(pCsr, pLeft, pRc); - }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ - fts3EvalNext(pCsr, pRight, pRc); - }else{ - fts3EvalNext(pCsr, pLeft, pRc); - fts3EvalNext(pCsr, pRight, pRc); - } - - pExpr->bEof = (pLeft->bEof && pRight->bEof); - iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - pExpr->iDocid = pLeft->iDocid; - }else{ - pExpr->iDocid = pRight->iDocid; - } - - break; - } - - case FTSQUERY_NOT: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - - if( pRight->bStart==0 ){ - fts3EvalNext(pCsr, pRight, pRc); - assert( *pRc!=SQLITE_OK || pRight->bStart ); - } - - fts3EvalNext(pCsr, pLeft, pRc); - if( pLeft->bEof==0 ){ - while( !*pRc - && !pRight->bEof - && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 - ){ - fts3EvalNext(pCsr, pRight, pRc); - } - } - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = pLeft->bEof; - break; - } - - default: { - Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3EvalZeroPoslist(pPhrase); - *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); - pExpr->iDocid = pPhrase->doclist.iDocid; - break; - } - } - } -} - -static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ - int bHit = 1; + int bHit = 1; /* Return value */ if( *pRc==SQLITE_OK ){ switch( pExpr->eType ){ case FTSQUERY_NEAR: case FTSQUERY_AND: bHit = ( - fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) - && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) && fts3EvalNearTest(pExpr, pRc) ); /* If the NEAR expression does not match any rows, zero the doclist for ** all phrases involved in the NEAR. This is because the snippet(), @@ -4041,31 +4365,31 @@ && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) ){ Fts3Expr *p; for(p=pExpr; p->pPhrase==0; p=p->pLeft){ if( p->pRight->iDocid==pCsr->iPrevId ){ - fts3EvalZeroPoslist(p->pRight->pPhrase); + fts3EvalInvalidatePoslist(p->pRight->pPhrase); } } if( p->iDocid==pCsr->iPrevId ){ - fts3EvalZeroPoslist(p->pPhrase); + fts3EvalInvalidatePoslist(p->pPhrase); } } break; case FTSQUERY_OR: { - int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc); - int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc); + int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); bHit = bHit1 || bHit2; break; } case FTSQUERY_NOT: bHit = ( - fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) - && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) ); break; default: { if( pCsr->pDeferred @@ -4072,11 +4396,11 @@ && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) ){ Fts3Phrase *pPhrase = pExpr->pPhrase; assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); if( pExpr->bDeferred ){ - fts3EvalZeroPoslist(pPhrase); + fts3EvalInvalidatePoslist(pPhrase); } *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); bHit = (pPhrase->doclist.pList!=0); pExpr->iDocid = pCsr->iPrevId; }else{ @@ -4088,31 +4412,53 @@ } return bHit; } /* -** Return 1 if both of the following are true: +** This function is called as the second part of each xNext operation when +** iterating through the results of a full-text query. At this point the +** cursor points to a row that matches the query expression, with the +** following caveats: +** +** * Up until this point, "NEAR" operators in the expression have been +** treated as "AND". +** +** * Deferred tokens have not yet been considered. +** +** If *pRc is not SQLITE_OK when this function is called, it immediately +** returns 0. Otherwise, it tests whether or not after considering NEAR +** operators and deferred tokens the current row is still a match for the +** expression. It returns 1 if both of the following are true: ** ** 1. *pRc is SQLITE_OK when this function returns, and ** ** 2. After scanning the current FTS table row for the deferred tokens, -** it is determined that the row does not match the query. +** it is determined that the row does *not* match the query. ** ** Or, if no error occurs and it seems the current row does match the FTS ** query, return 0. */ -static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ +static int fts3EvalTestDeferredAndNear(Fts3Cursor *pCsr, int *pRc){ int rc = *pRc; int bMiss = 0; if( rc==SQLITE_OK ){ + + /* If there are one or more deferred tokens, load the current row into + ** memory and scan it to determine the position list for each deferred + ** token. Then, see if this row is really a match, considering deferred + ** tokens and NEAR operators (neither of which were taken into account + ** earlier, by fts3EvalNextRow()). + */ if( pCsr->pDeferred ){ rc = fts3CursorSeek(0, pCsr); if( rc==SQLITE_OK ){ rc = sqlite3Fts3CacheDeferredDoclists(pCsr); } } - bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc)); + bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); + + /* Free the position-lists accumulated for each deferred token above. */ sqlite3Fts3FreeDeferredDoclists(pCsr); *pRc = rc; } return (rc==SQLITE_OK && bMiss); } @@ -4119,11 +4465,11 @@ /* ** Advance to the next document that matches the FTS expression in ** Fts3Cursor.pExpr. */ -int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){ +static int fts3EvalNext(Fts3Cursor *pCsr){ int rc = SQLITE_OK; /* Return Code */ Fts3Expr *pExpr = pCsr->pExpr; assert( pCsr->isEof==0 ); if( pExpr==0 ){ pCsr->isEof = 1; @@ -4131,23 +4477,23 @@ do { if( pCsr->isRequireSeek==0 ){ sqlite3_reset(pCsr->pStmt); } assert( sqlite3_data_count(pCsr->pStmt)==0 ); - fts3EvalNext(pCsr, pExpr, &rc); + fts3EvalNextRow(pCsr, pExpr, &rc); pCsr->isEof = pExpr->bEof; pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; pCsr->iPrevId = pExpr->iDocid; - }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) ); + }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) ); } return rc; } /* ** Restart interation for expression pExpr so that the next call to -** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental +** fts3EvalNext() visits the first row. Do not allow incremental ** loading or merging of phrase doclists for this iteration. ** ** If *pRc is other than SQLITE_OK when this function is called, it is ** a no-op. If an error occurs within this function, *pRc is set to an ** SQLite error code before returning. @@ -4159,11 +4505,11 @@ ){ if( pExpr && *pRc==SQLITE_OK ){ Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase ){ - fts3EvalZeroPoslist(pPhrase); + fts3EvalInvalidatePoslist(pPhrase); if( pPhrase->bIncr ){ assert( pPhrase->nToken==1 ); assert( pPhrase->aToken[0].pSegcsr ); sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr); *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); @@ -4275,18 +4621,18 @@ /* Ensure the %_content statement is reset. */ if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); assert( sqlite3_data_count(pCsr->pStmt)==0 ); /* Advance to the next document */ - fts3EvalNext(pCsr, pRoot, &rc); + fts3EvalNextRow(pCsr, pRoot, &rc); pCsr->isEof = pRoot->bEof; pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; pCsr->iPrevId = pRoot->iDocid; }while( pCsr->isEof==0 && pRoot->eType==FTSQUERY_NEAR - && fts3EvalLoadDeferred(pCsr, &rc) + && fts3EvalTestDeferredAndNear(pCsr, &rc) ); if( rc==SQLITE_OK && pCsr->isEof==0 ){ fts3EvalUpdateCounts(pRoot); } @@ -4304,14 +4650,14 @@ ** ** do {...} while( pRoot->iDocidbEof==0 ); }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); - fts3EvalLoadDeferred(pCsr, &rc); + fts3EvalTestDeferredAndNear(pCsr, &rc); } } return rc; } @@ -4438,15 +4784,29 @@ */ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ if( pPhrase ){ int i; sqlite3_free(pPhrase->doclist.aAll); - fts3EvalZeroPoslist(pPhrase); + fts3EvalInvalidatePoslist(pPhrase); memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); for(i=0; inToken; i++){ fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); pPhrase->aToken[i].pSegcsr = 0; } } } + +#if !SQLITE_CORE +/* +** Initialize API pointer table, if required. +*/ +int sqlite3_extension_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3Fts3Init(db); +} +#endif #endif Index: ext/fts3/fts3Int.h ================================================================== --- ext/fts3/fts3Int.h +++ ext/fts3/fts3Int.h @@ -25,11 +25,18 @@ */ #if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) # define SQLITE_ENABLE_FTS3 #endif -#ifdef SQLITE_ENABLE_FTS3 +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* If not building as part of the core, include sqlite3ext.h. */ +#ifndef SQLITE_CORE +# include "sqlite3ext.h" +extern const sqlite3_api_routines *sqlite3_api; +#endif + #include "sqlite3.h" #include "fts3_tokenizer.h" #include "fts3_hash.h" /* @@ -288,11 +295,11 @@ sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ int bFreeList; /* True if pList should be sqlite3_free()d */ char *pList; /* Pointer to position list following iDocid */ int nList; /* Length of position list */ -} doclist; +}; /* ** A "phrase" is a sequence of one or more tokens that must match in ** sequence. A single token is the base case and the most common case. ** For a sequence of tokens contained in double-quotes (i.e. "one two three") @@ -488,23 +495,12 @@ #endif /* fts3_aux.c */ int sqlite3Fts3InitAux(sqlite3 *db); -int sqlite3Fts3TermSegReaderCursor( - Fts3Cursor *pCsr, /* Virtual table cursor handle */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ - Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ -); - void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); -int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int); -int sqlite3Fts3EvalNext(Fts3Cursor *pCsr); - int sqlite3Fts3MsrIncrStart( Fts3Table*, Fts3MultiSegReader*, int, const char*, int); int sqlite3Fts3MsrIncrNext( Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); @@ -511,7 +507,7 @@ int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); -#endif /* SQLITE_ENABLE_FTS3 */ +#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ #endif /* _FTSINT_H */ Index: ext/fts3/fts3_hash.c ================================================================== --- ext/fts3/fts3_hash.c +++ ext/fts3/fts3_hash.c @@ -28,11 +28,10 @@ #include #include #include -#include "sqlite3.h" #include "fts3_hash.h" /* ** Malloc and Free functions */ Index: ext/fts3/fts3_test.c ================================================================== --- ext/fts3/fts3_test.c +++ ext/fts3/fts3_test.c @@ -16,10 +16,12 @@ */ #include #include #include + +#ifdef SQLITE_TEST /* Required so that the "ifdef SQLITE_ENABLE_FTS3" below works */ #include "fts3Int.h" #define NM_MAX_TOKEN 12 @@ -317,5 +319,6 @@ Tcl_CreateObjCommand(interp, "fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0 ); return TCL_OK; } +#endif /* ifdef SQLITE_TEST */ Index: ext/fts3/fts3_tokenizer.c ================================================================== --- ext/fts3/fts3_tokenizer.c +++ ext/fts3/fts3_tokenizer.c @@ -21,16 +21,11 @@ ** (in which case SQLITE_CORE is not defined), or ** ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#include "sqlite3ext.h" -#ifndef SQLITE_CORE - SQLITE_EXTENSION_INIT1 -#endif #include "fts3Int.h" - #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include #include Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -383,10 +383,13 @@ tclsh $(TOP)/tool/mksqlite3c.tcl echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c cat sqlite3.c >>tclsqlite3.c echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c + +sqlite3-all.c: sqlite3.c $(TOP)/tool/split-sqlite3c.tcl + tclsh $(TOP)/tool/split-sqlite3c.tcl fts2amal.c: target_source $(TOP)/ext/fts2/mkfts2amal.tcl tclsh $(TOP)/ext/fts2/mkfts2amal.tcl fts3amal.c: target_source $(TOP)/ext/fts3/mkfts3amal.tcl @@ -565,10 +568,17 @@ $(TEST_EXTENSION): $(TOP)/src/test_loadext.c $(MKSHLIB) $(TOP)/src/test_loadext.c -o $(TEST_EXTENSION) extensiontest: testfixture$(EXE) $(TEST_EXTENSION) ./testfixture$(EXE) $(TOP)/test/loadext.test + +# This target will fail if the SQLite amalgamation contains any exported +# symbols that do not begin with "sqlite3_". It is run as part of the +# releasetest.tcl script. +# +checksymbols: sqlite3.o + nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0 # Standard install and cleanup targets # install: sqlite3 libsqlite3.a sqlite3.h Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -2789,11 +2789,11 @@ if( pStart ){ assert( pEnd!=0 ); /* A named index with an explicit CREATE INDEX statement */ zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s", onError==OE_None ? "" : " UNIQUE", - pEnd->z - pName->z + 1, + (int)(pEnd->z - pName->z) + 1, pName->z); }else{ /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ /* zStmt = sqlite3MPrintf(""); */ zStmt = 0; Index: src/date.c ================================================================== --- src/date.c +++ src/date.c @@ -425,11 +425,13 @@ static int osLocaltime(time_t *t, struct tm *pTm){ int rc; #if (!defined(HAVE_LOCALTIME_R) || !HAVE_LOCALTIME_R) \ && (!defined(HAVE_LOCALTIME_S) || !HAVE_LOCALTIME_S) struct tm *pX; +#if SQLITE_THREADSAFE>0 sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif sqlite3_mutex_enter(mutex); pX = localtime(t); #ifndef SQLITE_OMIT_BUILTIN_TEST if( sqlite3GlobalConfig.bLocaltimeFault ) pX = 0; #endif Index: src/delete.c ================================================================== --- src/delete.c +++ src/delete.c @@ -376,11 +376,13 @@ int regRowid; /* Actual register containing rowids */ /* Collect rowids of every row to be deleted. */ sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,WHERE_DUPLICATES_OK); + pWInfo = sqlite3WhereBegin( + pParse, pTabList, pWhere, 0, 0, WHERE_DUPLICATES_OK + ); if( pWInfo==0 ) goto delete_from_cleanup; regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid); sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid); if( db->flags & SQLITE_CountRows ){ sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -900,10 +900,11 @@ pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); pNewItem->jointype = pOldItem->jointype; pNewItem->iCursor = pOldItem->iCursor; pNewItem->isPopulated = pOldItem->isPopulated; + pNewItem->isCorrelated = pOldItem->isCorrelated; pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); pNewItem->notIndexed = pOldItem->notIndexed; pNewItem->pIndex = pOldItem->pIndex; pTab = pNewItem->pTab = pOldItem->pTab; if( pTab ){ Index: src/fkey.c ================================================================== --- src/fkey.c +++ src/fkey.c @@ -558,11 +558,11 @@ /* Create VDBE to loop through the entries in pSrc that match the WHERE ** clause. If the constraint is not deferred, throw an exception for ** each row found. Otherwise, for deferred constraints, increment the ** deferred constraint counter by nIncr for each row selected. */ - pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0); + pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0); if( nIncr>0 && pFKey->isDeferred==0 ){ sqlite3ParseToplevel(pParse)->mayAbort = 1; } sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); if( pWInfo ){ Index: src/loadext.c ================================================================== --- src/loadext.c +++ src/loadext.c @@ -82,10 +82,12 @@ #ifdef SQLITE_OMIT_VIRTUALTABLE # define sqlite3_create_module 0 # define sqlite3_create_module_v2 0 # define sqlite3_declare_vtab 0 +# define sqlite3_vtab_config 0 +# define sqlite3_vtab_on_conflict 0 #endif #ifdef SQLITE_OMIT_SHARED_CACHE # define sqlite3_enable_shared_cache 0 #endif @@ -105,10 +107,11 @@ #define sqlite3_blob_bytes 0 #define sqlite3_blob_close 0 #define sqlite3_blob_open 0 #define sqlite3_blob_read 0 #define sqlite3_blob_write 0 +#define sqlite3_blob_reopen 0 #endif /* ** The following structure contains pointers to all SQLite API routines. ** A pointer to this structure is passed into extensions when they are @@ -370,10 +373,13 @@ #else 0, 0, 0, #endif + sqlite3_blob_reopen, + sqlite3_vtab_config, + sqlite3_vtab_on_conflict, }; /* ** Attempt to load an SQLite extension library contained in the file ** zFile. The entry point is zProc. zProc may be 0 in which case a Index: src/mem3.c ================================================================== --- src/mem3.c +++ src/mem3.c @@ -431,11 +431,11 @@ ** Free an outstanding memory allocation. ** ** This function assumes that the necessary mutexes, if any, are ** already held by the caller. Hence "Unsafe". */ -void memsys3FreeUnsafe(void *pOld){ +static void memsys3FreeUnsafe(void *pOld){ Mem3Block *p = (Mem3Block*)pOld; int i; u32 size, x; assert( sqlite3_mutex_held(mem3.mutex) ); assert( p>mem3.aPool && p<&mem3.aPool[mem3.nPool] ); @@ -506,21 +506,21 @@ } /* ** Free memory. */ -void memsys3Free(void *pPrior){ +static void memsys3Free(void *pPrior){ assert( pPrior ); memsys3Enter(); memsys3FreeUnsafe(pPrior); memsys3Leave(); } /* ** Change the size of an existing memory allocation */ -void *memsys3Realloc(void *pPrior, int nBytes){ +static void *memsys3Realloc(void *pPrior, int nBytes){ int nOld; void *p; if( pPrior==0 ){ return sqlite3_malloc(nBytes); } Index: src/os.c ================================================================== --- src/os.c +++ src/os.c @@ -134,11 +134,11 @@ DO_OS_MALLOC_TEST(0); /* 0x87f3f is a mask of SQLITE_OPEN_ flags that are valid to be passed ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before ** reaching the VFS. */ - rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x87f3f, pFlagsOut); + rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x87f7f, pFlagsOut); assert( rc==SQLITE_OK || pFile->pMethods==0 ); return rc; } int sqlite3OsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ return pVfs->xDelete(pVfs, zPath, dirSync); Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -675,11 +675,13 @@ case EINVAL: case ENOTCONN: case ENODEV: case ENXIO: case ENOENT: +#ifdef ESTALE /* ESTALE is not defined on Interix systems */ case ESTALE: +#endif case ENOSYS: /* these should force the client to close the file and reconnect */ default: return sqliteIOErr; @@ -3670,11 +3672,11 @@ unixShmNode *p = pFd->pInode->pShmNode; assert( unixMutexHeld() ); if( p && p->nRef==0 ){ int i; assert( p->pInode==pFd->pInode ); - if( p->mutex ) sqlite3_mutex_free(p->mutex); + sqlite3_mutex_free(p->mutex); for(i=0; inRegion; i++){ if( p->h>=0 ){ munmap(p->apRegion[i], p->szRegion); }else{ sqlite3_free(p->apRegion[i]); Index: src/os_win.c ================================================================== --- src/os_win.c +++ src/os_win.c @@ -399,10 +399,58 @@ iLine, iErrno, zFunc, zPath, zMsg ); return errcode; } + +/* +** The number of times that a ReadFile(), WriteFile(), and DeleteFile() +** will be retried following a locking error - probably caused by +** antivirus software. Also the initial delay before the first retry. +** The delay increases linearly with each retry. +*/ +#ifndef SQLITE_WIN32_IOERR_RETRY +# define SQLITE_WIN32_IOERR_RETRY 10 +#endif +#ifndef SQLITE_WIN32_IOERR_RETRY_DELAY +# define SQLITE_WIN32_IOERR_RETRY_DELAY 25 +#endif +static int win32IoerrRetry = SQLITE_WIN32_IOERR_RETRY; +static int win32IoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; + +/* +** If a ReadFile() or WriteFile() error occurs, invoke this routine +** to see if it should be retried. Return TRUE to retry. Return FALSE +** to give up with an error. +*/ +static int retryIoerr(int *pnRetry){ + DWORD e; + if( *pnRetry>=win32IoerrRetry ){ + return 0; + } + e = GetLastError(); + if( e==ERROR_ACCESS_DENIED || + e==ERROR_LOCK_VIOLATION || + e==ERROR_SHARING_VIOLATION ){ + Sleep(win32IoerrRetryDelay*(1+*pnRetry)); + ++*pnRetry; + return 1; + } + return 0; +} + +/* +** Log a I/O error retry episode. +*/ +static void logIoerr(int nRetry){ + if( nRetry ){ + sqlite3_log(SQLITE_IOERR, + "delayed %dms for lock/sharing conflict", + win32IoerrRetryDelay*nRetry*(nRetry+1)/2 + ); + } +} #if SQLITE_OS_WINCE /************************************************************************* ** This section contains code for WinCE only. */ @@ -818,22 +866,25 @@ int amt, /* Number of bytes to read */ sqlite3_int64 offset /* Begin reading at this offset */ ){ winFile *pFile = (winFile*)id; /* file handle */ DWORD nRead; /* Number of bytes actually read from file */ + int nRetry = 0; /* Number of retrys */ assert( id!=0 ); SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); if( seekWinFile(pFile, offset) ){ return SQLITE_FULL; } - if( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ + while( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ + if( retryIoerr(&nRetry) ) continue; pFile->lastErrno = GetLastError(); return winLogError(SQLITE_IOERR_READ, "winRead", pFile->zPath); } + logIoerr(nRetry); if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ memset(&((char*)pBuf)[nRead], 0, amt-nRead); return SQLITE_IOERR_SHORT_READ; } @@ -851,10 +902,11 @@ int amt, /* Number of bytes to write */ sqlite3_int64 offset /* Offset into the file to begin writing at */ ){ int rc; /* True if error has occured, else false */ winFile *pFile = (winFile*)id; /* File handle */ + int nRetry = 0; /* Number of retries */ assert( amt>0 ); assert( pFile ); SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); @@ -865,11 +917,16 @@ if( rc==0 ){ u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ int nRem = amt; /* Number of bytes yet to be written */ DWORD nWrite; /* Bytes written by each WriteFile() call */ - while( nRem>0 && WriteFile(pFile->h, aRem, nRem, &nWrite, 0) && nWrite>0 ){ + while( nRem>0 ){ + if( !WriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ + if( retryIoerr(&nRetry) ) continue; + break; + } + if( nWrite<=0 ) break; aRem += nWrite; nRem -= nWrite; } if( nRem>0 ){ pFile->lastErrno = GetLastError(); @@ -881,10 +938,12 @@ if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ) || ( pFile->lastErrno==ERROR_DISK_FULL )){ return SQLITE_FULL; } return winLogError(SQLITE_IOERR_WRITE, "winWrite", pFile->zPath); + }else{ + logIoerr(nRetry); } return SQLITE_OK; } /* @@ -1296,10 +1355,24 @@ SimulateIOErrorBenign(0); return SQLITE_OK; } case SQLITE_FCNTL_SYNC_OMITTED: { return SQLITE_OK; + } + case SQLITE_FCNTL_WIN32_AV_RETRY: { + int *a = (int*)pArg; + if( a[0]>0 ){ + win32IoerrRetry = a[0]; + }else{ + a[0] = win32IoerrRetry; + } + if( a[1]>0 ){ + win32IoerrRetryDelay = a[1]; + }else{ + a[1] = win32IoerrRetryDelay; + } + return SQLITE_OK; } } return SQLITE_NOTFOUND; } @@ -2314,19 +2387,17 @@ ** file open, we will be unable to delete it. To work around this ** problem, we delay 100 milliseconds and try to delete again. Up ** to MX_DELETION_ATTEMPTs deletion attempts are run before giving ** up and returning an error. */ -#define MX_DELETION_ATTEMPTS 5 static int winDelete( sqlite3_vfs *pVfs, /* Not used on win32 */ const char *zFilename, /* Name of file to delete */ int syncDir /* Not used on win32 */ ){ int cnt = 0; - DWORD rc; - DWORD error = 0; + int rc; void *zConverted; UNUSED_PARAMETER(pVfs); UNUSED_PARAMETER(syncDir); SimulateIOError(return SQLITE_IOERR_DELETE); @@ -2333,38 +2404,34 @@ zConverted = convertUtf8Filename(zFilename); if( zConverted==0 ){ return SQLITE_NOMEM; } if( isNT() ){ - do{ - DeleteFileW(zConverted); - }while( ( ((rc = GetFileAttributesW(zConverted)) != INVALID_FILE_ATTRIBUTES) - || ((error = GetLastError()) == ERROR_ACCESS_DENIED)) - && (++cnt < MX_DELETION_ATTEMPTS) - && (Sleep(100), 1) ); + rc = 1; + while( GetFileAttributesW(zConverted)!=INVALID_FILE_ATTRIBUTES && + (rc = DeleteFileW(zConverted))==0 && retryIoerr(&cnt) ){} + rc = rc ? SQLITE_OK : SQLITE_ERROR; /* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. ** Since the ASCII version of these Windows API do not exist for WINCE, ** it's important to not reference them for WINCE builds. */ #if SQLITE_OS_WINCE==0 }else{ - do{ - DeleteFileA(zConverted); - }while( ( ((rc = GetFileAttributesA(zConverted)) != INVALID_FILE_ATTRIBUTES) - || ((error = GetLastError()) == ERROR_ACCESS_DENIED)) - && (++cnt < MX_DELETION_ATTEMPTS) - && (Sleep(100), 1) ); + rc = 1; + while( GetFileAttributesA(zConverted)!=INVALID_FILE_ATTRIBUTES && + (rc = DeleteFileA(zConverted))==0 && retryIoerr(&cnt) ){} + rc = rc ? SQLITE_OK : SQLITE_ERROR; #endif } + if( rc ){ + rc = winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename); + }else{ + logIoerr(cnt); + } free(zConverted); - OSTRACE(("DELETE \"%s\" %s\n", zFilename, - ( (rc==INVALID_FILE_ATTRIBUTES) && (error==ERROR_FILE_NOT_FOUND)) ? - "ok" : "failed" )); - - return ( (rc == INVALID_FILE_ATTRIBUTES) - && (error == ERROR_FILE_NOT_FOUND)) ? SQLITE_OK : - winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename); + OSTRACE(("DELETE \"%s\" %s\n", zFilename, (rc ? "failed" : "ok" ))); + return rc; } /* ** Check the existance and status of a file. */ Index: src/resolve.c ================================================================== --- src/resolve.c +++ src/resolve.c @@ -994,15 +994,29 @@ /* Recursively resolve names in all subqueries */ for(i=0; ipSrc->nSrc; i++){ struct SrcList_item *pItem = &p->pSrc->a[i]; if( pItem->pSelect ){ + NameContext *pNC; /* Used to iterate name contexts */ + int nRef = 0; /* Refcount for pOuterNC and outer contexts */ const char *zSavedContext = pParse->zAuthContext; + + /* Count the total number of references to pOuterNC and all of its + ** parent contexts. After resolving references to expressions in + ** pItem->pSelect, check if this value has changed. If so, then + ** SELECT statement pItem->pSelect must be correlated. Set the + ** pItem->isCorrelated flag if this is the case. */ + for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef += pNC->nRef; + if( pItem->zName ) pParse->zAuthContext = pItem->zName; sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC); pParse->zAuthContext = zSavedContext; if( pParse->nErr || db->mallocFailed ) return WRC_Abort; + + for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef -= pNC->nRef; + assert( pItem->isCorrelated==0 && nRef<=0 ); + pItem->isCorrelated = (nRef!=0); } } /* If there are no aggregate functions in the result-set, and no GROUP BY ** expression, do not allow aggregates in any of the other expressions. Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -3719,10 +3719,11 @@ Expr *pHaving; /* The HAVING clause. May be NULL */ int isDistinct; /* True if the DISTINCT keyword is present */ int distinct; /* Table to use for the distinct set */ int rc = 1; /* Value to return from this function */ int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */ + int addrDistinctIndex; /* Address of an OP_OpenEphemeral instruction */ AggInfo sAggInfo; /* Information used by aggregate queries */ int iEnd; /* Address of the end of the query */ sqlite3 *db; /* The database connection */ #ifndef SQLITE_OMIT_EXPLAIN @@ -3845,20 +3846,10 @@ explainSetInteger(pParse->iSelectId, iRestoreSelectId); return rc; } #endif - /* If possible, rewrite the query to use GROUP BY instead of DISTINCT. - ** GROUP BY might use an index, DISTINCT never does. - */ - assert( p->pGroupBy==0 || (p->selFlags & SF_Aggregate)!=0 ); - if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ){ - p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); - pGroupBy = p->pGroupBy; - p->selFlags &= ~SF_Distinct; - } - /* If there is both a GROUP BY and an ORDER BY clause and they are ** identical, then disable the ORDER BY clause since the GROUP BY ** will cause elements to come out in the correct order. This is ** an optimization - the correct answer should result regardless. ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER @@ -3866,10 +3857,34 @@ */ if( sqlite3ExprListCompare(p->pGroupBy, pOrderBy)==0 && (db->flags & SQLITE_GroupByOrder)==0 ){ pOrderBy = 0; } + + /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and + ** if the select-list is the same as the ORDER BY list, then this query + ** can be rewritten as a GROUP BY. In other words, this: + ** + ** SELECT DISTINCT xyz FROM ... ORDER BY xyz + ** + ** is transformed to: + ** + ** SELECT xyz FROM ... GROUP BY xyz + ** + ** The second form is preferred as a single index (or temp-table) may be + ** used for both the ORDER BY and DISTINCT processing. As originally + ** written the query must use a temp-table for at least one of the ORDER + ** BY and DISTINCT, and an index or separate temp-table for the other. + */ + if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct + && sqlite3ExprListCompare(pOrderBy, p->pEList)==0 + ){ + p->selFlags &= ~SF_Distinct; + p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); + pGroupBy = p->pGroupBy; + pOrderBy = 0; + } /* If there is an ORDER BY clause, then this sorting ** index might end up being unused if the data can be ** extracted in pre-sorted order. If that is the case, then the ** OP_OpenEphemeral instruction will be changed to an OP_Noop once @@ -3902,26 +3917,25 @@ /* Open a virtual index to use for the distinct set. */ if( p->selFlags & SF_Distinct ){ KeyInfo *pKeyInfo; - assert( isAgg || pGroupBy ); distinct = pParse->nTab++; pKeyInfo = keyInfoFromExprList(pParse, p->pEList); - sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, - (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + addrDistinctIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, + (char*)pKeyInfo, P4_KEYINFO_HANDOFF); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); }else{ - distinct = -1; + distinct = addrDistinctIndex = -1; } /* Aggregate and non-aggregate queries are handled differently */ if( !isAgg && pGroupBy==0 ){ - /* This case is for non-aggregate queries - ** Begin the database scan - */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0); + ExprList *pDist = (isDistinct ? p->pEList : 0); + + /* Begin the database scan. */ + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, pDist, 0); if( pWInfo==0 ) goto select_end; if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut; /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral @@ -3930,14 +3944,56 @@ if( addrSortIndex>=0 && pOrderBy==0 ){ sqlite3VdbeChangeToNoop(v, addrSortIndex, 1); p->addrOpenEphm[2] = -1; } - /* Use the standard inner loop - */ - assert(!isDistinct); - selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, -1, pDest, + if( pWInfo->eDistinct ){ + VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ + + assert( addrDistinctIndex>0 ); + pOp = sqlite3VdbeGetOp(v, addrDistinctIndex); + + assert( isDistinct ); + assert( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED + || pWInfo->eDistinct==WHERE_DISTINCT_UNIQUE + ); + distinct = -1; + if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED ){ + int iJump; + int iExpr; + int iFlag = ++pParse->nMem; + int iBase = pParse->nMem+1; + int iBase2 = iBase + pEList->nExpr; + pParse->nMem += (pEList->nExpr*2); + + /* Change the OP_OpenEphemeral coded earlier to an OP_Integer. The + ** OP_Integer initializes the "first row" flag. */ + pOp->opcode = OP_Integer; + pOp->p1 = 1; + pOp->p2 = iFlag; + + sqlite3ExprCodeExprList(pParse, pEList, iBase, 1); + iJump = sqlite3VdbeCurrentAddr(v) + 1 + pEList->nExpr + 1 + 1; + sqlite3VdbeAddOp2(v, OP_If, iFlag, iJump-1); + for(iExpr=0; iExprnExpr; iExpr++){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[iExpr].pExpr); + sqlite3VdbeAddOp3(v, OP_Ne, iBase+iExpr, iJump, iBase2+iExpr); + sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iContinue); + + sqlite3VdbeAddOp2(v, OP_Integer, 0, iFlag); + assert( sqlite3VdbeCurrentAddr(v)==iJump ); + sqlite3VdbeAddOp3(v, OP_Move, iBase, iBase2, pEList->nExpr); + }else{ + pOp->opcode = OP_Noop; + } + } + + /* Use the standard inner loop. */ + selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, distinct, pDest, pWInfo->iContinue, pWInfo->iBreak); /* End the database scan loop. */ sqlite3WhereEnd(pWInfo); @@ -4043,11 +4099,11 @@ ** This might involve two separate loops with an OP_Sort in between, or ** it might be a single loop that uses an index to extract information ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0, 0); if( pWInfo==0 ) goto select_end; if( pGroupBy==0 ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be ** cancelled later because we still need to use the pKeyInfo @@ -4305,11 +4361,11 @@ /* This case runs if the aggregate has no GROUP BY clause. The ** processing is much simpler since there is only a single row ** of output. */ resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, flag); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, 0, flag); if( pWInfo==0 ){ sqlite3ExprListDelete(db, pDel); goto select_end; } updateAccumulator(pParse, &sAggInfo); Index: src/shell.c ================================================================== --- src/shell.c +++ src/shell.c @@ -2630,10 +2630,13 @@ " -version show SQLite version\n" " -vfs NAME use NAME as the default VFS\n" #ifdef SQLITE_ENABLE_VFSTRACE " -vfstrace enable tracing of all VFS calls\n" #endif +#ifdef SQLITE_ENABLE_MULTIPLEX + " -multiplex enable the multiplexor VFS\n" +#endif ; static void usage(int showDetail){ fprintf(stderr, "Usage: %s [OPTIONS] FILENAME [SQL]\n" "FILENAME is the name of an SQLite database. A new database is created\n" @@ -2731,10 +2734,15 @@ void *pOutArg, int makeDefault ); vfstrace_register("trace",0,(int(*)(const char*,void*))fputs,stderr,1); #endif +#ifdef SQLITE_ENABLE_MULTIPLEX + }else if( strcmp(argv[i],"-multiplex")==0 ){ + extern int sqlite3_multiple_initialize(const char*,int); + sqlite3_multiplex_initialize(0, 1); +#endif }else if( strcmp(argv[i],"-vfs")==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(argv[++i]); if( pVfs ){ sqlite3_vfs_register(pVfs, 1); }else{ @@ -2849,12 +2857,18 @@ stdin_is_interactive = 0; }else if( strcmp(z,"-heap")==0 ){ i++; }else if( strcmp(z,"-vfs")==0 ){ i++; +#ifdef SQLITE_ENABLE_VFSTRACE }else if( strcmp(z,"-vfstrace")==0 ){ i++; +#endif +#ifdef SQLITE_ENABLE_MULTIPLEX + }else if( strcmp(z,"-multiplex")==0 ){ + i++; +#endif }else if( strcmp(z,"-help")==0 || strcmp(z, "--help")==0 ){ usage(1); }else{ fprintf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); fprintf(stderr,"Use -help for a list of options.\n"); Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -734,20 +734,37 @@ ** when [PRAGMA synchronous | PRAGMA synchronous=OFF] is set, but most ** VFSes do not need this signal and should silently ignore this opcode. ** Applications should not call [sqlite3_file_control()] with this ** opcode as doing so may disrupt the operation of the specialized VFSes ** that do require it. +** +** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic +** retry counts and intervals for certain disk I/O operations for the +** windows [VFS] in order to work to provide robustness against +** anti-virus programs. By default, the windows VFS will retry file read, +** file write, and file delete opertions up to 10 times, with a delay +** of 25 milliseconds before the first retry and with the delay increasing +** by an additional 25 milliseconds with each subsequent retry. This +** opcode allows those to values (10 retries and 25 milliseconds of delay) +** to be adjusted. The values are changed for all database connections +** within the same process. The argument is a pointer to an array of two +** integers where the first integer i the new retry count and the second +** integer is the delay. If either integer is negative, then the setting +** is not changed but instead the prior value of that setting is written +** into the array entry, allowing the current retry settings to be +** interrogated. The zDbName parameter is ignored. +** */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_GET_LOCKPROXYFILE 2 #define SQLITE_SET_LOCKPROXYFILE 3 #define SQLITE_LAST_ERRNO 4 #define SQLITE_FCNTL_SIZE_HINT 5 #define SQLITE_FCNTL_CHUNK_SIZE 6 #define SQLITE_FCNTL_FILE_POINTER 7 #define SQLITE_FCNTL_SYNC_OMITTED 8 - +#define SQLITE_FCNTL_WIN32_AV_RETRY 9 /* ** CAPI3REF: Mutex Handle ** ** The mutex module within SQLite defines [sqlite3_mutex] to be an Index: src/sqlite3ext.h ================================================================== --- src/sqlite3ext.h +++ src/sqlite3ext.h @@ -210,10 +210,13 @@ int (*strnicmp)(const char*,const char*,int); int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*); int (*wal_autocheckpoint)(sqlite3*,int); int (*wal_checkpoint)(sqlite3*,const char*); void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*); + int (*blob_reopen)(sqlite3_blob*,sqlite3_int64); + int (*vtab_config)(sqlite3*,int op,...); + int (*vtab_on_conflict)(sqlite3*); }; /* ** The following macros redefine the API routines so that they are ** redirected throught the global sqlite3_api structure. @@ -410,11 +413,14 @@ #define sqlite3_strnicmp sqlite3_api->strnicmp #define sqlite3_unlock_notify sqlite3_api->unlock_notify #define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint #define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint #define sqlite3_wal_hook sqlite3_api->wal_hook +#define sqlite3_blob_reopen sqlite3_api->blob_reopen +#define sqlite3_vtab_config sqlite3_api->vtab_config +#define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict #endif /* SQLITE_CORE */ #define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0; #define SQLITE_EXTENSION_INIT2(v) sqlite3_api = v; #endif /* _SQLITE3EXT_H_ */ Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -962,10 +962,11 @@ #define SQLITE_IndexSearch 0x08 /* Disable indexes for searching */ #define SQLITE_IndexCover 0x10 /* Disable index covering table */ #define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */ #define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */ #define SQLITE_IdxRealAsInt 0x80 /* Store REAL as INT in indices */ +#define SQLITE_DistinctOpt 0x80 /* DISTINCT using indexes */ #define SQLITE_OptMask 0xff /* Mask of all disablable opts */ /* ** Possible values for the sqlite.magic field. ** The numbers are obtained at random and have no special meaning, other @@ -1853,10 +1854,11 @@ Table *pTab; /* An SQL table corresponding to zName */ Select *pSelect; /* A SELECT statement used in place of a table name */ u8 isPopulated; /* Temporary table associated with SELECT is populated */ u8 jointype; /* Type of join between this able and the previous */ u8 notIndexed; /* True if there is a NOT INDEXED clause */ + u8 isCorrelated; /* True if sub-query is correlated */ #ifndef SQLITE_OMIT_EXPLAIN u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ #endif int iCursor; /* The VDBE cursor number used to access this table */ Expr *pOn; /* The ON clause of a join */ @@ -1972,10 +1974,11 @@ struct WhereInfo { Parse *pParse; /* Parsing and code generating context */ u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE or DELETE */ u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ + u8 eDistinct; SrcList *pTabList; /* List of tables in the join */ int iTop; /* The very beginning of the WHERE loop */ int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ int nLevel; /* Number of nested loop */ @@ -1983,10 +1986,13 @@ double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ double nRowOut; /* Estimated number of output rows */ WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; +#define WHERE_DISTINCT_UNIQUE 1 +#define WHERE_DISTINCT_ORDERED 2 + /* ** A NameContext defines a context in which to resolve table and column ** names. The context consists of a list of tables (the pSrcList) field and ** a list of named expression (pEList). The named expression list may ** be NULL. The pSrc corresponds to the FROM clause of a SELECT or @@ -2745,11 +2751,11 @@ #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) Expr *sqlite3LimitWhere(Parse *, SrcList *, Expr *, ExprList *, Expr *, Expr *, char *); #endif void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); -WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**, u16); +WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**,ExprList*,u16); void sqlite3WhereEnd(WhereInfo*); int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int); void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); void sqlite3ExprCodeMove(Parse*, int, int, int); void sqlite3ExprCodeCopy(Parse*, int, int, int); Index: src/tclsqlite.c ================================================================== --- src/tclsqlite.c +++ src/tclsqlite.c @@ -105,10 +105,15 @@ typedef struct IncrblobChannel IncrblobChannel; /* ** There is one instance of this structure for each SQLite database ** that has been opened by the SQLite TCL interface. +** +** If this module is built with SQLITE_TEST defined (to create the SQLite +** testfixture executable), then it may be configured to use either +** sqlite3_prepare_v2() or sqlite3_prepare() to prepare SQL statements. +** If SqliteDb.bLegacyPrepare is true, sqlite3_prepare() is used. */ typedef struct SqliteDb SqliteDb; struct SqliteDb { sqlite3 *db; /* The "real" database structure. MUST BE FIRST */ Tcl_Interp *interp; /* The interpreter used for this database */ @@ -134,10 +139,13 @@ int maxStmt; /* The next maximum number of stmtList */ int nStmt; /* Number of statements in stmtList */ IncrblobChannel *pIncrblob;/* Linked list of open incrblob channels */ int nStep, nSort, nIndex; /* Statistics for most recent operation */ int nTransaction; /* Number of nested [transaction] methods */ +#ifdef SQLITE_TEST + int bLegacyPrepare; /* True to use sqlite3_prepare() */ +#endif }; struct IncrblobChannel { sqlite3_blob *pBlob; /* sqlite3 blob handle */ SqliteDb *pDb; /* Associated database connection */ @@ -427,25 +435,38 @@ pNew->pScript = 0; pNew->pNext = pDb->pFunc; pDb->pFunc = pNew; return pNew; } + +/* +** Free a single SqlPreparedStmt object. +*/ +static void dbFreeStmt(SqlPreparedStmt *pStmt){ +#ifdef SQLITE_TEST + if( sqlite3_sql(pStmt->pStmt)==0 ){ + Tcl_Free((char *)pStmt->zSql); + } +#endif + sqlite3_finalize(pStmt->pStmt); + Tcl_Free((char *)pStmt); +} /* ** Finalize and free a list of prepared statements */ -static void flushStmtCache( SqliteDb *pDb ){ +static void flushStmtCache(SqliteDb *pDb){ SqlPreparedStmt *pPreStmt; + SqlPreparedStmt *pNext; - while( pDb->stmtList ){ - sqlite3_finalize( pDb->stmtList->pStmt ); - pPreStmt = pDb->stmtList; - pDb->stmtList = pDb->stmtList->pNext; - Tcl_Free( (char*)pPreStmt ); + for(pPreStmt = pDb->stmtList; pPreStmt; pPreStmt=pNext){ + pNext = pPreStmt->pNext; + dbFreeStmt(pPreStmt); } pDb->nStmt = 0; pDb->stmtLast = 0; + pDb->stmtList = 0; } /* ** TCL calls this procedure when an sqlite3 database command is ** deleted. @@ -1071,10 +1092,31 @@ } pDb->disableAuth--; return rc; } + +/* +** Unless SQLITE_TEST is defined, this function is a simple wrapper around +** sqlite3_prepare_v2(). If SQLITE_TEST is defined, then it uses either +** sqlite3_prepare_v2() or legacy interface sqlite3_prepare(), depending +** on whether or not the [db_use_legacy_prepare] command has been used to +** configure the connection. +*/ +static int dbPrepare( + SqliteDb *pDb, /* Database object */ + const char *zSql, /* SQL to compile */ + sqlite3_stmt **ppStmt, /* OUT: Prepared statement */ + const char **pzOut /* OUT: Pointer to next SQL statement */ +){ +#ifdef SQLITE_TEST + if( pDb->bLegacyPrepare ){ + return sqlite3_prepare(pDb->db, zSql, -1, ppStmt, pzOut); + } +#endif + return sqlite3_prepare_v2(pDb->db, zSql, -1, ppStmt, pzOut); +} /* ** Search the cache for a prepared-statement object that implements the ** first SQL statement in the buffer pointed to by parameter zIn. If ** no such prepared-statement can be found, allocate and prepare a new @@ -1142,11 +1184,11 @@ /* If no prepared statement was found. Compile the SQL text. Also allocate ** a new SqlPreparedStmt structure. */ if( pPreStmt==0 ){ int nByte; - if( SQLITE_OK!=sqlite3_prepare_v2(pDb->db, zSql, -1, &pStmt, pzOut) ){ + if( SQLITE_OK!=dbPrepare(pDb, zSql, &pStmt, pzOut) ){ Tcl_SetObjResult(interp, dbTextToObj(sqlite3_errmsg(pDb->db))); return TCL_ERROR; } if( pStmt==0 ){ if( SQLITE_OK!=sqlite3_errcode(pDb->db) ){ @@ -1169,10 +1211,18 @@ pPreStmt->pStmt = pStmt; pPreStmt->nSql = (*pzOut - zSql); pPreStmt->zSql = sqlite3_sql(pStmt); pPreStmt->apParm = (Tcl_Obj **)&pPreStmt[1]; +#ifdef SQLITE_TEST + if( pPreStmt->zSql==0 ){ + char *zCopy = Tcl_Alloc(pPreStmt->nSql + 1); + memcpy(zCopy, zSql, pPreStmt->nSql); + zCopy[pPreStmt->nSql] = '\0'; + pPreStmt->zSql = zCopy; + } +#endif } assert( pPreStmt ); assert( strlen30(pPreStmt->zSql)==pPreStmt->nSql ); assert( 0==memcmp(pPreStmt->zSql, zSql, pPreStmt->nSql) ); @@ -1222,11 +1272,10 @@ *ppPreStmt = pPreStmt; return TCL_OK; } - /* ** Release a statement reference obtained by calling dbPrepareAndBind(). ** There should be exactly one call to this function for each call to ** dbPrepareAndBind(). ** @@ -1247,12 +1296,11 @@ } pPreStmt->nParm = 0; if( pDb->maxStmt<=0 || discard ){ /* If the cache is turned off, deallocated the statement */ - sqlite3_finalize(pPreStmt->pStmt); - Tcl_Free((char *)pPreStmt); + dbFreeStmt(pPreStmt); }else{ /* Add the prepared statement to the beginning of the cache list. */ pPreStmt->pNext = pDb->stmtList; pPreStmt->pPrev = 0; if( pDb->stmtList ){ @@ -1268,15 +1316,15 @@ pDb->nStmt++; /* If we have too many statement in cache, remove the surplus from ** the end of the cache list. */ while( pDb->nStmt>pDb->maxStmt ){ - sqlite3_finalize(pDb->stmtLast->pStmt); - pDb->stmtLast = pDb->stmtLast->pPrev; - Tcl_Free((char*)pDb->stmtLast->pNext); + SqlPreparedStmt *pLast = pDb->stmtLast; + pDb->stmtLast = pLast->pPrev; pDb->stmtLast->pNext = 0; pDb->nStmt--; + dbFreeStmt(pLast); } } } /* @@ -1405,13 +1453,16 @@ ** is analogous to a return of SQLITE_ROW from sqlite3_step(). If TCL_BREAK ** is returned, then the SQL script has finished executing and there are ** no further rows available. This is similar to SQLITE_DONE. */ static int dbEvalStep(DbEvalContext *p){ + const char *zPrevSql = 0; /* Previous value of p->zSql */ + while( p->zSql[0] || p->pPreStmt ){ int rc; if( p->pPreStmt==0 ){ + zPrevSql = (p->zSql==zPrevSql ? 0 : p->zSql); rc = dbPrepareAndBind(p->pDb, p->zSql, &p->zSql, &p->pPreStmt); if( rc!=TCL_OK ) return rc; }else{ int rcs; SqliteDb *pDb = p->pDb; @@ -1434,12 +1485,23 @@ p->pPreStmt = 0; if( rcs!=SQLITE_OK ){ /* If a run-time error occurs, report the error and stop reading ** the SQL. */ - Tcl_SetObjResult(pDb->interp, dbTextToObj(sqlite3_errmsg(pDb->db))); dbReleaseStmt(pDb, pPreStmt, 1); +#if SQLITE_TEST + if( p->pDb->bLegacyPrepare && rcs==SQLITE_SCHEMA && zPrevSql ){ + /* If the runtime error was an SQLITE_SCHEMA, and the database + ** handle is configured to use the legacy sqlite3_prepare() + ** interface, retry prepare()/step() on the same SQL statement. + ** This only happens once. If there is a second SQLITE_SCHEMA + ** error, the error will be returned to the caller. */ + p->zSql = zPrevSql; + continue; + } +#endif + Tcl_SetObjResult(pDb->interp, dbTextToObj(sqlite3_errmsg(pDb->db))); return TCL_ERROR; }else{ dbReleaseStmt(pDb, pPreStmt, 0); } } @@ -3070,11 +3132,11 @@ flags |= SQLITE_OPEN_NOMUTEX; flags &= ~SQLITE_OPEN_FULLMUTEX; }else{ flags &= ~SQLITE_OPEN_NOMUTEX; } - }else if( strcmp(zArg, "-fullmutex")==0 ){ + }else if( strcmp(zArg, "-fullmutex")==0 ){ int b; if( Tcl_GetBooleanFromObj(interp, objv[i+1], &b) ) return TCL_ERROR; if( b ){ flags |= SQLITE_OPEN_FULLMUTEX; flags &= ~SQLITE_OPEN_NOMUTEX; @@ -3671,10 +3733,48 @@ } init_all(slave); return TCL_OK; } + +/* +** Tclcmd: db_use_legacy_prepare DB BOOLEAN +** +** The first argument to this command must be a database command created by +** [sqlite3]. If the second argument is true, then the handle is configured +** to use the sqlite3_prepare_v2() function to prepare statements. If it +** is false, sqlite3_prepare(). +*/ +static int db_use_legacy_prepare_cmd( + ClientData cd, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + Tcl_CmdInfo cmdInfo; + SqliteDb *pDb; + int bPrepare; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB BOOLEAN"); + return TCL_ERROR; + } + + if( !Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){ + Tcl_AppendResult(interp, "no such db: ", Tcl_GetString(objv[1]), (char*)0); + return TCL_ERROR; + } + pDb = (SqliteDb*)cmdInfo.objClientData; + if( Tcl_GetBooleanFromObj(interp, objv[2], &bPrepare) ){ + return TCL_ERROR; + } + + pDb->bLegacyPrepare = bPrepare; + + Tcl_ResetResult(interp); + return TCL_OK; +} #endif /* ** Configure the interpreter passed as the first argument to have access ** to the commands and linked variables that make up: @@ -3781,11 +3881,16 @@ #endif #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) Sqlitetestfts3_Init(interp); #endif - Tcl_CreateObjCommand(interp,"load_testfixture_extensions",init_all_cmd,0,0); + Tcl_CreateObjCommand( + interp, "load_testfixture_extensions", init_all_cmd, 0, 0 + ); + Tcl_CreateObjCommand( + interp, "db_use_legacy_prepare", db_use_legacy_prepare_cmd, 0, 0 + ); #ifdef SQLITE_SSE Sqlitetestsse_Init(interp); #endif } Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -5094,10 +5094,43 @@ } #endif return TCL_OK; } +/* +** tclcmd: file_control_win32_av_retry DB NRETRY DELAY +** +** This TCL command runs the sqlite3_file_control interface with +** the SQLITE_FCNTL_WIN32_AV_RETRY opcode. +*/ +static int file_control_win32_av_retry( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + int rc; + int a[2]; + char z[100]; + + if( objc!=4 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " DB NRETRY DELAY", 0); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[2], &a[0]) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &a[1]) ) return TCL_ERROR; + rc = sqlite3_file_control(db, NULL, SQLITE_FCNTL_WIN32_AV_RETRY, (void*)a); + sqlite3_snprintf(sizeof(z), z, "%d %d %d", rc, a[0], a[1]); + Tcl_AppendResult(interp, z, (char*)0); + return TCL_OK; +} + /* ** tclcmd: sqlite3_vfs_list ** ** Return a tcl list containing the names of all registered vfs's. @@ -5572,10 +5605,98 @@ Tcl_ResetResult(interp); return TCL_OK; } +#if SQLITE_OS_WIN +/* +** Information passed from the main thread into the windows file locker +** background thread. +*/ +struct win32FileLocker { + HANDLE h; /* Handle of the file to be locked */ + int delay1; /* Delay before locking */ + int delay2; /* Delay before unlocking */ + int ok; /* Finished ok */ + int err; /* True if an error occurs */ +}; +#endif + + +#if SQLITE_OS_WIN +/* +** The background thread that does file locking. +*/ +static void win32_file_locker(void *pAppData){ + struct win32FileLocker *p = (struct win32FileLocker*)pAppData; + if( p->delay1 ) Sleep(p->delay1); + if( LockFile(p->h, 0, 0, 100000000, 0) ){ + Sleep(p->delay2); + UnlockFile(p->h, 0, 0, 100000000, 0); + p->ok = 1; + }else{ + p->err = 1; + } + CloseHandle(p->h); + p->h = 0; + p->delay1 = 0; + p->delay2 = 0; +} +#endif + +#if SQLITE_OS_WIN +/* +** lock_win32_file FILENAME DELAY1 DELAY2 +** +** Get an exclusive manditory lock on file for DELAY2 milliseconds. +** Wait DELAY1 milliseconds before acquiring the lock. +*/ +static int win32_file_lock( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + static struct win32FileLocker x = { 0, 0, 0 }; + const char *zFilename; + int retry = 0; + + if( objc!=4 && objc!=1 ){ + Tcl_WrongNumArgs(interp, 1, objv, "FILENAME DELAY1 DELAY2"); + return TCL_ERROR; + } + if( objc==1 ){ + char zBuf[200]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "%d %d %d %d %d", + x.ok, x.err, x.delay1, x.delay2, x.h); + Tcl_AppendResult(interp, zBuf, (char*)0); + return TCL_OK; + } + while( x.h && retry<30 ){ + retry++; + Sleep(100); + } + if( x.h ){ + Tcl_AppendResult(interp, "busy", (char*)0); + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[2], &x.delay1) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &x.delay2) ) return TCL_ERROR; + zFilename = Tcl_GetString(objv[1]); + x.h = CreateFile(zFilename, GENERIC_READ|GENERIC_WRITE, + FILE_SHARE_READ|FILE_SHARE_WRITE, 0, OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, 0); + if( !x.h ){ + Tcl_AppendResult(interp, "cannot open file: ", zFilename, (char*)0); + return TCL_ERROR; + } + _beginthread(win32_file_locker, 0, (void*)&x); + Sleep(0); + return TCL_OK; +} +#endif + /* ** optimization_control DB OPT BOOLEAN ** ** Enable or disable query optimizations using the sqlite3_test_control() @@ -5752,10 +5873,13 @@ { "save_prng_state", save_prng_state, 0 }, { "restore_prng_state", restore_prng_state, 0 }, { "reset_prng_state", reset_prng_state, 0 }, { "optimization_control", optimization_control,0}, +#if SQLITE_OS_WIN + { "lock_win32_file", win32_file_lock, 0 }, +#endif { "tcl_objproc", runAsObjProc, 0 }, /* sqlite3_column_*() API */ { "sqlite3_column_count", test_column_count ,0 }, { "sqlite3_data_count", test_data_count ,0 }, @@ -5800,11 +5924,12 @@ { "vfs_reregister_all", vfs_reregister_all, 0 }, { "file_control_test", file_control_test, 0 }, { "file_control_lasterrno_test", file_control_lasterrno_test, 0 }, { "file_control_lockproxy_test", file_control_lockproxy_test, 0 }, { "file_control_chunksize_test", file_control_chunksize_test, 0 }, - { "file_control_sizehint_test", file_control_sizehint_test, 0 }, + { "file_control_sizehint_test", file_control_sizehint_test, 0 }, + { "file_control_win32_av_retry", file_control_win32_av_retry, 0 }, { "sqlite3_vfs_list", vfs_list, 0 }, { "sqlite3_create_function_v2", test_create_function_v2, 0 }, /* Functions from os.h */ #ifndef SQLITE_OMIT_UTF16 Index: src/test_config.c ================================================================== --- src/test_config.c +++ src/test_config.c @@ -581,10 +581,11 @@ LINKVAR( DEFAULT_TEMP_CACHE_SIZE ); LINKVAR( DEFAULT_CACHE_SIZE ); LINKVAR( DEFAULT_PAGE_SIZE ); LINKVAR( DEFAULT_FILE_FORMAT ); LINKVAR( MAX_ATTACHED ); + LINKVAR( MAX_DEFAULT_PAGE_SIZE ); { static const int cv_TEMP_STORE = SQLITE_TEMP_STORE; Tcl_LinkVar(interp, "TEMP_STORE", (char *)&(cv_TEMP_STORE), TCL_LINK_INT | TCL_LINK_READ_ONLY); Index: src/test_multiplex.c ================================================================== --- src/test_multiplex.c +++ src/test_multiplex.c @@ -43,10 +43,11 @@ ** that do not support large files. */ #include "sqlite3.h" #include #include +#include #include "test_multiplex.h" #ifndef SQLITE_CORE #define SQLITE_CORE 1 /* Disable the API redefinition in sqlite3ext.h */ #endif @@ -76,24 +77,30 @@ #endif /* SQLITE_THREADSAFE==0 */ /************************ Shim Definitions ******************************/ -#define SQLITE_MULTIPLEX_VFS_NAME "multiplex" +#ifndef SQLITE_MULTIPLEX_VFS_NAME +# define SQLITE_MULTIPLEX_VFS_NAME "multiplex" +#endif /* This is the limit on the chunk size. It may be changed by calling ** the xFileControl() interface. It will be rounded up to a -** multiple of MAX_PAGE_SIZE. We default it here to 1GB. +** multiple of MAX_PAGE_SIZE. We default it here to 2GiB less 64KiB. */ -#define SQLITE_MULTIPLEX_CHUNK_SIZE (MAX_PAGE_SIZE*16384) +#ifndef SQLITE_MULTIPLEX_CHUNK_SIZE +# define SQLITE_MULTIPLEX_CHUNK_SIZE 2147418112 +#endif /* Default limit on number of chunks. Care should be taken ** so that values for chunks numbers fit in the SQLITE_MULTIPLEX_EXT_FMT ** format specifier. It may be changed by calling ** the xFileControl() interface. */ -#define SQLITE_MULTIPLEX_MAX_CHUNKS 32 +#ifndef SQLITE_MULTIPLEX_MAX_CHUNKS +# define SQLITE_MULTIPLEX_MAX_CHUNKS 32 +#endif /* If SQLITE_MULTIPLEX_EXT_OVWR is defined, the ** last SQLITE_MULTIPLEX_EXT_SZ characters of the ** filename will be overwritten, otherwise, the ** multiplex extension is simply appended to the filename. @@ -117,17 +124,19 @@ ** ** There is an instance of the following object for each defined multiplex ** group. */ struct multiplexGroup { - sqlite3_file **pReal; /* Handles to each chunk */ - char *bOpen; /* array of bools - 0 if chunk not opened */ + struct multiplexReal { /* For each chunk */ + sqlite3_file *p; /* Handle for the chunk */ + char *z; /* Name of this chunk */ + } *aReal; /* list of all chunks */ + int nReal; /* Number of chunks */ char *zName; /* Base filename of this group */ int nName; /* Length of base filename */ int flags; /* Flags used for original opening */ - int nChunkSize; /* Chunk size used for this group */ - int nMaxChunks; /* Max number of chunks for this group */ + unsigned int szChunk; /* Chunk size used for this group */ int bEnabled; /* TRUE to use Multiplex VFS for this file */ multiplexGroup *pNext, *pPrev; /* Doubly linked list of all group objects */ }; /* @@ -182,16 +191,10 @@ sqlite3_mutex *pMutex; /* List of multiplexGroup objects. */ multiplexGroup *pGroups; - - /* Storage for temp file names. Allocated during - ** initialization to the max pathname of the underlying VFS. - */ - char *zName; - } gMultiplex; /************************* Utility Routines *********************************/ /* ** Acquire and release the mutex used to serialize access to the @@ -263,11 +266,12 @@ /* Make 3 attempts to generate a unique name. */ do { attempts++; sqlite3_randomness(8, &zBuf[j]); for(i=0; i<8; i++){ - zBuf[j+i] = (char)zChars[ ((unsigned char)zBuf[j+i])%(sizeof(zChars)-1) ]; + unsigned char uc = (unsigned char)zBuf[j+i]; + zBuf[j+i] = (char)zChars[uc%(sizeof(zChars)-1)]; } memcpy(&zBuf[j+i], ".tmp", 5); rc = pOrigVfs->xAccess(pOrigVfs, zBuf, SQLITE_ACCESS_EXISTS, &exists); } while ( (rc==SQLITE_OK) && exists && (attempts<3) ); if( rc==SQLITE_OK && exists ){ @@ -276,40 +280,74 @@ } } return rc; } + +/* Compute the filename for the iChunk-th chunk +*/ +static int multiplexSubFilename(multiplexGroup *pGroup, int iChunk){ + if( iChunk>=pGroup->nReal ){ + struct multiplexReal *p; + p = sqlite3_realloc(pGroup->aReal, (iChunk+1)*sizeof(*p)); + if( p==0 ){ + return SQLITE_NOMEM; + } + memset(&p[pGroup->nReal], 0, sizeof(p[0])*(iChunk+1-pGroup->nReal)); + pGroup->aReal = p; + pGroup->nReal = iChunk+1; + } + if( pGroup->aReal[iChunk].z==0 ){ + char *z; + int n = pGroup->nName; + pGroup->aReal[iChunk].z = z = sqlite3_malloc( n+3 ); + if( z==0 ){ + return SQLITE_NOMEM; + } + memcpy(z, pGroup->zName, n+1); + if( iChunk>0 ){ +#ifdef SQLITE_ENABLE_8_3_NAMES + if( n>3 && z[n-3]=='.' ){ + n--; + }else if( n>4 && z[n-4]=='.' ){ + n -= 2; + } +#endif + sqlite3_snprintf(3,&z[n],"%02d",iChunk); + } + } + return SQLITE_OK; +} /* Translate an sqlite3_file* that is really a multiplexGroup* into ** the sqlite3_file* for the underlying original VFS. */ -static sqlite3_file *multiplexSubOpen(multiplexConn *pConn, int iChunk, int *rc, int *pOutFlags){ - multiplexGroup *pGroup = pConn->pGroup; +static sqlite3_file *multiplexSubOpen( + multiplexGroup *pGroup, + int iChunk, + int *rc, + int *pOutFlags +){ + sqlite3_file *pSubOpen = 0; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ - if( iChunknMaxChunks ){ - sqlite3_file *pSubOpen = pGroup->pReal[iChunk]; /* Real file descriptor */ - if( !pGroup->bOpen[iChunk] ){ - memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); - if( iChunk ){ -#ifdef SQLITE_MULTIPLEX_EXT_OVWR - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, SQLITE_MULTIPLEX_EXT_FMT, iChunk); -#else - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName, SQLITE_MULTIPLEX_EXT_FMT, iChunk); -#endif - } - *rc = pOrigVfs->xOpen(pOrigVfs, gMultiplex.zName, pSubOpen, pGroup->flags, pOutFlags); - if( *rc==SQLITE_OK ){ - pGroup->bOpen[iChunk] = -1; - return pSubOpen; - } - return NULL; + *rc = multiplexSubFilename(pGroup, iChunk); + if( (*rc)==SQLITE_OK && (pSubOpen = pGroup->aReal[iChunk].p)==0 ){ + pSubOpen = sqlite3_malloc( pOrigVfs->szOsFile ); + if( pSubOpen==0 ){ + *rc = SQLITE_NOMEM; + return 0; } - *rc = SQLITE_OK; - return pSubOpen; + pGroup->aReal[iChunk].p = pSubOpen; + *rc = pOrigVfs->xOpen(pOrigVfs, pGroup->aReal[iChunk].z, pSubOpen, + pGroup->flags, pOutFlags); + if( *rc!=SQLITE_OK ){ + sqlite3_free(pSubOpen); + pGroup->aReal[iChunk].p = 0; + return 0; + } } - *rc = SQLITE_FULL; - return NULL; + return pSubOpen; } /* ** This is the implementation of the multiplex_control() SQL function. */ @@ -363,10 +401,40 @@ int rc; rc = sqlite3_create_function(db, "multiplex_control", 2, SQLITE_ANY, 0, multiplexControlFunc, 0, 0); return rc; } + +/* +** Close a single sub-file in the connection group. +*/ +static void multiplexSubClose( + multiplexGroup *pGroup, + int iChunk, + sqlite3_vfs *pOrigVfs +){ + sqlite3_file *pSubOpen = pGroup->aReal[iChunk].p; + if( pSubOpen ){ + if( pOrigVfs ) pOrigVfs->xDelete(pOrigVfs, pGroup->aReal[iChunk].z, 0); + pSubOpen->pMethods->xClose(pSubOpen); + sqlite3_free(pGroup->aReal[iChunk].p); + } + sqlite3_free(pGroup->aReal[iChunk].z); + memset(&pGroup->aReal[iChunk], 0, sizeof(pGroup->aReal[iChunk])); +} + +/* +** Deallocate memory held by a multiplexGroup +*/ +static void multiplexFreeComponents(multiplexGroup *pGroup){ + int i; + for(i=0; inReal; i++){ multiplexSubClose(pGroup, i, 0); } + sqlite3_free(pGroup->aReal); + pGroup->aReal = 0; + pGroup->nReal = 0; +} + /************************* VFS Method Wrappers *****************************/ /* ** This is the xOpen method used for the "multiplex" VFS. @@ -380,20 +448,21 @@ const char *zName, /* Name of file to be opened */ sqlite3_file *pConn, /* Fill in this file descriptor */ int flags, /* Flags to control the opening */ int *pOutFlags /* Flags showing results of opening */ ){ - int rc = SQLITE_OK; /* Result code */ - multiplexConn *pMultiplexOpen; /* The new multiplex file descriptor */ - multiplexGroup *pGroup; /* Corresponding multiplexGroup object */ - sqlite3_file *pSubOpen; /* Real file descriptor */ + int rc = SQLITE_OK; /* Result code */ + multiplexConn *pMultiplexOpen; /* The new multiplex file descriptor */ + multiplexGroup *pGroup; /* Corresponding multiplexGroup object */ + sqlite3_file *pSubOpen = 0; /* Real file descriptor */ sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ int nName; - int i; int sz; + char *zToFree = 0; UNUSED_PARAMETER(pVfs); + memset(pConn, 0, pVfs->szOsFile); /* We need to create a group structure and manage ** access to this group of files. */ multiplexEnter(); @@ -403,67 +472,87 @@ ** temporary file name to use. This will be handled by the ** original xOpen method. We just need to allocate space for ** it. */ if( !zName ){ - rc = multiplexGetTempname(pOrigVfs, pOrigVfs->mxPathname, gMultiplex.zName); - zName = gMultiplex.zName; + zName = zToFree = sqlite3_malloc( pOrigVfs->mxPathname + 10 ); + if( zName==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = multiplexGetTempname(pOrigVfs, pOrigVfs->mxPathname, zToFree); + } } if( rc==SQLITE_OK ){ /* allocate space for group */ nName = multiplexStrlen30(zName); - sz = sizeof(multiplexGroup) /* multiplexGroup */ - + (sizeof(sqlite3_file *)*SQLITE_MULTIPLEX_MAX_CHUNKS) /* pReal[] */ - + (pOrigVfs->szOsFile*SQLITE_MULTIPLEX_MAX_CHUNKS) /* *pReal */ - + SQLITE_MULTIPLEX_MAX_CHUNKS /* bOpen[] */ - + nName + 1; /* zName */ -#ifndef SQLITE_MULTIPLEX_EXT_OVWR - sz += SQLITE_MULTIPLEX_EXT_SZ; - assert(nName+SQLITE_MULTIPLEX_EXT_SZ < pOrigVfs->mxPathname); -#else - assert(nName >= SQLITE_MULTIPLEX_EXT_SZ); - assert(nName < pOrigVfs->mxPathname); -#endif + sz = sizeof(multiplexGroup) /* multiplexGroup */ + + nName + 1; /* zName */ pGroup = sqlite3_malloc( sz ); if( pGroup==0 ){ - rc=SQLITE_NOMEM; + rc = SQLITE_NOMEM; } } if( rc==SQLITE_OK ){ /* assign pointers to extra space allocated */ char *p = (char *)&pGroup[1]; pMultiplexOpen->pGroup = pGroup; memset(pGroup, 0, sz); pGroup->bEnabled = -1; - pGroup->nChunkSize = SQLITE_MULTIPLEX_CHUNK_SIZE; - pGroup->nMaxChunks = SQLITE_MULTIPLEX_MAX_CHUNKS; - pGroup->pReal = (sqlite3_file **)p; - p += (sizeof(sqlite3_file *)*pGroup->nMaxChunks); - for(i=0; inMaxChunks; i++){ - pGroup->pReal[i] = (sqlite3_file *)p; - p += pOrigVfs->szOsFile; + pGroup->szChunk = SQLITE_MULTIPLEX_CHUNK_SIZE; + if( flags & SQLITE_OPEN_URI ){ + const char *zChunkSize; + zChunkSize = sqlite3_uri_parameter(zName, "chunksize"); + if( zChunkSize ){ + unsigned int n = 0; + int i; + for(i=0; zChunkSize[i]>='0' && zChunkSize[i]<='9'; i++){ + n = n*10 + zChunkSize[i] - '0'; + } + if( n>0 ){ + pGroup->szChunk = (n+0xffff)&~0xffff; + }else{ + /* A zero or negative chunksize disabled the multiplexor */ + pGroup->bEnabled = 0; + } + } } - /* bOpen[] vals should all be zero from memset above */ - pGroup->bOpen = p; - p += pGroup->nMaxChunks; pGroup->zName = p; /* save off base filename, name length, and original open flags */ memcpy(pGroup->zName, zName, nName+1); pGroup->nName = nName; pGroup->flags = flags; - pSubOpen = multiplexSubOpen(pMultiplexOpen, 0, &rc, pOutFlags); + rc = multiplexSubFilename(pGroup, 1); + if( rc==SQLITE_OK ){ + pSubOpen = multiplexSubOpen(pGroup, 0, &rc, pOutFlags); + } if( pSubOpen ){ - /* if this file is already larger than chunk size, disable - ** the multiplex feature. - */ + int exists, rc2, rc3; sqlite3_int64 sz; - int rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); - if( (rc2==SQLITE_OK) && (sz>pGroup->nChunkSize) ){ - pGroup->bEnabled = 0; + + rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); + if( rc2==SQLITE_OK ){ + /* If the first overflow file exists and if the size of the main file + ** is different from the chunk size, that means the chunk size is set + ** set incorrectly. So fix it. + ** + ** Or, if the first overflow file does not exist and the main file is + ** larger than the chunk size, that means the chunk size is too small. + ** But we have no way of determining the intended chunk size, so + ** just disable the multiplexor all togethre. + */ + rc3 = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[1].z, + SQLITE_ACCESS_EXISTS, &exists); + if( rc3==SQLITE_OK && exists && sz==(sz&0xffff0000) && sz>0 + && sz!=pGroup->szChunk ){ + pGroup->szChunk = sz; + }else if( rc3==SQLITE_OK && !exists && sz>pGroup->szChunk ){ + pGroup->bEnabled = 0; + } } + if( pSubOpen->pMethods->iVersion==1 ){ pMultiplexOpen->base.pMethods = &gMultiplex.sIoMethodsV1; }else{ pMultiplexOpen->base.pMethods = &gMultiplex.sIoMethodsV2; } @@ -470,63 +559,30 @@ /* place this group at the head of our list */ pGroup->pNext = gMultiplex.pGroups; if( gMultiplex.pGroups ) gMultiplex.pGroups->pPrev = pGroup; gMultiplex.pGroups = pGroup; }else{ + multiplexFreeComponents(pGroup); sqlite3_free(pGroup); } } multiplexLeave(); + sqlite3_free(zToFree); return rc; } /* ** This is the xDelete method used for the "multiplex" VFS. -** It attempts to delete the filename specified, as well -** as additional files with the SQLITE_MULTIPLEX_EXT_FMT extension. +** It attempts to delete the filename specified. */ static int multiplexDelete( sqlite3_vfs *pVfs, /* The multiplex VFS */ const char *zName, /* Name of file to delete */ int syncDir ){ sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ - int rc = SQLITE_OK; - int nName = multiplexStrlen30(zName); - int i; - - UNUSED_PARAMETER(pVfs); - - multiplexEnter(); - memcpy(gMultiplex.zName, zName, nName+1); - for(i=0; ixAccess(pOrigVfs, gMultiplex.zName, - SQLITE_ACCESS_EXISTS, &exists); - if( rc2==SQLITE_OK && exists ){ - /* if it exists, delete it */ - rc2 = pOrigVfs->xDelete(pOrigVfs, gMultiplex.zName, syncDir); - if( rc2!=SQLITE_OK ) rc = rc2; - }else{ - /* stop at first "gap" */ - break; - } - } - multiplexLeave(); - return rc; + return pOrigVfs->xDelete(pOrigVfs, zName, syncDir); } static int multiplexAccess(sqlite3_vfs *a, const char *b, int c, int *d){ return gMultiplex.pOrigVfs->xAccess(gMultiplex.pOrigVfs, b, c, d); } @@ -570,21 +626,12 @@ */ static int multiplexClose(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; multiplexGroup *pGroup = p->pGroup; int rc = SQLITE_OK; - int i; multiplexEnter(); - /* close any open handles */ - for(i=0; inMaxChunks; i++){ - if( pGroup->bOpen[i] ){ - sqlite3_file *pSubOpen = pGroup->pReal[i]; - int rc2 = pSubOpen->pMethods->xClose(pSubOpen); - if( rc2!=SQLITE_OK ) rc = rc2; - pGroup->bOpen[i] = 0; - } - } + multiplexFreeComponents(pGroup); /* remove from linked list */ if( pGroup->pNext ) pGroup->pNext->pPrev = pGroup->pPrev; if( pGroup->pPrev ){ pGroup->pPrev->pNext = pGroup->pNext; }else{ @@ -608,21 +655,26 @@ multiplexConn *p = (multiplexConn*)pConn; multiplexGroup *pGroup = p->pGroup; int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); - rc = ( !pSubOpen ) ? SQLITE_IOERR_READ : pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + if( pSubOpen==0 ){ + rc = SQLITE_IOERR_READ; + }else{ + rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst); + } }else{ while( iAmt > 0 ){ - int i = (int)(iOfst / pGroup->nChunkSize); - sqlite3_file *pSubOpen = multiplexSubOpen(p, i, &rc, NULL); + int i = (int)(iOfst / pGroup->szChunk); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); if( pSubOpen ){ - int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - pGroup->nChunkSize; + int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) - pGroup->szChunk; if( extra<0 ) extra = 0; iAmt -= extra; - rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst % pGroup->nChunkSize); + rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, + iOfst % pGroup->szChunk); if( rc!=SQLITE_OK ) break; pBuf = (char *)pBuf + iAmt; iOfst += iAmt; iAmt = extra; }else{ @@ -648,21 +700,27 @@ multiplexConn *p = (multiplexConn*)pConn; multiplexGroup *pGroup = p->pGroup; int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); - rc = ( !pSubOpen ) ? SQLITE_IOERR_WRITE : pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + if( pSubOpen==0 ){ + rc = SQLITE_IOERR_WRITE; + }else{ + rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst); + } }else{ while( iAmt > 0 ){ - int i = (int)(iOfst / pGroup->nChunkSize); - sqlite3_file *pSubOpen = multiplexSubOpen(p, i, &rc, NULL); + int i = (int)(iOfst / pGroup->szChunk); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); if( pSubOpen ){ - int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - pGroup->nChunkSize; + int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) - + pGroup->szChunk; if( extra<0 ) extra = 0; iAmt -= extra; - rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst % pGroup->nChunkSize); + rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, + iOfst % pGroup->szChunk); if( rc!=SQLITE_OK ) break; pBuf = (char *)pBuf + iAmt; iOfst += iAmt; iAmt = extra; }else{ @@ -683,42 +741,28 @@ multiplexConn *p = (multiplexConn*)pConn; multiplexGroup *pGroup = p->pGroup; int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); - rc = ( !pSubOpen ) ? SQLITE_IOERR_TRUNCATE : pSubOpen->pMethods->xTruncate(pSubOpen, size); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + if( pSubOpen==0 ){ + rc = SQLITE_IOERR_TRUNCATE; + }else{ + rc = pSubOpen->pMethods->xTruncate(pSubOpen, size); + } }else{ int rc2; int i; sqlite3_file *pSubOpen; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ - memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); /* delete the chunks above the truncate limit */ - for(i=(int)(size / pGroup->nChunkSize)+1; inMaxChunks; i++){ - /* close any open chunks before deleting them */ - if( pGroup->bOpen[i] ){ - pSubOpen = pGroup->pReal[i]; - rc2 = pSubOpen->pMethods->xClose(pSubOpen); - if( rc2!=SQLITE_OK ) rc = SQLITE_IOERR_TRUNCATE; - pGroup->bOpen[i] = 0; - } -#ifdef SQLITE_MULTIPLEX_EXT_OVWR - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, - gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, - SQLITE_MULTIPLEX_EXT_FMT, i); -#else - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, - gMultiplex.zName+pGroup->nName, - SQLITE_MULTIPLEX_EXT_FMT, i); -#endif - rc2 = pOrigVfs->xDelete(pOrigVfs, gMultiplex.zName, 0); - if( rc2!=SQLITE_OK ) rc = SQLITE_IOERR_TRUNCATE; + for(i=(int)(size / pGroup->szChunk)+1; inReal; i++){ + multiplexSubClose(pGroup, i, pOrigVfs); } - pSubOpen = multiplexSubOpen(p, (int)(size / pGroup->nChunkSize), &rc2, NULL); + pSubOpen = multiplexSubOpen(pGroup, (int)(size/pGroup->szChunk), &rc2,0); if( pSubOpen ){ - rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->nChunkSize); + rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->szChunk); if( rc2!=SQLITE_OK ) rc = rc2; }else{ rc = SQLITE_IOERR_TRUNCATE; } } @@ -732,14 +776,13 @@ multiplexConn *p = (multiplexConn*)pConn; multiplexGroup *pGroup = p->pGroup; int rc = SQLITE_OK; int i; multiplexEnter(); - for(i=0; inMaxChunks; i++){ - /* if we don't have it open, we don't need to sync it */ - if( pGroup->bOpen[i] ){ - sqlite3_file *pSubOpen = pGroup->pReal[i]; + for(i=0; inReal; i++){ + sqlite3_file *pSubOpen = pGroup->aReal[i].p; + if( pSubOpen ){ int rc2 = pSubOpen->pMethods->xSync(pSubOpen, flags); if( rc2!=SQLITE_OK ) rc = rc2; } } multiplexLeave(); @@ -755,51 +798,40 @@ int rc = SQLITE_OK; int rc2; int i; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); - rc = ( !pSubOpen ) ? SQLITE_IOERR_FSTAT : pSubOpen->pMethods->xFileSize(pSubOpen, pSize); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + if( pSubOpen==0 ){ + rc = SQLITE_IOERR_FSTAT; + }else{ + rc = pSubOpen->pMethods->xFileSize(pSubOpen, pSize); + } }else{ + sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; *pSize = 0; - for(i=0; inMaxChunks; i++){ - sqlite3_file *pSubOpen = NULL; - /* if not opened already, check to see if the chunk exists */ - if( pGroup->bOpen[i] ){ - pSubOpen = pGroup->pReal[i]; + for(i=0; 1; i++){ + sqlite3_file *pSubOpen = 0; + int exists = 0; + rc = multiplexSubFilename(pGroup, i); + if( rc ) break; + rc2 = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[i].z, + SQLITE_ACCESS_EXISTS, &exists); + if( rc2==SQLITE_OK && exists){ + /* if it exists, open it */ + pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); }else{ - sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ - int exists = 0; - memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); - if( i ){ -#ifdef SQLITE_MULTIPLEX_EXT_OVWR - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, - gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, - SQLITE_MULTIPLEX_EXT_FMT, i); -#else - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, - gMultiplex.zName+pGroup->nName, - SQLITE_MULTIPLEX_EXT_FMT, i); -#endif - } - rc2 = pOrigVfs->xAccess(pOrigVfs, gMultiplex.zName, - SQLITE_ACCESS_EXISTS, &exists); - if( rc2==SQLITE_OK && exists){ - /* if it exists, open it */ - pSubOpen = multiplexSubOpen(p, i, &rc, NULL); - }else{ - /* stop at first "gap" */ - break; - } + /* stop at first "gap" */ + break; } if( pSubOpen ){ sqlite3_int64 sz; rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); if( rc2!=SQLITE_OK ){ rc = rc2; }else{ - if( sz>pGroup->nChunkSize ){ + if( sz>pGroup->szChunk ){ rc = SQLITE_IOERR_FSTAT; } *pSize += sz; } }else{ @@ -814,11 +846,11 @@ /* Pass xLock requests through to the original VFS unchanged. */ static int multiplexLock(sqlite3_file *pConn, int lock){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xLock(pSubOpen, lock); } return SQLITE_BUSY; } @@ -826,11 +858,11 @@ /* Pass xUnlock requests through to the original VFS unchanged. */ static int multiplexUnlock(sqlite3_file *pConn, int lock){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xUnlock(pSubOpen, lock); } return SQLITE_IOERR_UNLOCK; } @@ -838,11 +870,11 @@ /* Pass xCheckReservedLock requests through to the original VFS unchanged. */ static int multiplexCheckReservedLock(sqlite3_file *pConn, int *pResOut){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xCheckReservedLock(pSubOpen, pResOut); } return SQLITE_IOERR_CHECKRESERVEDLOCK; } @@ -865,40 +897,32 @@ rc = SQLITE_OK; } break; case MULTIPLEX_CTRL_SET_CHUNK_SIZE: if( pArg ) { - int nChunkSize = *(int *)pArg; - if( nChunkSize<1 ){ + unsigned int szChunk = *(unsigned*)pArg; + if( szChunk<1 ){ rc = SQLITE_MISUSE; }else{ /* Round up to nearest multiple of MAX_PAGE_SIZE. */ - nChunkSize = (nChunkSize + (MAX_PAGE_SIZE-1)); - nChunkSize &= ~(MAX_PAGE_SIZE-1); - pGroup->nChunkSize = nChunkSize; + szChunk = (szChunk + (MAX_PAGE_SIZE-1)); + szChunk &= ~(MAX_PAGE_SIZE-1); + pGroup->szChunk = szChunk; rc = SQLITE_OK; } } break; case MULTIPLEX_CTRL_SET_MAX_CHUNKS: - if( pArg ) { - int nMaxChunks = *(int *)pArg; - if(( nMaxChunks<1 ) || ( nMaxChunks>SQLITE_MULTIPLEX_MAX_CHUNKS )){ - rc = SQLITE_MISUSE; - }else{ - pGroup->nMaxChunks = nMaxChunks; - rc = SQLITE_OK; - } - } + rc = SQLITE_OK; break; case SQLITE_FCNTL_SIZE_HINT: case SQLITE_FCNTL_CHUNK_SIZE: /* no-op these */ rc = SQLITE_OK; break; default: - pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); if( pSubOpen ){ rc = pSubOpen->pMethods->xFileControl(pSubOpen, op, pArg); } break; } @@ -908,11 +932,11 @@ /* Pass xSectorSize requests through to the original VFS unchanged. */ static int multiplexSectorSize(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xSectorSize(pSubOpen); } return DEFAULT_SECTOR_SIZE; } @@ -920,11 +944,11 @@ /* Pass xDeviceCharacteristics requests through to the original VFS unchanged. */ static int multiplexDeviceCharacteristics(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xDeviceCharacteristics(pSubOpen); } return 0; } @@ -938,13 +962,13 @@ int bExtend, /* True to extend file if necessary */ void volatile **pp /* OUT: Mapped memory */ ){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ - return pSubOpen->pMethods->xShmMap(pSubOpen, iRegion, szRegion, bExtend, pp); + return pSubOpen->pMethods->xShmMap(pSubOpen, iRegion, szRegion, bExtend,pp); } return SQLITE_IOERR; } /* Pass xShmLock requests through to the original VFS unchanged. @@ -955,11 +979,11 @@ int n, /* Number of locks to acquire or release */ int flags /* What to do with the lock */ ){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xShmLock(pSubOpen, ofst, n, flags); } return SQLITE_BUSY; } @@ -967,11 +991,11 @@ /* Pass xShmBarrier requests through to the original VFS unchanged. */ static void multiplexShmBarrier(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ pSubOpen->pMethods->xShmBarrier(pSubOpen); } } @@ -978,11 +1002,11 @@ /* Pass xShmUnmap requests through to the original VFS unchanged. */ static int multiplexShmUnmap(sqlite3_file *pConn, int deleteFlag){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xShmUnmap(pSubOpen, deleteFlag); } return SQLITE_OK; } @@ -1008,15 +1032,10 @@ assert( pOrigVfs!=&gMultiplex.sThisVfs ); gMultiplex.pMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); if( !gMultiplex.pMutex ){ return SQLITE_NOMEM; } - gMultiplex.zName = sqlite3_malloc(pOrigVfs->mxPathname); - if( !gMultiplex.zName ){ - sqlite3_mutex_free(gMultiplex.pMutex); - return SQLITE_NOMEM; - } gMultiplex.pGroups = NULL; gMultiplex.isInitialized = 1; gMultiplex.pOrigVfs = pOrigVfs; gMultiplex.sThisVfs = *pOrigVfs; gMultiplex.sThisVfs.szOsFile += sizeof(multiplexConn); @@ -1045,11 +1064,12 @@ gMultiplex.sIoMethodsV1.xLock = multiplexLock; gMultiplex.sIoMethodsV1.xUnlock = multiplexUnlock; gMultiplex.sIoMethodsV1.xCheckReservedLock = multiplexCheckReservedLock; gMultiplex.sIoMethodsV1.xFileControl = multiplexFileControl; gMultiplex.sIoMethodsV1.xSectorSize = multiplexSectorSize; - gMultiplex.sIoMethodsV1.xDeviceCharacteristics = multiplexDeviceCharacteristics; + gMultiplex.sIoMethodsV1.xDeviceCharacteristics = + multiplexDeviceCharacteristics; gMultiplex.sIoMethodsV2 = gMultiplex.sIoMethodsV1; gMultiplex.sIoMethodsV2.iVersion = 2; gMultiplex.sIoMethodsV2.xShmMap = multiplexShmMap; gMultiplex.sIoMethodsV2.xShmLock = multiplexShmLock; gMultiplex.sIoMethodsV2.xShmBarrier = multiplexShmBarrier; @@ -1072,11 +1092,10 @@ */ int sqlite3_multiplex_shutdown(void){ if( gMultiplex.isInitialized==0 ) return SQLITE_MISUSE; if( gMultiplex.pGroups ) return SQLITE_MISUSE; gMultiplex.isInitialized = 0; - sqlite3_free(gMultiplex.zName); sqlite3_mutex_free(gMultiplex.pMutex); sqlite3_vfs_unregister(&gMultiplex.sThisVfs); memset(&gMultiplex, 0, sizeof(gMultiplex)); return SQLITE_OK; } @@ -1174,20 +1193,20 @@ Tcl_NewIntObj(pGroup->nName)); Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewIntObj(pGroup->flags)); /* count number of chunks with open handles */ - for(i=0; inMaxChunks; i++){ - if( pGroup->bOpen[i] ) nChunks++; + for(i=0; inReal; i++){ + if( pGroup->aReal[i].p!=0 ) nChunks++; } Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewIntObj(nChunks)); Tcl_ListObjAppendElement(interp, pGroupTerm, - Tcl_NewIntObj(pGroup->nChunkSize)); + Tcl_NewIntObj(pGroup->szChunk)); Tcl_ListObjAppendElement(interp, pGroupTerm, - Tcl_NewIntObj(pGroup->nMaxChunks)); + Tcl_NewIntObj(pGroup->nReal)); Tcl_ListObjAppendElement(interp, pResult, pGroupTerm); } multiplexLeave(); Tcl_SetObjResult(interp, pResult); Index: src/trigger.c ================================================================== --- src/trigger.c +++ src/trigger.c @@ -115,19 +115,32 @@ iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); if( iDb<0 ){ goto trigger_cleanup; } } + if( !pTableName || db->mallocFailed ){ + goto trigger_cleanup; + } + + /* A long-standing parser bug is that this syntax was allowed: + ** + ** CREATE TRIGGER attached.demo AFTER INSERT ON attached.tab .... + ** ^^^^^^^^ + ** + ** To maintain backwards compatibility, ignore the database + ** name on pTableName if we are reparsing our of SQLITE_MASTER. + */ + if( db->init.busy && iDb!=1 ){ + sqlite3DbFree(db, pTableName->a[0].zDatabase); + pTableName->a[0].zDatabase = 0; + } /* If the trigger name was unqualified, and the table is a temp table, ** then set iDb to 1 to create the trigger in the temporary database. ** If sqlite3SrcListLookup() returns 0, indicating the table does not ** exist, the error is caught by the block below. */ - if( !pTableName || db->mallocFailed ){ - goto trigger_cleanup; - } pTab = sqlite3SrcListLookup(pParse, pTableName); if( db->init.busy==0 && pName2->n==0 && pTab && pTab->pSchema==db->aDb[1].pSchema ){ iDb = 1; } Index: src/update.c ================================================================== --- src/update.c +++ src/update.c @@ -309,11 +309,13 @@ } /* Begin the database scan */ sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0, WHERE_ONEPASS_DESIRED); + pWInfo = sqlite3WhereBegin( + pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED + ); if( pWInfo==0 ) goto update_cleanup; okOnePass = pWInfo->okOnePass; /* Remember the rowid of every item to be updated. */ Index: src/util.c ================================================================== --- src/util.c +++ src/util.c @@ -1147,28 +1147,34 @@ return -x; } #ifdef SQLITE_ENABLE_8_3_NAMES /* -** If SQLITE_ENABLE_8_3_NAME is set at compile-time and if the database +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database ** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and ** if filename in z[] has a suffix (a.k.a. "extension") that is longer than ** three characters, then shorten the suffix on z[] to be the last three ** characters of the original suffix. +** +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always +** do the suffix shortening regardless of URI parameter. ** ** Examples: ** ** test.db-journal => test.nal ** test.db-wal => test.wal ** test.db-shm => test.shm */ void sqlite3FileSuffix3(const char *zBaseFilename, char *z){ +#if SQLITE_ENABLE_8_3_NAMES<2 const char *zOk; zOk = sqlite3_uri_parameter(zBaseFilename, "8_3_names"); - if( zOk && sqlite3GetBoolean(zOk) ){ + if( zOk && sqlite3GetBoolean(zOk) ) +#endif + { int i, sz; sz = sqlite3Strlen30(z); for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} if( z[i]=='.' && ALWAYS(sz>i+4) ) memcpy(&z[i+1], &z[sz-3], 4); } } #endif Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -3180,11 +3180,11 @@ ** automatically created table with root-page 1 (an BLOB_INTKEY table). */ if( pOp->p4.pKeyInfo ){ int pgno; assert( pOp->p4type==P4_KEYINFO ); - rc = sqlite3BtreeCreateTable(pCx->pBt, &pgno, BTREE_BLOBKEY); + rc = sqlite3BtreeCreateTable(pCx->pBt, &pgno, BTREE_BLOBKEY | pOp->p5); if( rc==SQLITE_OK ){ assert( pgno==MASTER_ROOT+1 ); rc = sqlite3BtreeCursor(pCx->pBt, pgno, 1, (KeyInfo*)pOp->p4.z, pCx->pCursor); pCx->pKeyInfo = pOp->p4.pKeyInfo; Index: src/vdbeapi.c ================================================================== --- src/vdbeapi.c +++ src/vdbeapi.c @@ -486,11 +486,11 @@ sqlite3_mutex_enter(db->mutex); while( (rc = sqlite3Step(v))==SQLITE_SCHEMA && cnt++ < SQLITE_MAX_SCHEMA_RETRY && (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){ sqlite3_reset(pStmt); - v->expired = 0; + assert( v->expired==0 ); } if( rc2!=SQLITE_OK && ALWAYS(v->isPrepareV2) && ALWAYS(db->pErr) ){ /* This case occurs after failing to recompile an sql statement. ** The error message from the SQL compiler has already been loaded ** into the database handle. This block copies the error message Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -1504,10 +1504,11 @@ nMem = 10; } memset(zCsr, 0, zEnd-zCsr); zCsr += (zCsr - (u8*)0)&7; assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); + p->expired = 0; /* Memory for registers, parameters, cursor, etc, is allocated in two ** passes. On the first pass, we try to reuse unused space at the ** end of the opcode array. If we are unable to satisfy all memory ** requirements by reusing the opcode array tail, then the second Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -251,10 +251,11 @@ #define WHERE_REVERSE 0x02000000 /* Scan in reverse order */ #define WHERE_UNIQUE 0x04000000 /* Selects no more than one row */ #define WHERE_VIRTUALTABLE 0x08000000 /* Use virtual-table processing */ #define WHERE_MULTI_OR 0x10000000 /* OR using multiple indices */ #define WHERE_TEMP_INDEX 0x20000000 /* Uses an ephemeral index */ +#define WHERE_DISTINCT 0x40000000 /* Correct order for DISTINCT */ /* ** Initialize a preallocated WhereClause structure. */ static void whereClauseInit( @@ -1395,10 +1396,166 @@ } } return 0; } +/* +** This function searches the expression list passed as the second argument +** for an expression of type TK_COLUMN that refers to the same column and +** uses the same collation sequence as the iCol'th column of index pIdx. +** Argument iBase is the cursor number used for the table that pIdx refers +** to. +** +** If such an expression is found, its index in pList->a[] is returned. If +** no expression is found, -1 is returned. +*/ +static int findIndexCol( + Parse *pParse, /* Parse context */ + ExprList *pList, /* Expression list to search */ + int iBase, /* Cursor for table associated with pIdx */ + Index *pIdx, /* Index to match column of */ + int iCol /* Column of index to match */ +){ + int i; + const char *zColl = pIdx->azColl[iCol]; + + for(i=0; inExpr; i++){ + Expr *p = pList->a[i].pExpr; + if( p->op==TK_COLUMN + && p->iColumn==pIdx->aiColumn[iCol] + && p->iTable==iBase + ){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, p); + if( ALWAYS(pColl) && 0==sqlite3StrICmp(pColl->zName, zColl) ){ + return i; + } + } + } + + return -1; +} + +/* +** This routine determines if pIdx can be used to assist in processing a +** DISTINCT qualifier. In other words, it tests whether or not using this +** index for the outer loop guarantees that rows with equal values for +** all expressions in the pDistinct list are delivered grouped together. +** +** For example, the query +** +** SELECT DISTINCT a, b, c FROM tbl WHERE a = ? +** +** can benefit from any index on columns "b" and "c". +*/ +static int isDistinctIndex( + Parse *pParse, /* Parsing context */ + WhereClause *pWC, /* The WHERE clause */ + Index *pIdx, /* The index being considered */ + int base, /* Cursor number for the table pIdx is on */ + ExprList *pDistinct, /* The DISTINCT expressions */ + int nEqCol /* Number of index columns with == */ +){ + Bitmask mask = 0; /* Mask of unaccounted for pDistinct exprs */ + int i; /* Iterator variable */ + + if( pIdx->zName==0 || pDistinct==0 || pDistinct->nExpr>=BMS ) return 0; + testcase( pDistinct->nExpr==BMS-1 ); + + /* Loop through all the expressions in the distinct list. If any of them + ** are not simple column references, return early. Otherwise, test if the + ** WHERE clause contains a "col=X" clause. If it does, the expression + ** can be ignored. If it does not, and the column does not belong to the + ** same table as index pIdx, return early. Finally, if there is no + ** matching "col=X" expression and the column is on the same table as pIdx, + ** set the corresponding bit in variable mask. + */ + for(i=0; inExpr; i++){ + WhereTerm *pTerm; + Expr *p = pDistinct->a[i].pExpr; + if( p->op!=TK_COLUMN ) return 0; + pTerm = findTerm(pWC, p->iTable, p->iColumn, ~(Bitmask)0, WO_EQ, 0); + if( pTerm ){ + Expr *pX = pTerm->pExpr; + CollSeq *p1 = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); + CollSeq *p2 = sqlite3ExprCollSeq(pParse, p); + if( p1==p2 ) continue; + } + if( p->iTable!=base ) return 0; + mask |= (((Bitmask)1) << i); + } + + for(i=nEqCol; mask && inColumn; i++){ + int iExpr = findIndexCol(pParse, pDistinct, base, pIdx, i); + if( iExpr<0 ) break; + mask &= ~(((Bitmask)1) << iExpr); + } + + return (mask==0); +} + + +/* +** Return true if the DISTINCT expression-list passed as the third argument +** is redundant. A DISTINCT list is redundant if the database contains a +** UNIQUE index that guarantees that the result of the query will be distinct +** anyway. +*/ +static int isDistinctRedundant( + Parse *pParse, + SrcList *pTabList, + WhereClause *pWC, + ExprList *pDistinct +){ + Table *pTab; + Index *pIdx; + int i; + int iBase; + + /* If there is more than one table or sub-select in the FROM clause of + ** this query, then it will not be possible to show that the DISTINCT + ** clause is redundant. */ + if( pTabList->nSrc!=1 ) return 0; + iBase = pTabList->a[0].iCursor; + pTab = pTabList->a[0].pTab; + + /* If any of the expressions is an IPK column on table iBase, then return + ** true. Note: The (p->iTable==iBase) part of this test may be false if the + ** current SELECT is a correlated sub-query. + */ + for(i=0; inExpr; i++){ + Expr *p = pDistinct->a[i].pExpr; + if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; + } + + /* Loop through all indices on the table, checking each to see if it makes + ** the DISTINCT qualifier redundant. It does so if: + ** + ** 1. The index is itself UNIQUE, and + ** + ** 2. All of the columns in the index are either part of the pDistinct + ** list, or else the WHERE clause contains a term of the form "col=X", + ** where X is a constant value. The collation sequences of the + ** comparison and select-list expressions must match those of the index. + */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->onError==OE_None ) continue; + for(i=0; inColumn; i++){ + int iCol = pIdx->aiColumn[i]; + if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) + && 0>findIndexCol(pParse, pDistinct, iBase, pIdx, i) + ){ + break; + } + } + if( i==pIdx->nColumn ){ + /* This index implies that the DISTINCT qualifier is redundant. */ + return 1; + } + } + + return 0; +} /* ** This routine decides if pIdx can be used to satisfy the ORDER BY ** clause. If it can, it returns 1. If pIdx cannot satisfy the ** ORDER BY clause, this routine returns 0. @@ -1431,11 +1588,14 @@ int sortOrder = 0; /* XOR of index and ORDER BY sort direction */ int nTerm; /* Number of ORDER BY terms */ struct ExprList_item *pTerm; /* A term of the ORDER BY clause */ sqlite3 *db = pParse->db; - assert( pOrderBy!=0 ); + if( !pOrderBy ) return 0; + if( wsFlags & WHERE_COLUMN_IN ) return 0; + if( pIdx->bUnordered ) return 0; + nTerm = pOrderBy->nExpr; assert( nTerm>0 ); /* Argument pIdx must either point to a 'real' named index structure, ** or an index structure allocated on the stack by bestBtreeIndex() to @@ -1744,10 +1904,14 @@ double costTempIdx; /* per-query cost of the transient index */ WhereTerm *pTerm; /* A single term of the WHERE clause */ WhereTerm *pWCEnd; /* End of pWC->a[] */ Table *pTable; /* Table tht might be indexed */ + if( pParse->nQueryLoop<=(double)1 ){ + /* There is no point in building an automatic index for a single scan */ + return; + } if( (pParse->db->flags & SQLITE_AutoIndex)==0 ){ /* Automatic indices are disabled at run-time */ return; } if( (pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 ){ @@ -1755,10 +1919,14 @@ return; } if( pSrc->notIndexed ){ /* The NOT INDEXED clause appears in the SQL. */ return; + } + if( pSrc->isCorrelated ){ + /* The source is a correlated sub-query. No point in indexing it. */ + return; } assert( pParse->nQueryLoop >= (double)1 ); pTable = pSrc->pTab; nTableRow = pTable->nRowEst; @@ -2687,10 +2855,11 @@ WhereClause *pWC, /* The WHERE clause */ struct SrcList_item *pSrc, /* The FROM clause term to search */ Bitmask notReady, /* Mask of cursors not available for indexing */ Bitmask notValid, /* Cursors not available for any purpose */ ExprList *pOrderBy, /* The ORDER BY clause */ + ExprList *pDistinct, /* The select-list if query is DISTINCT */ WhereCost *pCost /* Lowest cost query plan */ ){ int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ Index *pProbe; /* An index we are evaluating */ Index *pIdx; /* Copy of pProbe, or zero for IPK index */ @@ -2827,11 +2996,12 @@ int nEq; /* Number of == or IN terms matching index */ int bInEst = 0; /* True if "x IN (SELECT...)" seen */ int nInMul = 1; /* Number of distinct equalities to lookup */ int estBound = 100; /* Estimated reduction in search space */ int nBound = 0; /* Number of range constraints seen */ - int bSort = 0; /* True if external sort required */ + int bSort = !!pOrderBy; /* True if external sort required */ + int bDist = !!pDistinct; /* True if index cannot help with DISTINCT */ int bLookup = 0; /* True if not a covering index */ WhereTerm *pTerm; /* A single term of the WHERE clause */ #ifdef SQLITE_ENABLE_STAT2 WhereTerm *pFirstTerm = 0; /* First term matching the index */ #endif @@ -2891,21 +3061,24 @@ /* If there is an ORDER BY clause and the index being considered will ** naturally scan rows in the required order, set the appropriate flags ** in wsFlags. Otherwise, if there is an ORDER BY clause but the index ** will scan rows in a different order, set the bSort variable. */ - if( pOrderBy ){ - if( (wsFlags & WHERE_COLUMN_IN)==0 - && pProbe->bUnordered==0 - && isSortingIndex(pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, - nEq, wsFlags, &rev) - ){ - wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; - wsFlags |= (rev ? WHERE_REVERSE : 0); - }else{ - bSort = 1; - } + if( isSortingIndex( + pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, nEq, wsFlags, &rev) + ){ + bSort = 0; + wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; + wsFlags |= (rev ? WHERE_REVERSE : 0); + } + + /* If there is a DISTINCT qualifier and this index will scan rows in + ** order of the DISTINCT expressions, clear bDist and set the appropriate + ** flags in wsFlags. */ + if( isDistinctIndex(pParse, pWC, pProbe, iCur, pDistinct, nEq) ){ + bDist = 0; + wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT; } /* If currently calculating the cost of using an index (not the IPK ** index), determine if all required column data may be obtained without ** using the main table (i.e. if the index is a covering @@ -2936,16 +3109,17 @@ nRow = aiRowEst[0]/2; nInMul = (int)(nRow / aiRowEst[nEq]); } #ifdef SQLITE_ENABLE_STAT2 - /* If the constraint is of the form x=VALUE and histogram + /* If the constraint is of the form x=VALUE or x IN (E1,E2,...) + ** and we do not think that values of x are unique and if histogram ** data is available for column x, then it might be possible ** to get a better estimate on the number of rows based on ** VALUE and how common that value is according to the histogram. */ - if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 ){ + if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 && aiRowEst[1]>1 ){ if( pFirstTerm->eOperator & (WO_EQ|WO_ISNULL) ){ testcase( pFirstTerm->eOperator==WO_EQ ); testcase( pFirstTerm->eOperator==WO_ISNULL ); whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, &nRow); }else if( pFirstTerm->eOperator==WO_IN && bInEst==0 ){ @@ -3018,10 +3192,13 @@ ** difference and select C of 3.0. */ if( bSort ){ cost += nRow*estLog(nRow)*3; } + if( bDist ){ + cost += nRow*estLog(nRow)*3; + } /**** Cost of using this index has now been computed ****/ /* If there are additional constraints on this table that cannot ** be used with the current index, but which might lower the number @@ -3163,11 +3340,11 @@ } sqlite3DbFree(pParse->db, p); }else #endif { - bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); + bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, 0, pCost); } } /* ** Disable a term in the WHERE clause. Except, do not disable the term @@ -4125,11 +4302,11 @@ for(ii=0; iinTerm; ii++){ WhereTerm *pOrTerm = &pOrWc->a[ii]; if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){ WhereInfo *pSubWInfo; /* Info for single OR-term scan */ /* Loop through table entries that match term pOrTerm. */ - pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, + pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, 0, WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE | WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY); if( pSubWInfo ){ explainOneScan( pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 @@ -4366,10 +4543,11 @@ WhereInfo *sqlite3WhereBegin( Parse *pParse, /* The parser context */ SrcList *pTabList, /* A list of all tables to be scanned */ Expr *pWhere, /* The WHERE clause */ ExprList **ppOrderBy, /* An ORDER BY clause, or NULL */ + ExprList *pDistinct, /* The select-list for DISTINCT queries - or NULL */ u16 wctrlFlags /* One of the WHERE_* flags defined in sqliteInt.h */ ){ int i; /* Loop counter */ int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ int nTabList; /* Number of elements in pTabList */ @@ -4425,10 +4603,14 @@ pWInfo->iBreak = sqlite3VdbeMakeLabel(v); pWInfo->pWC = pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo]; pWInfo->wctrlFlags = wctrlFlags; pWInfo->savedNQueryLoop = pParse->nQueryLoop; pMaskSet = (WhereMaskSet*)&pWC[1]; + + /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via + ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ + if( db->flags & SQLITE_DistinctOpt ) pDistinct = 0; /* Split the WHERE clause into separate subexpressions where each ** subexpression is separated by an AND operator. */ initMaskSet(pMaskSet); @@ -4492,10 +4674,19 @@ */ exprAnalyzeAll(pTabList, pWC); if( db->mallocFailed ){ goto whereBeginError; } + + /* Check if the DISTINCT qualifier, if there is one, is redundant. + ** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to + ** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT. + */ + if( pDistinct && isDistinctRedundant(pParse, pTabList, pWC, pDistinct) ){ + pDistinct = 0; + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } /* Chose the best index to use for each table in the FROM clause. ** ** This loop fills in the following fields: ** @@ -4576,10 +4767,11 @@ Bitmask mask; /* Mask of tables not yet ready */ for(j=iFrom, pTabItem=&pTabList->a[j]; jjointype & (JT_LEFT|JT_CROSS))!=0; if( j!=iFrom && doNotReorder ) break; m = getMask(pMaskSet, pTabItem->iCursor); if( (m & notReady)==0 ){ @@ -4586,10 +4778,11 @@ if( j==iFrom ) iFrom++; continue; } mask = (isOptimal ? m : notReady); pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0); + pDist = (i==0 ? pDistinct : 0); if( pTabItem->pIndex==0 ) nUnconstrained++; WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", j, isOptimal)); assert( pTabItem->pTab ); @@ -4600,11 +4793,11 @@ &sCost, pp); }else #endif { bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, - &sCost); + pDist, &sCost); } assert( isOptimal || (sCost.used¬Ready)==0 ); /* If an INDEXED BY clause is present, then the plan must use that ** index if it uses any index at all */ @@ -4660,10 +4853,14 @@ WHERETRACE(("*** Optimizer selects table %d for loop %d" " with cost=%g and nRow=%g\n", bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow)); if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){ *ppOrderBy = 0; + } + if( (bestPlan.plan.wsFlags & WHERE_DISTINCT)!=0 ){ + assert( pWInfo->eDistinct==0 ); + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } andFlags &= bestPlan.plan.wsFlags; pLevel->plan = bestPlan.plan; testcase( bestPlan.plan.wsFlags & WHERE_INDEXED ); testcase( bestPlan.plan.wsFlags & WHERE_TEMP_INDEX ); Index: test/all.test ================================================================== --- test/all.test +++ test/all.test @@ -36,10 +36,11 @@ run_test_suite pcache0 run_test_suite pcache10 run_test_suite pcache50 run_test_suite pcache90 run_test_suite pcache100 +run_test_suite prepare if {$::tcl_platform(platform)=="unix"} { ifcapable !default_autovacuum { run_test_suite autovacuum_crash } Index: test/alter2.test ================================================================== --- test/alter2.test +++ test/alter2.test @@ -135,10 +135,11 @@ } } {8 {} 1 10} do_test alter2-1.9 { # ALTER TABLE abc ADD COLUMN d; alter_table abc {CREATE TABLE abc(a, b, c, d);} + if {[permutation] == "prepare"} { db cache flush } execsql { SELECT * FROM abc; } execsql { UPDATE abc SET d = 11 WHERE c IS NULL AND a<4; SELECT * FROM abc; } Index: test/autoindex1.test ================================================================== --- test/autoindex1.test +++ test/autoindex1.test @@ -245,7 +245,17 @@ 1 0 0 {EXECUTE CORRELATED SCALAR SUBQUERY 2} 2 0 0 {SEARCH TABLE flock_owner AS later USING COVERING INDEX sqlite_autoindex_flock_owner_1 (flock_no=? AND owner_change_date>? AND owner_change_date=f2.word AND streetname.n<=(f2.word || x'F7BFBFBF') } -} {steelewood tallia tallu talwyn taymouth thelema trailer {tyler finley}} +} {{tyler finley} trailer taymouth steelewood tallia tallu talwyn thelema} finish_test Index: test/incrblob_err.test ================================================================== --- test/incrblob_err.test +++ test/incrblob_err.test @@ -33,10 +33,11 @@ CREATE TABLE blobs(k, v BLOB); INSERT INTO blobs VALUES(1, zeroblob($::bytes)); } } -tclbody { set ::blob [db incrblob blobs v 1] + fconfigure $::blob -translation binary set rc [catch {puts -nonewline $::blob $::data}] if {$rc} { error "out of memory" } } do_malloc_test 2 -tclprep { @@ -69,12 +70,11 @@ } set rc [catch {close $::blob}] if {$rc} { error "out of memory" } -} - +} do_ioerr_test incrblob_err-4 -cksum 1 -sqlprep { CREATE TABLE blobs(k, v BLOB); INSERT INTO blobs VALUES(1, $::data); } -tclbody { @@ -85,19 +85,21 @@ do_ioerr_test incrblob_err-5 -cksum 1 -sqlprep { CREATE TABLE blobs(k, v BLOB); INSERT INTO blobs VALUES(1, zeroblob(length(CAST($::data AS BLOB)))); } -tclbody { set ::blob [db incrblob blobs v 1] + fconfigure $::blob -translation binary puts -nonewline $::blob $::data close $::blob } do_ioerr_test incrblob_err-6 -cksum 1 -sqlprep { CREATE TABLE blobs(k, v BLOB); INSERT INTO blobs VALUES(1, $::data || $::data || $::data); } -tclbody { set ::blob [db incrblob blobs v 1] + fconfigure $::blob -translation binary seek $::blob -20 end puts -nonewline $::blob "12345678900987654321" close $::blob } Index: test/insert4.test ================================================================== --- test/insert4.test +++ test/insert4.test @@ -110,11 +110,11 @@ execsql { DELETE FROM t3; INSERT INTO t3 SELECT DISTINCT * FROM t2; SELECT * FROM t3; } -} {1 9 9 1} +} {9 1 1 9} xferopt_test insert4-2.4.2 0 do_test insert4-2.4.3 { catchsql { DELETE FROM t1; INSERT INTO t1 SELECT DISTINCT * FROM t2; Index: test/like.test ================================================================== --- test/like.test +++ test/like.test @@ -68,16 +68,19 @@ execsql { SELECT x FROM t1 WHERE x LIKE 'aBc' ORDER BY 1; } } {ABC abc} do_test like-1.5.1 { + # Use sqlite3_exec() to verify fix for ticket [25ee81271091] 2011-06-26 + sqlite3_exec db {PRAGMA case_sensitive_like=on} +} {0 {}} +do_test like-1.5.2 { execsql { - PRAGMA case_sensitive_like=on; SELECT x FROM t1 WHERE x LIKE 'abc' ORDER BY 1; } } {abc} -do_test like-1.5.2 { +do_test like-1.5.3 { execsql { PRAGMA case_sensitive_like; -- no argument; does not change setting SELECT x FROM t1 WHERE x LIKE 'abc' ORDER BY 1; } } {abc} Index: test/misc5.test ================================================================== --- test/misc5.test +++ test/misc5.test @@ -503,11 +503,11 @@ ) ) ) ORDER BY LOWER(artist) ASC; } - } {one} + } {two} } # Ticket #1370. Do not overwrite small files (less than 1024 bytes) # when trying to open them as a database. # Index: test/multiplex.test ================================================================== --- test/multiplex.test +++ test/multiplex.test @@ -47,10 +47,11 @@ # This attempts to delete the base file and # and files with the chunk extension. proc multiplex_delete {name} { global g_max_chunks + forcedelete $name for {set i 0} {$i<$g_max_chunks} {incr i} { forcedelete [multiplex_name $name $i] forcedelete [multiplex_name $name-journal $i] forcedelete [multiplex_name $name-wal $i] } @@ -76,25 +77,23 @@ do_test multiplex-1.9.1 { sqlite3_multiplex_initialize "" 1 } {SQLITE_OK} do_test multiplex-1.9.2 { sqlite3 db test.db } {} do_test multiplex-1.9.3 { multiplex_set db main 32768 16 } {SQLITE_OK} -do_test multiplex-1.9.4 { multiplex_set db main 32768 -1 } {SQLITE_MISUSE} -do_test multiplex-1.9.5 { multiplex_set db main -1 16 } {SQLITE_MISUSE} +do_test multiplex-1.9.4 { multiplex_set db main 32768 -1 } {SQLITE_OK} do_test multiplex-1.9.6 { multiplex_set db main 31 16 } {SQLITE_OK} -do_test multiplex-1.9.7 { multiplex_set db main 32768 100 } {SQLITE_MISUSE} +do_test multiplex-1.9.7 { multiplex_set db main 32768 100 } {SQLITE_OK} do_test multiplex-1.9.8 { multiplex_set db main 1073741824 1 } {SQLITE_OK} do_test multiplex-1.9.9 { db close } {} do_test multiplex-1.9.10 { sqlite3_multiplex_shutdown } {SQLITE_OK} do_test multiplex-1.10.1 { sqlite3_multiplex_initialize "" 1 } {SQLITE_OK} do_test multiplex-1.10.2 { sqlite3 db test.db } {} do_test multiplex-1.10.3 { lindex [ catchsql { SELECT multiplex_control(2, 32768); } ] 0 } {0} -do_test multiplex-1.10.4 { lindex [ catchsql { SELECT multiplex_control(3, -1); } ] 0 } {1} -do_test multiplex-1.10.5 { lindex [ catchsql { SELECT multiplex_control(2, -1); } ] 0 } {1} +do_test multiplex-1.10.4 { lindex [ catchsql { SELECT multiplex_control(3, -1); } ] 0 } {0} do_test multiplex-1.10.6 { lindex [ catchsql { SELECT multiplex_control(2, 31); } ] 0 } {0} -do_test multiplex-1.10.7 { lindex [ catchsql { SELECT multiplex_control(3, 100); } ] 0 } {1} +do_test multiplex-1.10.7 { lindex [ catchsql { SELECT multiplex_control(3, 100); } ] 0 } {0} do_test multiplex-1.10.8 { lindex [ catchsql { SELECT multiplex_control(2, 1073741824); } ] 0 } {0} do_test multiplex-1.10.9 { db close } {} do_test multiplex-1.10.10 { sqlite3_multiplex_shutdown } {SQLITE_OK} do_test multiplex-1.11.1 { sqlite3_multiplex_initialize "" 1 } {SQLITE_OK} @@ -144,12 +143,13 @@ # sqlite3_multiplex_initialize "" 1 multiplex_set db main 32768 16 +file delete -force test.x do_test multiplex-2.1.2 { - sqlite3 db test.db + sqlite3 db test.x execsql { PRAGMA page_size=1024; PRAGMA auto_vacuum=OFF; PRAGMA journal_mode=DELETE; } @@ -157,22 +157,22 @@ CREATE TABLE t1(a, b); INSERT INTO t1 VALUES(1, randomblob(1100)); INSERT INTO t1 VALUES(2, randomblob(1100)); } } {} -do_test multiplex-2.1.3 { file size [multiplex_name test.db 0] } {4096} +do_test multiplex-2.1.3 { file size [multiplex_name test.x 0] } {4096} do_test multiplex-2.1.4 { execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) } } {} do_test multiplex-2.2.1 { execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) } } {} -do_test multiplex-2.2.3 { file size [multiplex_name test.db 0] } {6144} +do_test multiplex-2.2.3 { file size [multiplex_name test.x 0] } {6144} do_test multiplex-2.3.1 { - sqlite3 db2 test2.db + sqlite3 db2 test2.x db2 close } {} do_test multiplex-2.4.1 { @@ -179,21 +179,21 @@ sqlite3_multiplex_shutdown } {SQLITE_MISUSE} do_test multiplex-2.4.2 { execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) } } {} -do_test multiplex-2.4.4 { file size [multiplex_name test.db 0] } {7168} +do_test multiplex-2.4.4 { file size [multiplex_name test.x 0] } {7168} do_test multiplex-2.4.99 { db close sqlite3_multiplex_shutdown } {SQLITE_OK} do_test multiplex-2.5.1 { - multiplex_delete test.db + multiplex_delete test.x sqlite3_multiplex_initialize "" 1 - sqlite3 db test.db + sqlite3 db test.x multiplex_set db main 4096 16 } {SQLITE_OK} do_test multiplex-2.5.2 { execsql { @@ -234,12 +234,12 @@ do_test multiplex-2.5.8 { db eval {SELECT a,length(b) FROM t1 WHERE a=4} } {4 4000} -do_test multiplex-2.5.9 { file size [multiplex_name test.db 0] } [list $g_chunk_size] -do_test multiplex-2.5.10 { file size [multiplex_name test.db 1] } [list $g_chunk_size] +do_test multiplex-2.5.9 { file size [multiplex_name test.x 0] } [list $g_chunk_size] +do_test multiplex-2.5.10 { file size [multiplex_name test.x 1] } [list $g_chunk_size] do_test multiplex-2.5.99 { db close sqlite3_multiplex_shutdown } {SQLITE_OK} @@ -521,64 +521,21 @@ } -body { sqlite3_multiplex_initialize "" 1 multiplex_set db main 32768 16 } -# test that mismatch filesize is detected -# -# Do not run this test if $::G(perm:presql) is set. If it is set, then the -# expected IO error will occur within the Tcl [sqlite3] wrapper, not within -# the first SQL statement executed below. This breaks the test case. -# -if {0==[info exists ::G(perm:presql)] || $::G(perm:presql) == ""} { - set all_journal_modes {delete persist truncate memory off} - foreach jmode $all_journal_modes { - do_test multiplex-5.6.1.$jmode { - sqlite3_multiplex_shutdown - multiplex_delete test.db - sqlite3 db test.db - db eval { - PRAGMA page_size = 1024; - PRAGMA auto_vacuum = off; - } - db eval "PRAGMA journal_mode = $jmode;" - } $jmode - do_test multiplex-5.6.2.$jmode { - execsql { - CREATE TABLE t1(a, b); - INSERT INTO t1 VALUES(1, randomblob(15000)); - INSERT INTO t1 VALUES(2, randomblob(15000)); - INSERT INTO t1 VALUES(3, randomblob(15000)); - INSERT INTO t1 VALUES(4, randomblob(15000)); - INSERT INTO t1 VALUES(5, randomblob(15000)); - } - db close - sqlite3_multiplex_initialize "" 1 - sqlite3 db test.db - multiplex_set db main 4096 16 - } {SQLITE_OK} - do_test multiplex-5.6.3.$jmode { - catchsql { - INSERT INTO t1 VALUES(6, randomblob(15000)); - } - } {1 {disk I/O error}} - do_test multiplex-5.6.4.$jmode { - db close - } {} - } -} - #------------------------------------------------------------------------- # Test that you can vacuum a multiplex'ed DB. ifcapable vacuum { sqlite3_multiplex_shutdown do_test multiplex-6.0.0 { multiplex_delete test.db + multiplex_delete test.x sqlite3_multiplex_initialize "" 1 - sqlite3 db test.db + sqlite3 db test.x multiplex_set db main 4096 16 } {SQLITE_OK} do_test multiplex-6.1.0 { execsql { @@ -590,23 +547,23 @@ CREATE TABLE t1(a, b); INSERT INTO t1 VALUES(1, randomblob($g_chunk_size)); INSERT INTO t1 VALUES(2, randomblob($g_chunk_size)); } } {} -do_test multiplex-6.2.1 { file size [multiplex_name test.db 0] } [list $g_chunk_size] -do_test multiplex-6.2.2 { file size [multiplex_name test.db 1] } [list $g_chunk_size] +do_test multiplex-6.2.1 { file size [multiplex_name test.x 0] } [list $g_chunk_size] +do_test multiplex-6.2.2 { file size [multiplex_name test.x 1] } [list $g_chunk_size] do_test multiplex-6.3.0 { execsql { VACUUM } } {} do_test multiplex-6.99 { db close - multiplex_delete test.db + multiplex_delete test.x sqlite3_multiplex_shutdown } {SQLITE_OK} } catch { sqlite3_multiplex_shutdown } finish_test Index: test/permutations.test ================================================================== --- test/permutations.test +++ test/permutations.test @@ -780,10 +780,19 @@ select1.test select2.test select3.test select4.test select5.test select7.test select8.test selectA.test selectC.test } -dbconfig { optimization_control $::dbhandle all 0 } + +test_suite "prepare" -description { + Run tests with the db connection using sqlite3_prepare() instead of _v2(). +} -dbconfig { + db_use_legacy_prepare $::dbhandle 1 + #$::dbhandle cache size 0 +} -files [ + test_set $allquicktests -exclude *malloc* *ioerr* *fault* +] # End of tests ############################################################################# # run_tests NAME OPTIONS Index: test/releasetest.tcl ================================================================== --- test/releasetest.tcl +++ test/releasetest.tcl @@ -153,14 +153,14 @@ } } array set ::Platforms { Linux-x86_64 { + "Debug-One" "checksymbols test" "Secure-Delete" test "Unlock-Notify" "QUICKTEST_INCLUDE=notify2.test test" "Update-Delete-Limit" test - "Debug-One" test "Extra-Robustness" test "Device-Two" test "Ftrapv" test "Default" "threadtest test" "Device-One" fulltest @@ -174,10 +174,11 @@ Darwin-i386 { "Locking-Style" test "OS-X" "threadtest fulltest" } } + # End of configuration section. ######################################################################### ######################################################################### Index: test/selectB.test ================================================================== --- test/selectB.test +++ test/selectB.test @@ -353,11 +353,11 @@ execsql { SELECT * FROM ( SELECT DISTINCT (a/10) FROM t1 UNION ALL SELECT DISTINCT(d%2) FROM t2 ) } - } {0 1 0 1} + } {0 1 1 0} do_test selectB-$ii.20 { execsql { SELECT DISTINCT * FROM ( SELECT DISTINCT (a/10) FROM t1 UNION ALL SELECT DISTINCT(d%2) FROM t2 Index: test/sqllimits1.test ================================================================== --- test/sqllimits1.test +++ test/sqllimits1.test @@ -317,22 +317,26 @@ set np1 [expr {$SQLITE_LIMIT_LENGTH + 1}] set ::str1 [string repeat A $np1] catch {sqlite3_bind_text $::STMT 1 $::str1 -1} res set res } {SQLITE_TOOBIG} -do_test sqllimits1-5.14.5 { - catch {sqlite3_bind_text16 $::STMT 1 $::str1 -1} res - set res -} {SQLITE_TOOBIG} +ifcapable utf16 { + do_test sqllimits1-5.14.5 { + catch {sqlite3_bind_text16 $::STMT 1 $::str1 -1} res + set res + } {SQLITE_TOOBIG} +} do_test sqllimits1-5.14.6 { catch {sqlite3_bind_text $::STMT 1 $::str1 $np1} res set res } {SQLITE_TOOBIG} -do_test sqllimits1-5.14.7 { - catch {sqlite3_bind_text16 $::STMT 1 $::str1 $np1} res - set res -} {SQLITE_TOOBIG} +ifcapable utf16 { + do_test sqllimits1-5.14.7 { + catch {sqlite3_bind_text16 $::STMT 1 $::str1 $np1} res + set res + } {SQLITE_TOOBIG} +} do_test sqllimits1-5.14.8 { set n [expr {$np1-1}] catch {sqlite3_bind_text $::STMT 1 $::str1 $n} res set res } {} Index: test/temptable.test ================================================================== --- test/temptable.test +++ test/temptable.test @@ -290,10 +290,11 @@ # do_test temptable-5.1 { execsql { CREATE TEMP TABLE mask(a,b,c) } db2 + if {[permutation]=="prepare"} { db2 cache flush } execsql { CREATE INDEX mask ON t2(x); SELECT * FROM t2; } } {3 4} ADDED test/tkt-54844eea3f.test Index: test/tkt-54844eea3f.test ================================================================== --- test/tkt-54844eea3f.test +++ test/tkt-54844eea3f.test @@ -0,0 +1,67 @@ +# 2011 July 8 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing that bug [54844eea3f] has been fixed. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set ::testprefix tkt-54844eea3f + +do_test 1.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY); + INSERT INTO t1 VALUES(1); + INSERT INTO t1 VALUES(4); + + CREATE TABLE t2(b INTEGER PRIMARY KEY); + INSERT INTO t2 VALUES(1); + INSERT INTO t2 VALUES(2); + INSERT INTO t2 SELECT b+2 FROM t2; + INSERT INTO t2 SELECT b+4 FROM t2; + INSERT INTO t2 SELECT b+8 FROM t2; + INSERT INTO t2 SELECT b+16 FROM t2; + + CREATE TABLE t3(c INTEGER PRIMARY KEY); + INSERT INTO t3 VALUES(1); + INSERT INTO t3 VALUES(2); + INSERT INTO t3 VALUES(3); + } +} {} + +do_test 1.1 { + execsql { + SELECT 'test-2', t3.c, ( + SELECT count(*) + FROM t1 JOIN (SELECT DISTINCT t3.c AS p FROM t2) AS x ON t1.a=x.p + ) + FROM t3; + } +} {test-2 1 1 test-2 2 0 test-2 3 0} + +do_test 1.2 { + execsql { + CREATE TABLE t4(a, b, c); + INSERT INTO t4 VALUES('a', 1, 'one'); + INSERT INTO t4 VALUES('a', 2, 'two'); + INSERT INTO t4 VALUES('b', 1, 'three'); + INSERT INTO t4 VALUES('b', 2, 'four'); + SELECT ( + SELECT c FROM ( + SELECT * FROM t4 WHERE a=out.a ORDER BY b LIMIT 10 OFFSET 1 + ) WHERE b=out.b + ) FROM t4 AS out; + } +} {{} two {} four} + + +finish_test Index: test/triggerD.test ================================================================== --- test/triggerD.test +++ test/triggerD.test @@ -12,10 +12,16 @@ # Verify that when columns named "rowid", "oid", and "_rowid_" appear # in a table as ordinary columns (not as the INTEGER PRIMARY KEY) then # the use of these columns in triggers will refer to the column and not # to the actual ROWID. Ticket [34d2ae1c6d08b5271ba5e5592936d4a1d913ffe3] # +# Also, verify that triggers created like this: +# +# CREATE TRIGGER attached.trig AFTER INSERT ON attached.tab ... +# +# can be reparsed as a main database. Ticket [d6ddba6706353915ceedc56b4e3] +# set testdir [file dirname $argv0] source $testdir/tester.tcl ifcapable {!trigger} { finish_test @@ -168,7 +174,46 @@ INSERT INTO temp.t300 VALUES(4); SELECT * FROM t301; } } {10003 20004} + +############################################################################# +# +# Ticket [d6ddba6706353915ceedc56b4e3e72ecb4d77ba4] +# +# The following syntax really should not be allowed: +# +# CREATE TRIGGER xyz.trig BEFORE UPDATE ON xyz.tab BEGIN ... +# +# But a long-standing bug does allow it. And the "xyz.tab" slips into +# the sqlite_master table. We cannot fix the bug simply by disallowing +# "xyz.tab" since that could break legacy applications. We have to +# fix the system so that the "xyz." on "xyz.tab" is ignored. +# Verify that this is the case. +# +do_test triggerD-4.1 { + db close + file delete -force test.db test2.db + sqlite3 db test.db + db eval { + CREATE TABLE t1(x); + ATTACH 'test2.db' AS db2; + CREATE TABLE db2.t2(y); + CREATE TABLE db2.log(z); + CREATE TRIGGER db2.trig AFTER INSERT ON db2.t2 BEGIN + INSERT INTO log(z) VALUES(new.y); + END; + INSERT INTO t2 VALUES(123); + SELECT * FROM log; + } +} {123} +do_test triggerD-4.2 { + sqlite3 db2 test2.db + db2 eval { + INSERT INTO t2 VALUES(234); + SELECT * FROM log; + } +} {123 234} +db2 close finish_test Index: test/wal3.test ================================================================== --- test/wal3.test +++ test/wal3.test @@ -705,11 +705,11 @@ # on any aReadMark[] slot (because there are already several readers), # the client takes a shared-lock on a slot without modifying the value # and continues. # set nConn 50 -if { [string match *BSD $tcl_platform(os)] } { set nConn 35 } +if { [string match *BSD $tcl_platform(os)] } { set nConn 25 } do_test wal3-9.0 { file delete -force test.db test.db-journal test.db wal sqlite3 db test.db execsql { PRAGMA page_size = 1024; @@ -782,6 +782,5 @@ sql2 {PRAGMA integrity_check} } {ok} } finish_test - Index: test/wal6.test ================================================================== --- test/wal6.test +++ test/wal6.test @@ -41,23 +41,15 @@ SELECT * FROM t1; } } {1 2} # Under Windows, you'll get an error trying to delete -# a file this is already opened. For now, make sure -# we get that error, then close the first connection +# a file this is already opened. Close the first connection # so the other tests work. if {$tcl_platform(platform)=="windows"} { if {$jmode=="persist" || $jmode=="truncate"} { - do_test wal6-1.2.$jmode.win { - sqlite3 db2 test.db - catchsql { - PRAGMA journal_mode=WAL; - } db2 - } {1 {disk I/O error}} - db2 close - db close + db close } } do_test wal6-1.2.$jmode { sqlite3 db2 test.db @@ -85,6 +77,5 @@ forcedelete test.db } finish_test - ADDED test/win32lock.test Index: test/win32lock.test ================================================================== --- test/win32lock.test +++ test/win32lock.test @@ -0,0 +1,108 @@ +# 2011 July 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this script is recovery from transient manditory locks +# that sometimes appear on database files due to anti-virus software. +# + +if {$tcl_platform(platform)!="windows"} return + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix win32lock + +db close +sqlite3_shutdown +test_sqlite3_log xLog +proc xLog {error_code msg} { + lappend ::log $msg +} +sqlite3 db test.db + +do_test win32lock-1.1 { + db eval { + PRAGMA cache_size=10; + CREATE TABLE t1(x,y); + INSERT INTO t1 VALUES(1,randomblob(100000)); + INSERT INTO t1 VALUES(2,randomblob(50000)); + INSERT INTO t1 VALUES(3,randomblob(25000)); + INSERT INTO t1 VALUES(4,randomblob(12500)); + SELECT x, length(y) FROM t1 ORDER BY rowid; + } +} {1 100000 2 50000 3 25000 4 12500} + +unset -nocomplain delay1 rc msg +set delay1 50 +set rc 0 +set old_pending_byte [sqlite3_test_control_pending_byte 0x40000000] +while {1} { + sqlite3_sleep 10 + lock_win32_file test.db 0 $::delay1 + set rc [catch {db eval {SELECT x, length(y) FROM t1 ORDER BY rowid}} msg] + if {$rc} { + do_test win32lock-1.2-$delay1-fin { + set ::msg + } {disk I/O error} + break + } else { + do_test win32lock-1.2-$delay1 { + set ::msg + } {1 100000 2 50000 3 25000 4 12500} + if {$::log!=""} { + do_test win32lock-1.2-$delay1-log1 { + regsub {\d+} $::log # x + set x + } {{delayed #ms for lock/sharing conflict}} + } + incr delay1 50 + } + set ::log {} +} + +do_test win32lock-2.0 { + file_control_win32_av_retry db -1 -1 +} {0 10 25} +do_test win32lock-2.1 { + file_control_win32_av_retry db 1 1 +} {0 1 1} + +set delay1 50 +while {1} { + sqlite3_sleep 10 + lock_win32_file test.db 0 $::delay1 + set rc [catch {db eval {SELECT x, length(y) FROM t1 ORDER BY rowid}} msg] + if {$rc} { + do_test win32lock-2.2-$delay1-fin { + set ::msg + } {disk I/O error} + break + } else { + do_test win32lock-2.2-$delay1 { + set ::msg + } {1 100000 2 50000 3 25000 4 12500} + if {$::log!=""} { + do_test win32lock-2.2-$delay1-log1 { + regsub {\d+} $::log # x + set x + } {{delayed #ms for lock/sharing conflict}} + } + incr delay1 50 + } + set ::log {} +} + +file_control_win32_av_retry db 10 25 +sqlite3_test_control_pending_byte $old_pending_byte +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize +finish_test Index: tool/symbols.sh ================================================================== --- tool/symbols.sh +++ tool/symbols.sh @@ -10,14 +10,14 @@ -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \ -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ -DSQLITE_ENABLE_ICU \ sqlite3.c -nm sqlite3.o | grep ' T ' | sort -k 3 +nm sqlite3.o | grep ' [TD] ' | sort -k 3 echo '****** Surplus symbols from a build including RTREE, FTS4 & ICU ******' -nm sqlite3.o | grep ' T ' | grep -v ' sqlite3_' +nm sqlite3.o | grep ' [TD] ' | grep -v ' .*sqlite3_' echo '****** Dependencies of the core. No extensions. No OS interface *******' gcc -c -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \ -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ Index: tool/warnings.sh ================================================================== --- tool/warnings.sh +++ tool/warnings.sh @@ -6,9 +6,13 @@ make sqlite3.c echo '********** No optimizations. Includes FTS4 and RTREE *********' gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ sqlite3.c -echo '********** Optimized -O3. Includes FTS4 and RTREE *********' +echo '********** No optimizations. ENABLE_STAT2. THREADSAFE=0 *******' +gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ + -ansi -DSQLITE_ENABLE_STAT2 -DSQLITE_THREADSAFE=0 \ + sqlite3.c +echo '********** Optimized -O3. Includes FTS4 and RTREE ************' gcc -O3 -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ sqlite3.c