Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -1304,10 +1304,13 @@ $(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS) showshm$(TEXE): $(TOP)/tool/showshm.c $(LTLINK) -o $@ $(TOP)/tool/showshm.c +index_usage$(TEXE): $(TOP)/tool/index_usage.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/tool/index_usage.c sqlite3.lo $(TLIBS) + changeset$(TEXE): $(TOP)/ext/session/changeset.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS) changesetfuzz$(TEXE): $(TOP)/ext/session/changesetfuzz.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/ext/session/changesetfuzz.c sqlite3.lo $(TLIBS) Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -2442,10 +2442,14 @@ $(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showshm.exe: $(TOP)\tool\showshm.c $(LTLINK) $(NO_WARN) $(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS) +index_usage.exe: $(TOP)\tool\index_usage.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ + $(TOP)\tool\index_usage.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + changeset.exe: $(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \ $(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) Index: VERSION ================================================================== --- VERSION +++ VERSION @@ -1,1 +1,1 @@ -3.26.0 +3.27.0 Index: autoconf/Makefile.msc ================================================================== --- autoconf/Makefile.msc +++ autoconf/Makefile.msc @@ -281,10 +281,11 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1 !ENDIF OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 !ENDIF # Should the session extension be enabled? If so, add compilation options @@ -935,10 +936,11 @@ # !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1 !ENDIF # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. Index: configure ================================================================== --- configure +++ configure @@ -1,8 +1,8 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.26.0. +# Generated by GNU Autoconf 2.69 for sqlite 3.27.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. # # @@ -724,12 +724,12 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.26.0' -PACKAGE_STRING='sqlite 3.26.0' +PACKAGE_VERSION='3.27.0' +PACKAGE_STRING='sqlite 3.27.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' # Factoring default headers for most tests. ac_includes_default="\ @@ -1464,11 +1464,11 @@ # if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.26.0 to adapt to many kinds of systems. +\`configure' configures sqlite 3.27.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... To assign environment variables (e.g., CC, CFLAGS...), specify them as VAR=VALUE. See below for descriptions of some of the useful variables. @@ -1529,11 +1529,11 @@ _ACEOF fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.26.0:";; + short | recursive ) echo "Configuration of sqlite 3.27.0:";; esac cat <<\_ACEOF Optional Features: --disable-option-checking ignore unrecognized --enable/--with options @@ -1655,11 +1655,11 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.26.0 +sqlite configure 3.27.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it. @@ -2074,11 +2074,11 @@ } # ac_fn_c_check_header_mongrel cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.26.0, which was +It was created by sqlite $as_me 3.27.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ _ACEOF @@ -12230,11 +12230,11 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # Save the log message, to keep $0 and so on meaningful, and to # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.26.0, which was +This file was extended by sqlite $as_me 3.27.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS CONFIG_LINKS = $CONFIG_LINKS @@ -12296,11 +12296,11 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.26.0 +sqlite config.status 3.27.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" Copyright (C) 2012 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation Index: ext/fts3/fts3_unicode.c ================================================================== --- ext/fts3/fts3_unicode.c +++ ext/fts3/fts3_unicode.c @@ -80,11 +80,11 @@ typedef struct unicode_tokenizer unicode_tokenizer; typedef struct unicode_cursor unicode_cursor; struct unicode_tokenizer { sqlite3_tokenizer base; - int bRemoveDiacritic; + int eRemoveDiacritic; int nException; int *aiException; }; struct unicode_cursor { @@ -225,21 +225,24 @@ int rc = SQLITE_OK; pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer)); if( pNew==NULL ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(unicode_tokenizer)); - pNew->bRemoveDiacritic = 1; + pNew->eRemoveDiacritic = 1; for(i=0; rc==SQLITE_OK && ibRemoveDiacritic = 1; + pNew->eRemoveDiacritic = 1; } else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){ - pNew->bRemoveDiacritic = 0; + pNew->eRemoveDiacritic = 0; + } + else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){ + pNew->eRemoveDiacritic = 2; } else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){ rc = unicodeAddExceptions(pNew, 1, &z[11], n-11); } else if( n>=11 && memcmp("separators=", z, 11)==0 ){ @@ -348,11 +351,11 @@ pCsr->nAlloc += 64; } /* Write the folded case of the last character read to the output */ zEnd = z; - iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic); + iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic); if( iOut ){ WRITE_UTF8(zOut, iOut); } /* If the cursor is not at EOF, read the next character */ Index: ext/fts3/fts3_unicode2.c ================================================================== --- ext/fts3/fts3_unicode2.c +++ ext/fts3/fts3_unicode2.c @@ -157,36 +157,51 @@ ** of the ASCII letter only. For example, if passed 235 - "LATIN ** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER ** E"). The resuls of passing a codepoint that corresponds to an ** uppercase letter are undefined. */ -static int remove_diacritic(int c){ +static int remove_diacritic(int c, int bComplex){ unsigned short aDia[] = { 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, - 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, - 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, - 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, - 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, - 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, - 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, - 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, - 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, - 62924, 63050, 63082, 63274, 63390, + 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896, + 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106, + 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344, + 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198, + 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468, + 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, + 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, + 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, + 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, + 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, + 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, + 63182, 63242, 63274, 63310, 63368, 63390, }; char aChar[] = { - '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', - 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', - 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', - 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', - 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', - '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', - 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', - 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', - 'e', 'i', 'o', 'u', 'y', + '\0', 'a'|0x00, 'c'|0x00, 'e'|0x00, 'i'|0x00, 'n'|0x00, + 'o'|0x00, 'u'|0x00, 'y'|0x00, 'y'|0x00, 'a'|0x00, 'c'|0x00, + 'd'|0x00, 'e'|0x00, 'e'|0x00, 'g'|0x00, 'h'|0x00, 'i'|0x00, + 'j'|0x00, 'k'|0x00, 'l'|0x00, 'n'|0x00, 'o'|0x00, 'r'|0x00, + 's'|0x00, 't'|0x00, 'u'|0x00, 'u'|0x00, 'w'|0x00, 'y'|0x00, + 'z'|0x00, 'o'|0x00, 'u'|0x00, 'a'|0x00, 'i'|0x00, 'o'|0x00, + 'u'|0x00, 'u'|0x80, 'a'|0x80, 'g'|0x00, 'k'|0x00, 'o'|0x00, + 'o'|0x80, 'j'|0x00, 'g'|0x00, 'n'|0x00, 'a'|0x80, 'a'|0x00, + 'e'|0x00, 'i'|0x00, 'o'|0x00, 'r'|0x00, 'u'|0x00, 's'|0x00, + 't'|0x00, 'h'|0x00, 'a'|0x00, 'e'|0x00, 'o'|0x80, 'o'|0x00, + 'o'|0x80, 'y'|0x00, '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a'|0x00, 'b'|0x00, + 'c'|0x80, 'd'|0x00, 'd'|0x00, 'e'|0x80, 'e'|0x00, 'e'|0x80, + 'f'|0x00, 'g'|0x00, 'h'|0x00, 'h'|0x00, 'i'|0x00, 'i'|0x80, + 'k'|0x00, 'l'|0x00, 'l'|0x80, 'l'|0x00, 'm'|0x00, 'n'|0x00, + 'o'|0x80, 'p'|0x00, 'r'|0x00, 'r'|0x80, 'r'|0x00, 's'|0x00, + 's'|0x80, 't'|0x00, 'u'|0x00, 'u'|0x80, 'v'|0x00, 'w'|0x00, + 'w'|0x00, 'x'|0x00, 'y'|0x00, 'z'|0x00, 'h'|0x00, 't'|0x00, + 'w'|0x00, 'y'|0x00, 'a'|0x00, 'a'|0x80, 'a'|0x80, 'a'|0x80, + 'e'|0x00, 'e'|0x80, 'e'|0x80, 'i'|0x00, 'o'|0x00, 'o'|0x80, + 'o'|0x80, 'o'|0x80, 'u'|0x00, 'u'|0x80, 'u'|0x80, 'y'|0x00, }; unsigned int key = (((unsigned int)c)<<3) | 0x00000007; int iRes = 0; int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; @@ -199,11 +214,12 @@ }else{ iHi = iTest-1; } } assert( key>=aDia[iRes] ); - return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F); } /* ** Return true if the argument interpreted as a unicode codepoint @@ -226,11 +242,11 @@ ** Otherwise, return a copy of the argument. ** ** The results are undefined if the value passed to this function ** is less than zero. */ -int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ +int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){ /* Each entry in the following array defines a rule for folding a range ** of codepoints to lower case. The rule applies to a range of nRange ** codepoints starting at codepoint iCode. ** ** If the least significant bit in flags is clear, then the rule applies @@ -349,11 +365,13 @@ if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; assert( ret>0 ); } - if( bRemoveDiacritic ) ret = remove_diacritic(ret); + if( eRemoveDiacritic ){ + ret = remove_diacritic(ret, eRemoveDiacritic==2); + } } else if( c>=66560 && c<66600 ){ ret = c + 40; } Index: ext/fts3/unicode/mkunicode.tcl ================================================================== --- ext/fts3/unicode/mkunicode.tcl +++ ext/fts3/unicode/mkunicode.tcl @@ -7,15 +7,16 @@ set lRange [list] set nRange 1 set iFirst [lindex $map 0 0] set cPrev [lindex $map 0 1] + set fPrev [lindex $map 0 2] foreach m [lrange $map 1 end] { - foreach {i c} $m {} + foreach {i c f} $m {} - if {$cPrev == $c} { + if {$cPrev == $c && $fPrev==$f} { for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} { if {[info exists tl_lookup_table($j)]==0} break } if {$j==$i} { @@ -27,27 +28,30 @@ } } lappend lRange [list $iFirst $nRange] lappend aChar $cPrev + lappend aFlag $fPrev set iFirst $i set cPrev $c + set fPrev $f set nRange 1 } lappend lRange [list $iFirst $nRange] lappend aChar $cPrev + lappend aFlag $fPrev puts "/*" puts "** If the argument is a codepoint corresponding to a lowercase letter" puts "** in the ASCII range with a diacritic added, return the codepoint" puts "** of the ASCII letter only. For example, if passed 235 - \"LATIN" puts "** SMALL LETTER E WITH DIAERESIS\" - return 65 (\"LATIN SMALL LETTER" puts "** E\"). The resuls of passing a codepoint that corresponds to an" puts "** uppercase letter are undefined." puts "*/" - puts "static int ${::remove_diacritic}(int c)\{" + puts "static int ${::remove_diacritic}(int c, int bComplex)\{" puts " unsigned short aDia\[\] = \{" puts -nonewline " 0, " set i 1 foreach r $lRange { foreach {iCode nRange} $r {} @@ -58,17 +62,21 @@ puts -nonewline ", " } puts "" puts " \};" puts " char aChar\[\] = \{" - puts -nonewline " '\\0', " + puts -nonewline " '\\0', " set i 1 - foreach c $aChar { - set str "'$c', " - if {$c == ""} { set str "'\\0', " } + foreach c $aChar f $aFlag { + if { $f } { + set str "'$c'|0x80, " + } else { + set str "'$c'|0x00, " + } + if {$c == ""} { set str "'\\0', " } - if {($i % 12)==0} {puts "" ; puts -nonewline " " } + if {($i % 6)==0} {puts "" ; puts -nonewline " " } incr i puts -nonewline "$str" } puts "" puts " \};" @@ -85,19 +93,21 @@ }else{ iHi = iTest-1; } } assert( key>=aDia[iRes] ); - return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);} + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);} puts "\}" } proc print_isdiacritic {zFunc map} { set lCode [list] foreach m $map { - foreach {code char} $m {} + foreach {code char flag} $m {} + if {$flag} continue if {$code && $char == ""} { lappend lCode $code } } set lCode [lsort -integer $lCode] set iFirst [lindex $lCode 0] set iLast [lindex $lCode end] @@ -470,11 +480,11 @@ puts "** Otherwise, return a copy of the argument." puts "**" puts "** The results are undefined if the value passed to this function" puts "** is less than zero." puts "*/" - puts "int ${zFunc}\(int c, int bRemoveDiacritic)\{" + puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{" set liOff [tl_generate_ioff_table $lRecord] tl_print_table_header foreach entry $lRecord { if {[tl_print_table_entry toggle $entry $liOff]} { @@ -514,11 +524,13 @@ if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; assert( ret>0 ); } - if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret); + if( eRemoveDiacritic ){ + ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2); + } } }] foreach entry $lHigh { tl_print_if_entry $entry @@ -603,14 +615,10 @@ set caseS [categories_switch C S {c k m o}] set caseZ [categories_switch C Z {l p s}] set nCat [expr [llength [array names C]] + 1] puts [code { - int sqlite3Fts5UnicodeNCat(void) { - return $nCat; - } - int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ aArray[0] = 1; switch( zCat[0] ){ $caseC $caseL Index: ext/fts3/unicode/parseunicode.tcl ================================================================== --- ext/fts3/unicode/parseunicode.tcl +++ ext/fts3/unicode/parseunicode.tcl @@ -5,15 +5,27 @@ # diacritical marks from a unicode string. Each mapping is itself a list # consisting of two elements - the unicode codepoint and the single ASCII # character that it should be replaced with, or an empty string if the # codepoint should simply be removed from the input. Examples: # -# { 224 a } (replace codepoint 224 to "a") -# { 769 "" } (remove codepoint 769 from input) +# { 224 a 0 } (replace codepoint 224 to "a") +# { 769 "" 0 } (remove codepoint 769 from input) # # Mappings are only returned for non-upper case codepoints. It is assumed # that the input has already been folded to lower case. +# +# The third value in the list is always either 0 or 1. 0 if the +# UnicodeData.txt file maps the codepoint to a single ASCII character and +# a diacritic, or 1 if the mapping is indirect. For example, consider the +# two entries: +# +# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC +# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8 +# +# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a +# diacritic). The second is an indirect mapping, as it maps to the +# first codepoint plus 0302 (a diacritic). # proc rd_load_unicodedata_text {zName} { global tl_lookup_table set fd [open $zName] @@ -51,22 +63,33 @@ set iCode [expr "0x$code"] set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"] set iDia [expr "0x[lindex $character_decomposition_mapping 1]"] + # Filter out upper-case characters, as they will be mapped to their + # lower-case equivalents before this data is used. if {[info exists tl_lookup_table($iCode)]} continue + + # Check if this is an indirect mapping. If so, set bIndirect to true + # and change $iAscii to the indirectly mappped ASCII character. + set bIndirect 0 + if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} { + set iAscii $mapping($iAscii) + set bIndirect 1 + } if { ($iAscii >= 97 && $iAscii <= 122) || ($iAscii >= 65 && $iAscii <= 90) } { - lappend lRet [list $iCode [string tolower [format %c $iAscii]]] + lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect] + set mapping($iCode) $iAscii set dia($iDia) 1 } } foreach d [array names dia] { - lappend lRet [list $d ""] + lappend lRet [list $d "" 0] } set lRet [lsort -integer -index 0 $lRet] close $fd set lRet Index: ext/fts5/fts5_tokenize.c ================================================================== --- ext/fts5/fts5_tokenize.c +++ ext/fts5/fts5_tokenize.c @@ -232,16 +232,21 @@ typedef struct Unicode61Tokenizer Unicode61Tokenizer; struct Unicode61Tokenizer { unsigned char aTokenChar[128]; /* ASCII range token characters */ char *aFold; /* Buffer to fold text into */ int nFold; /* Size of aFold[] in bytes */ - int bRemoveDiacritic; /* True if remove_diacritics=1 is set */ + int eRemoveDiacritic; /* True if remove_diacritics=1 is set */ int nException; int *aiException; unsigned char aCategory[32]; /* True for token char categories */ }; + +/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */ +#define FTS5_REMOVE_DIACRITICS_NONE 0 +#define FTS5_REMOVE_DIACRITICS_SIMPLE 1 +#define FTS5_REMOVE_DIACRITICS_COMPLEX 2 static int fts5UnicodeAddExceptions( Unicode61Tokenizer *p, /* Tokenizer object */ const char *z, /* Characters to treat as exceptions */ int bTokenChars /* 1 for 'tokenchars', 0 for 'separators' */ @@ -359,11 +364,11 @@ if( p ){ const char *zCat = "L* N* Co"; int i; memset(p, 0, sizeof(Unicode61Tokenizer)); - p->bRemoveDiacritic = 1; + p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE; p->nFold = 64; p->aFold = sqlite3_malloc(p->nFold * sizeof(char)); if( p->aFold==0 ){ rc = SQLITE_NOMEM; } @@ -380,14 +385,19 @@ } for(i=0; rc==SQLITE_OK && ieRemoveDiacritic = (zArg[0] - '0'); + assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE + || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE + || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX + ); } - p->bRemoveDiacritic = (zArg[0]=='1'); }else if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){ rc = fts5UnicodeAddExceptions(p, zArg, 1); }else if( 0==sqlite3_stricmp(azArg[i], "separators") ){ @@ -497,11 +507,11 @@ /* An non-ascii-range character. Fold it into the output buffer if ** it is a token character, or break out of the loop if it is not. */ READ_UTF8(zCsr, zTerm, iCode); if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){ non_ascii_tokenchar: - iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic); + iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic); if( iCode ) WRITE_UTF8(zOut, iCode); }else{ break; } }else if( a[*zCsr]==0 ){ Index: ext/fts5/fts5_unicode2.c ================================================================== --- ext/fts5/fts5_unicode2.c +++ ext/fts5/fts5_unicode2.c @@ -26,36 +26,51 @@ ** of the ASCII letter only. For example, if passed 235 - "LATIN ** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER ** E"). The resuls of passing a codepoint that corresponds to an ** uppercase letter are undefined. */ -static int fts5_remove_diacritic(int c){ +static int fts5_remove_diacritic(int c, int bComplex){ unsigned short aDia[] = { 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, - 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, - 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, - 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, - 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, - 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, - 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, - 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, - 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, - 62924, 63050, 63082, 63274, 63390, + 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896, + 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106, + 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344, + 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198, + 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468, + 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, + 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, + 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, + 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, + 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, + 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, + 63182, 63242, 63274, 63310, 63368, 63390, }; char aChar[] = { - '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', - 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', - 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', - 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', - 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', - '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', - 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', - 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', - 'e', 'i', 'o', 'u', 'y', + '\0', 'a'|0x00, 'c'|0x00, 'e'|0x00, 'i'|0x00, 'n'|0x00, + 'o'|0x00, 'u'|0x00, 'y'|0x00, 'y'|0x00, 'a'|0x00, 'c'|0x00, + 'd'|0x00, 'e'|0x00, 'e'|0x00, 'g'|0x00, 'h'|0x00, 'i'|0x00, + 'j'|0x00, 'k'|0x00, 'l'|0x00, 'n'|0x00, 'o'|0x00, 'r'|0x00, + 's'|0x00, 't'|0x00, 'u'|0x00, 'u'|0x00, 'w'|0x00, 'y'|0x00, + 'z'|0x00, 'o'|0x00, 'u'|0x00, 'a'|0x00, 'i'|0x00, 'o'|0x00, + 'u'|0x00, 'u'|0x80, 'a'|0x80, 'g'|0x00, 'k'|0x00, 'o'|0x00, + 'o'|0x80, 'j'|0x00, 'g'|0x00, 'n'|0x00, 'a'|0x80, 'a'|0x00, + 'e'|0x00, 'i'|0x00, 'o'|0x00, 'r'|0x00, 'u'|0x00, 's'|0x00, + 't'|0x00, 'h'|0x00, 'a'|0x00, 'e'|0x00, 'o'|0x80, 'o'|0x00, + 'o'|0x80, 'y'|0x00, '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a'|0x00, 'b'|0x00, + 'c'|0x80, 'd'|0x00, 'd'|0x00, 'e'|0x80, 'e'|0x00, 'e'|0x80, + 'f'|0x00, 'g'|0x00, 'h'|0x00, 'h'|0x00, 'i'|0x00, 'i'|0x80, + 'k'|0x00, 'l'|0x00, 'l'|0x80, 'l'|0x00, 'm'|0x00, 'n'|0x00, + 'o'|0x80, 'p'|0x00, 'r'|0x00, 'r'|0x80, 'r'|0x00, 's'|0x00, + 's'|0x80, 't'|0x00, 'u'|0x00, 'u'|0x80, 'v'|0x00, 'w'|0x00, + 'w'|0x00, 'x'|0x00, 'y'|0x00, 'z'|0x00, 'h'|0x00, 't'|0x00, + 'w'|0x00, 'y'|0x00, 'a'|0x00, 'a'|0x80, 'a'|0x80, 'a'|0x80, + 'e'|0x00, 'e'|0x80, 'e'|0x80, 'i'|0x00, 'o'|0x00, 'o'|0x80, + 'o'|0x80, 'o'|0x80, 'u'|0x00, 'u'|0x80, 'u'|0x80, 'y'|0x00, }; unsigned int key = (((unsigned int)c)<<3) | 0x00000007; int iRes = 0; int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; @@ -68,11 +83,12 @@ }else{ iHi = iTest-1; } } assert( key>=aDia[iRes] ); - return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F); } /* ** Return true if the argument interpreted as a unicode codepoint @@ -95,11 +111,11 @@ ** Otherwise, return a copy of the argument. ** ** The results are undefined if the value passed to this function ** is less than zero. */ -int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){ +int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){ /* Each entry in the following array defines a rule for folding a range ** of codepoints to lower case. The rule applies to a range of nRange ** codepoints starting at codepoint iCode. ** ** If the least significant bit in flags is clear, then the rule applies @@ -218,27 +234,22 @@ if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; assert( ret>0 ); } - if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret); + if( eRemoveDiacritic ){ + ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2); + } } else if( c>=66560 && c<66600 ){ ret = c + 40; } return ret; } - -#if 0 -int sqlite3Fts5UnicodeNCat(void) { - return 32; -} -#endif - int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ aArray[0] = 1; switch( zCat[0] ){ case 'C': switch( zCat[1] ){ @@ -754,11 +765,10 @@ int iTbl = 0; while( i<128 ){ int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ]; int n = (aFts5UnicodeData[iTbl] >> 5) + i; for(; i<128 && i59 AND x1 < 59 } {2 3} - do_rbu_vacuum_test 3.5 $step + do_rbu_vacuum_test 3.5 $step $state do_execsql_test 3.6.1 { SELECT rowid FROM rt WHERE x2>51 AND x1 < 51 } {1 2} do_execsql_test 3.6.2 { @@ -145,19 +148,20 @@ table t1 t1 2 {CREATE TABLE t1(a, b, c)} view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1} trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END} } - do_rbu_vacuum_test 4.3 $step + do_rbu_vacuum_test 4.3 $step $state do_execsql_test 4.4 { SELECT * FROM sqlite_master; } { table t1 t1 2 {CREATE TABLE t1(a, b, c)} view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1} trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END} } } +} } #------------------------------------------------------------------------- # Test that passing a NULL value as the second argument to # sqlite3rbu_vacuum() causes it to: @@ -229,6 +233,11 @@ while {[rbu step]!="SQLITE_DONE"} { rbu step } rbu close execsql { PRAGMA integrity_check } } {ok} +do_test 6.4 { + sqlite3rbu_vacuum rbu test.db test.db-vactmp + list [catch { rbu close } msg] $msg +} {1 SQLITE_MISUSE} + finish_test Index: ext/rbu/sqlite3rbu.c ================================================================== --- ext/rbu/sqlite3rbu.c +++ ext/rbu/sqlite3rbu.c @@ -2475,11 +2475,11 @@ if( *zExtra++=='?' ) break; } if( *zExtra=='\0' ) zExtra = 0; } - zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", + zTarget = sqlite3_mprintf("file:%s-vactmp?rbu_memory=1%s%s", sqlite3_db_filename(p->dbRbu, "main"), (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra) ); if( zTarget==0 ){ @@ -3741,10 +3741,16 @@ sqlite3rbu *sqlite3rbu_vacuum( const char *zTarget, const char *zState ){ if( zTarget==0 ){ return rbuMisuseError(); } + if( zState ){ + int n = strlen(zState); + if( n>=7 && 0==memcmp("-vactmp", &zState[n-7], 7) ){ + return rbuMisuseError(); + } + } /* TODO: Check that both arguments are non-NULL */ return openRbuHandle(0, zTarget, zState); } /* Index: ext/rbu/sqlite3rbu.h ================================================================== --- ext/rbu/sqlite3rbu.h +++ ext/rbu/sqlite3rbu.h @@ -331,11 +331,15 @@ ** ** If the second argument passed to this function is NULL, then the ** name of the state database is "-vacuum", where ** is the name of the target database file. In this case, on UNIX, if the ** state database is not already present in the file-system, it is created -** with the same permissions as the target db is made. +** with the same permissions as the target db is made. +** +** With an RBU vacuum, it is an SQLITE_MISUSE error if the name of the +** state database ends with "-vactmp". This name is reserved for internal +** use. ** ** This function does not delete the state database after an RBU vacuum ** is completed, even if it created it. However, if the call to ** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents ** of the state tables within the state database are zeroed. This way, Index: ext/rbu/test_rbu.c ================================================================== --- ext/rbu/test_rbu.c +++ ext/rbu/test_rbu.c @@ -271,10 +271,11 @@ return TCL_ERROR; } zCmd = Tcl_GetString(objv[1]); zTarget = Tcl_GetString(objv[2]); if( objc==4 ) zStateDb = Tcl_GetString(objv[3]); + if( zStateDb && zStateDb[0]=='\0' ) zStateDb = 0; pRbu = sqlite3rbu_vacuum(zTarget, zStateDb); Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0); Tcl_SetObjResult(interp, objv[1]); return TCL_OK; Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -997,10 +997,14 @@ $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \ $(TOP)/tool/showwal.c sqlite3.o $(THREADLIB) showshm$(EXE): $(TOP)/tool/showshm.c $(TCC) -o showshm$(EXE) $(TOP)/tool/showshm.c + +index_usage$(EXE): $(TOP)/tool/index_usage.c sqlite3.o + $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_DEPRECATED -o index_usage$(EXE) \ + $(TOP)/tool/index_usage.c sqlite3.o $(THREADLIB) changeset$(EXE): $(TOP)/ext/session/changeset.c sqlite3.o $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \ $(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB) Index: src/alter.c ================================================================== --- src/alter.c +++ src/alter.c @@ -777,19 +777,36 @@ pCtx->nList++; break; } } } + +/* +** Iterate through the Select objects that are part of WITH clauses attached +** to select statement pSelect. +*/ +static void renameWalkWith(Walker *pWalker, Select *pSelect){ + if( pSelect->pWith ){ + int i; + for(i=0; ipWith->nCte; i++){ + Select *p = pSelect->pWith->a[i].pSelect; + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pWalker->pParse; + sqlite3SelectPrep(sNC.pParse, p, &sNC); + sqlite3WalkSelect(pWalker, p); + } + } +} /* ** This is a Walker select callback. It does nothing. It is only required ** because without a dummy callback, sqlite3WalkExpr() and similar do not ** descend into sub-select statements. */ static int renameColumnSelectCb(Walker *pWalker, Select *p){ - UNUSED_PARAMETER(pWalker); - UNUSED_PARAMETER(p); + renameWalkWith(pWalker, p); return WRC_Continue; } /* ** This is a Walker expression callback. @@ -1356,16 +1373,21 @@ */ static int renameTableSelectCb(Walker *pWalker, Select *pSelect){ int i; RenameCtx *p = pWalker->u.pRename; SrcList *pSrc = pSelect->pSrc; + if( pSrc==0 ){ + assert( pWalker->pParse->db->mallocFailed ); + return WRC_Abort; + } for(i=0; inSrc; i++){ struct SrcList_item *pItem = &pSrc->a[i]; if( pItem->pTab==p->pTab ){ renameTokenFind(pWalker->pParse, p, pItem->zName); } } + renameWalkWith(pWalker, pSelect); return WRC_Continue; } Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -10327,11 +10327,11 @@ "incremental_vacuum enabled with a max rootpage of zero" ); } #endif testcase( pBt->db->flags & SQLITE_CellSizeCk ); - pBt->db->flags &= ~SQLITE_CellSizeCk; + pBt->db->flags &= ~(u64)SQLITE_CellSizeCk; for(i=0; (int)iautoVacuum && aRoot[i]>1 ){ Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -227,11 +227,11 @@ /* Get the VDBE program ready for execution */ if( v && pParse->nErr==0 && !db->mallocFailed ){ /* A minimum of one cursor is required if autoincrement is used * See ticket [a696379c1f08866] */ - if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; + assert( pParse->pAinc==0 || pParse->nTab>0 ); sqlite3VdbeMakeReady(v, pParse); pParse->rc = SQLITE_DONE; }else{ pParse->rc = SQLITE_ERROR; } @@ -636,16 +636,10 @@ /* Delete any foreign keys attached to this table. */ sqlite3FkDelete(db, pTable); /* Delete the Table structure itself. */ -#ifdef SQLITE_ENABLE_NORMALIZE - if( pTable->pColHash ){ - sqlite3HashClear(pTable->pColHash); - sqlite3_free(pTable->pColHash); - } -#endif sqlite3DeleteColumnNames(db, pTable); sqlite3DbFree(db, pTable->zName); sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); sqlite3ExprListDelete(db, pTable->pCheck); Index: src/callback.c ================================================================== --- src/callback.c +++ src/callback.c @@ -281,11 +281,11 @@ /* ** Search a FuncDefHash for a function with the given name. Return ** a pointer to the matching FuncDef if found, or 0 if there is no match. */ -static FuncDef *functionSearch( +FuncDef *sqlite3FunctionSearch( int h, /* Hash of the name */ const char *zFunc /* Name of function */ ){ FuncDef *p; for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){ @@ -293,25 +293,10 @@ return p; } } return 0; } -#ifdef SQLITE_ENABLE_NORMALIZE -FuncDef *sqlite3FunctionSearchN( - int h, /* Hash of the name */ - const char *zFunc, /* Name of function */ - int nFunc /* Length of the name */ -){ - FuncDef *p; - for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){ - if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 ){ - return p; - } - } - return 0; -} -#endif /* SQLITE_ENABLE_NORMALIZE */ /* ** Insert a new FuncDef into a FuncDefHash hash table. */ void sqlite3InsertBuiltinFuncs( @@ -323,11 +308,11 @@ FuncDef *pOther; const char *zName = aDef[i].zName; int nName = sqlite3Strlen30(zName); int h = SQLITE_FUNC_HASH(zName[0], nName); assert( zName[0]>='a' && zName[0]<='z' ); - pOther = functionSearch(h, zName); + pOther = sqlite3FunctionSearch(h, zName); if( pOther ){ assert( pOther!=&aDef[i] && pOther->pNext!=&aDef[i] ); aDef[i].pNext = pOther->pNext; pOther->pNext = &aDef[i]; }else{ @@ -401,11 +386,11 @@ ** So we must not search for built-ins when creating a new function. */ if( !createFlag && (pBest==0 || (db->mDbFlags & DBFLAG_PreferBuiltin)!=0) ){ bestScore = 0; h = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zName[0]], nName); - p = functionSearch(h, zName); + p = sqlite3FunctionSearch(h, zName); while( p ){ int score = matchQuality(p, nArg, enc); if( score>bestScore ){ pBest = p; bestScore = score; Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -1327,10 +1327,39 @@ return pRet; } #else # define withDup(x,y) 0 #endif + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** The gatherSelectWindows() procedure and its helper routine +** gatherSelectWindowsCallback() are used to scan all the expressions +** an a newly duplicated SELECT statement and gather all of the Window +** objects found there, assembling them onto the linked list at Select->pWin. +*/ +static int gatherSelectWindowsCallback(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_FUNCTION && pExpr->y.pWin!=0 ){ + assert( ExprHasProperty(pExpr, EP_WinFunc) ); + pExpr->y.pWin->pNextWin = pWalker->u.pSelect->pWin; + pWalker->u.pSelect->pWin = pExpr->y.pWin; + } + return WRC_Continue; +} +static int gatherSelectWindowsSelectCallback(Walker *pWalker, Select *p){ + return p==pWalker->u.pSelect ? WRC_Continue : WRC_Prune; +} +static void gatherSelectWindows(Select *p){ + Walker w; + w.xExprCallback = gatherSelectWindowsCallback; + w.xSelectCallback = gatherSelectWindowsSelectCallback; + w.xSelectCallback2 = 0; + w.u.pSelect = p; + sqlite3WalkSelect(&w, p); +} +#endif + /* ** The following group of routines make deep copies of expressions, ** expression lists, ID lists, and select statements. The copies can ** be deleted (by being passed to their respective ...Delete() routines) @@ -1495,10 +1524,11 @@ pNew->nSelectRow = p->nSelectRow; pNew->pWith = withDup(db, p->pWith); #ifndef SQLITE_OMIT_WINDOWFUNC pNew->pWin = 0; pNew->pWinDefn = sqlite3WindowListDup(db, p->pWinDefn); + if( p->pWin ) gatherSelectWindows(pNew); #endif pNew->selId = p->selId; *pp = pNew; pp = &pNew->pPrior; pNext = pNew; @@ -2147,18 +2177,10 @@ if( sqlite3StrICmp(z, "_ROWID_")==0 ) return 1; if( sqlite3StrICmp(z, "ROWID")==0 ) return 1; if( sqlite3StrICmp(z, "OID")==0 ) return 1; return 0; } -#ifdef SQLITE_ENABLE_NORMALIZE -int sqlite3IsRowidN(const char *z, int n){ - if( sqlite3StrNICmp(z, "_ROWID_", n)==0 ) return 1; - if( sqlite3StrNICmp(z, "ROWID", n)==0 ) return 1; - if( sqlite3StrNICmp(z, "OID", n)==0 ) return 1; - return 0; -} -#endif /* ** pX is the RHS of an IN operator. If pX is a SELECT statement ** that can be simplified to a direct table access, then return ** a pointer to the SELECT statement. If pX is not a SELECT statement, @@ -2419,10 +2441,11 @@ /* Search for an existing index that will work for this IN operator */ for(pIdx=pTab->pIndex; pIdx && eType==0; pIdx=pIdx->pNext){ Bitmask colUsed; /* Columns of the index used */ Bitmask mCol; /* Mask for the current column */ if( pIdx->nColumnpPartIdxWhere!=0 ) continue; /* Maximum nColumn is BMS-2, not BMS-1, so that we can compute ** BITMASK(nExpr) without overflowing */ testcase( pIdx->nColumn==BMS-2 ); testcase( pIdx->nColumn==BMS-1 ); if( pIdx->nColumn>=BMS-1 ) continue; Index: src/hash.c ================================================================== --- src/hash.c +++ src/hash.c @@ -62,24 +62,10 @@ h += sqlite3UpperToLower[c]; h *= 0x9e3779b1; } return h; } -#ifdef SQLITE_ENABLE_NORMALIZE -static unsigned int strHashN(const char *z, int n){ - unsigned int h = 0; - int i; - for(i=0; inext; } return &nullElement; } -#ifdef SQLITE_ENABLE_NORMALIZE -static HashElem *findElementWithHashN( - const Hash *pH, /* The pH to be searched */ - const char *pKey, /* The key we are searching for */ - int nKey, /* Number of key bytes to use */ - unsigned int *pHash /* Write the hash value here */ -){ - HashElem *elem; /* Used to loop thru the element list */ - int count; /* Number of elements left to test */ - unsigned int h; /* The computed hash */ - static HashElem nullElement = { 0, 0, 0, 0 }; - - if( pH->ht ){ /*OPTIMIZATION-IF-TRUE*/ - struct _ht *pEntry; - h = strHashN(pKey, nKey) % pH->htsize; - pEntry = &pH->ht[h]; - elem = pEntry->chain; - count = pEntry->count; - }else{ - h = 0; - elem = pH->first; - count = pH->count; - } - if( pHash ) *pHash = h; - while( count-- ){ - assert( elem!=0 ); - if( sqlite3StrNICmp(elem->pKey,pKey,nKey)==0 ){ - return elem; - } - elem = elem->next; - } - return &nullElement; -} -#endif /* SQLITE_ENABLE_NORMALIZE */ /* Remove a single entry from the hash table given a pointer to that ** element and a hash on the element's key. */ static void removeElementGivenHash( @@ -265,18 +217,10 @@ void *sqlite3HashFind(const Hash *pH, const char *pKey){ assert( pH!=0 ); assert( pKey!=0 ); return findElementWithHash(pH, pKey, 0)->data; } -#ifdef SQLITE_ENABLE_NORMALIZE -void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey){ - assert( pH!=0 ); - assert( pKey!=0 ); - assert( nKey>=0 ); - return findElementWithHashN(pH, pKey, nKey, 0)->data; -} -#endif /* SQLITE_ENABLE_NORMALIZE */ /* Insert an element into the hash table pH. The key is pKey ** and the data is "data". ** ** If no element exists with a matching key, then a new Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -317,10 +317,11 @@ aOp[5].p3 = memId; aOp[6].p1 = memId; aOp[7].p2 = memId+2; aOp[7].p1 = memId; aOp[10].p2 = memId; + if( pParse->nTab==0 ) pParse->nTab = 1; } } /* ** Update the maximum rowid for an autoincrement calculation. Index: src/loadext.c ================================================================== --- src/loadext.c +++ src/loadext.c @@ -648,11 +648,11 @@ int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ sqlite3_mutex_enter(db->mutex); if( onoff ){ db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc; }else{ - db->flags &= ~(SQLITE_LoadExtension|SQLITE_LoadExtFunc); + db->flags &= ~(u64)(SQLITE_LoadExtension|SQLITE_LoadExtFunc); } sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -841,15 +841,15 @@ rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ for(i=0; iflags; + u64 oldFlags = db->flags; if( onoff>0 ){ db->flags |= aFlagOp[i].mask; }else if( onoff==0 ){ - db->flags &= ~aFlagOp[i].mask; + db->flags &= ~(u64)aFlagOp[i].mask; } if( oldFlags!=db->flags ){ sqlite3ExpirePreparedStatements(db, 0); } if( pRes ){ @@ -1308,11 +1308,11 @@ sqlite3BtreeLeaveAll(db); /* Any deferred constraint violations have now been resolved. */ db->nDeferredCons = 0; db->nDeferredImmCons = 0; - db->flags &= ~SQLITE_DeferFKs; + db->flags &= ~(u64)SQLITE_DeferFKs; /* If one has been configured, invoke the rollback-hook callback */ if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ db->xRollbackCallback(db->pRollbackArg); } @@ -2050,10 +2050,12 @@ #endif sqlite3_mutex_enter(db->mutex); pOld = db->pProfileArg; db->xProfile = xProfile; db->pProfileArg = pArg; + db->mTrace &= SQLITE_TRACE_NONLEGACY_MASK; + if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE; sqlite3_mutex_leave(db->mutex); return pOld; } #endif /* SQLITE_OMIT_DEPRECATED */ #endif /* SQLITE_OMIT_TRACE */ @@ -2401,11 +2403,11 @@ sqlite3_mutex_enter(db->mutex); if( db->mallocFailed ){ z = sqlite3ErrStr(SQLITE_NOMEM_BKPT); }else{ testcase( db->pErr==0 ); - z = (char*)sqlite3_value_text(db->pErr); + z = db->errCode ? (char*)sqlite3_value_text(db->pErr) : 0; assert( !db->mallocFailed ); if( z==0 ){ z = sqlite3ErrStr(db->errCode); } } Index: src/parse.y ================================================================== --- src/parse.y +++ src/parse.y @@ -677,10 +677,16 @@ struct SrcList_item *pNew = &A->a[A->nSrc-1]; struct SrcList_item *pOld = F->a; pNew->zName = pOld->zName; pNew->zDatabase = pOld->zDatabase; pNew->pSelect = pOld->pSelect; + if( pOld->fg.isTabFunc ){ + pNew->u1.pFuncArg = pOld->u1.pFuncArg; + pOld->u1.pFuncArg = 0; + pOld->fg.isTabFunc = 0; + pNew->fg.isTabFunc = 1; + } pOld->zName = pOld->zDatabase = 0; pOld->pSelect = 0; } sqlite3SrcListDelete(pParse->db, F); }else{ @@ -1374,12 +1380,16 @@ ///////////////////////////// The VACUUM command ///////////////////////////// // %ifndef SQLITE_OMIT_VACUUM %ifndef SQLITE_OMIT_ATTACH -cmd ::= VACUUM. {sqlite3Vacuum(pParse,0);} -cmd ::= VACUUM nm(X). {sqlite3Vacuum(pParse,&X);} +%type vinto {Expr*} +%destructor vinto {sqlite3ExprDelete(pParse->db, $$);} +cmd ::= VACUUM vinto(Y). {sqlite3Vacuum(pParse,0,Y);} +cmd ::= VACUUM nm(X) vinto(Y). {sqlite3Vacuum(pParse,&X,Y);} +vinto(A) ::= INTO expr(X). {A = X;} +vinto(A) ::= . {A = 0;} %endif SQLITE_OMIT_ATTACH %endif SQLITE_OMIT_VACUUM ///////////////////////////// The PRAGMA command ///////////////////////////// // Index: src/pcache1.c ================================================================== --- src/pcache1.c +++ src/pcache1.c @@ -100,10 +100,11 @@ u8 isAnchor; /* This is the PGroup.lru element */ PgHdr1 *pNext; /* Next in hash table chain */ PCache1 *pCache; /* Cache that currently owns this page */ PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ + /* NB: pLruPrev is only valid if pLruNext!=0 */ }; /* ** A page is pinned if it is not on the LRU list. To be "pinned" means ** that the page is in active use and must not be deallocated. @@ -568,11 +569,12 @@ assert( pPage->pLruPrev ); assert( sqlite3_mutex_held(pPage->pCache->pGroup->mutex) ); pPage->pLruPrev->pLruNext = pPage->pLruNext; pPage->pLruNext->pLruPrev = pPage->pLruPrev; pPage->pLruNext = 0; - pPage->pLruPrev = 0; + /* pPage->pLruPrev = 0; + ** No need to clear pLruPrev as it is never accessed if pLruNext is 0 */ assert( pPage->isAnchor==0 ); assert( pPage->pCache->pGroup->lru.isAnchor==1 ); pPage->pCache->nRecyclable--; return pPage; } @@ -906,12 +908,13 @@ unsigned int h = iKey % pCache->nHash; pCache->nPage++; pPage->iKey = iKey; pPage->pNext = pCache->apHash[h]; pPage->pCache = pCache; - pPage->pLruPrev = 0; pPage->pLruNext = 0; + /* pPage->pLruPrev = 0; + ** No need to clear pLruPrev since it is not accessed when pLruNext==0 */ *(void **)pPage->page.pExtra = 0; pCache->apHash[h] = pPage; if( iKey>pCache->iMaxKey ){ pCache->iMaxKey = iKey; } @@ -1067,11 +1070,11 @@ pcache1EnterMutex(pGroup); /* It is an error to call this function if the page is already ** part of the PGroup LRU list. */ - assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); + assert( pPage->pLruNext==0 ); assert( PAGE_IS_PINNED(pPage) ); if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){ pcache1RemoveFromHash(pPage, 1); }else{ Index: src/pragma.c ================================================================== --- src/pragma.c +++ src/pragma.c @@ -815,11 +815,11 @@ sqlite3BtreeSetSpillSize(pDb->pBt, size); } if( sqlite3GetBoolean(zRight, size!=0) ){ db->flags |= SQLITE_CacheSpill; }else{ - db->flags &= ~SQLITE_CacheSpill; + db->flags &= ~(u64)SQLITE_CacheSpill; } setAllPagerFlags(db); } break; } Index: src/prepare.c ================================================================== --- src/prepare.c +++ src/prepare.c @@ -291,11 +291,11 @@ ** clear the legacy_file_format pragma flag so that a VACUUM will ** not downgrade the database and thus invalidate any descending ** indices that the user might have created. */ if( iDb==0 && meta[BTREE_FILE_FORMAT-1]>=4 ){ - db->flags &= ~SQLITE_LegacyFileFmt; + db->flags &= ~(u64)SQLITE_LegacyFileFmt; } /* Read the schema information out of the schema tables */ assert( db->init.busy ); @@ -707,297 +707,10 @@ assert( (rc&db->errMask)==rc ); sqlite3_mutex_leave(db->mutex); return rc; } -#ifdef SQLITE_ENABLE_NORMALIZE -/* -** Checks if the specified token is a table, column, or function name, -** based on the databases associated with the statement being prepared. -** If the function fails, zero is returned and pRc is filled with the -** error code. -*/ -static int shouldTreatAsIdentifier( - sqlite3 *db, /* Database handle. */ - const char *zToken, /* Pointer to start of token to be checked */ - int nToken, /* Length of token to be checked */ - int *pRc /* Pointer to error code upon failure */ -){ - int bFound = 0; /* Non-zero if token is an identifier name. */ - int i, j; /* Database and column loop indexes. */ - Schema *pSchema; /* Schema for current database. */ - Hash *pHash; /* Hash table of tables for current database. */ - HashElem *e; /* Hash element for hash table iteration. */ - Table *pTab; /* Database table for columns being checked. */ - - if( sqlite3IsRowidN(zToken, nToken) ){ - return 1; - } - if( nToken>0 ){ - int hash = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zToken[0]], nToken); - if( sqlite3FunctionSearchN(hash, zToken, nToken) ) return 1; - } - assert( db!=0 ); - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeEnterAll(db); - for(i=0; inDb; i++){ - pHash = &db->aFunc; - if( sqlite3HashFindN(pHash, zToken, nToken) ){ - bFound = 1; - break; - } - pSchema = db->aDb[i].pSchema; - if( pSchema==0 ) continue; - pHash = &pSchema->tblHash; - if( sqlite3HashFindN(pHash, zToken, nToken) ){ - bFound = 1; - break; - } - for(e=sqliteHashFirst(pHash); e; e=sqliteHashNext(e)){ - pTab = sqliteHashData(e); - if( pTab==0 ) continue; - pHash = pTab->pColHash; - if( pHash==0 ){ - pTab->pColHash = pHash = sqlite3_malloc(sizeof(Hash)); - if( pHash ){ - sqlite3HashInit(pHash); - for(j=0; jnCol; j++){ - Column *pCol = &pTab->aCol[j]; - sqlite3HashInsert(pHash, pCol->zName, pCol); - } - }else{ - *pRc = SQLITE_NOMEM_BKPT; - bFound = 0; - goto done; - } - } - if( pHash && sqlite3HashFindN(pHash, zToken, nToken) ){ - bFound = 1; - goto done; - } - } - } -done: - sqlite3BtreeLeaveAll(db); - sqlite3_mutex_leave(db->mutex); - return bFound; -} - -/* -** Attempt to estimate the final output buffer size needed for the fully -** normalized version of the specified SQL string. This should take into -** account any potential expansion that could occur (e.g. via IN clauses -** being expanded, etc). This size returned is the total number of bytes -** including the NUL terminator. -*/ -static int estimateNormalizedSize( - const char *zSql, /* The original SQL string */ - int nSql, /* Length of original SQL string */ - u8 prepFlags /* The flags passed to sqlite3_prepare_v3() */ -){ - int nOut = nSql + 4; - const char *z = zSql; - while( nOut0 ){ - zOut[j++] = '"'; - continue; - }else if( k==nToken-1 ){ - zOut[j++] = '"'; - continue; - } - } - if( bKeyword ){ - zOut[j++] = sqlite3Toupper(zSql[iIn+k]); - }else{ - zOut[j++] = sqlite3Tolower(zSql[iIn+k]); - } - } - *piOut = j; -} - -/* -** Perform normalization of the SQL contained in the prepared statement and -** store the result in the zNormSql field. The schema for the associated -** databases are consulted while performing the normalization in order to -** determine if a token appears to be an identifier. All identifiers are -** left intact in the normalized SQL and all literals are replaced with a -** single '?'. -*/ -void sqlite3Normalize( - Vdbe *pVdbe, /* VM being reprepared */ - const char *zSql, /* The original SQL string */ - int nSql, /* Size of the input string in bytes */ - u8 prepFlags /* The flags passed to sqlite3_prepare_v3() */ -){ - sqlite3 *db; /* Database handle. */ - char *z; /* The output string */ - int nZ; /* Size of the output string in bytes */ - int i; /* Next character to read from zSql[] */ - int j; /* Next character to fill in on z[] */ - int tokenType = 0; /* Type of the next token */ - int prevTokenType = 0; /* Type of the previous token, except spaces */ - int n; /* Size of the next token */ - int nParen = 0; /* Nesting level of parenthesis */ - Hash inHash; /* Table of parenthesis levels to output index. */ - - db = sqlite3VdbeDb(pVdbe); - assert( db!=0 ); - assert( pVdbe->zNormSql==0 ); - if( zSql==0 ) return; - nZ = estimateNormalizedSize(zSql, nSql, prepFlags); - z = sqlite3DbMallocRawNN(db, nZ); - if( z==0 ) return; - sqlite3HashInit(&inHash); - for(i=j=0; i0 ){ - sqlite3HashInsert(&inHash, zSql+nParen, 0); - assert( jj+6=0 ); - assert( nZ-1-j=0 ); - /* Fall through */ - } - case TK_MINUS: - case TK_SEMI: - case TK_PLUS: - case TK_STAR: - case TK_SLASH: - case TK_REM: - case TK_EQ: - case TK_LE: - case TK_NE: - case TK_LSHIFT: - case TK_LT: - case TK_RSHIFT: - case TK_GT: - case TK_GE: - case TK_BITOR: - case TK_CONCAT: - case TK_COMMA: - case TK_BITAND: - case TK_BITNOT: - case TK_DOT: - case TK_IN: - case TK_IS: - case TK_NOT: - case TK_NULL: - case TK_ID: { - if( tokenType==TK_NULL ){ - if( prevTokenType==TK_IS || prevTokenType==TK_NOT ){ - /* NULL is a keyword in this case, not a literal value */ - }else{ - /* Here the NULL is a literal value */ - z[j++] = '?'; - break; - } - } - if( j>0 && sqlite3IsIdChar(z[j-1]) && sqlite3IsIdChar(zSql[i]) ){ - z[j++] = ' '; - } - if( tokenType==TK_ID ){ - int i2 = i, n2 = n, rc = SQLITE_OK; - if( nParen>0 ){ - assert( nParen0 && z[j-1]==' ' ){ j--; } - if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; } - z[j] = 0; - assert( jzNormSql = z; - sqlite3HashClear(&inHash); -} -#endif /* SQLITE_ENABLE_NORMALIZE */ /* ** Rerun the compilation of a statement after a schema change. ** ** If the statement is successfully recompiled, return SQLITE_OK. Otherwise, Index: src/resolve.c ================================================================== --- src/resolve.c +++ src/resolve.c @@ -78,11 +78,11 @@ if( pDup!=0 ){ if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery); if( pExpr->op==TK_COLLATE ){ pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } - ExprSetProperty(pDup, EP_Alias); +// ExprSetProperty(pDup, EP_Alias); /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This ** prevents ExprDelete() from deleting the Expr structure itself, ** allowing it to be repopulated by the memcpy() on the following line. ** The pExpr->u.zToken might point into memory that will be freed by the @@ -472,10 +472,29 @@ ** fields are not changed in any context. */ if( cnt==0 && zTab==0 ){ assert( pExpr->op==TK_ID ); if( ExprHasProperty(pExpr,EP_DblQuoted) ){ + /* If a double-quoted identifier does not match any known column name, + ** then treat it as a string. + ** + ** This hack was added in the early days of SQLite in a misguided attempt + ** to be compatible with MySQL 3.x, which used double-quotes for strings. + ** I now sorely regret putting in this hack. The effect of this hack is + ** that misspelled identifier names are silently converted into strings + ** rather than causing an error, to the frustration of countless + ** programmers. To all those frustrated programmers, my apologies. + ** + ** Someday, I hope to get rid of this hack. Unfortunately there is + ** a huge amount of legacy SQL that uses it. So for now, we just + ** issue a warning. + */ + sqlite3_log(SQLITE_WARNING, + "double-quoted string literal: \"%w\"", zCol); +#ifdef SQLITE_ENABLE_NORMALIZE + sqlite3VdbeAddDblquoteStr(db, pParse->pVdbe, zCol); +#endif pExpr->op = TK_STRING; pExpr->y.pTab = 0; return WRC_Prune; } if( sqlite3ExprIdToTrueFalse(pExpr) ){ Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -2075,14 +2075,14 @@ ** the result set of that SELECT. */ Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ Table *pTab; sqlite3 *db = pParse->db; - int savedFlags; + u64 savedFlags; savedFlags = db->flags; - db->flags &= ~SQLITE_FullColNames; + db->flags &= ~(u64)SQLITE_FullColNames; db->flags |= SQLITE_ShortColNames; sqlite3SelectPrep(pParse, pSelect, 0); if( pParse->nErr ) return 0; while( pSelect->pPrior ) pSelect = pSelect->pPrior; db->flags = savedFlags; @@ -3459,10 +3459,11 @@ ifNullRow.op = TK_IF_NULL_ROW; ifNullRow.pLeft = pCopy; ifNullRow.iTable = pSubst->iNewTable; pCopy = &ifNullRow; } + testcase( ExprHasProperty(pCopy, EP_Subquery) ); pNew = sqlite3ExprDup(db, pCopy, 0); if( pNew && pSubst->isLeftJoin ){ ExprSetProperty(pNew, EP_CanBeNull); } if( pNew && ExprHasProperty(pExpr,EP_FromJoin) ){ @@ -4023,11 +4024,12 @@ } assert( pParent->pOrderBy==0 ); pParent->pOrderBy = pOrderBy; pSub->pOrderBy = 0; } - pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); + pWhere = pSub->pWhere; + pSub->pWhere = 0; if( isLeftJoin>0 ){ setJoinExpr(pWhere, iNewParent); } pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere); if( db->mallocFailed==0 ){ Index: src/shell.c.in ================================================================== --- src/shell.c.in +++ src/shell.c.in @@ -1005,10 +1005,11 @@ u8 statsOn; /* True to display memory stats before each finalize */ u8 scanstatsOn; /* True to display scan stats before each finalize */ u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */ u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */ u8 nEqpLevel; /* Depth of the EQP output graph */ + u8 eTraceType; /* SHELL_TRACE_* value for type of trace */ unsigned mEqpLines; /* Mask of veritical lines in the EQP output graph */ int outCount; /* Revert to stdout when reaching zero */ int cnt; /* Number of records displayed so far */ FILE *out; /* Write results here */ FILE *traceOut; /* Output for sqlite3_trace() */ @@ -1064,10 +1065,16 @@ #define SHELL_OPEN_APPENDVFS 2 /* Use appendvfs */ #define SHELL_OPEN_ZIPFILE 3 /* Use the zipfile virtual table */ #define SHELL_OPEN_READONLY 4 /* Open a normal database read-only */ #define SHELL_OPEN_DESERIALIZE 5 /* Open using sqlite3_deserialize() */ +/* Allowed values for ShellState.eTraceType +*/ +#define SHELL_TRACE_PLAIN 0 /* Show input SQL text */ +#define SHELL_TRACE_EXPANDED 1 /* Show expanded SQL text */ +#define SHELL_TRACE_NORMALIZED 2 /* Show normalized SQL text */ + /* ** These are the allowed shellFlgs values */ #define SHFLG_Pagecache 0x00000001 /* The --pagecache option is used */ #define SHFLG_Lookaside 0x00000002 /* Lookaside memory is used */ @@ -3368,10 +3375,11 @@ #ifndef SQLITE_OMIT_AUTHORIZATION ".auth ON|OFF Show authorizer callbacks", #endif ".backup ?DB? FILE Backup DB (default \"main\") to FILE", " --append Use the appendvfs", + " --async Write to FILE without a journal and without fsync()", ".bail on|off Stop after hitting an error. Default OFF", ".binary on|off Turn binary output on or off. Default OFF", ".cd DIRECTORY Change the working directory to DIRECTORY", ".changes on|off Show number of rows changed by SQL", ".check GLOB Fail if output since .testcase does not match", @@ -3490,11 +3498,26 @@ #endif ".tables ?TABLE? List names of tables matching LIKE pattern TABLE", ".testcase NAME Begin redirecting output to 'testcase-out.txt'", ".timeout MS Try opening locked tables for MS milliseconds", ".timer on|off Turn SQL timer on or off", - ".trace FILE|off Output each SQL statement as it is run", +#ifndef SQLITE_OMIT_TRACE + ".trace ?OPTIONS? Output each SQL statement as it is run", + " FILE Send output to FILE", + " stdout Send output to stdout", + " stderr Send output to stderr", + " off Disable tracing", + " --expanded Expand query parameters", +#ifdef SQLITE_ENABLE_NORMALIZE + " --normalized Normal the SQL statements", +#endif + " --plain Show SQL as it is input", + " --stmt Trace statement execution (SQLITE_TRACE_STMT)", + " --profile Profile statements (SQLITE_TRACE_PROFILE)", + " --row Trace each row (SQLITE_TRACE_ROW)", + " --close Trace connection close (SQLITE_TRACE_CLOSE)", +#endif /* SQLITE_OMIT_TRACE */ ".vfsinfo ?AUX? Information about the top-level VFS", ".vfslist List all available VFSes", ".vfsname ?AUX? Print the name of the VFS stack", ".width NUM1 NUM2 ... Set column widths for \"column\" mode", " Negative values right-justify", @@ -3997,28 +4020,64 @@ } } return f; } -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) +#ifndef SQLITE_OMIT_TRACE /* ** A routine for handling output from sqlite3_trace(). */ static int sql_trace_callback( - unsigned mType, - void *pArg, - void *pP, - void *pX + unsigned mType, /* The trace type */ + void *pArg, /* The ShellState pointer */ + void *pP, /* Usually a pointer to sqlite_stmt */ + void *pX /* Auxiliary output */ ){ - FILE *f = (FILE*)pArg; - UNUSED_PARAMETER(mType); - UNUSED_PARAMETER(pP); - if( f ){ - const char *z = (const char*)pX; - int i = strlen30(z); - while( i>0 && z[i-1]==';' ){ i--; } - utf8_printf(f, "%.*s;\n", i, z); + ShellState *p = (ShellState*)pArg; + sqlite3_stmt *pStmt; + const char *zSql; + int nSql; + if( p->traceOut==0 ) return 0; + if( mType==SQLITE_TRACE_CLOSE ){ + utf8_printf(p->traceOut, "-- closing database connection\n"); + return 0; + } + if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){ + zSql = (const char*)pX; + }else{ + pStmt = (sqlite3_stmt*)pP; + switch( p->eTraceType ){ + case SHELL_TRACE_EXPANDED: { + zSql = sqlite3_expanded_sql(pStmt); + break; + } +#ifdef SQLITE_ENABLE_NORMALIZE + case SHELL_TRACE_NORMALIZED: { + zSql = sqlite3_normalized_sql(pStmt); + break; + } +#endif + default: { + zSql = sqlite3_sql(pStmt); + break; + } + } + } + if( zSql==0 ) return 0; + nSql = strlen30(zSql); + while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; } + switch( mType ){ + case SQLITE_TRACE_ROW: + case SQLITE_TRACE_STMT: { + utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql); + break; + } + case SQLITE_TRACE_PROFILE: { + sqlite3_int64 nNanosec = *(sqlite3_int64*)pX; + utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec); + break; + } } return 0; } #endif @@ -5780,17 +5839,21 @@ const char *zDestFile = 0; const char *zDb = 0; sqlite3 *pDest; sqlite3_backup *pBackup; int j; + int bAsync = 0; const char *zVfs = 0; for(j=1; jdb, zDb); if( pBackup==0 ){ utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest)); close_db(pDest); @@ -7836,27 +7903,59 @@ raw_printf(stderr, "Usage: .timer on|off\n"); rc = 1; } }else +#ifndef SQLITE_OMIT_TRACE if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){ + int mType = 0; + int jj; open_db(p, 0); - if( nArg!=2 ){ - raw_printf(stderr, "Usage: .trace FILE|off\n"); - rc = 1; - goto meta_command_exit; - } - output_file_close(p->traceOut); - p->traceOut = output_file_open(azArg[1], 0); -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) + for(jj=1; jjeTraceType = SHELL_TRACE_EXPANDED; + } +#ifdef SQLITE_ENABLE_NORMALIZE + else if( optionMatch(z, "normalized") ){ + p->eTraceType = SHELL_TRACE_NORMALIZED; + } +#endif + else if( optionMatch(z, "plain") ){ + p->eTraceType = SHELL_TRACE_PLAIN; + } + else if( optionMatch(z, "profile") ){ + mType |= SQLITE_TRACE_PROFILE; + } + else if( optionMatch(z, "row") ){ + mType |= SQLITE_TRACE_ROW; + } + else if( optionMatch(z, "stmt") ){ + mType |= SQLITE_TRACE_STMT; + } + else if( optionMatch(z, "close") ){ + mType |= SQLITE_TRACE_CLOSE; + } + else { + raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z); + rc = 1; + goto meta_command_exit; + } + }else{ + output_file_close(p->traceOut); + p->traceOut = output_file_open(azArg[1], 0); + } + } if( p->traceOut==0 ){ sqlite3_trace_v2(p->db, 0, 0, 0); }else{ - sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut); + if( mType==0 ) mType = SQLITE_TRACE_STMT; + sqlite3_trace_v2(p->db, mType, sql_trace_callback, p); } -#endif }else +#endif /* !defined(SQLITE_OMIT_TRACE) */ #if SQLITE_USER_AUTHENTICATION if( c=='u' && strncmp(azArg[0], "user", n)==0 ){ if( nArg<2 ){ raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n"); Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -2989,13 +2989,13 @@ ** the original statement text and an estimate of wall-clock time ** of how long that statement took to run. ^The profile callback ** time is in units of nanoseconds, however the current implementation ** is only capable of millisecond resolution so the six least significant ** digits in the time are meaningless. Future versions of SQLite -** might provide greater resolution on the profiler callback. The -** sqlite3_profile() function is considered experimental and is -** subject to change in future versions of SQLite. +** might provide greater resolution on the profiler callback. Invoking +** either [sqlite3_trace()] or [sqlite3_trace_v2()] will cancel the +** profile callback. */ SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*, void(*xProfile)(void*,const char*,sqlite3_uint64), void*); @@ -3627,18 +3627,17 @@ ** [sqlite3_finalize()] relatively soon. The current implementation acts ** on this hint by avoiding the use of [lookaside memory] so as not to ** deplete the limited store of lookaside memory. Future versions of ** SQLite may act on this hint differently. ** -** [[SQLITE_PREPARE_NORMALIZE]] ^(
SQLITE_PREPARE_NORMALIZE
-**
The SQLITE_PREPARE_NORMALIZE flag indicates that a normalized -** representation of the SQL statement should be calculated and then -** associated with the prepared statement, which can be obtained via -** the [sqlite3_normalized_sql()] interface.)^ The semantics used to -** normalize a SQL statement are unspecified and subject to change. -** At a minimum, literal values will be replaced with suitable -** placeholders. +** [[SQLITE_PREPARE_NORMALIZE]]
SQLITE_PREPARE_NORMALIZE
+**
The SQLITE_PREPARE_NORMALIZE flag is a no-op. This flag used +** to be required for any prepared statement that wanted to use the +** [sqlite3_normalized_sql()] interface. However, the +** [sqlite3_normalized_sql()] interface is now available to all +** prepared statements, regardless of whether or not they use this +** flag. ** */ #define SQLITE_PREPARE_PERSISTENT 0x01 #define SQLITE_PREPARE_NORMALIZE 0x02 Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -1354,14 +1354,17 @@ #ifndef SQLITE_OMIT_DEPRECATED /* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing ** in the style of sqlite3_trace() */ -#define SQLITE_TRACE_LEGACY 0x80 +#define SQLITE_TRACE_LEGACY 0x40 /* Use the legacy xTrace */ +#define SQLITE_TRACE_XPROFILE 0x80 /* Use the legacy xProfile */ #else -#define SQLITE_TRACE_LEGACY 0 +#define SQLITE_TRACE_LEGACY 0 +#define SQLITE_TRACE_XPROFILE 0 #endif /* SQLITE_OMIT_DEPRECATED */ +#define SQLITE_TRACE_NONLEGACY_MASK 0x0f /* Normal flags */ /* ** Each database connection is an instance of the following structure. */ @@ -1417,12 +1420,14 @@ int nVDestroy; /* Number of active OP_VDestroy operations */ int nExtension; /* Number of loaded extensions */ void **aExtension; /* Array of shared library handles */ int (*xTrace)(u32,void*,void*,void*); /* Trace function */ void *pTraceArg; /* Argument to the trace function */ +#ifndef SQLITE_OMIT_DEPRECATED void (*xProfile)(void*,const char*,u64); /* Profiling function */ void *pProfileArg; /* Argument to profile function */ +#endif void *pCommitArg; /* Argument to xCommitCallback() */ int (*xCommitCallback)(void*); /* Invoked at every commit. */ void *pRollbackArg; /* Argument to xRollbackCallback() */ void (*xRollbackCallback)(void*); /* Invoked at every commit. */ void *pUpdateArg; @@ -1951,13 +1956,10 @@ ** by an instance of the following structure. */ struct Table { char *zName; /* Name of the table or view */ Column *aCol; /* Information about each column */ -#ifdef SQLITE_ENABLE_NORMALIZE - Hash *pColHash; /* All columns indexed by name */ -#endif Index *pIndex; /* List of SQL indexes on this table. */ Select *pSelect; /* NULL for tables. Points to definition if a view. */ FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ ExprList *pCheck; /* All CHECK constraints */ @@ -3982,12 +3984,12 @@ Table *sqlite3LocateTable(Parse*,u32 flags,const char*, const char*); Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *); Index *sqlite3FindIndex(sqlite3*,const char*, const char*); void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*); void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*); -void sqlite3Vacuum(Parse*,Token*); -int sqlite3RunVacuum(char**, sqlite3*, int); +void sqlite3Vacuum(Parse*,Token*,Expr*); +int sqlite3RunVacuum(char**, sqlite3*, int, sqlite3_value*); char *sqlite3NameFromToken(sqlite3*, Token*); int sqlite3ExprCompare(Parse*,Expr*, Expr*, int); int sqlite3ExprCompareSkip(Expr*, Expr*, int); int sqlite3ExprListCompare(ExprList*, ExprList*, int); int sqlite3ExprImpliesExpr(Parse*,Expr*, Expr*, int); @@ -4021,13 +4023,10 @@ #endif int sqlite3ExprIsInteger(Expr*, int*); int sqlite3ExprCanBeNull(const Expr*); int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); int sqlite3IsRowid(const char*); -#ifdef SQLITE_ENABLE_NORMALIZE -int sqlite3IsRowidN(const char*, int); -#endif void sqlite3GenerateRowDelete( Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int); void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int); int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); void sqlite3ResolvePartIdxLabel(Parse*,int); @@ -4050,13 +4049,11 @@ Expr *sqlite3ExprDup(sqlite3*,Expr*,int); ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int); SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int); IdList *sqlite3IdListDup(sqlite3*,IdList*); Select *sqlite3SelectDup(sqlite3*,Select*,int); -#ifdef SQLITE_ENABLE_NORMALIZE -FuncDef *sqlite3FunctionSearchN(int,const char*,int); -#endif +FuncDef *sqlite3FunctionSearch(int,const char*); void sqlite3InsertBuiltinFuncs(FuncDef*,int); FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8); void sqlite3RegisterBuiltinFunctions(void); void sqlite3RegisterDateTimeFunctions(void); void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*); @@ -4257,13 +4254,10 @@ void sqlite3Reindex(Parse*, Token*, Token*); void sqlite3AlterFunctions(void); void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*); int sqlite3GetToken(const unsigned char *, int *); -#ifdef SQLITE_ENABLE_NORMALIZE -int sqlite3GetTokenNormalized(const unsigned char *, int *, int *); -#endif void sqlite3NestedParse(Parse*, const char*, ...); void sqlite3ExpirePreparedStatements(sqlite3*, int); int sqlite3CodeSubselect(Parse*, Expr *, int, int); void sqlite3SelectPrep(Parse*, Select*, NameContext*); void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p); @@ -4418,11 +4412,11 @@ sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*); int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); void sqlite3ParserReset(Parse*); #ifdef SQLITE_ENABLE_NORMALIZE -void sqlite3Normalize(Vdbe*, const char*, int, u8); +char *sqlite3Normalize(Vdbe*, const char*); #endif int sqlite3Reprepare(Vdbe*); void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*); CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *); int sqlite3TempInMemory(const sqlite3*); Index: src/test_hexio.c ================================================================== --- src/test_hexio.c +++ src/test_hexio.c @@ -188,11 +188,11 @@ Tcl_SetObjResult(interp, Tcl_NewIntObj(written)); return TCL_OK; } /* -** USAGE: hexio_get_int HEXDATA +** USAGE: hexio_get_int [-littleendian] HEXDATA ** ** Interpret the HEXDATA argument as a big-endian integer. Return ** the value of that integer. HEXDATA can contain between 2 and 8 ** hexadecimal digits. */ @@ -205,16 +205,24 @@ int val; int nIn, nOut; const unsigned char *zIn; unsigned char *aOut; unsigned char aNum[4]; + int bLittle = 0; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "HEXDATA"); + if( objc==3 ){ + int n; + char *z = Tcl_GetStringFromObj(objv[1], &n); + if( n>=2 && n<=13 && memcmp(z, "-littleendian", n)==0 ){ + bLittle = 1; + } + } + if( (objc-bLittle)!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "[-littleendian] HEXDATA"); return TCL_ERROR; } - zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1], &nIn); + zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1+bLittle], &nIn); aOut = sqlite3_malloc( nIn/2 ); if( aOut==0 ){ return TCL_ERROR; } nOut = sqlite3TestHexToBin(zIn, nIn, aOut); @@ -223,11 +231,15 @@ }else{ memset(aNum, 0, sizeof(aNum)); memcpy(&aNum[4-nOut], aOut, nOut); } sqlite3_free(aOut); - val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3]; + if( bLittle ){ + val = (aNum[3]<<24) | (aNum[2]<<16) | (aNum[1]<<8) | aNum[0]; + }else{ + val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3]; + } Tcl_SetObjResult(interp, Tcl_NewIntObj(val)); return TCL_OK; } Index: src/tokenize.c ================================================================== --- src/tokenize.c +++ src/tokenize.c @@ -543,77 +543,10 @@ while( IdChar(z[i]) ){ i++; } *tokenType = TK_ID; return i; } -#ifdef SQLITE_ENABLE_NORMALIZE -/* -** Return the length (in bytes) of the token that begins at z[0]. -** Store the token type in *tokenType before returning. If flags has -** SQLITE_TOKEN_NORMALIZE flag enabled, use the identifier token type -** for keywords. Add SQLITE_TOKEN_QUOTED to flags if the token was -** actually a quoted identifier. Add SQLITE_TOKEN_KEYWORD to flags -** if the token was recognized as a keyword; this is useful when the -** SQLITE_TOKEN_NORMALIZE flag is used, because it enables the caller -** to differentiate between a keyword being treated as an identifier -** (for normalization purposes) and an actual identifier. -*/ -int sqlite3GetTokenNormalized( - const unsigned char *z, - int *tokenType, - int *flags -){ - int n; - unsigned char iClass = aiClass[*z]; - if( iClass==CC_KYWD ){ - int i; - for(i=1; aiClass[z[i]]<=CC_KYWD; i++){} - if( IdChar(z[i]) ){ - /* This token started out using characters that can appear in keywords, - ** but z[i] is a character not allowed within keywords, so this must - ** be an identifier instead */ - i++; - while( IdChar(z[i]) ){ i++; } - *tokenType = TK_ID; - return i; - } - *tokenType = TK_ID; - n = keywordCode((char*)z, i, tokenType); - /* If the token is no longer considered to be an identifier, then it is a - ** keyword of some kind. Make the token back into an identifier and then - ** set the SQLITE_TOKEN_KEYWORD flag. Several non-identifier tokens are - ** used verbatim, including IN, IS, NOT, and NULL. */ - switch( *tokenType ){ - case TK_ID: { - /* do nothing, handled by caller */ - break; - } - case TK_IN: - case TK_IS: - case TK_NOT: - case TK_NULL: { - *flags |= SQLITE_TOKEN_KEYWORD; - break; - } - default: { - *tokenType = TK_ID; - *flags |= SQLITE_TOKEN_KEYWORD; - break; - } - } - }else{ - n = sqlite3GetToken(z, tokenType); - /* If the token is considered to be an identifier and the character class - ** of the first character is a quote, set the SQLITE_TOKEN_QUOTED flag. */ - if( *tokenType==TK_ID && (iClass==CC_QUOTE || iClass==CC_QUOTE2) ){ - *flags |= SQLITE_TOKEN_QUOTED; - } - } - return n; -} -#endif /* SQLITE_ENABLE_NORMALIZE */ - /* ** Run the parser on the given SQL string. The parser structure is ** passed in. An SQLITE_ status code is returned. If an error occurs ** then an and attempt is made to write an error message into ** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that @@ -779,5 +712,140 @@ sqlite3DeleteTable(db, p); } assert( nErr==0 || pParse->rc!=SQLITE_OK ); return nErr; } + + +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** Insert a single space character into pStr if the current string +** ends with an identifier +*/ +static void addSpaceSeparator(sqlite3_str *pStr){ + if( pStr->nChar && sqlite3IsIdChar(pStr->zText[pStr->nChar-1]) ){ + sqlite3_str_append(pStr, " ", 1); + } +} + +/* +** Compute a normalization of the SQL given by zSql[0..nSql-1]. Return +** the normalization in space obtained from sqlite3DbMalloc(). Or return +** NULL if anything goes wrong or if zSql is NULL. +*/ +char *sqlite3Normalize( + Vdbe *pVdbe, /* VM being reprepared */ + const char *zSql /* The original SQL string */ +){ + sqlite3 *db; /* The database connection */ + int i; /* Next unread byte of zSql[] */ + int n; /* length of current token */ + int tokenType; /* type of current token */ + int prevType; /* Previous non-whitespace token */ + int nParen; /* Number of nested levels of parentheses */ + int iStartIN; /* Start of RHS of IN operator in z[] */ + int nParenAtIN; /* Value of nParent at start of RHS of IN operator */ + int j; /* Bytes of normalized SQL generated so far */ + sqlite3_str *pStr; /* The normalized SQL string under construction */ + + db = sqlite3VdbeDb(pVdbe); + tokenType = -1; + nParen = iStartIN = nParenAtIN = 0; + pStr = sqlite3_str_new(db); + assert( pStr!=0 ); /* sqlite3_str_new() never returns NULL */ + for(i=0; zSql[i] && pStr->accError==0; i+=n){ + if( tokenType!=TK_SPACE ){ + prevType = tokenType; + } + n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType); + if( NEVER(n<=0) ) break; + switch( tokenType ){ + case TK_SPACE: { + break; + } + case TK_NULL: { + if( prevType==TK_IS || prevType==TK_NOT ){ + sqlite3_str_append(pStr, " NULL", 5); + break; + } + /* Fall through */ + } + case TK_STRING: + case TK_INTEGER: + case TK_FLOAT: + case TK_VARIABLE: + case TK_BLOB: { + sqlite3_str_append(pStr, "?", 1); + break; + } + case TK_LP: { + nParen++; + if( prevType==TK_IN ){ + iStartIN = pStr->nChar; + nParenAtIN = nParen; + } + sqlite3_str_append(pStr, "(", 1); + break; + } + case TK_RP: { + if( iStartIN>0 && nParen==nParenAtIN ){ + assert( pStr->nChar>=iStartIN ); + pStr->nChar = iStartIN+1; + sqlite3_str_append(pStr, "?,?,?", 5); + iStartIN = 0; + } + nParen--; + sqlite3_str_append(pStr, ")", 1); + break; + } + case TK_ID: { + iStartIN = 0; + j = pStr->nChar; + if( sqlite3Isquote(zSql[i]) ){ + char *zId = sqlite3DbStrNDup(db, zSql+i, n); + int nId; + int eType = 0; + if( zId==0 ) break; + sqlite3Dequote(zId); + if( zSql[i]=='"' && sqlite3VdbeUsesDoubleQuotedString(pVdbe, zId) ){ + sqlite3_str_append(pStr, "?", 1); + sqlite3DbFree(db, zId); + break; + } + nId = sqlite3Strlen30(zId); + if( sqlite3GetToken((u8*)zId, &eType)==nId && eType==TK_ID ){ + addSpaceSeparator(pStr); + sqlite3_str_append(pStr, zId, nId); + }else{ + sqlite3_str_appendf(pStr, "\"%w\"", zId); + } + sqlite3DbFree(db, zId); + }else{ + addSpaceSeparator(pStr); + sqlite3_str_append(pStr, zSql+i, n); + } + while( jnChar ){ + pStr->zText[j] = sqlite3Tolower(pStr->zText[j]); + j++; + } + break; + } + case TK_SELECT: { + iStartIN = 0; + /* fall through */ + } + default: { + if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr); + j = pStr->nChar; + sqlite3_str_append(pStr, zSql+i, n); + while( jnChar ){ + pStr->zText[j] = sqlite3Toupper(pStr->zText[j]); + j++; + } + break; + } + } + } + if( tokenType!=TK_SEMI ) sqlite3_str_append(pStr, ";", 1); + return sqlite3_str_finish(pStr); +} +#endif /* SQLITE_ENABLE_NORMALIZE */ Index: src/vacuum.c ================================================================== --- src/vacuum.c +++ src/vacuum.c @@ -100,20 +100,20 @@ ** not work if other processes are attached to the original database. ** And a power loss in between deleting the original and renaming the ** transient would cause the database file to appear to be deleted ** following reboot. */ -void sqlite3Vacuum(Parse *pParse, Token *pNm){ +void sqlite3Vacuum(Parse *pParse, Token *pNm, Expr *pInto){ Vdbe *v = sqlite3GetVdbe(pParse); int iDb = 0; - if( v==0 ) return; + if( v==0 ) goto build_vacuum_end; if( pNm ){ #ifndef SQLITE_BUG_COMPATIBLE_20160819 /* Default behavior: Report an error if the argument to VACUUM is ** not recognized */ iDb = sqlite3TwoPartName(pParse, pNm, pNm, &pNm); - if( iDb<0 ) return; + if( iDb<0 ) goto build_vacuum_end; #else /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments ** to VACUUM are silently ignored. This is a back-out of a bug fix that ** occurred on 2016-08-19 (https://www.sqlite.org/src/info/083f9e6270). ** The buggy behavior is required for binary compatibility with some @@ -121,42 +121,64 @@ iDb = sqlite3FindDb(pParse->db, pNm); if( iDb<0 ) iDb = 0; #endif } if( iDb!=1 ){ - sqlite3VdbeAddOp1(v, OP_Vacuum, iDb); + int iIntoReg = 0; + if( pInto ){ + iIntoReg = ++pParse->nMem; + sqlite3ExprCode(pParse, pInto, iIntoReg); + } + sqlite3VdbeAddOp2(v, OP_Vacuum, iDb, iIntoReg); sqlite3VdbeUsesBtree(v, iDb); } +build_vacuum_end: + sqlite3ExprDelete(pParse->db, pInto); return; } /* ** This routine implements the OP_Vacuum opcode of the VDBE. */ -int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ +int sqlite3RunVacuum( + char **pzErrMsg, /* Write error message here */ + sqlite3 *db, /* Database connection */ + int iDb, /* Which attached DB to vacuum */ + sqlite3_value *pOut /* Write results here, if not NULL */ +){ int rc = SQLITE_OK; /* Return code from service routines */ Btree *pMain; /* The database being vacuumed */ Btree *pTemp; /* The temporary database we vacuum into */ - u16 saved_mDbFlags; /* Saved value of db->mDbFlags */ - u32 saved_flags; /* Saved value of db->flags */ + u32 saved_mDbFlags; /* Saved value of db->mDbFlags */ + u64 saved_flags; /* Saved value of db->flags */ int saved_nChange; /* Saved value of db->nChange */ int saved_nTotalChange; /* Saved value of db->nTotalChange */ u8 saved_mTrace; /* Saved trace settings */ Db *pDb = 0; /* Database to detach at end of vacuum */ int isMemDb; /* True if vacuuming a :memory: database */ int nRes; /* Bytes of reserved space at the end of each page */ int nDb; /* Number of attached databases */ const char *zDbMain; /* Schema name of database to vacuum */ + const char *zOut; /* Name of output file */ if( !db->autoCommit ){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); return SQLITE_ERROR; } if( db->nVdbeActive>1 ){ sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress"); return SQLITE_ERROR; } + if( pOut ){ + if( sqlite3_value_type(pOut)!=SQLITE_TEXT ){ + sqlite3SetString(pzErrMsg, db, "non-text filename"); + return SQLITE_ERROR; + } + zOut = (const char*)sqlite3_value_text(pOut); + }else{ + zOut = ""; + } /* Save the current value of the database flags so that it can be ** restored before returning. Then set the writable-schema flag, and ** disable CHECK and foreign key constraints. */ saved_flags = db->flags; @@ -164,11 +186,11 @@ saved_nChange = db->nChange; saved_nTotalChange = db->nTotalChange; saved_mTrace = db->mTrace; db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks; db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum; - db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder + db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder | SQLITE_Defensive | SQLITE_CountRows); db->mTrace = 0; zDbMain = db->aDb[iDb].zDbSName; pMain = db->aDb[iDb].pBt; @@ -187,23 +209,25 @@ ** empty. Only the journal header is written. Apparently it takes more ** time to parse and run the PRAGMA to turn journalling off than it does ** to write the journal header file. */ nDb = db->nDb; - rc = execSql(db, pzErrMsg, "ATTACH''AS vacuum_db"); + rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS vacuum_db", zOut); if( rc!=SQLITE_OK ) goto end_of_vacuum; assert( (db->nDb-1)==nDb ); pDb = &db->aDb[nDb]; assert( strcmp(pDb->zDbSName,"vacuum_db")==0 ); pTemp = pDb->pBt; - - /* The call to execSql() to attach the temp database has left the file - ** locked (as there was more than one active statement when the transaction - ** to read the schema was concluded. Unlock it here so that this doesn't - ** cause problems for the call to BtreeSetPageSize() below. */ - sqlite3BtreeCommit(pTemp); - + if( pOut ){ + sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp)); + i64 sz = 0; + if( id->pMethods!=0 && (sqlite3OsFileSize(id, &sz)!=SQLITE_OK || sz>0) ){ + rc = SQLITE_ERROR; + sqlite3SetString(pzErrMsg, db, "output file already exists"); + goto end_of_vacuum; + } + } nRes = sqlite3BtreeGetOptimalReserve(pMain); /* A VACUUM cannot change the pagesize of an encrypted database. */ #ifdef SQLITE_HAS_CODEC if( db->nextPagesize ){ @@ -223,11 +247,11 @@ ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below, ** to ensure that we do not try to change the page-size on a WAL database. */ rc = execSql(db, pzErrMsg, "BEGIN"); if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = sqlite3BtreeBeginTrans(pMain, 2, 0); + rc = sqlite3BtreeBeginTrans(pMain, pOut==0 ? 2 : 0, 0); if( rc!=SQLITE_OK ) goto end_of_vacuum; /* Do not attempt to change the page size for a WAL database */ if( sqlite3PagerGetJournalMode(sqlite3BtreePager(pMain)) ==PAGER_JOURNALMODE_WAL ){ @@ -318,11 +342,11 @@ BTREE_USER_VERSION, 0, /* Preserve the user version */ BTREE_APPLICATION_ID, 0, /* Preserve the application id */ }; assert( 1==sqlite3BtreeIsInTrans(pTemp) ); - assert( 1==sqlite3BtreeIsInTrans(pMain) ); + assert( pOut!=0 || 1==sqlite3BtreeIsInTrans(pMain) ); /* Copy Btree meta values */ for(i=0; iflags */ db->init.iDb = 0; db->mDbFlags = saved_mDbFlags; Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -6725,18 +6725,23 @@ break; }; #endif /* SQLITE_OMIT_PRAGMA */ #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) -/* Opcode: Vacuum P1 * * * * +/* Opcode: Vacuum P1 P2 * * * ** ** Vacuum the entire database P1. P1 is 0 for "main", and 2 or more ** for an attached database. The "temp" database may not be vacuumed. +** +** If P2 is not zero, then it is a register holding a string which is +** the file into which the result of vacuum should be written. When +** P2 is zero, the vacuum overwrites the original database. */ case OP_Vacuum: { assert( p->readOnly==0 ); - rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1); + rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1, + pOp->p2 ? &aMem[pOp->p2] : 0); if( rc ) goto abort_due_to_error; break; } #endif @@ -7135,11 +7140,11 @@ testcase( pName->enc==SQLITE_UTF16BE ); testcase( pName->enc==SQLITE_UTF16LE ); rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8); if( rc ) goto abort_due_to_error; rc = pVtab->pModule->xRename(pVtab, pName->z); - if( isLegacy==0 ) db->flags &= ~SQLITE_LegacyAlter; + if( isLegacy==0 ) db->flags &= ~(u64)SQLITE_LegacyAlter; sqlite3VtabImportErrmsg(p, pVtab); p->expired = 0; if( rc ) goto abort_due_to_error; break; } Index: src/vdbe.h ================================================================== --- src/vdbe.h +++ src/vdbe.h @@ -249,10 +249,14 @@ int sqlite3VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*)); void sqlite3VdbeCountChanges(Vdbe*); sqlite3 *sqlite3VdbeDb(Vdbe*); u8 sqlite3VdbePrepareFlags(Vdbe*); void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, u8); +#ifdef SQLITE_ENABLE_NORMALIZE +void sqlite3VdbeAddDblquoteStr(sqlite3*,Vdbe*,const char*); +int sqlite3VdbeUsesDoubleQuotedString(Vdbe*,const char*); +#endif void sqlite3VdbeSwap(Vdbe*,Vdbe*); VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*); sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8); void sqlite3VdbeSetVarmask(Vdbe*, int); #ifndef SQLITE_OMIT_TRACE Index: src/vdbeInt.h ================================================================== --- src/vdbeInt.h +++ src/vdbeInt.h @@ -333,19 +333,35 @@ /* A bitfield type for use inside of structures. Always follow with :N where ** N is the number of bits. */ typedef unsigned bft; /* Bit Field Type */ +/* The ScanStatus object holds a single value for the +** sqlite3_stmt_scanstatus() interface. +*/ typedef struct ScanStatus ScanStatus; struct ScanStatus { int addrExplain; /* OP_Explain for loop */ int addrLoop; /* Address of "loops" counter */ int addrVisit; /* Address of "rows visited" counter */ int iSelectID; /* The "Select-ID" for this loop */ LogEst nEst; /* Estimated output rows per loop */ char *zName; /* Name of table or index */ }; + +/* The DblquoteStr object holds the text of a double-quoted +** string for a prepared statement. A linked list of these objects +** is constructed during statement parsing and is held on Vdbe.pDblStr. +** When computing a normalized SQL statement for an SQL statement, that +** list is consulted for each double-quoted identifier to see if the +** identifier should really be a string literal. +*/ +typedef struct DblquoteStr DblquoteStr; +struct DblquoteStr { + DblquoteStr *pNextStr; /* Next string literal in the list */ + char z[8]; /* Dequoted value for the string */ +}; /* ** An instance of the virtual machine. This structure contains the complete ** state of the virtual machine. ** @@ -362,11 +378,11 @@ int nCursor; /* Number of slots in apCsr[] */ u32 cacheCtr; /* VdbeCursor row cache generation counter */ int pc; /* The program counter */ int rc; /* Value to return */ int nChange; /* Number of db changes made since last reset */ - int iStatement; /* Statement number (or 0 if has not opened stmt) */ + int iStatement; /* Statement number (or 0 if has no opened stmt) */ i64 iCurrentTime; /* Value of julianday('now') for this statement */ i64 nFkConstraint; /* Number of imm. FK constraints this VM */ i64 nStmtDefCons; /* Number of def. constraints when stmt started */ i64 nStmtDefImmCons; /* Number of def. imm constraints when stmt started */ @@ -406,10 +422,11 @@ yDbMask lockMask; /* Subset of btreeMask that requires a lock */ u32 aCounter[7]; /* Counters used by sqlite3_stmt_status() */ char *zSql; /* Text of the SQL statement that generated this */ #ifdef SQLITE_ENABLE_NORMALIZE char *zNormSql; /* Normalization of the associated SQL statement */ + DblquoteStr *pDblStr; /* List of double-quoted string literals */ #endif void *pFree; /* Free this when deleting the vdbe */ VdbeFrame *pFrame; /* Parent frame */ VdbeFrame *pDelFrame; /* List of frame objects to free on VM reset */ int nFrame; /* Number of frames in pFrame list */ Index: src/vdbeapi.c ================================================================== --- src/vdbeapi.c +++ src/vdbeapi.c @@ -60,18 +60,20 @@ */ static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){ sqlite3_int64 iNow; sqlite3_int64 iElapse; assert( p->startTime>0 ); - assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 ); + assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 ); assert( db->init.busy==0 ); assert( p->zSql!=0 ); sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); iElapse = (iNow - p->startTime)*1000000; +#ifndef SQLITE_OMIT_DEPRECATED if( db->xProfile ){ db->xProfile(db->pProfileArg, p->zSql, iElapse); } +#endif if( db->mTrace & SQLITE_TRACE_PROFILE ){ db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse); } p->startTime = 0; } @@ -581,11 +583,11 @@ if( db->mallocFailed ){ p->rc = SQLITE_NOMEM; return SQLITE_NOMEM_BKPT; } - if( p->pc<=0 && p->expired ){ + if( p->pc<0 && p->expired ){ p->rc = SQLITE_SCHEMA; rc = SQLITE_ERROR; goto end_of_step; } if( p->pc<0 ){ @@ -600,11 +602,11 @@ assert( db->nVdbeWrite>0 || db->autoCommit==0 || (db->nDeferredCons==0 && db->nDeferredImmCons==0) ); #ifndef SQLITE_OMIT_TRACE - if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0) + if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 && !db->init.busy && p->zSql ){ sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); }else{ assert( p->startTime==0 ); } @@ -627,20 +629,22 @@ db->nVdbeExec++; rc = sqlite3VdbeExec(p); db->nVdbeExec--; } + if( rc!=SQLITE_ROW ){ #ifndef SQLITE_OMIT_TRACE - /* If the statement completed successfully, invoke the profile callback */ - if( rc!=SQLITE_ROW ) checkProfileCallback(db, p); + /* If the statement completed successfully, invoke the profile callback */ + checkProfileCallback(db, p); #endif - if( rc==SQLITE_DONE && db->autoCommit ){ - assert( p->rc==SQLITE_OK ); - p->rc = doWalCallbacks(db); - if( p->rc!=SQLITE_OK ){ - rc = SQLITE_ERROR; + if( rc==SQLITE_DONE && db->autoCommit ){ + assert( p->rc==SQLITE_OK ); + p->rc = doWalCallbacks(db); + if( p->rc!=SQLITE_OK ){ + rc = SQLITE_ERROR; + } } } db->errCode = rc; if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){ @@ -656,13 +660,13 @@ */ assert( rc==SQLITE_ROW || rc==SQLITE_DONE || rc==SQLITE_ERROR || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE ); assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp ); - if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 - && rc!=SQLITE_ROW - && rc!=SQLITE_DONE + if( rc!=SQLITE_ROW + && rc!=SQLITE_DONE + && (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 ){ /* If this statement was prepared using saved SQL and an ** error has occurred, then return the error code in p->rc to the ** caller. Set the error code in the database handle to the same value. */ @@ -1280,11 +1284,11 @@ } i--; pVar = &p->aVar[i]; sqlite3VdbeMemRelease(pVar); pVar->flags = MEM_Null; - sqlite3Error(p->db, SQLITE_OK); + p->db->errCode = SQLITE_OK; /* If the bit corresponding to this variable in Vdbe.expmask is set, then ** binding a new value to this variable invalidates the current query plan. ** ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host @@ -1706,11 +1710,17 @@ /* ** Return the normalized SQL associated with a prepared statement. */ const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe *)pStmt; - return p ? p->zNormSql : 0; + if( p==0 ) return 0; + if( p->zNormSql==0 && ALWAYS(p->zSql!=0) ){ + sqlite3_mutex_enter(p->db->mutex); + p->zNormSql = sqlite3Normalize(p, p->zSql); + sqlite3_mutex_leave(p->db->mutex); + } + return p->zNormSql; } #endif /* SQLITE_ENABLE_NORMALIZE */ #ifdef SQLITE_ENABLE_PREUPDATE_HOOK /* Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -62,18 +62,48 @@ if( (prepFlags & SQLITE_PREPARE_SAVESQL)==0 ){ p->expmask = 0; } assert( p->zSql==0 ); p->zSql = sqlite3DbStrNDup(p->db, z, n); +} + +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** Add a new element to the Vdbe->pDblStr list. +*/ +void sqlite3VdbeAddDblquoteStr(sqlite3 *db, Vdbe *p, const char *z){ + if( p ){ + int n = sqlite3Strlen30(z); + DblquoteStr *pStr = sqlite3DbMallocRawNN(db, + sizeof(*pStr)+n+1-sizeof(pStr->z)); + if( pStr ){ + pStr->pNextStr = p->pDblStr; + p->pDblStr = pStr; + memcpy(pStr->z, z, n+1); + } + } +} +#endif + #ifdef SQLITE_ENABLE_NORMALIZE - assert( p->zNormSql==0 ); - if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){ - sqlite3Normalize(p, p->zSql, n, prepFlags); - assert( p->zNormSql!=0 || p->db->mallocFailed ); +/* +** zId of length nId is a double-quoted identifier. Check to see if +** that identifier is really used as a string literal. +*/ +int sqlite3VdbeUsesDoubleQuotedString( + Vdbe *pVdbe, /* The prepared statement */ + const char *zId /* The double-quoted identifier, already dequoted */ +){ + DblquoteStr *pStr; + assert( zId!=0 ); + if( pVdbe->pDblStr==0 ) return 0; + for(pStr=pVdbe->pDblStr; pStr; pStr=pStr->pNextStr){ + if( strcmp(zId, pStr->z)==0 ) return 1; } + return 0; +} #endif -} /* ** Swap all content between two VDBE structures. */ void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ @@ -90,11 +120,11 @@ pA->pPrev = pB->pPrev; pB->pPrev = pTmp; zTmp = pA->zSql; pA->zSql = pB->zSql; pB->zSql = zTmp; -#ifdef SQLITE_ENABLE_NORMALIZE +#if 0 zTmp = pA->zNormSql; pA->zNormSql = pB->zNormSql; pB->zNormSql = zTmp; #endif pB->expmask = pA->expmask; @@ -2871,11 +2901,11 @@ sqlite3RollbackAll(db, SQLITE_OK); p->nChange = 0; }else{ db->nDeferredCons = 0; db->nDeferredImmCons = 0; - db->flags &= ~SQLITE_DeferFKs; + db->flags &= ~(u64)SQLITE_DeferFKs; sqlite3CommitInternalChanges(db); } }else{ sqlite3RollbackAll(db, SQLITE_OK); p->nChange = 0; @@ -3188,10 +3218,17 @@ vdbeFreeOpArray(db, p->aOp, p->nOp); sqlite3DbFree(db, p->aColName); sqlite3DbFree(db, p->zSql); #ifdef SQLITE_ENABLE_NORMALIZE sqlite3DbFree(db, p->zNormSql); + { + DblquoteStr *pThis, *pNext; + for(pThis=p->pDblStr; pThis; pThis=pNext){ + pNext = pThis->pNextStr; + sqlite3DbFree(db, pThis); + } + } #endif #ifdef SQLITE_ENABLE_STMT_SCANSTATUS { int i; for(i=0; inScan; i++){ Index: src/wal.c ================================================================== --- src/wal.c +++ src/wal.c @@ -360,11 +360,10 @@ ** ** There is still a single WRITER and a single CHECKPOINTER lock. The ** recovery procedure still takes the same exclusive lock on the entire ** range of SQLITE_SHM_NLOCK shm-locks. This works because the read-locks ** above use four of the six read-locking slots used by legacy wal mode. -** See the header comment for function walLockReader() for details. ** ** STARTUP/RECOVERY ** ** The read and write version fields of the database header in a wal2 ** database are set to 0x03, instead of 0x02 as in legacy wal mode. @@ -471,20 +470,24 @@ ** ** In wal mode, the Wal.readLock member is set to -1 when no read-lock ** is held, or else is the index of the read-mark on which a lock is ** held. ** -** In wal2 mode, Wal.readLock must be set to one of the following values. -** A value of -1 still indicates that no read-lock is held, but the other -** values are symbolic. See the implementation of walLockReader() for -** details of how the symbols map to OS level locks. +** In wal2 mode, a value of -1 still indicates that no read-lock is held. +** And a non-zero value still represents the index of the read-mark on +** which a lock is held. There are two differences: +** +** 1. wal2 mode never uses read-mark 0. +** +** 2. locks on each read-mark have a different interpretation, as +** indicated by the symbolic names below. */ #define WAL_LOCK_NONE -1 #define WAL_LOCK_PART1 1 #define WAL_LOCK_PART1_FULL2 2 -#define WAL_LOCK_PART2 3 -#define WAL_LOCK_PART2_FULL1 4 +#define WAL_LOCK_PART2_FULL1 3 +#define WAL_LOCK_PART2 4 /* ** This constant is used in wal2 mode only. ** ** In wal2 mode, when committing a transaction, if the current wal file @@ -1110,40 +1113,10 @@ SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE); WALTRACE(("WAL%p: release EXCLUSIVE-%s cnt=%d\n", pWal, walLockName(lockIdx), n)); } -/* -** This function is used to take and release read-locks in wal2 mode. -** -** Use of WAL_READ_LOCK(x) slots for (1<=x<=4). -** -** 1) Partial read of *-wal-1 (blocks checkpointer from checkpointing) -** 2) Full read of *-wal-2 (blocks writer from writing) -** 3) Partial read of *-wal-2 (blocks checkpointer from checkpointing) -** 4) Full read of *-wal-1 (blocks writer from writing) -*/ -static int walLockReader(Wal *pWal, int eLock, int bLock){ - int i; /* Index of first readmark to lock */ - int n; /* Number of readmarks to lock */ - - assert( pWal->hdr.iVersion==WAL_VERSION2 ); - if( pWal->exclusiveMode ) return SQLITE_OK; - - switch( eLock ){ - case WAL_LOCK_PART1 : i = 1; n = 1; break; - case WAL_LOCK_PART1_FULL2: i = 1; n = 2; break; - case WAL_LOCK_PART2 : i = 3; n = 1; break; - case WAL_LOCK_PART2_FULL1: i = 3; n = 2; break; - default: assert( !"cannot happen" ); - } - - return sqlite3OsShmLock(pWal->pDbFd, WAL_READ_LOCK(i), n, - SQLITE_SHM_SHARED | (bLock ? SQLITE_SHM_LOCK : SQLITE_SHM_UNLOCK) - ); -} - /* ** Compute a hash on a page number. The resulting hash value must land ** between 0 and (HASHTABLE_NSLOT-1). The walHashNext() function advances ** the hash to the next value in the event of a collision. */ @@ -1230,18 +1203,13 @@ /* A frame in wal file 0 */ *piRead = (iExternal <= HASHTABLE_NPAGE_ONE) ? iExternal : iExternal - (iHash/2) * HASHTABLE_NPAGE; return 0; } - if( iHash==0 ){ - *piRead = iExternal; - return 0; - }else{ - *piRead = iExternal - HASHTABLE_NPAGE_ONE - ((iHash-1)/2) * HASHTABLE_NPAGE; - } - - return (iHash % 2); + + *piRead = iExternal - HASHTABLE_NPAGE_ONE - ((iHash-1)/2) * HASHTABLE_NPAGE; + return 1; } /* ** Return the number of the wal-index page that contains the hash-table ** and page-number array that contain entries corresponding to WAL frame @@ -1563,11 +1531,11 @@ aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; } } sqlite3_free(aFrame); - }else if( pbZero && nSize==0 ){ + }else if( pbZero ){ *pbZero = 1; } } pWal->hdr.aFrameCksum[0] = aFrameCksum[0]; @@ -1687,12 +1655,12 @@ if( isOpen(pWal->apWalFd[1]) ){ /* The case where *-wal2 may follow *-wal */ if( nCkpt2<=0x0F && nCkpt2==nCkpt1+1 ){ if( sqlite3Get4byte((u8*)(&pWal->hdr.aSalt[0]))==hdr.aFrameCksum[0] - && sqlite3Get4byte((u8*)(&pWal->hdr.aSalt[1]))==hdr.aFrameCksum[1] - ){ + && sqlite3Get4byte((u8*)(&pWal->hdr.aSalt[1]))==hdr.aFrameCksum[1] + ){ walidxSetFile(&pWal->hdr, 1); walidxSetMxFrame(&pWal->hdr, 1, pWal->hdr.mxFrame); walidxSetMxFrame(&pWal->hdr, 0, hdr.mxFrame); }else{ pWal->hdr = hdr; @@ -1746,13 +1714,13 @@ ** problems caused by applications routinely shutting down without ** checkpointing the log file. */ if( pWal->hdr.nPage ){ if( isWalMode2(pWal) ){ sqlite3_log(SQLITE_NOTICE_RECOVER_WAL, - "recovered (%d,%d) frames from WAL files %s[2] (%s mode)", + "recovered (%d,%d) frames from WAL files %s[2] (wal2 mode)", walidxGetMxFrame(&pWal->hdr, 0), walidxGetMxFrame(&pWal->hdr, 1), - pWal->zWalName, isWalMode2(pWal) ? "wal2" : "wal" + pWal->zWalName ); }else{ sqlite3_log(SQLITE_NOTICE_RECOVER_WAL, "recovered %d frames from WAL file %s", pWal->hdr.mxFrame, pWal->zWalName @@ -2249,10 +2217,72 @@ pInfo->nBackfillAttempted = 0; pInfo->aReadMark[1] = 0; for(i=2; iaReadMark[i] = READMARK_NOT_USED; assert( pInfo->aReadMark[0]==0 ); } + +/* +** This function is used in wal2 mode. +** +** This function is called when writer pWal is just about to start +** writing out frames. Parameter iApp is the current wal file. The "other" wal +** file (wal file !iApp) has been fully checkpointed. This function returns +** SQLITE_OK if there are no readers preventing the writer from switching to +** the other wal file. Or SQLITE_BUSY if there are. +*/ +static int wal2RestartOk(Wal *pWal, int iApp){ + /* The other wal file (wal file !iApp) can be overwritten if there + ** are no readers reading from it - no "full" or "partial" locks. + ** Technically speaking it is not possible for any reader to hold + ** a "part" lock, as this would have prevented the file from being + ** checkpointed. But checking anyway doesn't hurt. The following + ** is equivalent to: + ** + ** if( iApp==0 ) eLock = WAL_LOCK_PART1_FULL2; + ** if( iApp==1 ) eLock = WAL_LOCK_PART1; + */ + int eLock = 1 + (iApp==0); + + assert( WAL_LOCK_PART1==1 ); + assert( WAL_LOCK_PART1_FULL2==2 ); + assert( WAL_LOCK_PART2_FULL1==3 ); + assert( WAL_LOCK_PART2==4 ); + + assert( iApp!=0 || eLock==WAL_LOCK_PART1_FULL2 ); + assert( iApp!=1 || eLock==WAL_LOCK_PART1 ); + + return walLockExclusive(pWal, WAL_READ_LOCK(eLock), 3); +} +static void wal2RestartFinished(Wal *pWal, int iApp){ + walUnlockExclusive(pWal, WAL_READ_LOCK(1 + (iApp==0)), 3); +} + +/* +** This function is used in wal2 mode. +** +** This function is called when a checkpointer wishes to checkpoint wal +** file iCkpt. It takes the required lock and, if successful, returns +** SQLITE_OK. Otherwise, an SQLite error code (e.g. SQLITE_BUSY). If this +** function returns SQLITE_OK, it is the responsibility of the caller +** to invoke wal2CheckpointFinished() to release the lock. +*/ +static int wal2CheckpointOk(Wal *pWal, int iCkpt){ + int eLock = 1 + (iCkpt*2); + + assert( WAL_LOCK_PART1==1 ); + assert( WAL_LOCK_PART1_FULL2==2 ); + assert( WAL_LOCK_PART2_FULL1==3 ); + assert( WAL_LOCK_PART2==4 ); + + assert( iCkpt!=0 || eLock==WAL_LOCK_PART1 ); + assert( iCkpt!=1 || eLock==WAL_LOCK_PART2_FULL1 ); + + return walLockExclusive(pWal, WAL_READ_LOCK(eLock), 2); +} +static void wal2CheckpointFinished(Wal *pWal, int iCkpt){ + walUnlockExclusive(pWal, WAL_READ_LOCK(1 + (iCkpt*2)), 2); +} /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. ** @@ -2317,11 +2347,11 @@ /* If this is a wal2 system, check for a reader holding a lock ** preventing this checkpoint operation. If one is found, return ** early. */ if( bWal2 ){ - rc = walLockExclusive(pWal, WAL_READ_LOCK(1 + iCkpt*2), 1); + rc = wal2CheckpointOk(pWal, iCkpt); if( rc!=SQLITE_OK ) return rc; } /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ @@ -2366,13 +2396,13 @@ assert( bWal2==0 || pInfo->nBackfill==0 ); rc = walIteratorInit(pWal, iCkpt, pInfo->nBackfill, &pIter); assert( rc==SQLITE_OK || pIter==0 ); } - if( pIter - && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0),1))==SQLITE_OK - ){ + if( pIter && (bWal2 + || (rc = walBusyLock(pWal, xBusy, pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK + )){ u32 nBackfill = pInfo->nBackfill; assert( bWal2==0 || nBackfill==0 ); pInfo->nBackfillAttempted = mxSafeFrame; @@ -2433,21 +2463,21 @@ if( rc==SQLITE_OK ){ pInfo->nBackfill = bWal2 ? 1 : mxSafeFrame; } /* Release the reader lock held while backfilling */ - if( bWal2 ){ - walUnlockExclusive(pWal, WAL_READ_LOCK(1 + iCkpt*2), 1); + if( bWal2==0 ){ + walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); } - walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); } if( rc==SQLITE_BUSY ){ /* Reset the return code so as not to report a checkpoint failure ** just because there are active readers. */ rc = SQLITE_OK; } + if( bWal2 ) wal2CheckpointFinished(pWal, iCkpt); } /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the ** entire wal file has been copied into the database file, then block ** until all readers have finished using the wal file. This ensures that @@ -3078,22 +3108,31 @@ assert( pWal->nWiData>0 ); assert( pWal->apWiData[0]!=0 ); pInfo = walCkptInfo(pWal); if( isWalMode2(pWal) ){ - int eLock = 1 + (walidxGetFile(&pWal->hdr)*2); - if( pInfo->nBackfill==0 ){ - eLock += walidxGetMxFrame(&pWal->hdr, !walidxGetFile(&pWal->hdr))>0; - } - rc = walLockReader(pWal, eLock, 1); + /* This connection needs a "part" lock on the current wal file and, + ** unless pInfo->nBackfill is set to indicate that it has already been + ** checkpointed, a "full" lock on the other wal file. */ + int iWal = walidxGetFile(&pWal->hdr); + int nBackfill = pInfo->nBackfill || walidxGetMxFrame(&pWal->hdr, !iWal)==0; + int eLock = 1 + (iWal*2) + (nBackfill==iWal); + + assert( nBackfill==0 || nBackfill==1 ); + assert( iWal==0 || iWal==1 ); + assert( iWal!=0 || nBackfill!=1 || eLock==WAL_LOCK_PART1 ); + assert( iWal!=0 || nBackfill!=0 || eLock==WAL_LOCK_PART1_FULL2 ); + assert( iWal!=1 || nBackfill!=1 || eLock==WAL_LOCK_PART2 ); + assert( iWal!=1 || nBackfill!=0 || eLock==WAL_LOCK_PART2_FULL1 ); + + rc = walLockShared(pWal, WAL_READ_LOCK(eLock)); if( rc!=SQLITE_OK ){ - return rc; + return (rc==SQLITE_BUSY ? WAL_RETRY : rc); } - walShmBarrier(pWal); if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ - walLockReader(pWal, eLock, 0); + walUnlockShared(pWal, WAL_READ_LOCK(eLock)); return WAL_RETRY; }else{ pWal->readLock = eLock; } assert( pWal->minFrame==0 && walFramePage(pWal->minFrame)==0 ); @@ -3422,15 +3461,11 @@ ** read-lock. */ void sqlite3WalEndReadTransaction(Wal *pWal){ sqlite3WalEndWriteTransaction(pWal); if( pWal->readLock!=WAL_LOCK_NONE ){ - if( isWalMode2(pWal) ){ - (void)walLockReader(pWal, pWal->readLock, 0); - }else{ - walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); - } + walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); pWal->readLock = WAL_LOCK_NONE; } } /* Search hash table iHash for an entry matching page number @@ -4061,37 +4096,10 @@ } return rc; } -/* -** This function is used in wal2 mode. -** -** This function is called when writer pWal is just about to start -** writing out frames. The "other" wal file (wal file !pWal->hdr.iAppend) -** has been fully checkpointed. This function returns SQLITE_OK if there -** are no readers preventing the writer from switching to the other wal -** file. Or SQLITE_BUSY if there are. -*/ -static int walRestartOk(Wal *pWal){ - int rc; /* Return code */ - int iApp = walidxGetFile(&pWal->hdr); /* Current WAL file */ - - /* No reader can be doing a "partial" read of wal file !iApp - in that - ** case it would not have been possible to checkpoint the file. So - ** it is only necessary to test for "full" readers. See the comment - ** above walLockReader() function for exactly what this means in terms - ** of locks. */ - int i = (iApp==0) ? 2 : 4; - - rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1); - if( rc==SQLITE_OK ){ - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - } - return rc; -} - /* ** This function is called just before writing a set of frames to the log ** file (see sqlite3WalFrames()). It checks to see if, instead of appending ** to the current log file, it is possible and desirable to switch to the ** other log file and write the new transaction to the start of it. @@ -4115,23 +4123,24 @@ } if( walidxGetMxFrame(&pWal->hdr, iApp)>=nWalSize ){ volatile WalCkptInfo *pInfo = walCkptInfo(pWal); if( walidxGetMxFrame(&pWal->hdr, !iApp)==0 || pInfo->nBackfill ){ - rc = walRestartOk(pWal); + rc = wal2RestartOk(pWal, iApp); if( rc==SQLITE_OK ){ - iApp = !iApp; + int iNew = !iApp; pWal->nCkpt++; - walidxSetFile(&pWal->hdr, iApp); - walidxSetMxFrame(&pWal->hdr, iApp, 0); + walidxSetFile(&pWal->hdr, iNew); + walidxSetMxFrame(&pWal->hdr, iNew, 0); sqlite3Put4byte((u8*)&pWal->hdr.aSalt[0], pWal->hdr.aFrameCksum[0]); sqlite3Put4byte((u8*)&pWal->hdr.aSalt[1], pWal->hdr.aFrameCksum[1]); walIndexWriteHdr(pWal); pInfo->nBackfill = 0; - walLockReader(pWal, pWal->readLock, 0); - pWal->readLock = iApp ? WAL_LOCK_PART2_FULL1 : WAL_LOCK_PART1_FULL2; - rc = walLockReader(pWal, pWal->readLock, 1); + wal2RestartFinished(pWal, iApp); + walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); + pWal->readLock = iNew ? WAL_LOCK_PART2_FULL1 : WAL_LOCK_PART1_FULL2; + rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock)); }else if( rc==SQLITE_BUSY ){ rc = SQLITE_OK; } } } @@ -4756,15 +4765,11 @@ assert( pWal->readLock!=WAL_LOCK_NONE || (op<=0 && pWal->exclusiveMode==0) ); if( op==0 ){ if( pWal->exclusiveMode ){ pWal->exclusiveMode = WAL_NORMAL_MODE; - if( isWalMode2(pWal) ){ - rc = walLockReader(pWal, pWal->readLock, 1); - }else{ - rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock)); - } + rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock)); if( rc!=SQLITE_OK ){ pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; } rc = pWal->exclusiveMode==WAL_NORMAL_MODE; }else{ @@ -4772,15 +4777,11 @@ rc = 0; } }else if( op>0 ){ assert( pWal->exclusiveMode==WAL_NORMAL_MODE ); assert( pWal->readLock>=0 ); - if( isWalMode2(pWal) ){ - walLockReader(pWal, pWal->readLock, 0); - }else{ - walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); - } + walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; rc = 1; }else{ rc = pWal->exclusiveMode==WAL_NORMAL_MODE; } Index: src/window.c ================================================================== --- src/window.c +++ src/window.c @@ -2131,10 +2131,11 @@ if( ALWAYS(p) ){ pNew = sqlite3DbMallocZero(db, sizeof(Window)); if( pNew ){ pNew->zName = sqlite3DbStrDup(db, p->zName); pNew->pFilter = sqlite3ExprDup(db, p->pFilter, 0); + pNew->pFunc = p->pFunc; pNew->pPartition = sqlite3ExprListDup(db, p->pPartition, 0); pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, 0); pNew->eType = p->eType; pNew->eEnd = p->eEnd; pNew->eStart = p->eStart; Index: test/altertab2.test ================================================================== --- test/altertab2.test +++ test/altertab2.test @@ -10,11 +10,11 @@ #************************************************************************* # set testdir [file dirname $argv0] source $testdir/tester.tcl -set testprefix altertab +set testprefix altertab2 # If SQLITE_OMIT_ALTERTABLE is defined, omit this file. ifcapable !altertable { finish_test return @@ -83,7 +83,63 @@ {CREATE TABLE c1(x REFERENCES "p3")} {CREATE TABLE c2(x, FOREIGN KEY (x) REFERENCES "p3")} {CREATE TABLE c3(x, FOREIGN KEY (x) REFERENCES "p3"(a))} } +#------------------------------------------------------------------------- +# Table name in WITH clauses that are part of views or triggers. +# +foreach {tn schema} { + 1 { + CREATE TABLE log_entry(col1, y); + CREATE INDEX i1 ON log_entry(col1); + } + + 2 { + CREATE TABLE t1(a, b, c); + CREATE TABLE t2(x); + CREATE TABLE log_entry(col1); + CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN + INSERT INTO t2 SELECT col1 FROM log_entry; + END; + } + + 3 { + CREATE TABLE t1(a, b, c); + CREATE TABLE t2(x); + CREATE TABLE log_entry(col1); + CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN + INSERT INTO t2 + WITH xyz(x) AS (SELECT col1 FROM log_entry) + SELECT x FROM xyz; + END; + } + + 4 { + CREATE TABLE log_entry(col1); + CREATE VIEW ttt AS + WITH xyz(x) AS (SELECT col1 FROM log_entry) + SELECT x FROM xyz; + } +} { + reset_db + do_execsql_test 3.$tn.1 $schema + set expect [db eval "SELECT sql FROM sqlite_master"] + set expect [string map {log_entry {"newname"}} $expect] + + do_execsql_test 3.$tn.2 { + ALTER TABLE log_entry RENAME TO newname; + SELECT sql FROM sqlite_master; + } $expect + + reset_db + do_execsql_test 3.$tn.3 $schema + set expect [db eval "SELECT sql FROM sqlite_master"] + set expect [string map {col1 newname} $expect] + + do_execsql_test 3.$tn.4 { + ALTER TABLE log_entry RENAME col1 TO newname; + SELECT sql FROM sqlite_master; + } $expect +} finish_test Index: test/dbfuzz2.c ================================================================== --- test/dbfuzz2.c +++ test/dbfuzz2.c @@ -76,10 +76,11 @@ if( eVerbosity>=1 ){ printf("************** nByte=%d ***************\n", (int)nByte); fflush(stdout); } + if( sqlite3_initialize() ) return 0; rc = sqlite3_open(0, &db); if( rc ) return 1; a = sqlite3_malloc64(nByte+1); if( a==0 ) return 1; memcpy(a, aData, nByte); ADDED test/fts4umlaut.test Index: test/fts4umlaut.test ================================================================== --- /dev/null +++ test/fts4umlaut.test @@ -0,0 +1,65 @@ +# 2018 December 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts4umlaut + +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + CREATE VIRTUAL TABLE t2 USING fts4( + x, + tokenize=unicode61 "remove_diacritics=2" + ); +} + +foreach {tn q res1 res2} { + 1 "Hà Nội" 0 1 + 2 "Hà Noi" 1 1 + 3 "Ha Noi" 1 1 + 4 "Ha N\u1ed9i" 0 1 + 5 "Ha N\u006fi" 1 1 + 6 "Ha N\u006f\u0302i" 1 1 + 7 "Ha N\u006f\u0323\u0302i" 1 1 +} { + do_execsql_test 1.$tn.1 { + DELETE FROM t1; + INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi'); + SELECT count(*) FROM t1 WHERE t1 MATCH $q + } $res1 + do_execsql_test 1.$tn.2 { + DELETE FROM t1; + INSERT INTO t1(rowid, x) VALUES (1, $q); + SELECT count(*) FROM t1 WHERE t1 MATCH 'Ha Noi' + } $res1 + + do_execsql_test 1.$tn.2 { + DELETE FROM t2; + INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi'); + SELECT count(*) FROM t2 WHERE t2 MATCH $q + } $res2 + do_execsql_test 1.$tn.2 { + DELETE FROM t2; + INSERT INTO t2(rowid, x) VALUES (1, $q); + SELECT count(*) FROM t2 WHERE t2 MATCH 'Ha Noi' + } $res2 +} + +finish_test + Index: test/index6.test ================================================================== --- test/index6.test +++ test/index6.test @@ -387,8 +387,27 @@ EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99; } {/USING INDEX t11x/} do_execsql_test index6-11.2 { EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99 AND c<>98; } {/USING INDEX t11x/} - +# 2018-12-08 +# Ticket https://www.sqlite.org/src/info/1d958d90596593a7 +# NOT IN operator fails when using a partial index. +# +do_execsql_test index6-12.1 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(a,b); + INSERT INTO t1 VALUES(1,1); + INSERT INTO t1 VALUES(2,2); + CREATE TABLE t2(x); + INSERT INTO t2 VALUES(1); + INSERT INTO t2 VALUES(2); + SELECT 'one', * FROM t2 WHERE x NOT IN (SELECT a FROM t1); + CREATE INDEX t1a ON t1(a) WHERE b=1; + SELECT 'two', * FROM t2 WHERE x NOT IN (SELECT a FROM t1); +} {} +do_execsql_test index6-12.2 { + SELECT x FROM t2 WHERE x IN (SELECT a FROM t1) ORDER BY +x; +} {1 2} finish_test Index: test/json101.test ================================================================== --- test/json101.test +++ test/json101.test @@ -811,8 +811,25 @@ } {$} do_execsql_test json-14.170 { SELECT fullkey FROM json_tree('null'); } {$} - +# 2018-12-03 +# Make sure the table-valued functions contained within parentheses +# work correctly. +# +# Bug reported via private email. See TH3 for more information. +# +do_execsql_test json-15.100 { + SELECT * FROM JSON_EACH('{"a":1, "b":2}'); +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}} +do_execsql_test json-15.110 { + SELECT xyz.* FROM JSON_EACH('{"a":1, "b":2}') AS xyz; +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}} +do_execsql_test json-15.120 { + SELECT * FROM (JSON_EACH('{"a":1, "b":2}')); +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}} +do_execsql_test json-15.130 { + SELECT xyz.* FROM (JSON_EACH('{"a":1, "b":2}')) AS xyz; +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}} finish_test Index: test/normalize.test ================================================================== --- test/normalize.test +++ test/normalize.test @@ -84,11 +84,11 @@ sqlite3_bind_null $STMT 1 } {} do_test 202 { sqlite3_normalized_sql $STMT -} {} +} {SELECT a,b FROM t1 WHERE b=?ORDER BY a;} do_test 203 { sqlite3_finalize $STMT } {SQLITE_OK} do_test 210 { @@ -205,21 +205,21 @@ {0 {SELECT a FROM t1 WHERE x IN(?,?,?)AND hex8(?);}} 430 {SELECT "a" FROM t1 WHERE "x" IN ("1","2",'3');} 0x2 - {0 {SELECT"a"FROM t1 WHERE"x"IN(?,?,?);}} + {0 {SELECT a FROM t1 WHERE x IN(?,?,?);}} 440 {SELECT 'a' FROM t1 WHERE 'x';} 0x2 {0 {SELECT?FROM t1 WHERE?;}} 450 {SELECT [a] FROM t1 WHERE [x];} 0x2 - {0 {SELECT"a"FROM t1 WHERE"x";}} + {0 {SELECT a FROM t1 WHERE x;}} 460 {SELECT * FROM t1 WHERE x IN (x);} 0x2 {0 {SELECT*FROM t1 WHERE x IN(x);}} @@ -230,31 +230,31 @@ {0 {SELECT*FROM t1 WHERE x IN(x,a);}} 480 {SELECT * FROM t1 WHERE x IN ([x],"a");} 0x2 - {0 {SELECT*FROM t1 WHERE x IN("x","a");}} + {0 {SELECT*FROM t1 WHERE x IN(x,a);}} 500 {SELECT * FROM t1 WHERE x IN ([x],"a",'b',sqlite_version());} 0x2 - {0 {SELECT*FROM t1 WHERE x IN("x","a",?,sqlite_version());}} + {0 {SELECT*FROM t1 WHERE x IN(x,a,?,sqlite_version());}} 520 {SELECT * FROM t1 WHERE x IN (SELECT x FROM t1);} 0x2 {0 {SELECT*FROM t1 WHERE x IN(SELECT x FROM t1);}} 540 {SELECT * FROM t1 WHERE x IN ((SELECT x FROM t1));} 0x2 - {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}} + {0 {SELECT*FROM t1 WHERE x IN((SELECT x FROM t1));}} 550 {SELECT a, a+1, a||'b', a+"b" FROM t1;} 0x2 - {0 {SELECT a,a+?,a||?,a+"b"FROM t1;}} + {0 {SELECT a,a+?,a||?,a+b FROM t1;}} 570 {SELECT * FROM t1 WHERE x IN (1);} 0x2 {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}} @@ -314,11 +314,11 @@ {0 {SELECT"col f","col f"FROM t1;}} 680 {SELECT a, "col f" FROM t1 LEFT OUTER JOIN t2 ON [t1].[col f] == [t2].[col y];} 0x2 - {0 {SELECT a,"col f"FROM t1 LEFT OUTER JOIN t2 ON"t1"."col f"=="t2"."col y";}} + {0 {SELECT a,"col f"FROM t1 LEFT OUTER JOIN t2 ON t1."col f"==t2."col y";}} 690 {SELECT * FROM ( WITH x AS ( SELECT * FROM t1 WHERE x IN ( 1)) SELECT 10);} 0x2 {0 {SELECT*FROM(WITH x AS(SELECT*FROM t1 WHERE x IN(?,?,?))SELECT?);}} @@ -344,11 +344,11 @@ {0 {SELECT x FROM t1 WHERE x=?;}} 760 {SELECT x FROM t1 WHERE x IN ([x] IS NOT NULL, NULL, 1, 'a', "b", x'00');} 0x2 - {0 {SELECT x FROM t1 WHERE x IN("x"IS NOT NULL,?,?,?,"b",?);}} + {0 {SELECT x FROM t1 WHERE x IN(x IS NOT NULL,?,?,?,b,?);}} } { do_test $tnum { set code [catch { set STMT [sqlite3_prepare_v3 $DB $sql -1 $flags TAIL] sqlite3_normalized_sql $STMT Index: test/permutations.test ================================================================== --- test/permutations.test +++ test/permutations.test @@ -428,14 +428,20 @@ # coverage-wal # test_suite "coverage-wal" -description { Coverage tests for file wal.c. } -files { - wal.test wal2.test wal3.test walmode.test - walbak.test walhook.test walcrash2.test walcksum.test - walfault.test walbig.test walnoshm.test - wal5.test +wal2big.test wal2recover.test wal2rewrite.test +wal2simple.test wal2snapshot.test wal2.test +wal3.test wal4.test wal5.test +wal64k.test wal6.test wal7.test wal8.test wal9.test +walbak.test walbig.test walblock.test walcksum.test +walfault.test walhook.test walmode.test walnoshm.test +waloverwrite.test walpersist.test walprotocol2.test +walprotocol.test walro2.test walrofault.test walro.test +walshared.test walslow.test wal.test +wal2savepoint.test wal2lock.test wal2recover2.test } test_suite "coverage-pager" -description { Coverage tests for file pager.c. } -files { Index: test/shell1.test ================================================================== --- test/shell1.test +++ test/shell1.test @@ -254,11 +254,11 @@ catchcmd "test.db" ".backup FOO BAR" } {1 {Error: unknown database FOO}} do_test shell1-3.1.4 { # too many arguments catchcmd "test.db" ".backup FOO BAR BAD" -} {1 {Usage: .backup ?DB? ?--append? FILENAME}} +} {1 {Usage: .backup ?DB? ?OPTIONS? FILENAME}} # .bail ON|OFF Stop after hitting an error. Default OFF do_test shell1-3.2.1 { catchcmd "test.db" ".bail" } {1 {Usage: .bail on|off}} Index: test/shell4.test ================================================================== --- test/shell4.test +++ test/shell4.test @@ -105,18 +105,18 @@ [regexp {Heap Usage} $res] \ [regexp {Autoindex Inserts} $res] } {1 1 1} do_test shell4-2.1 { - catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace" -} {1 {Usage: .trace FILE|off}} + catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace --unknown" +} {1 {Unknown option "--unknown" on ".trace"}} do_test shell4-2.2 { catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace off\n.trace off\n" } {0 {}} do_test shell4-2.3 { - catchcmd ":memory:" ".trace stdout\n.trace\n.trace off\n.dump\n" -} {/^1 {PRAGMA.*Usage:.*}$/} + catchcmd ":memory:" ".trace stdout\n.dump\n.trace off\n" +} {/^0 {PRAGMA.*}$/} ifcapable trace { do_test shell4-2.4 { catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;" } {0 {CREATE TABLE t1(x); SELECT * FROM t1;}} Index: test/tabfunc01.test ================================================================== --- test/tabfunc01.test +++ test/tabfunc01.test @@ -122,10 +122,15 @@ do_execsql_test tabfunc01-4.3 { ATTACH ':memory:' AS aux1; CREATE TABLE aux1.t1(a,b,c); SELECT * FROM aux1.generate_series(1,4) } {1 2 3 4} + +# 2018-12-03: Fix bug reported by by private email. +do_execsql_test tabfunc01-4.4 { + SELECT * FROM (generate_series(1,5,2)) AS x LIMIT 10; +} {1 3 5} # The next series of tests is verifying that virtual table are able # to optimize the IN operator, even on terms that are not marked "omit". # When the generate_series virtual table is compiled for the testfixture, # the special -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 option is used, which ADDED test/vacuum-into.test Index: test/vacuum-into.test ================================================================== --- /dev/null +++ test/vacuum-into.test @@ -0,0 +1,69 @@ +# 2018-12-07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the VACUUM INTO statement. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If the VACUUM statement is disabled in the current build, skip all +# the tests in this file. +# +ifcapable {!vacuum} { + omit_test vacuum.test {Compiled with SQLITE_OMIT_VACUUM} + finish_test + return +} + +forcedelete out.db +do_execsql_test vacuum-into-100 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) + INSERT INTO t1(a,b) SELECT x, randomblob(600) FROM c; + CREATE INDEX t1b ON t1(b); + DELETE FROM t1 WHERE a%2; + SELECT count(*), sum(a), sum(length(b)) FROM t1; +} {50 2550 30000} +do_execsql_test vacuum-into-110 { + VACUUM main INTO 'out.db'; +} {} +sqlite3 db2 out.db +do_test vacuum-into-120 { + db2 eval {SELECT count(*), sum(a), sum(length(b)) FROM t1} +} {50 2550 30000} +do_catchsql_test vacuum-into-130 { + VACUUM INTO 'out.db'; +} {1 {output file already exists}} +forcedelete out2.db +do_catchsql_test vacuum-into-140 { + VACUUM INTO 'out2.db'; +} {0 {}} +do_catchsql_test vacuum-into-150 { + VACUUM INTO 'out2.db'; +} {1 {output file already exists}} + +do_catchsql_test vacuum-into-200 { + VACUUM main INTO ':memory:'; +} {0 {}} + +# The INTO argument can be an arbitrary expression. +# +do_execsql_test vacuum-into-300 { + CREATE TABLE t2(name TEXT); + INSERT INTO t2 VALUES(':memory:'); + VACUUM main INTO (SELECT name FROM t2); +} {} +do_catchsql_test vacuum-into-310 { + VACUUM INTO null; +} {1 {non-text filename}} + +finish_test ADDED test/wal2big.test Index: test/wal2big.test ================================================================== --- /dev/null +++ test/wal2big.test @@ -0,0 +1,71 @@ +# 2017 September 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix wal2big +ifcapable !wal {finish_test ; return } + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c); + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 10000000; + + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200000 + ) + INSERT INTO t1 SELECT random(), random(), random() FROM s; +} {wal2 10000000} + +do_execsql_test 1.1 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200000 + ) + INSERT INTO t1 SELECT random(), random(), random() FROM s; +} + +do_test 1.2 { + list [expr [file size test.db-wal]>10000000] \ + [expr [file size test.db-wal2]>10000000] +} {1 1} + +do_test 1.3 { + sqlite3 db2 test.db + execsql { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } db2 +} {400000 ok} + +do_test 1.4 { + db2 close + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-wal2 test.db2-wal2 + sqlite3 db2 test.db2 + execsql { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } +} {400000 ok} + +finish_test ADDED test/wal2lock.test Index: test/wal2lock.test ================================================================== --- /dev/null +++ test/wal2lock.test @@ -0,0 +1,106 @@ +# 2018 December 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix wal2lock +ifcapable !wal {finish_test ; return } + +db close +testvfs tvfs +sqlite3 db test.db -vfs tvfs + +do_execsql_test 1.0 { + PRAGMA journal_mode = wal2; + CREATE TABLE y1(y, yy); + CREATE INDEX y1y ON y1(y); + CREATE INDEX y1yy ON y1(yy); + INSERT INTO y1 VALUES(1, 2), (3, 4), (5, 6); +} {wal2} + +tvfs script vfs_callback +tvfs filter xShmLock + +set ::lock [list] +proc vfs_callback {func file name lock} { + lappend ::lock $lock + return SQLITE_OK +} + +do_execsql_test 1.1.1 { + SELECT * FROM y1 +} {1 2 3 4 5 6} +do_test 1.1.2 { + set ::lock +} {{4 1 lock shared} {4 1 unlock shared}} + +set ::bFirst 1 +proc vfs_callback {func file name lock} { + if {$::bFirst} { + set ::bFirst 0 + return SQLITE_BUSY + } + return SQLITE_OK +} +do_execsql_test 1.2 { + SELECT * FROM y1 +} {1 2 3 4 5 6} + +set ::bFirst 1 +proc vfs_callback {func file name lock} { + if {$::bFirst} { + set ::bFirst 0 + return SQLITE_IOERR + } + return SQLITE_OK +} +do_catchsql_test 1.3 { + SELECT * FROM y1 +} {1 {disk I/O error}} + +puts "# Warning: This next test case causes SQLite to call xSleep(1) 100 times." +puts "# Normally this equates to a delay of roughly 10 seconds, but if SQLite" +puts "# is built on unix without HAVE_USLEEP defined, it may be much longer." +proc vfs_callback {func file name lock} { return SQLITE_BUSY } +do_catchsql_test 1.4 { + SELECT * FROM y1 +} {1 {locking protocol}} +proc vfs_callback {func file name lock} { return SQLITE_OK } + +sqlite3 db2 test.db -vfs tvfs +set ::bFirst 1 + +proc vfs_callback {func file name lock} { + if {$::bFirst} { + set ::bFirst 0 + db2 eval { INSERT INTO y1 VALUES(7, 8) } + } +} + +do_execsql_test 1.5.1 { + SELECT * FROM y1 +} {1 2 3 4 5 6 7 8} +do_execsql_test 1.5.2 { + SELECT * FROM y1 +} {1 2 3 4 5 6 7 8} + +db close +db2 close +tvfs delete +finish_test ADDED test/wal2recover.test Index: test/wal2recover.test ================================================================== --- /dev/null +++ test/wal2recover.test @@ -0,0 +1,232 @@ +# 2018 December 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix wal2recover +ifcapable !wal {finish_test ; return } + +proc db_copy {from to} { + forcecopy $from $to + forcecopy ${from}-wal ${to}-wal + forcecopy ${from}-wal2 ${to}-wal2 +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c); + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 15000; + PRAGMA wal_autocheckpoint = 0; +} {wal2 15000 0} + +do_test 1.1 { + for {set i 1} {$i <= 1000} {incr i} { + execsql { INSERT INTO t1 VALUES(random(), random(), random()) } + db_copy test.db test.db2 + sqlite3 db2 test.db + set res [execsql { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } db2] + db2 close + if {$res != [list $i ok]} { + error "failure on iteration $i" + } + } + set {} {} +} {} + +#-------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(x UNIQUE); + CREATE TABLE t2(x UNIQUE); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 10000; + PRAGMA wal_autocheckpoint = 0; + BEGIN; + INSERT INTO t1 VALUES(randomblob(4000)); + INSERT INTO t1 VALUES(randomblob(4000)); + INSERT INTO t1 VALUES(randomblob(4000)); + COMMIT; + BEGIN; + INSERT INTO t2 VALUES(randomblob(4000)); + INSERT INTO t2 VALUES(randomblob(4000)); + INSERT INTO t2 VALUES(randomblob(4000)); + COMMIT; +} {wal2 10000 0} +do_test 2.0.1 { + list [file size test.db] [file size test.db-wal] [file size test.db-wal2] +} {5120 28328 28328} + +# Test recovery with both wal files intact. +# +do_test 2.1 { + db_copy test.db test.db2 + sqlite3 db2 test.db2 + execsql { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; + PRAGMA integrity_check; + } db2 +} {3 3 ok} + +do_test 2.2 { + db2 close + db_copy test.db test.db2 + hexio_write test.db2-wal 16 12345678 + sqlite3 db2 test.db2 + execsql { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; + } db2 +} {0 3} + +do_test 2.3 { + db2 close + db_copy test.db test.db2 + hexio_write test.db2-wal2 16 12345678 + sqlite3 db2 test.db2 + execsql { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; + PRAGMA integrity_check; + } db2 +} {3 0 ok} + +do_test 2.4 { + db2 close + db_copy test.db test.db2 + forcecopy test.db-wal test.db2-wal2 + sqlite3 db2 test.db2 + execsql { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; + PRAGMA integrity_check; + } db2 +} {3 0 ok} + +do_test 2.5 { + db2 close + db_copy test.db test.db2 + forcecopy test.db-wal test.db2-wal2 + forcecopy test.db-wal2 test.db2-wal + sqlite3 db2 test.db2 + execsql { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; + PRAGMA integrity_check; + } db2 +} {3 3 ok} + +do_test 2.6 { + db2 close + db_copy test.db test.db2 + forcecopy test.db-wal test.db2-wal2 + close [open test.db-wal w] + sqlite3 db2 test.db2 + execsql { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; + PRAGMA integrity_check; + } db2 +} {3 0 ok} + +do_test 2.7 { + db2 close + db_copy test.db test.db2 + forcedelete test.db2-wal + sqlite3 db2 test.db2 + execsql { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; + PRAGMA integrity_check; + } db2 +} {0 0 ok} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a TEXT, b TEXT, c TEXT); + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 10000; + PRAGMA wal_autocheckpoint = 0; + PRAGMA cache_size = 5; +} {wal2 10000 0} + +do_execsql_test 3.1 { + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200) + INSERT INTO t1 SELECT i, i, i FROM s; + + INSERT INTO t1 VALUES(201, 201, 201); +} {} + +do_test 3.2 { + list [file size test.db] [file size test.db-wal] [file size test.db-wal2] +} {5120 15752 4224} + +do_test 3.3 { + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-wal2 test.db2-wal2 + sqlite3 db2 test.db2 + execsql { + PRAGMA journal_size_limit = 10000; + PRAGMA wal_autocheckpoint = 0; + PRAGMA cache_size = 5; + BEGIN; + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200) + INSERT INTO t1 SELECT i, i, i FROM s; + } db2 + list [file size test.db2] [file size test.db2-wal] [file size test.db2-wal2] +} {5120 15752 23088} + +do_test 3.4 { + set fd [open test.db2-shm] + fconfigure $fd -encoding binary -translation binary + set data [read $fd] + close $fd + + set fd [open test.db-shm w] + fconfigure $fd -encoding binary -translation binary + puts -nonewline $fd $data + close $fd + + execsql { + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 10) + INSERT INTO t1 SELECT i, i, i FROM s; + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } +} {211 ok} + +do_test 3.5 { + list [file size test.db] [file size test.db-wal] [file size test.db-wal2] +} {5120 15752 18896} + + +finish_test + ADDED test/wal2recover2.test Index: test/wal2recover2.test ================================================================== --- /dev/null +++ test/wal2recover2.test @@ -0,0 +1,227 @@ +# 2018 December 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix wal2recover2 +ifcapable !wal {finish_test ; return } + +do_execsql_test 1.0 { + CREATE TABLE t1(x); + CREATE TABLE t2(x); + WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 ) + INSERT INTO t1 SELECT i FROM s; + WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 ) + INSERT INTO t2 SELECT i FROM s; + + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 10000; +} {wal2 10000} + +set ::L 1125750 +set ::M 1126500 +set ::H 1127250 + +do_execsql_test 1.1 { + UPDATE t1 SET x=x+1; + UPDATE t2 SET x=x+1 WHERE rowid<=750; + + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; +} [list $H $M] + +do_test 1.2 { + list [file size test.db] [file size test.db-wal] [file size test.db-wal2] +} {31744 14704 7368} + +proc cksum {zIn data} { + if {[string length $zIn]==0} { + set s0 0 + set s1 0 + } else { + set s0 [hexio_get_int [string range $zIn 0 7]] + set s1 [hexio_get_int [string range $zIn 8 15]] + } + set n [expr [string length $data] / 8] + + for {set i 0} {$i < $n} {incr i 2} { + set x0 [hexio_get_int -l [string range $data [expr $i*8] [expr $i*8+7]]] + set x1 [hexio_get_int -l [string range $data [expr $i*8+8] [expr $i*8+8+7]]] + + set s0 [expr ($s0 + $x0 + $s1) & 0xFFFFFFFF] + set s1 [expr ($s1 + $x1 + $s0) & 0xFFFFFFFF] + } + + return "[hexio_render_int32 $s0][hexio_render_int32 $s1]" +} + +proc fix_wal_cksums {file} { + # Fix the checksum on the wal header. + set data [hexio_read $file 0 32] + set cksum [cksum {} [string range $data 0 47]] + set salt [hexio_read $file 16 8] + hexio_write $file 24 $cksum + + # Fix the checksums for all pages in the wal file. + set pgsz [hexio_get_int [hexio_read $file 8 4]] + set sz [file size $file] + for {set off 32} {$off < $sz} {incr off [expr $pgsz+24]} { + set e [hexio_read $file $off 8] + set cksum [cksum $cksum $e] + + set p [hexio_read $file [expr $off+24] $pgsz] + set cksum [cksum $cksum $p] + + hexio_write $file [expr $off+8] $salt + hexio_write $file [expr $off+16] $cksum + } +} + +proc wal_incr_hdrfield {file field} { + switch -- $field { + nCkpt { set offset 12 } + salt0 { set offset 16 } + salt1 { set offset 20 } + default { + error "unknown field $field - should be \"nCkpt\", \"salt0\" or \"salt1\"" + } + } + + # Increment the value in the wal header. + set v [hexio_get_int [hexio_read $file $offset 4]] + incr v + hexio_write $file $offset [hexio_render_int32 $v] + + # Fix various checksums + fix_wal_cksums $file +} + +proc wal_set_nckpt {file val} { + # Increment the value in the wal header. + hexio_write $file 12 [hexio_render_int32 $val] + + # Fix various checksums + fix_wal_cksums $file +} + +proc wal_set_follow {file prevfile} { + set pgsz [hexio_get_int [hexio_read $prevfile 8 4]] + set sz [file size $prevfile] + set cksum [hexio_read $prevfile [expr $sz-$pgsz-8] 8] + + hexio_write $file 16 $cksum + fix_wal_cksums $file +} + +foreach {tn file field} { + 1 test.db2-wal salt0 + 2 test.db2-wal salt1 + 3 test.db2-wal nCkpt + 4 test.db2-wal2 salt0 + 5 test.db2-wal2 salt1 + 6 test.db2-wal2 nCkpt +} { + do_test 1.3.$tn { + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-wal2 test.db2-wal2 + wal_incr_hdrfield $file $field + sqlite3 db2 test.db2 + execsql { + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; + } db2 + } [list $H $L] + db2 close +} + +do_test 1.4 { + forcecopy test.db test.db2 + forcecopy test.db-wal2 test.db2-wal + forcedelete test.db2-wal2 + sqlite3 db2 test.db2 + execsql { + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; + } db2 +} [list $L $M] + +do_test 1.5 { + forcecopy test.db test.db2 + forcecopy test.db-wal2 test.db2-wal + forcecopy test.db-wal test.db2-wal2 + sqlite3 db2 test.db2 + execsql { + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; + } db2 +} [list $H $M] + +foreach {tn file field} { + 1 test.db2-wal salt0 + 2 test.db2-wal salt1 + 3 test.db2-wal2 salt0 + 4 test.db2-wal2 salt1 +} { + do_test 1.6.$tn { + forcecopy test.db test.db2 + forcecopy test.db-wal2 test.db2-wal + forcecopy test.db-wal test.db2-wal2 + wal_incr_hdrfield $file $field + breakpoint + sqlite3 db2 test.db2 + execsql { + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; + } db2 + } [list $H $L] + db2 close +} + +foreach {tn nCkpt1 nCkpt2 res} [list \ + 1 2 1 "$H $M" \ + 2 2 2 "$L $M" \ + 3 3 1 "$H $L" \ + 4 15 14 "$H $M" \ + 5 0 15 "$H $M" \ + 6 1 15 "$L $M" \ +] { + do_test 1.7.$tn { + forcecopy test.db test.db2 + forcecopy test.db-wal2 test.db2-wal + forcecopy test.db-wal test.db2-wal2 + + wal_set_nckpt test.db2-wal2 $nCkpt2 + wal_set_nckpt test.db2-wal $nCkpt1 + wal_set_follow test.db2-wal test.db2-wal2 + + if {$tn==1} breakpoint + + sqlite3 db2 test.db2 + execsql { + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; + } db2 + } $res + db2 close +} + + +finish_test + ADDED test/wal2savepoint.test Index: test/wal2savepoint.test ================================================================== --- /dev/null +++ test/wal2savepoint.test @@ -0,0 +1,60 @@ +# 2018 December 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix wal2savepoint +ifcapable !wal {finish_test ; return } + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c); + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 15000; + PRAGMA wal_autocheckpoint = 0; + PRAGMA cache_size = 5; +} {wal2 15000 0} + +do_execsql_test 1.1 { + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200) + INSERT INTO t1 SELECT random(), random(), random() FROM s; +} {} + +do_test 1.2 { + list [file size test.db] [file size test.db-wal] [file size test.db-wal2] +} {5120 23088 0} + +do_execsql_test 1.3 { + BEGIN; + SAVEPOINT abc; + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 100) + INSERT INTO t1 SELECT random(), random(), random() FROM s; + ROLLBACK TO abc; + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 10) + INSERT INTO t1 SELECT random(), random(), random() FROM s; + COMMIT; + SELECT count(*) FROM t1; + PRAGMA integrity_check; +} {210 ok} + + +finish_test + Index: test/wal2simple.test ================================================================== --- test/wal2simple.test +++ test/wal2simple.test @@ -112,11 +112,10 @@ CREATE INDEX i1 ON t1(x); PRAGMA cache_size = 5; PRAGMA journal_mode = wal2; } {wal2} -breakpoint do_test 3.1 { execsql BEGIN for {set i 1} {$i < 1000} {incr i} { execsql { INSERT INTO t1 VALUES(randomblob(800), $i) } } @@ -241,12 +240,10 @@ for {set i 0} {$i < 10} {incr i} { execsql "CREATE TABLE t$i (x);" } } {} -puts "[file size test.db-wal] [file size test.db-wal2]" - do_test 6.2.1 { foreach f [glob -nocomplain test.db2*] { forcedelete $f } forcecopy test.db-wal2 test.db2-wal2 sqlite3 db2 test.db2 db2 eval { SELECT * FROM sqlite_master } @@ -286,7 +283,192 @@ PRAGMA journal_mode = wal2; SELECT * FROM sqlite_master; } } {wal2} db2 close + +#------------------------------------------------------------------------- +reset_db +sqlite3 db2 test.db +do_execsql_test 7.0 { + PRAGMA journal_size_limit = 10000; + PRAGMA journal_mode = wal2; + PRAGMA wal_autocheckpoint = 0; + BEGIN; + CREATE TABLE t1(a); + INSERT INTO t1 VALUES( randomblob(8000) ); + COMMIT; +} {10000 wal2 0} + +do_test 7.1 { + list [file size test.db-wal] [file size test.db-wal2] +} {9464 0} + +# Connection db2 is holding a PART1 lock. +# +# 7.2.2: Test that the PART1 does not prevent db from switching to the +# other wal file. +# +# 7.2.3: Test that the PART1 does prevent a checkpoint of test.db-wal. +# +# 7.2.4: Test that after the PART1 is released the checkpoint is possible. +# +do_test 7.2.1 { + execsql { + BEGIN; + SELECT count(*) FROM t1; + } db2 +} {1} +do_test 7.2.2 { + execsql { + INSERT INTO t1 VALUES( randomblob(800) ); + INSERT INTO t1 VALUES( randomblob(800) ); + } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {13656 3176 1024} +do_test 7.2.3 { + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {13656 3176 1024} +do_test 7.2.4 { + execsql { END } db2 + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {13656 3176 11264} + +# Connection db2 is holding a PART2_FULL1 lock. +# +# 7.3.2: Test that the lock does not prevent checkpointing. +# +# 7.3.3: Test that the lock does prevent the writer from overwriting +# test.db-wal. +# +# 7.3.4: Test that after the PART2_FULL1 is released the writer can +# switch wal files and overwrite test.db-wal +# +db close +db2 close +sqlite3 db test.db +sqlite3 db2 test.db +do_test 7.3.1 { + execsql { + PRAGMA wal_autocheckpoint = 0; + PRAGMA journal_size_limit = 10000; + INSERT INTO t1 VALUES(randomblob(10000)); + INSERT INTO t1 VALUES(randomblob(500)); + } + execsql { + BEGIN; + SELECT count(*) FROM t1; + } db2 + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 3176 11264} +do_test 7.3.2 { + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 3176 21504} +do_test 7.3.3 { + execsql { + INSERT INTO t1 VALUES(randomblob(10000)); + INSERT INTO t1 VALUES(randomblob(500)); + } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 18896 21504} +do_test 7.3.4 { + execsql END db2 + execsql { INSERT INTO t1 VALUES(randomblob(5000)); } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 18896 21504} + +# Connection db2 is holding a PART2 lock. +# +# 7.4.2: Test that the lock does not prevent writer switching to test.db-wal. +# +# 7.3.3: Test that the lock does prevent checkpointing of test.db-wal2. +# +# 7.3.4: Test that after the PART2 is released test.db-wal2 can be +# checkpointed. +# +db close +db2 close +sqlite3 db test.db +sqlite3 db2 test.db +do_test 7.4.1 { + execsql { + PRAGMA wal_autocheckpoint = 0; + PRAGMA journal_size_limit = 10000; + INSERT INTO t1 VALUES(randomblob(10000)); + INSERT INTO t1 VALUES(randomblob(10000)); + PRAGMA wal_checkpoint; + } + execsql { + BEGIN; + SELECT count(*) FROM t1; + } db2 + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 44032} +do_test 7.4.2 { + execsql { + INSERT INTO t1 VALUES(randomblob(5000)); + } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 44032} +do_test 7.4.3 { + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 44032} +do_test 7.4.4 { + execsql END db2 + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 54272} + +# Connection db2 is holding a PART1_FULL2 lock. +# +# 7.5.2: Test that the lock does not prevent a checkpoint of test.db-wal2. +# +# 7.5.3: Test that the lock does prevent the writer from overwriting +# test.db-wal2. +# +# 7.5.4: Test that after the PART1_FULL2 lock is released, the writer +# can switch to test.db-wal2. +# +db close +db2 close +sqlite3 db test.db +sqlite3 db2 test.db +do_test 7.5.1 { + execsql { + PRAGMA wal_autocheckpoint = 0; + PRAGMA journal_size_limit = 10000; + INSERT INTO t1 VALUES(randomblob(10000)); + INSERT INTO t1 VALUES(randomblob(10000)); + PRAGMA wal_checkpoint; + INSERT INTO t1 VALUES(randomblob(5000)); + } + execsql { + BEGIN; + SELECT count(*) FROM t1; + } db2 + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 64512} +do_test 7.5.2 { + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 75776} +do_test 7.5.3.1 { + execsql { INSERT INTO t1 VALUES(randomblob(5000)) } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {14704 12608 75776} +do_test 7.5.3.2 { + execsql { INSERT INTO t1 VALUES(randomblob(5000)) } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {22040 12608 75776} +do_test 7.5.4 { + execsql END db2 + execsql { INSERT INTO t1 VALUES(randomblob(5000)) } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {22040 12608 75776} + finish_test + ADDED test/wal2snapshot.test Index: test/wal2snapshot.test ================================================================== --- /dev/null +++ test/wal2snapshot.test @@ -0,0 +1,84 @@ +# 2018 December 5 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix wal2snapshot +ifcapable !wal {finish_test ; return } +ifcapable !snapshot {finish_test; return} + +foreach {tn mode} {1 wal 2 wal2} { + reset_db + do_execsql_test $tn.1 "PRAGMA journal_mode = $mode" $mode + + do_execsql_test $tn.2 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + BEGIN; + } + + # Check that sqlite3_snapshot_get() is an error for a wal2 db. + # + if {$tn==1} { + do_test 1.3 { + set S [sqlite3_snapshot_get db main] + sqlite3_snapshot_free $S + } {} + } else { + do_test 2.3 { + list [catch { sqlite3_snapshot_get db main } msg] $msg + } {1 SQLITE_ERROR} + } + + # Check that sqlite3_snapshot_recover() is an error for a wal2 db. + # + do_execsql_test $tn.4 COMMIT + if {$tn==1} { + do_test 1.5 { + sqlite3_snapshot_recover db main + } {} + } else { + do_test 2.5 { + list [catch { sqlite3_snapshot_recover db main } msg] $msg + } {1 SQLITE_ERROR} + } + + # Check that sqlite3_snapshot_open() is an error for a wal2 db. + # + if {$tn==1} { + do_test 1.6 { + execsql BEGIN + set SNAPSHOT [sqlite3_snapshot_get_blob db main] + sqlite3_snapshot_open_blob db main $SNAPSHOT + execsql COMMIT + } {} + } else { + do_test 2.6 { + execsql BEGIN + set res [ + list [catch { sqlite3_snapshot_open_blob db main $SNAPSHOT } msg] $msg + ] + execsql COMMIT + set res + } {1 SQLITE_ERROR} + } +} + + +finish_test + + Index: test/window1.test ================================================================== --- test/window1.test +++ test/window1.test @@ -591,7 +591,34 @@ SELECT a, rank() OVER(ORDER BY b) FROM t1 INTERSECT SELECT a, rank() OVER(ORDER BY b DESC) FROM t1; } { } + +# 2018-12-06 +# https://www.sqlite.org/src/info/f09fcd17810f65f7 +# Assertion fault when window functions are used. +# +# Root cause is the query flattener invoking sqlite3ExprDup() on +# expressions that contain subqueries with window functions. The +# sqlite3ExprDup() routine is not making correctly initializing +# Select.pWin field of the subqueries. +# +sqlite3 db :memory: +do_execsql_test 14.0 { + SELECT * FROM( + SELECT * FROM (SELECT 1 AS c) WHERE c IN ( + SELECT (row_number() OVER()) FROM (VALUES (0)) + ) + ); +} {1} +do_execsql_test 14.1 { + CREATE TABLE t1(x); INSERT INTO t1(x) VALUES(12345); + CREATE TABLE t2(c); INSERT INTO t2(c) VALUES(1); + SELECT y, y+1, y+2 FROM ( + SELECT c IN ( + SELECT (row_number() OVER()) FROM t1 + ) AS y FROM t2 + ); +} {1 2 3} finish_test ADDED tool/index_usage.c Index: tool/index_usage.c ================================================================== --- /dev/null +++ tool/index_usage.c @@ -0,0 +1,164 @@ +/* +** 2018-12-04 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a utility program used to help determine which +** indexes in a database schema are used and unused, and how often specific +** indexes are used. +*/ +#include "sqlite3.h" +#include +#include +#include +#include + +static void usage(const char *argv0){ + printf("Usage: %s DATABASE LOG\n\n", argv0); + printf( + "DATABASE is an SQLite database against which various statements\n" + "have been run. The SQL text is stored in LOG. LOG is an SQLite\n" + "database with this schema:\n" + "\n" + " CREATE TABLE sqllog(sql TEXT);\n" + "\n" + "This utility program analyzes statements contained in LOG and prints\n" + "a report showing how many times each index in DATABASE is used by the\n" + "statements in LOG.\n" + "\n" + "DATABASE only needs to contain the schema used by the statements in\n" + "LOG. The content can be removed from DATABASE.\n" + ); + printf("\nAnalysis will be done by SQLite version %s dated %.20s\n" + "checkin number %.40s. Different versions\n" + "of SQLite might use different indexes.\n", + sqlite3_libversion(), sqlite3_sourceid(), sqlite3_sourceid()+21); + exit(1); +} + +int main(int argc, char **argv){ + sqlite3 *db = 0; /* The main database */ + sqlite3_stmt *pStmt = 0; /* a query */ + char *zSql; + int nErr = 0; + int rc; + + if( argc!=3 ) usage(argv[0]); + rc = sqlite3_open_v2(argv[1], &db, SQLITE_OPEN_READONLY, 0); + if( rc ){ + printf("Cannot open \"%s\" for reading: %s\n", argv[1], sqlite3_errmsg(db)); + goto errorOut; + } + rc = sqlite3_prepare_v2(db, "SELECT * FROM sqlite_master", -1, &pStmt, 0); + if( rc ){ + printf("Cannot read the schema from \"%s\" - %s\n", argv[1], + sqlite3_errmsg(db)); + goto errorOut; + } + sqlite3_finalize(pStmt); + pStmt = 0; + rc = sqlite3_exec(db, + "CREATE TABLE temp.idxu(\n" + " tbl TEXT,\n" + " idx TEXT,\n" + " cnt INT,\n" + " PRIMARY KEY(idx)\n" + ") WITHOUT ROWID;", 0, 0, 0); + if( rc ){ + printf("Cannot create the result table - %s\n", + sqlite3_errmsg(db)); + goto errorOut; + } + rc = sqlite3_exec(db, + "INSERT INTO temp.idxu(tbl,idx,cnt)" + " SELECT tbl_name, name, 0 FROM sqlite_master" + " WHERE type='index' AND sql IS NOT NULL", 0, 0, 0); + + /* Open the LOG database */ + zSql = sqlite3_mprintf("ATTACH %Q AS log", argv[2]); + rc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + if( rc ){ + printf("Cannot open the LOG database \"%s\" - %s\n", + argv[2], sqlite3_errmsg(db)); + goto errorOut; + } + rc = sqlite3_prepare_v2(db, "SELECT sql, rowid FROM log.sqllog", + -1, &pStmt, 0); + if( rc ){ + printf("Cannot read the SQLLOG table in the LOG database \"%s\" - %s\n", + argv[2], sqlite3_errmsg(db)); + goto errorOut; + } + + /* Update the counts based on LOG */ + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zLog = (const char*)sqlite3_column_text(pStmt, 0); + sqlite3_stmt *pS2; + if( zLog==0 ) continue; + zSql = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zLog); + rc = sqlite3_prepare_v2(db, zSql, -1, &pS2, 0); + sqlite3_free(zSql); + if( rc ){ + printf("Cannot compile LOG entry %d (%s): %s\n", + sqlite3_column_int(pStmt, 1), zLog, sqlite3_errmsg(db)); + nErr++; + }else{ + while( sqlite3_step(pS2)==SQLITE_ROW ){ + const char *zExplain = (const char*)sqlite3_column_text(pS2,3); + const char *z1, *z2; + int n; + /* printf("EXPLAIN: %s\n", zExplain); */ + z1 = strstr(zExplain, " USING INDEX "); + if( z1==0 ) continue; + z1 += 13; + for(z2=z1+1; z2[1] && z2[1]!='('; z2++){} + n = z2 - z1; + zSql = sqlite3_mprintf( + "UPDATE temp.idxu SET cnt=cnt+1 WHERE idx='%.*q'", n, z1 + ); + /* printf("sql: %s\n", zSql); */ + sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } + } + sqlite3_finalize(pS2); + } + sqlite3_finalize(pStmt); + + /* Generate the report */ + rc = sqlite3_prepare_v2(db, + "SELECT tbl, idx, cnt, " + " (SELECT group_concat(name,',') FROM pragma_index_info(idx))" + " FROM temp.idxu, main.sqlite_master" + " WHERE temp.idxu.tbl=main.sqlite_master.tbl_name" + " AND temp.idxu.idx=main.sqlite_master.name" + " ORDER BY cnt DESC, tbl, idx", + -1, &pStmt, 0); + if( rc ){ + printf("Cannot query the result table - %s\n", + sqlite3_errmsg(db)); + goto errorOut; + } + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("%10d %s on %s(%s)\n", + sqlite3_column_int(pStmt, 2), + sqlite3_column_text(pStmt, 1), + sqlite3_column_text(pStmt, 0), + sqlite3_column_text(pStmt, 3)); + } + sqlite3_finalize(pStmt); + pStmt = 0; + +errorOut: + sqlite3_finalize(pStmt); + sqlite3_close(db); + return nErr; +} Index: tool/lemon.c ================================================================== --- tool/lemon.c +++ tool/lemon.c @@ -4588,17 +4588,24 @@ /* Generate code which executes whenever the parser stack overflows */ tplt_print(out,lemp,lemp->overflow,&lineno); tplt_xfer(lemp->name,in,out,&lineno); - /* Generate the table of rule information + /* Generate the tables of rule information. yyRuleInfoLhs[] and + ** yyRuleInfoNRhs[]. ** ** Note: This code depends on the fact that rules are number ** sequentually beginning with 0. */ for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ - fprintf(out," { %4d, %4d }, /* (%d) ",rp->lhs->index,-rp->nrhs,i); + fprintf(out," %4d, /* (%d) ", rp->lhs->index, i); + rule_print(out, rp); + fprintf(out," */\n"); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ + fprintf(out," %3d, /* (%d) ", -rp->nrhs, i); rule_print(out, rp); fprintf(out," */\n"); lineno++; } tplt_xfer(lemp->name,in,out,&lineno); Index: tool/lempar.c ================================================================== --- tool/lempar.c +++ tool/lempar.c @@ -684,17 +684,19 @@ yytos->major = yyMajor; yytos->minor.yy0 = yyMinor; yyTraceShift(yypParser, yyNewState, "Shift"); } -/* The following table contains information about every rule that -** is used during the reduce. -*/ -static const struct { - YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ - signed char nrhs; /* Negative of the number of RHS symbols in the rule */ -} yyRuleInfo[] = { +/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side +** of that rule */ +static const YYCODETYPE yyRuleInfoLhs[] = { +%% +}; + +/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number +** of symbols on the right-hand side of that rule. */ +static const signed char yyRuleInfoNRhs[] = { %% }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -723,11 +725,11 @@ (void)yyLookahead; (void)yyLookaheadToken; yymsp = yypParser->yytos; #ifndef NDEBUG if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfo[yyruleno].nrhs; + yysize = yyRuleInfoNRhs[yyruleno]; if( yysize ){ fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", yyTracePrompt, yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno); }else{ @@ -738,11 +740,11 @@ #endif /* NDEBUG */ /* Check that the stack is large enough to grow by a single entry ** if the RHS of the rule is empty. This ensures that there is room ** enough on the stack to push the LHS value */ - if( yyRuleInfo[yyruleno].nrhs==0 ){ + if( yyRuleInfoNRhs[yyruleno]==0 ){ #ifdef YYTRACKMAXSTACKDEPTH if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ yypParser->yyhwm++; assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack)); } @@ -780,13 +782,13 @@ */ /********** Begin reduce actions **********************************************/ %% /********** End reduce actions ************************************************/ }; - assert( yyrulenoYY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) );