Index: Makefile.in
==================================================================
--- Makefile.in
+++ Makefile.in
@@ -1304,10 +1304,13 @@
 	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)
 
 showshm$(TEXE):	$(TOP)/tool/showshm.c
 	$(LTLINK) -o $@ $(TOP)/tool/showshm.c
 
+index_usage$(TEXE): $(TOP)/tool/index_usage.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/tool/index_usage.c sqlite3.lo $(TLIBS)
+
 changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
 
 changesetfuzz$(TEXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/ext/session/changesetfuzz.c sqlite3.lo $(TLIBS)

Index: Makefile.msc
==================================================================
--- Makefile.msc
+++ Makefile.msc
@@ -2442,10 +2442,14 @@
 		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
 showshm.exe:	$(TOP)\tool\showshm.c
 	$(LTLINK) $(NO_WARN)	$(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS)
 
+index_usage.exe: $(TOP)\tool\index_usage.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
+		$(TOP)\tool\index_usage.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
 changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
 		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 

Index: VERSION
==================================================================
--- VERSION
+++ VERSION
@@ -1,1 +1,1 @@
-3.26.0
+3.27.0

Index: autoconf/Makefile.msc
==================================================================
--- autoconf/Makefile.msc
+++ autoconf/Makefile.msc
@@ -281,10 +281,11 @@
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF
 
 # Should the session extension be enabled?  If so, add compilation options
@@ -935,10 +936,11 @@
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF
 
 
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.

Index: configure
==================================================================
--- configure
+++ configure
@@ -1,8 +1,8 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.26.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.27.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
 #
 #
@@ -724,12 +724,12 @@
 MAKEFLAGS=
 
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.26.0'
-PACKAGE_STRING='sqlite 3.26.0'
+PACKAGE_VERSION='3.27.0'
+PACKAGE_STRING='sqlite 3.27.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
 # Factoring default headers for most tests.
 ac_includes_default="\
@@ -1464,11 +1464,11 @@
 #
 if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures sqlite 3.26.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.27.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
 To assign environment variables (e.g., CC, CFLAGS...), specify them as
 VAR=VALUE.  See below for descriptions of some of the useful variables.
@@ -1529,11 +1529,11 @@
 _ACEOF
 fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.26.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.27.0:";;
    esac
   cat <<\_ACEOF
 
 Optional Features:
   --disable-option-checking  ignore unrecognized --enable/--with options
@@ -1655,11 +1655,11 @@
 fi
 
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-sqlite configure 3.26.0
+sqlite configure 3.27.0
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
 This configure script is free software; the Free Software Foundation
 gives unlimited permission to copy, distribute and modify it.
@@ -2074,11 +2074,11 @@
 } # ac_fn_c_check_header_mongrel
 cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by sqlite $as_me 3.26.0, which was
+It was created by sqlite $as_me 3.27.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
 
 _ACEOF
@@ -12230,11 +12230,11 @@
 cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # Save the log message, to keep $0 and so on meaningful, and to
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.26.0, which was
+This file was extended by sqlite $as_me 3.27.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
   CONFIG_HEADERS  = $CONFIG_HEADERS
   CONFIG_LINKS    = $CONFIG_LINKS
@@ -12296,11 +12296,11 @@
 
 _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.26.0
+sqlite config.status 3.27.0
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 
 Copyright (C) 2012 Free Software Foundation, Inc.
 This config.status script is free software; the Free Software Foundation

Index: ext/fts3/fts3_unicode.c
==================================================================
--- ext/fts3/fts3_unicode.c
+++ ext/fts3/fts3_unicode.c
@@ -80,11 +80,11 @@
 typedef struct unicode_tokenizer unicode_tokenizer;
 typedef struct unicode_cursor unicode_cursor;
 
 struct unicode_tokenizer {
   sqlite3_tokenizer base;
-  int bRemoveDiacritic;
+  int eRemoveDiacritic;
   int nException;
   int *aiException;
 };
 
 struct unicode_cursor {
@@ -225,21 +225,24 @@
   int rc = SQLITE_OK;
 
   pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
   if( pNew==NULL ) return SQLITE_NOMEM;
   memset(pNew, 0, sizeof(unicode_tokenizer));
-  pNew->bRemoveDiacritic = 1;
+  pNew->eRemoveDiacritic = 1;
 
   for(i=0; rc==SQLITE_OK && i<nArg; i++){
     const char *z = azArg[i];
     int n = (int)strlen(z);
 
     if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
-      pNew->bRemoveDiacritic = 1;
+      pNew->eRemoveDiacritic = 1;
     }
     else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
-      pNew->bRemoveDiacritic = 0;
+      pNew->eRemoveDiacritic = 0;
+    }
+    else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){
+      pNew->eRemoveDiacritic = 2;
     }
     else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){
       rc = unicodeAddExceptions(pNew, 1, &z[11], n-11);
     }
     else if( n>=11 && memcmp("separators=", z, 11)==0 ){
@@ -348,11 +351,11 @@
       pCsr->nAlloc += 64;
     }
 
     /* Write the folded case of the last character read to the output */
     zEnd = z;
-    iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
+    iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic);
     if( iOut ){
       WRITE_UTF8(zOut, iOut);
     }
 
     /* If the cursor is not at EOF, read the next character */

Index: ext/fts3/fts3_unicode2.c
==================================================================
--- ext/fts3/fts3_unicode2.c
+++ ext/fts3/fts3_unicode2.c
@@ -157,36 +157,51 @@
 ** of the ASCII letter only. For example, if passed 235 - "LATIN
 ** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
 ** E"). The resuls of passing a codepoint that corresponds to an
 ** uppercase letter are undefined.
 */
-static int remove_diacritic(int c){
+static int remove_diacritic(int c, int bComplex){
   unsigned short aDia[] = {
         0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
      2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
      2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
      2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
-     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
-     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
-     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
-     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
-    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
-    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
-    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
-    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
-    62924, 63050, 63082, 63274, 63390, 
+     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
+     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
+     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
+     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
+     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
+    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
+    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
+    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
+    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
+    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
+    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
+    63182, 63242, 63274, 63310, 63368, 63390, 
   };
   char aChar[] = {
-    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
-    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
-    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
-    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
-    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
-    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
-    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
-    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
-    'e',  'i',  'o',  'u',  'y',  
+    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
+    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
+    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
+    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
+    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
+    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
+    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
+    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
+    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
+    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
+    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
+    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
+    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
+    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
+    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
+    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
+    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
+    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
+    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
+    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
+    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
   };
 
   unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
   int iRes = 0;
   int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
@@ -199,11 +214,12 @@
     }else{
       iHi = iTest-1;
     }
   }
   assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
 }
 
 
 /*
 ** Return true if the argument interpreted as a unicode codepoint
@@ -226,11 +242,11 @@
 ** Otherwise, return a copy of the argument.
 **
 ** The results are undefined if the value passed to this function
 ** is less than zero.
 */
-int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
+int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){
   /* Each entry in the following array defines a rule for folding a range
   ** of codepoints to lower case. The rule applies to a range of nRange
   ** codepoints starting at codepoint iCode.
   **
   ** If the least significant bit in flags is clear, then the rule applies
@@ -349,11 +365,13 @@
     if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
       ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
       assert( ret>0 );
     }
 
-    if( bRemoveDiacritic ) ret = remove_diacritic(ret);
+    if( eRemoveDiacritic ){
+      ret = remove_diacritic(ret, eRemoveDiacritic==2);
+    }
   }
   
   else if( c>=66560 && c<66600 ){
     ret = c + 40;
   }

Index: ext/fts3/unicode/mkunicode.tcl
==================================================================
--- ext/fts3/unicode/mkunicode.tcl
+++ ext/fts3/unicode/mkunicode.tcl
@@ -7,15 +7,16 @@
   set lRange [list]
 
   set nRange 1
   set iFirst  [lindex $map 0 0]
   set cPrev   [lindex $map 0 1]
+  set fPrev   [lindex $map 0 2]
 
   foreach m [lrange $map 1 end] {
-    foreach {i c} $m {}
+    foreach {i c f} $m {}
 
-    if {$cPrev == $c} {
+    if {$cPrev == $c && $fPrev==$f} {
       for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} {
         if {[info exists tl_lookup_table($j)]==0} break
       }
 
       if {$j==$i} {
@@ -27,27 +28,30 @@
       }
     }
 
     lappend lRange [list $iFirst $nRange]
     lappend aChar  $cPrev
+    lappend aFlag  $fPrev
 
     set iFirst $i
     set cPrev  $c
+    set fPrev  $f
     set nRange 1
   }
   lappend lRange [list $iFirst $nRange]
   lappend aChar $cPrev
+  lappend aFlag $fPrev
 
   puts "/*"
   puts "** If the argument is a codepoint corresponding to a lowercase letter"
   puts "** in the ASCII range with a diacritic added, return the codepoint"
   puts "** of the ASCII letter only. For example, if passed 235 - \"LATIN"
   puts "** SMALL LETTER E WITH DIAERESIS\" - return 65 (\"LATIN SMALL LETTER"
   puts "** E\"). The resuls of passing a codepoint that corresponds to an"
   puts "** uppercase letter are undefined."
   puts "*/"
-  puts "static int ${::remove_diacritic}(int c)\{"
+  puts "static int ${::remove_diacritic}(int c, int bComplex)\{"
   puts "  unsigned short aDia\[\] = \{"
   puts -nonewline "        0, "
   set i 1
   foreach r $lRange {
     foreach {iCode nRange} $r {}
@@ -58,17 +62,21 @@
     puts -nonewline ", "
   }
   puts ""
   puts "  \};"
   puts "  char aChar\[\] = \{"
-  puts -nonewline "    '\\0', "
+  puts -nonewline "    '\\0',      "
   set i 1
-  foreach c $aChar {
-    set str "'$c',  "
-    if {$c == ""} { set str "'\\0', " }
+  foreach c $aChar f $aFlag {
+    if { $f } {
+      set str "'$c'|0x80,  "
+    } else {
+      set str "'$c'|0x00,  "
+    }
+    if {$c == ""} { set str "'\\0',      " }
 
-    if {($i % 12)==0} {puts "" ; puts -nonewline "    " }
+    if {($i % 6)==0} {puts "" ; puts -nonewline "    " }
     incr i
     puts -nonewline "$str"
   }
   puts ""
   puts "  \};"
@@ -85,19 +93,21 @@
     }else{
       iHi = iTest-1;
     }
   }
   assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);}
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);}
   puts "\}"
 }
 
 proc print_isdiacritic {zFunc map} {
 
   set lCode [list]
   foreach m $map {
-    foreach {code char} $m {}
+    foreach {code char flag} $m {}
+    if {$flag} continue
     if {$code && $char == ""} { lappend lCode $code }
   }
   set lCode [lsort -integer $lCode]
   set iFirst [lindex $lCode 0]
   set iLast [lindex $lCode end]
@@ -470,11 +480,11 @@
   puts "** Otherwise, return a copy of the argument."
   puts "**"
   puts "** The results are undefined if the value passed to this function"
   puts "** is less than zero."
   puts "*/"
-  puts "int ${zFunc}\(int c, int bRemoveDiacritic)\{"
+  puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{"
 
   set liOff [tl_generate_ioff_table $lRecord]
   tl_print_table_header
   foreach entry $lRecord { 
     if {[tl_print_table_entry toggle $entry $liOff]} { 
@@ -514,11 +524,13 @@
     if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
       ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
       assert( ret>0 );
     }
 
-    if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);
+    if( eRemoveDiacritic ){
+      ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2);
+    }
   }
   }]
 
   foreach entry $lHigh {
     tl_print_if_entry $entry
@@ -603,14 +615,10 @@
   set caseS [categories_switch C S {c k m o}]
   set caseZ [categories_switch C Z {l p s}]
 
   set nCat [expr [llength [array names C]] + 1]
   puts [code {
-    int sqlite3Fts5UnicodeNCat(void) { 
-      return $nCat;
-    }
-
     int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
       aArray[0] = 1;
       switch( zCat[0] ){
         $caseC
         $caseL

Index: ext/fts3/unicode/parseunicode.tcl
==================================================================
--- ext/fts3/unicode/parseunicode.tcl
+++ ext/fts3/unicode/parseunicode.tcl
@@ -5,15 +5,27 @@
 # diacritical marks from a unicode string. Each mapping is itself a list
 # consisting of two elements - the unicode codepoint and the single ASCII
 # character that it should be replaced with, or an empty string if the 
 # codepoint should simply be removed from the input. Examples:
 #
-#   { 224 a  }     (replace codepoint 224 to "a")
-#   { 769 "" }     (remove codepoint 769 from input)
+#   { 224 a  0 }     (replace codepoint 224 to "a")
+#   { 769 "" 0 }     (remove codepoint 769 from input)
 #
 # Mappings are only returned for non-upper case codepoints. It is assumed
 # that the input has already been folded to lower case.
+#
+# The third value in the list is always either 0 or 1. 0 if the 
+# UnicodeData.txt file maps the codepoint to a single ASCII character and
+# a diacritic, or 1 if the mapping is indirect. For example, consider the 
+# two entries:
+#
+# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC
+# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8
+#
+# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a 
+# diacritic). The second is an indirect mapping, as it maps to the
+# first codepoint plus 0302 (a diacritic).
 #
 proc rd_load_unicodedata_text {zName} {
   global tl_lookup_table
 
   set fd [open $zName]
@@ -51,22 +63,33 @@
 
     set iCode  [expr "0x$code"]
     set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
     set iDia   [expr "0x[lindex $character_decomposition_mapping 1]"]
 
+    # Filter out upper-case characters, as they will be mapped to their
+    # lower-case equivalents before this data is used.
     if {[info exists tl_lookup_table($iCode)]} continue
+
+    # Check if this is an indirect mapping. If so, set bIndirect to true
+    # and change $iAscii to the indirectly mappped ASCII character.
+    set bIndirect 0
+    if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} {
+      set iAscii $mapping($iAscii)
+      set bIndirect 1
+    }
 
     if { ($iAscii >= 97 && $iAscii <= 122)
       || ($iAscii >= 65 && $iAscii <= 90)
     } {
-      lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
+      lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect]
+      set mapping($iCode) $iAscii
       set dia($iDia) 1
     }
   }
 
   foreach d [array names dia] {
-    lappend lRet [list $d ""]
+    lappend lRet [list $d "" 0]
   }
   set lRet [lsort -integer -index 0 $lRet]
 
   close $fd
   set lRet

Index: ext/fts5/fts5_tokenize.c
==================================================================
--- ext/fts5/fts5_tokenize.c
+++ ext/fts5/fts5_tokenize.c
@@ -232,16 +232,21 @@
 typedef struct Unicode61Tokenizer Unicode61Tokenizer;
 struct Unicode61Tokenizer {
   unsigned char aTokenChar[128];  /* ASCII range token characters */
   char *aFold;                    /* Buffer to fold text into */
   int nFold;                      /* Size of aFold[] in bytes */
-  int bRemoveDiacritic;           /* True if remove_diacritics=1 is set */
+  int eRemoveDiacritic;           /* True if remove_diacritics=1 is set */
   int nException;
   int *aiException;
 
   unsigned char aCategory[32];    /* True for token char categories */
 };
+
+/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */
+#define FTS5_REMOVE_DIACRITICS_NONE    0
+#define FTS5_REMOVE_DIACRITICS_SIMPLE  1
+#define FTS5_REMOVE_DIACRITICS_COMPLEX 2
 
 static int fts5UnicodeAddExceptions(
   Unicode61Tokenizer *p,          /* Tokenizer object */
   const char *z,                  /* Characters to treat as exceptions */
   int bTokenChars                 /* 1 for 'tokenchars', 0 for 'separators' */
@@ -359,11 +364,11 @@
     if( p ){
       const char *zCat = "L* N* Co";
       int i;
       memset(p, 0, sizeof(Unicode61Tokenizer));
 
-      p->bRemoveDiacritic = 1;
+      p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE;
       p->nFold = 64;
       p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
       if( p->aFold==0 ){
         rc = SQLITE_NOMEM;
       }
@@ -380,14 +385,19 @@
       }
 
       for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
         const char *zArg = azArg[i+1];
         if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
-          if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
+          if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
             rc = SQLITE_ERROR;
+          }else{
+            p->eRemoveDiacritic = (zArg[0] - '0');
+            assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE
+                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE
+                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX
+            );
           }
-          p->bRemoveDiacritic = (zArg[0]=='1');
         }else
         if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
           rc = fts5UnicodeAddExceptions(p, zArg, 1);
         }else
         if( 0==sqlite3_stricmp(azArg[i], "separators") ){
@@ -497,11 +507,11 @@
         /* An non-ascii-range character. Fold it into the output buffer if
         ** it is a token character, or break out of the loop if it is not. */
         READ_UTF8(zCsr, zTerm, iCode);
         if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){
  non_ascii_tokenchar:
-          iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic);
+          iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic);
           if( iCode ) WRITE_UTF8(zOut, iCode);
         }else{
           break;
         }
       }else if( a[*zCsr]==0 ){

Index: ext/fts5/fts5_unicode2.c
==================================================================
--- ext/fts5/fts5_unicode2.c
+++ ext/fts5/fts5_unicode2.c
@@ -26,36 +26,51 @@
 ** of the ASCII letter only. For example, if passed 235 - "LATIN
 ** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
 ** E"). The resuls of passing a codepoint that corresponds to an
 ** uppercase letter are undefined.
 */
-static int fts5_remove_diacritic(int c){
+static int fts5_remove_diacritic(int c, int bComplex){
   unsigned short aDia[] = {
         0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
      2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
      2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
      2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
-     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
-     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
-     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
-     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
-    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
-    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
-    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
-    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
-    62924, 63050, 63082, 63274, 63390, 
+     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
+     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
+     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
+     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
+     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
+    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
+    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
+    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
+    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
+    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
+    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
+    63182, 63242, 63274, 63310, 63368, 63390, 
   };
   char aChar[] = {
-    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
-    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
-    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
-    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
-    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
-    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
-    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
-    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
-    'e',  'i',  'o',  'u',  'y',  
+    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
+    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
+    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
+    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
+    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
+    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
+    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
+    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
+    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
+    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
+    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
+    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
+    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
+    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
+    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
+    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
+    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
+    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
+    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
+    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
+    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
   };
 
   unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
   int iRes = 0;
   int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
@@ -68,11 +83,12 @@
     }else{
       iHi = iTest-1;
     }
   }
   assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
 }
 
 
 /*
 ** Return true if the argument interpreted as a unicode codepoint
@@ -95,11 +111,11 @@
 ** Otherwise, return a copy of the argument.
 **
 ** The results are undefined if the value passed to this function
 ** is less than zero.
 */
-int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
+int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){
   /* Each entry in the following array defines a rule for folding a range
   ** of codepoints to lower case. The rule applies to a range of nRange
   ** codepoints starting at codepoint iCode.
   **
   ** If the least significant bit in flags is clear, then the rule applies
@@ -218,27 +234,22 @@
     if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
       ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
       assert( ret>0 );
     }
 
-    if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret);
+    if( eRemoveDiacritic ){
+      ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2);
+    }
   }
   
   else if( c>=66560 && c<66600 ){
     ret = c + 40;
   }
 
   return ret;
 }
 
-
-#if 0
-int sqlite3Fts5UnicodeNCat(void) { 
-  return 32;
-}
-#endif
-
 int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
   aArray[0] = 1;
   switch( zCat[0] ){
     case 'C':
           switch( zCat[1] ){
@@ -754,11 +765,10 @@
   int iTbl = 0;
   while( i<128 ){
     int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
     int n = (aFts5UnicodeData[iTbl] >> 5) + i;
     for(; i<128 && i<n; i++){
-      aAscii[i] = (u8)bToken;
+      aAscii[i] = bToken;
     }
     iTbl++;
   }
 }
-

Index: ext/fts5/test/fts5tokenizer.test
==================================================================
--- ext/fts5/test/fts5tokenizer.test
+++ ext/fts5/test/fts5tokenizer.test
@@ -187,11 +187,11 @@
 do_catchsql_test 6.2 {
   CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'unicode61 a b');
 } {1 {error in tokenizer constructor}}
 do_catchsql_test 6.3 {
   CREATE VIRTUAL TABLE a3 USING fts5(
-    x, y, tokenize = 'unicode61 remove_diacritics 2'
+    x, y, tokenize = 'unicode61 remove_diacritics 3'
   );
 } {1 {error in tokenizer constructor}}
 do_catchsql_test 6.4 {
   CREATE VIRTUAL TABLE a3 USING fts5(
     x, y, tokenize = 'unicode61 remove_diacritics 10'

ADDED   ext/fts5/test/fts5umlaut.test
Index: ext/fts5/test/fts5umlaut.test
==================================================================
--- ext/fts5/test/fts5umlaut.test
+++ ext/fts5/test/fts5umlaut.test
@@ -0,0 +1,65 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5umlaut
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  CREATE VIRTUAL TABLE t2 USING fts5(
+      x, 
+      tokenize="unicode61 remove_diacritics 2"
+  );
+}
+
+foreach {tn q res1 res2} {
+  1 "Hà Nội"                  0 1
+  2 "Hà Noi"                  1 1
+  3 "Ha Noi"                  1 1
+  4 "Ha N\u1ed9i"             0 1
+  5 "Ha N\u006fi"             1 1
+  6 "Ha N\u006f\u0302i"       1 1
+  7 "Ha N\u006f\u0323\u0302i" 1 1
+} {
+  do_execsql_test 1.$tn.1 {
+    DELETE FROM t1;
+    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
+    SELECT count(*) FROM t1($q)
+  } $res1
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t1;
+    INSERT INTO t1(rowid, x) VALUES (1, $q);
+    SELECT count(*) FROM t1('Ha Noi')
+  } $res1
+
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t2;
+    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
+    SELECT count(*) FROM t2($q)
+  } $res2
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t2;
+    INSERT INTO t2(rowid, x) VALUES (1, $q);
+    SELECT count(*) FROM t2('Ha Noi')
+  } $res2
+}
+
+finish_test
+

Index: ext/fts5/test/fts5unicode3.test
==================================================================
--- ext/fts5/test/fts5unicode3.test
+++ ext/fts5/test/fts5unicode3.test
@@ -34,28 +34,30 @@
 foreach x [an_load_unicodedata_text $UD] {
   set aNotAlnum($x) 1
 }
 
 foreach {y} [rd_load_unicodedata_text $UD] {
-  foreach {code ascii} $y {}
+  foreach {code ascii f} $y {}
   if {$ascii==""} {
     set int 0
   } else {
     binary scan $ascii c int
   }
-  set aDiacritic($code) $int
+  set aDiacritic($code,$f) $int
+  if {$f==0} { set aDiacritic($code,1) $int }
 }
 
 proc tcl_fold {i {bRemoveDiacritic 0}} {
   global tl_lookup_table
   global aDiacritic
+  set f [expr $bRemoveDiacritic==2]
 
   if {[info exists tl_lookup_table($i)]} {
     set i $tl_lookup_table($i)
   }
-  if {$bRemoveDiacritic && [info exists aDiacritic($i)]} {
-    set i $aDiacritic($i)
+  if {$bRemoveDiacritic && [info exists aDiacritic($i,$f)]} {
+    set i $aDiacritic($i,$f)
   }
   expr $i
 }
 db func tcl_fold tcl_fold
 
@@ -83,19 +85,29 @@
     SELECT i+1 FROM ii WHERE i<100000
   )
   SELECT count(*), min(i) FROM ii WHERE fts5_fold(i)!=CAST(tcl_fold(i) AS int);
 } {0 {}}
 
-do_execsql_test 1.2 {
+do_execsql_test 1.2.1 {
   WITH ii(i) AS (
     SELECT -1
     UNION ALL
     SELECT i+1 FROM ii WHERE i<100000
   )
   SELECT count(*), min(i) FROM ii 
   WHERE fts5_fold(i,1)!=CAST(tcl_fold(i,1) AS int);
 } {0 {}}
+
+do_execsql_test 1.2.2 {
+  WITH ii(i) AS (
+    SELECT -1
+    UNION ALL
+    SELECT i+1 FROM ii WHERE i<100000
+  )
+  SELECT count(*), min(i) FROM ii 
+  WHERE fts5_fold(i,2)!=CAST(tcl_fold(i,2) AS int);
+} {0 {}}
 
 do_execsql_test 1.3 {
   WITH ii(i) AS (
     SELECT -1
     UNION ALL

Index: ext/rbu/rbu_common.tcl
==================================================================
--- ext/rbu/rbu_common.tcl
+++ ext/rbu/rbu_common.tcl
@@ -84,26 +84,27 @@
     tmpdb close
   }
   set rc
 }
 
-proc do_rbu_vacuum_test {tn step} {
-  forcedelete state.db
-  uplevel [list do_test $tn.1 {
-    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
+proc do_rbu_vacuum_test {tn step {statedb state.db}} {
+  forcedelete $statedb
+  if {$statedb=="" && $step==1} breakpoint
+  uplevel [list do_test $tn.1 [string map [list %state% $statedb] {
+    if {$step==0} { sqlite3rbu_vacuum rbu test.db {%state%}}
     while 1 {
-      if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
+      if {$step==1} { sqlite3rbu_vacuum rbu test.db {%state%}}
       set state [rbu state]
       check_prestep_state test.db $state
       set rc [rbu step]
       check_poststep_state $rc test.db $state
       if {$rc!="SQLITE_OK"} break
       if {$step==1} { rbu close }
     }
     rbu close
-  } {SQLITE_DONE}]
+  }] {SQLITE_DONE}]
 
   uplevel [list do_execsql_test $tn.2 {
     PRAGMA integrity_check
   } ok]
 }
 

Index: ext/rbu/rbuvacuum2.test
==================================================================
--- ext/rbu/rbuvacuum2.test
+++ ext/rbu/rbuvacuum2.test
@@ -14,12 +14,15 @@
 # expected.
 #
 
 source [file join [file dirname [info script]] rbu_common.tcl]
 
-foreach step {0 1} {
-  set ::testprefix rbuvacuum2-$step
+foreach {step} {0 1} {
+foreach {ttt state} {
+  s state.db t test.db-vacuum n {}
+} {
+  set ::testprefix rbuvacuum2-$step$ttt
   
   #-------------------------------------------------------------------------
   # Test that a database that contains fts3 tables can be vacuumed.
   #
   ifcapable fts3 {
@@ -27,11 +30,11 @@
     do_execsql_test 1.1 {
       CREATE VIRTUAL TABLE t1 USING fts3(z, y);
       INSERT INTO t1 VALUES('fix this issue', 'at some point');
     }
   
-    do_rbu_vacuum_test 1.2 $step
+    do_rbu_vacuum_test 1.2 $step $state
   
     do_execsql_test 1.3 {
       SELECT * FROM t1;
     } {{fix this issue} {at some point}}
   
@@ -44,11 +47,11 @@
       INSERT INTO t1 VALUES('l h i', 'd e f');
       DELETE FROM t1 WHERE docid = 2;
       INSERT INTO t1 VALUES('a b c', 'x y z');
     }
 
-    do_rbu_vacuum_test 1.6 $step
+    do_rbu_vacuum_test 1.6 $step $state
     do_execsql_test 1.7 {
       INSERT INTO t1(t1) VALUES('integrity-check');
       SELECT * FROM t1;
     } {
       {fix this issue} {at some point}
@@ -65,11 +68,11 @@
     do_execsql_test 2.1 {
       CREATE VIRTUAL TABLE t1 USING fts5(z, y);
       INSERT INTO t1 VALUES('fix this issue', 'at some point');
     }
   
-    do_rbu_vacuum_test 2.2 $step
+    do_rbu_vacuum_test 2.2 $step $state
   
     do_execsql_test 2.3 {
       SELECT * FROM t1;
     } {{fix this issue} {at some point}}
   
@@ -82,11 +85,11 @@
       INSERT INTO t1 VALUES('l h i', 'd e f');
       DELETE FROM t1 WHERE rowid = 2;
       INSERT INTO t1 VALUES('a b c', 'x y z');
     }
 
-    do_rbu_vacuum_test 2.6 $step
+    do_rbu_vacuum_test 2.6 $step $state
     do_execsql_test 2.7 {
       INSERT INTO t1(t1) VALUES('integrity-check');
       SELECT * FROM t1;
     } {
       {fix this issue} {at some point}
@@ -105,11 +108,11 @@
       INSERT INTO rt VALUES(1, 45, 55);
       INSERT INTO rt VALUES(2, 50, 60);
       INSERT INTO rt VALUES(3, 55, 65);
     }
   
-    do_rbu_vacuum_test 3.2 $step
+    do_rbu_vacuum_test 3.2 $step $state
   
     do_execsql_test 3.3 {
       SELECT * FROM rt;
     } {1 45.0 55.0 2 50.0 60.0 3 55.0 65.0}
   
@@ -118,11 +121,11 @@
     } {1 2}
     do_execsql_test 3.4.2 {
       SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
     } {2 3}
 
-    do_rbu_vacuum_test 3.5 $step
+    do_rbu_vacuum_test 3.5 $step $state
 
     do_execsql_test 3.6.1 {
       SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
     } {1 2}
     do_execsql_test 3.6.2 {
@@ -145,19 +148,20 @@
     table t1 t1 2 {CREATE TABLE t1(a, b, c)}
     view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
     trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
     }
 
-    do_rbu_vacuum_test 4.3 $step
+    do_rbu_vacuum_test 4.3 $step $state
     do_execsql_test 4.4 {
       SELECT * FROM sqlite_master;
     } {
     table t1 t1 2 {CREATE TABLE t1(a, b, c)}
     view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
     trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
     }
   }
+}
 }
   
 #-------------------------------------------------------------------------
 # Test that passing a NULL value as the second argument to 
 # sqlite3rbu_vacuum() causes it to:
@@ -229,6 +233,11 @@
   while {[rbu step]!="SQLITE_DONE"} { rbu step }
   rbu close
   execsql { PRAGMA integrity_check }
 } {ok}
 
+do_test 6.4 {
+  sqlite3rbu_vacuum rbu test.db test.db-vactmp
+  list [catch { rbu close } msg] $msg
+} {1 SQLITE_MISUSE}
+
 finish_test

Index: ext/rbu/sqlite3rbu.c
==================================================================
--- ext/rbu/sqlite3rbu.c
+++ ext/rbu/sqlite3rbu.c
@@ -2475,11 +2475,11 @@
           if( *zExtra++=='?' ) break;
         }
         if( *zExtra=='\0' ) zExtra = 0;
       }
 
-      zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", 
+      zTarget = sqlite3_mprintf("file:%s-vactmp?rbu_memory=1%s%s", 
           sqlite3_db_filename(p->dbRbu, "main"),
           (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra)
       );
 
       if( zTarget==0 ){
@@ -3741,10 +3741,16 @@
 sqlite3rbu *sqlite3rbu_vacuum(
   const char *zTarget, 
   const char *zState
 ){
   if( zTarget==0 ){ return rbuMisuseError(); }
+  if( zState ){
+    int n = strlen(zState);
+    if( n>=7 && 0==memcmp("-vactmp", &zState[n-7], 7) ){
+      return rbuMisuseError();
+    }
+  }
   /* TODO: Check that both arguments are non-NULL */
   return openRbuHandle(0, zTarget, zState);
 }
 
 /*

Index: ext/rbu/sqlite3rbu.h
==================================================================
--- ext/rbu/sqlite3rbu.h
+++ ext/rbu/sqlite3rbu.h
@@ -331,11 +331,15 @@
 **
 ** If the second argument passed to this function is NULL, then the
 ** name of the state database is "<database>-vacuum", where <database>
 ** is the name of the target database file. In this case, on UNIX, if the
 ** state database is not already present in the file-system, it is created
-** with the same permissions as the target db is made.
+** with the same permissions as the target db is made. 
+**
+** With an RBU vacuum, it is an SQLITE_MISUSE error if the name of the 
+** state database ends with "-vactmp". This name is reserved for internal 
+** use.
 **
 ** This function does not delete the state database after an RBU vacuum
 ** is completed, even if it created it. However, if the call to
 ** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents
 ** of the state tables within the state database are zeroed. This way,

Index: ext/rbu/test_rbu.c
==================================================================
--- ext/rbu/test_rbu.c
+++ ext/rbu/test_rbu.c
@@ -271,10 +271,11 @@
     return TCL_ERROR;
   }
   zCmd = Tcl_GetString(objv[1]);
   zTarget = Tcl_GetString(objv[2]);
   if( objc==4 ) zStateDb = Tcl_GetString(objv[3]);
+  if( zStateDb && zStateDb[0]=='\0' ) zStateDb = 0;
 
   pRbu = sqlite3rbu_vacuum(zTarget, zStateDb);
   Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0);
   Tcl_SetObjResult(interp, objv[1]);
   return TCL_OK;

Index: main.mk
==================================================================
--- main.mk
+++ main.mk
@@ -997,10 +997,14 @@
 	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \
 		$(TOP)/tool/showwal.c sqlite3.o $(THREADLIB)
 
 showshm$(EXE):	$(TOP)/tool/showshm.c
 	$(TCC) -o showshm$(EXE) $(TOP)/tool/showshm.c
+
+index_usage$(EXE): $(TOP)/tool/index_usage.c sqlite3.o
+	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_DEPRECATED -o index_usage$(EXE) \
+		$(TOP)/tool/index_usage.c sqlite3.o $(THREADLIB)
 
 changeset$(EXE):	$(TOP)/ext/session/changeset.c sqlite3.o
 	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \
 		$(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB)
 

Index: src/alter.c
==================================================================
--- src/alter.c
+++ src/alter.c
@@ -777,19 +777,36 @@
       pCtx->nList++;
       break;
     }
   }
 }
+
+/*
+** Iterate through the Select objects that are part of WITH clauses attached
+** to select statement pSelect.
+*/
+static void renameWalkWith(Walker *pWalker, Select *pSelect){
+  if( pSelect->pWith ){
+    int i;
+    for(i=0; i<pSelect->pWith->nCte; i++){
+      Select *p = pSelect->pWith->a[i].pSelect;
+      NameContext sNC;
+      memset(&sNC, 0, sizeof(sNC));
+      sNC.pParse = pWalker->pParse;
+      sqlite3SelectPrep(sNC.pParse, p, &sNC);
+      sqlite3WalkSelect(pWalker, p);
+    }
+  }
+}
 
 /*
 ** This is a Walker select callback. It does nothing. It is only required
 ** because without a dummy callback, sqlite3WalkExpr() and similar do not
 ** descend into sub-select statements.
 */
 static int renameColumnSelectCb(Walker *pWalker, Select *p){
-  UNUSED_PARAMETER(pWalker);
-  UNUSED_PARAMETER(p);
+  renameWalkWith(pWalker, p);
   return WRC_Continue;
 }
 
 /*
 ** This is a Walker expression callback.
@@ -1356,16 +1373,21 @@
 */
 static int renameTableSelectCb(Walker *pWalker, Select *pSelect){
   int i;
   RenameCtx *p = pWalker->u.pRename;
   SrcList *pSrc = pSelect->pSrc;
+  if( pSrc==0 ){
+    assert( pWalker->pParse->db->mallocFailed );
+    return WRC_Abort;
+  }
   for(i=0; i<pSrc->nSrc; i++){
     struct SrcList_item *pItem = &pSrc->a[i];
     if( pItem->pTab==p->pTab ){
       renameTokenFind(pWalker->pParse, p, pItem->zName);
     }
   }
+  renameWalkWith(pWalker, pSelect);
 
   return WRC_Continue;
 }
 
 

Index: src/btree.c
==================================================================
--- src/btree.c
+++ src/btree.c
@@ -10327,11 +10327,11 @@
       "incremental_vacuum enabled with a max rootpage of zero"
     );
   }
 #endif
   testcase( pBt->db->flags & SQLITE_CellSizeCk );
-  pBt->db->flags &= ~SQLITE_CellSizeCk;
+  pBt->db->flags &= ~(u64)SQLITE_CellSizeCk;
   for(i=0; (int)i<nRoot && sCheck.mxErr; i++){
     i64 notUsed;
     if( aRoot[i]==0 ) continue;
 #ifndef SQLITE_OMIT_AUTOVACUUM
     if( pBt->autoVacuum && aRoot[i]>1 ){

Index: src/build.c
==================================================================
--- src/build.c
+++ src/build.c
@@ -227,11 +227,11 @@
   /* Get the VDBE program ready for execution
   */
   if( v && pParse->nErr==0 && !db->mallocFailed ){
     /* A minimum of one cursor is required if autoincrement is used
     *  See ticket [a696379c1f08866] */
-    if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1;
+    assert( pParse->pAinc==0 || pParse->nTab>0 );
     sqlite3VdbeMakeReady(v, pParse);
     pParse->rc = SQLITE_DONE;
   }else{
     pParse->rc = SQLITE_ERROR;
   }
@@ -636,16 +636,10 @@
   /* Delete any foreign keys attached to this table. */
   sqlite3FkDelete(db, pTable);
 
   /* Delete the Table structure itself.
   */
-#ifdef SQLITE_ENABLE_NORMALIZE
-  if( pTable->pColHash ){
-    sqlite3HashClear(pTable->pColHash);
-    sqlite3_free(pTable->pColHash);
-  }
-#endif
   sqlite3DeleteColumnNames(db, pTable);
   sqlite3DbFree(db, pTable->zName);
   sqlite3DbFree(db, pTable->zColAff);
   sqlite3SelectDelete(db, pTable->pSelect);
   sqlite3ExprListDelete(db, pTable->pCheck);

Index: src/callback.c
==================================================================
--- src/callback.c
+++ src/callback.c
@@ -281,11 +281,11 @@
 
 /*
 ** Search a FuncDefHash for a function with the given name.  Return
 ** a pointer to the matching FuncDef if found, or 0 if there is no match.
 */
-static FuncDef *functionSearch(
+FuncDef *sqlite3FunctionSearch(
   int h,               /* Hash of the name */
   const char *zFunc    /* Name of function */
 ){
   FuncDef *p;
   for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){
@@ -293,25 +293,10 @@
       return p;
     }
   }
   return 0;
 }
-#ifdef SQLITE_ENABLE_NORMALIZE
-FuncDef *sqlite3FunctionSearchN(
-  int h,               /* Hash of the name */
-  const char *zFunc,   /* Name of function */
-  int nFunc            /* Length of the name */
-){
-  FuncDef *p;
-  for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){
-    if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 ){
-      return p;
-    }
-  }
-  return 0;
-}
-#endif /* SQLITE_ENABLE_NORMALIZE */
 
 /*
 ** Insert a new FuncDef into a FuncDefHash hash table.
 */
 void sqlite3InsertBuiltinFuncs(
@@ -323,11 +308,11 @@
     FuncDef *pOther;
     const char *zName = aDef[i].zName;
     int nName = sqlite3Strlen30(zName);
     int h = SQLITE_FUNC_HASH(zName[0], nName);
     assert( zName[0]>='a' && zName[0]<='z' );
-    pOther = functionSearch(h, zName);
+    pOther = sqlite3FunctionSearch(h, zName);
     if( pOther ){
       assert( pOther!=&aDef[i] && pOther->pNext!=&aDef[i] );
       aDef[i].pNext = pOther->pNext;
       pOther->pNext = &aDef[i];
     }else{
@@ -401,11 +386,11 @@
   ** So we must not search for built-ins when creating a new function.
   */ 
   if( !createFlag && (pBest==0 || (db->mDbFlags & DBFLAG_PreferBuiltin)!=0) ){
     bestScore = 0;
     h = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zName[0]], nName);
-    p = functionSearch(h, zName);
+    p = sqlite3FunctionSearch(h, zName);
     while( p ){
       int score = matchQuality(p, nArg, enc);
       if( score>bestScore ){
         pBest = p;
         bestScore = score;

Index: src/expr.c
==================================================================
--- src/expr.c
+++ src/expr.c
@@ -1327,10 +1327,39 @@
   return pRet;
 }
 #else
 # define withDup(x,y) 0
 #endif
+
+#ifndef SQLITE_OMIT_WINDOWFUNC
+/*
+** The gatherSelectWindows() procedure and its helper routine
+** gatherSelectWindowsCallback() are used to scan all the expressions
+** an a newly duplicated SELECT statement and gather all of the Window
+** objects found there, assembling them onto the linked list at Select->pWin.
+*/
+static int gatherSelectWindowsCallback(Walker *pWalker, Expr *pExpr){
+  if( pExpr->op==TK_FUNCTION && pExpr->y.pWin!=0 ){
+    assert( ExprHasProperty(pExpr, EP_WinFunc) );
+    pExpr->y.pWin->pNextWin = pWalker->u.pSelect->pWin;
+    pWalker->u.pSelect->pWin = pExpr->y.pWin;
+  }
+  return WRC_Continue;
+}
+static int gatherSelectWindowsSelectCallback(Walker *pWalker, Select *p){
+  return p==pWalker->u.pSelect ? WRC_Continue : WRC_Prune;
+}
+static void gatherSelectWindows(Select *p){
+  Walker w;
+  w.xExprCallback = gatherSelectWindowsCallback;
+  w.xSelectCallback = gatherSelectWindowsSelectCallback;
+  w.xSelectCallback2 = 0;
+  w.u.pSelect = p;
+  sqlite3WalkSelect(&w, p);
+}
+#endif
+
 
 /*
 ** The following group of routines make deep copies of expressions,
 ** expression lists, ID lists, and select statements.  The copies can
 ** be deleted (by being passed to their respective ...Delete() routines)
@@ -1495,10 +1524,11 @@
     pNew->nSelectRow = p->nSelectRow;
     pNew->pWith = withDup(db, p->pWith);
 #ifndef SQLITE_OMIT_WINDOWFUNC
     pNew->pWin = 0;
     pNew->pWinDefn = sqlite3WindowListDup(db, p->pWinDefn);
+    if( p->pWin ) gatherSelectWindows(pNew);
 #endif
     pNew->selId = p->selId;
     *pp = pNew;
     pp = &pNew->pPrior;
     pNext = pNew;
@@ -2147,18 +2177,10 @@
   if( sqlite3StrICmp(z, "_ROWID_")==0 ) return 1;
   if( sqlite3StrICmp(z, "ROWID")==0 ) return 1;
   if( sqlite3StrICmp(z, "OID")==0 ) return 1;
   return 0;
 }
-#ifdef SQLITE_ENABLE_NORMALIZE
-int sqlite3IsRowidN(const char *z, int n){
-  if( sqlite3StrNICmp(z, "_ROWID_", n)==0 ) return 1;
-  if( sqlite3StrNICmp(z, "ROWID", n)==0 ) return 1;
-  if( sqlite3StrNICmp(z, "OID", n)==0 ) return 1;
-  return 0;
-}
-#endif
 
 /*
 ** pX is the RHS of an IN operator.  If pX is a SELECT statement 
 ** that can be simplified to a direct table access, then return
 ** a pointer to the SELECT statement.  If pX is not a SELECT statement,
@@ -2419,10 +2441,11 @@
         /* Search for an existing index that will work for this IN operator */
         for(pIdx=pTab->pIndex; pIdx && eType==0; pIdx=pIdx->pNext){
           Bitmask colUsed;      /* Columns of the index used */
           Bitmask mCol;         /* Mask for the current column */
           if( pIdx->nColumn<nExpr ) continue;
+          if( pIdx->pPartIdxWhere!=0 ) continue;
           /* Maximum nColumn is BMS-2, not BMS-1, so that we can compute
           ** BITMASK(nExpr) without overflowing */
           testcase( pIdx->nColumn==BMS-2 );
           testcase( pIdx->nColumn==BMS-1 );
           if( pIdx->nColumn>=BMS-1 ) continue;

Index: src/hash.c
==================================================================
--- src/hash.c
+++ src/hash.c
@@ -62,24 +62,10 @@
     h += sqlite3UpperToLower[c];
     h *= 0x9e3779b1;
   }
   return h;
 }
-#ifdef SQLITE_ENABLE_NORMALIZE
-static unsigned int strHashN(const char *z, int n){
-  unsigned int h = 0;
-  int i;
-  for(i=0; i<n; i++){
-    /* Knuth multiplicative hashing.  (Sorting & Searching, p. 510).
-    ** 0x9e3779b1 is 2654435761 which is the closest prime number to
-    ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */
-    h += sqlite3UpperToLower[z[i]];
-    h *= 0x9e3779b1;
-  }
-  return h;
-}
-#endif /* SQLITE_ENABLE_NORMALIZE */
 
 
 /* Link pNew element into the hash table pH.  If pEntry!=0 then also
 ** insert pNew into the pEntry hash bucket.
 */
@@ -187,44 +173,10 @@
     }
     elem = elem->next;
   }
   return &nullElement;
 }
-#ifdef SQLITE_ENABLE_NORMALIZE
-static HashElem *findElementWithHashN(
-  const Hash *pH,     /* The pH to be searched */
-  const char *pKey,   /* The key we are searching for */
-  int nKey,           /* Number of key bytes to use */
-  unsigned int *pHash /* Write the hash value here */
-){
-  HashElem *elem;                /* Used to loop thru the element list */
-  int count;                     /* Number of elements left to test */
-  unsigned int h;                /* The computed hash */
-  static HashElem nullElement = { 0, 0, 0, 0 };
-
-  if( pH->ht ){   /*OPTIMIZATION-IF-TRUE*/
-    struct _ht *pEntry;
-    h = strHashN(pKey, nKey) % pH->htsize;
-    pEntry = &pH->ht[h];
-    elem = pEntry->chain;
-    count = pEntry->count;
-  }else{
-    h = 0;
-    elem = pH->first;
-    count = pH->count;
-  }
-  if( pHash ) *pHash = h;
-  while( count-- ){
-    assert( elem!=0 );
-    if( sqlite3StrNICmp(elem->pKey,pKey,nKey)==0 ){ 
-      return elem;
-    }
-    elem = elem->next;
-  }
-  return &nullElement;
-}
-#endif /* SQLITE_ENABLE_NORMALIZE */
 
 /* Remove a single entry from the hash table given a pointer to that
 ** element and a hash on the element's key.
 */
 static void removeElementGivenHash(
@@ -265,18 +217,10 @@
 void *sqlite3HashFind(const Hash *pH, const char *pKey){
   assert( pH!=0 );
   assert( pKey!=0 );
   return findElementWithHash(pH, pKey, 0)->data;
 }
-#ifdef SQLITE_ENABLE_NORMALIZE
-void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey){
-  assert( pH!=0 );
-  assert( pKey!=0 );
-  assert( nKey>=0 );
-  return findElementWithHashN(pH, pKey, nKey, 0)->data;
-}
-#endif /* SQLITE_ENABLE_NORMALIZE */
 
 /* Insert an element into the hash table pH.  The key is pKey
 ** and the data is "data".
 **
 ** If no element exists with a matching key, then a new

Index: src/insert.c
==================================================================
--- src/insert.c
+++ src/insert.c
@@ -317,10 +317,11 @@
     aOp[5].p3 = memId;
     aOp[6].p1 = memId;
     aOp[7].p2 = memId+2;
     aOp[7].p1 = memId;
     aOp[10].p2 = memId;
+    if( pParse->nTab==0 ) pParse->nTab = 1;
   }
 }
 
 /*
 ** Update the maximum rowid for an autoincrement calculation.

Index: src/loadext.c
==================================================================
--- src/loadext.c
+++ src/loadext.c
@@ -648,11 +648,11 @@
 int sqlite3_enable_load_extension(sqlite3 *db, int onoff){
   sqlite3_mutex_enter(db->mutex);
   if( onoff ){
     db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc;
   }else{
-    db->flags &= ~(SQLITE_LoadExtension|SQLITE_LoadExtFunc);
+    db->flags &= ~(u64)(SQLITE_LoadExtension|SQLITE_LoadExtFunc);
   }
   sqlite3_mutex_leave(db->mutex);
   return SQLITE_OK;
 }
 

Index: src/main.c
==================================================================
--- src/main.c
+++ src/main.c
@@ -841,15 +841,15 @@
       rc = SQLITE_ERROR; /* IMP: R-42790-23372 */
       for(i=0; i<ArraySize(aFlagOp); i++){
         if( aFlagOp[i].op==op ){
           int onoff = va_arg(ap, int);
           int *pRes = va_arg(ap, int*);
-          u32 oldFlags = db->flags;
+          u64 oldFlags = db->flags;
           if( onoff>0 ){
             db->flags |= aFlagOp[i].mask;
           }else if( onoff==0 ){
-            db->flags &= ~aFlagOp[i].mask;
+            db->flags &= ~(u64)aFlagOp[i].mask;
           }
           if( oldFlags!=db->flags ){
             sqlite3ExpirePreparedStatements(db, 0);
           }
           if( pRes ){
@@ -1308,11 +1308,11 @@
   sqlite3BtreeLeaveAll(db);
 
   /* Any deferred constraint violations have now been resolved. */
   db->nDeferredCons = 0;
   db->nDeferredImmCons = 0;
-  db->flags &= ~SQLITE_DeferFKs;
+  db->flags &= ~(u64)SQLITE_DeferFKs;
 
   /* If one has been configured, invoke the rollback-hook callback */
   if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){
     db->xRollbackCallback(db->pRollbackArg);
   }
@@ -2050,10 +2050,12 @@
 #endif
   sqlite3_mutex_enter(db->mutex);
   pOld = db->pProfileArg;
   db->xProfile = xProfile;
   db->pProfileArg = pArg;
+  db->mTrace &= SQLITE_TRACE_NONLEGACY_MASK;
+  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
   sqlite3_mutex_leave(db->mutex);
   return pOld;
 }
 #endif /* SQLITE_OMIT_DEPRECATED */
 #endif /* SQLITE_OMIT_TRACE */
@@ -2401,11 +2403,11 @@
   sqlite3_mutex_enter(db->mutex);
   if( db->mallocFailed ){
     z = sqlite3ErrStr(SQLITE_NOMEM_BKPT);
   }else{
     testcase( db->pErr==0 );
-    z = (char*)sqlite3_value_text(db->pErr);
+    z = db->errCode ? (char*)sqlite3_value_text(db->pErr) : 0;
     assert( !db->mallocFailed );
     if( z==0 ){
       z = sqlite3ErrStr(db->errCode);
     }
   }

Index: src/parse.y
==================================================================
--- src/parse.y
+++ src/parse.y
@@ -677,10 +677,16 @@
         struct SrcList_item *pNew = &A->a[A->nSrc-1];
         struct SrcList_item *pOld = F->a;
         pNew->zName = pOld->zName;
         pNew->zDatabase = pOld->zDatabase;
         pNew->pSelect = pOld->pSelect;
+        if( pOld->fg.isTabFunc ){
+          pNew->u1.pFuncArg = pOld->u1.pFuncArg;
+          pOld->u1.pFuncArg = 0;
+          pOld->fg.isTabFunc = 0;
+          pNew->fg.isTabFunc = 1;
+        }
         pOld->zName = pOld->zDatabase = 0;
         pOld->pSelect = 0;
       }
       sqlite3SrcListDelete(pParse->db, F);
     }else{
@@ -1374,12 +1380,16 @@
 
 ///////////////////////////// The VACUUM command /////////////////////////////
 //
 %ifndef SQLITE_OMIT_VACUUM
 %ifndef SQLITE_OMIT_ATTACH
-cmd ::= VACUUM.                {sqlite3Vacuum(pParse,0);}
-cmd ::= VACUUM nm(X).          {sqlite3Vacuum(pParse,&X);}
+%type vinto {Expr*}
+%destructor vinto {sqlite3ExprDelete(pParse->db, $$);}
+cmd ::= VACUUM vinto(Y).                {sqlite3Vacuum(pParse,0,Y);}
+cmd ::= VACUUM nm(X) vinto(Y).          {sqlite3Vacuum(pParse,&X,Y);}
+vinto(A) ::= INTO expr(X).              {A = X;}
+vinto(A) ::= .                          {A = 0;}
 %endif  SQLITE_OMIT_ATTACH
 %endif  SQLITE_OMIT_VACUUM
 
 ///////////////////////////// The PRAGMA command /////////////////////////////
 //

Index: src/pcache1.c
==================================================================
--- src/pcache1.c
+++ src/pcache1.c
@@ -100,10 +100,11 @@
   u8 isAnchor;                   /* This is the PGroup.lru element */
   PgHdr1 *pNext;                 /* Next in hash table chain */
   PCache1 *pCache;               /* Cache that currently owns this page */
   PgHdr1 *pLruNext;              /* Next in LRU list of unpinned pages */
   PgHdr1 *pLruPrev;              /* Previous in LRU list of unpinned pages */
+                                 /* NB: pLruPrev is only valid if pLruNext!=0 */
 };
 
 /*
 ** A page is pinned if it is not on the LRU list.  To be "pinned" means
 ** that the page is in active use and must not be deallocated.
@@ -568,11 +569,12 @@
   assert( pPage->pLruPrev );
   assert( sqlite3_mutex_held(pPage->pCache->pGroup->mutex) );
   pPage->pLruPrev->pLruNext = pPage->pLruNext;
   pPage->pLruNext->pLruPrev = pPage->pLruPrev;
   pPage->pLruNext = 0;
-  pPage->pLruPrev = 0;
+  /* pPage->pLruPrev = 0;
+  ** No need to clear pLruPrev as it is never accessed if pLruNext is 0 */
   assert( pPage->isAnchor==0 );
   assert( pPage->pCache->pGroup->lru.isAnchor==1 );
   pPage->pCache->nRecyclable--;
   return pPage;
 }
@@ -906,12 +908,13 @@
     unsigned int h = iKey % pCache->nHash;
     pCache->nPage++;
     pPage->iKey = iKey;
     pPage->pNext = pCache->apHash[h];
     pPage->pCache = pCache;
-    pPage->pLruPrev = 0;
     pPage->pLruNext = 0;
+    /* pPage->pLruPrev = 0;
+    ** No need to clear pLruPrev since it is not accessed when pLruNext==0 */
     *(void **)pPage->page.pExtra = 0;
     pCache->apHash[h] = pPage;
     if( iKey>pCache->iMaxKey ){
       pCache->iMaxKey = iKey;
     }
@@ -1067,11 +1070,11 @@
   pcache1EnterMutex(pGroup);
 
   /* It is an error to call this function if the page is already 
   ** part of the PGroup LRU list.
   */
-  assert( pPage->pLruPrev==0 && pPage->pLruNext==0 );
+  assert( pPage->pLruNext==0 );
   assert( PAGE_IS_PINNED(pPage) );
 
   if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){
     pcache1RemoveFromHash(pPage, 1);
   }else{

Index: src/pragma.c
==================================================================
--- src/pragma.c
+++ src/pragma.c
@@ -815,11 +815,11 @@
         sqlite3BtreeSetSpillSize(pDb->pBt, size);
       }
       if( sqlite3GetBoolean(zRight, size!=0) ){
         db->flags |= SQLITE_CacheSpill;
       }else{
-        db->flags &= ~SQLITE_CacheSpill;
+        db->flags &= ~(u64)SQLITE_CacheSpill;
       }
       setAllPagerFlags(db);
     }
     break;
   }

Index: src/prepare.c
==================================================================
--- src/prepare.c
+++ src/prepare.c
@@ -291,11 +291,11 @@
   ** clear the legacy_file_format pragma flag so that a VACUUM will
   ** not downgrade the database and thus invalidate any descending
   ** indices that the user might have created.
   */
   if( iDb==0 && meta[BTREE_FILE_FORMAT-1]>=4 ){
-    db->flags &= ~SQLITE_LegacyFileFmt;
+    db->flags &= ~(u64)SQLITE_LegacyFileFmt;
   }
 
   /* Read the schema information out of the schema tables
   */
   assert( db->init.busy );
@@ -707,297 +707,10 @@
   assert( (rc&db->errMask)==rc );
   sqlite3_mutex_leave(db->mutex);
   return rc;
 }
 
-#ifdef SQLITE_ENABLE_NORMALIZE
-/*
-** Checks if the specified token is a table, column, or function name,
-** based on the databases associated with the statement being prepared.
-** If the function fails, zero is returned and pRc is filled with the
-** error code.
-*/
-static int shouldTreatAsIdentifier(
-  sqlite3 *db,        /* Database handle. */
-  const char *zToken, /* Pointer to start of token to be checked */
-  int nToken,         /* Length of token to be checked */
-  int *pRc            /* Pointer to error code upon failure */
-){
-  int bFound = 0;     /* Non-zero if token is an identifier name. */
-  int i, j;           /* Database and column loop indexes. */
-  Schema *pSchema;    /* Schema for current database. */
-  Hash *pHash;        /* Hash table of tables for current database. */
-  HashElem *e;        /* Hash element for hash table iteration. */
-  Table *pTab;        /* Database table for columns being checked. */
-
-  if( sqlite3IsRowidN(zToken, nToken) ){
-    return 1;
-  }
-  if( nToken>0 ){
-    int hash = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zToken[0]], nToken);
-    if( sqlite3FunctionSearchN(hash, zToken, nToken) ) return 1;
-  }
-  assert( db!=0 );
-  sqlite3_mutex_enter(db->mutex);
-  sqlite3BtreeEnterAll(db);
-  for(i=0; i<db->nDb; i++){
-    pHash = &db->aFunc;
-    if( sqlite3HashFindN(pHash, zToken, nToken) ){
-      bFound = 1;
-      break;
-    }
-    pSchema = db->aDb[i].pSchema;
-    if( pSchema==0 ) continue;
-    pHash = &pSchema->tblHash;
-    if( sqlite3HashFindN(pHash, zToken, nToken) ){
-      bFound = 1;
-      break;
-    }
-    for(e=sqliteHashFirst(pHash); e; e=sqliteHashNext(e)){
-      pTab = sqliteHashData(e);
-      if( pTab==0 ) continue;
-      pHash = pTab->pColHash;
-      if( pHash==0 ){
-        pTab->pColHash = pHash = sqlite3_malloc(sizeof(Hash));
-        if( pHash ){
-          sqlite3HashInit(pHash);
-          for(j=0; j<pTab->nCol; j++){
-            Column *pCol = &pTab->aCol[j];
-            sqlite3HashInsert(pHash, pCol->zName, pCol);
-          }
-        }else{
-          *pRc = SQLITE_NOMEM_BKPT;
-          bFound = 0;
-          goto done;
-        }
-      }
-      if( pHash && sqlite3HashFindN(pHash, zToken, nToken) ){
-        bFound = 1;
-        goto done;
-      }
-    }
-  }
-done:
-  sqlite3BtreeLeaveAll(db);
-  sqlite3_mutex_leave(db->mutex);
-  return bFound;
-}
-
-/*
-** Attempt to estimate the final output buffer size needed for the fully
-** normalized version of the specified SQL string.  This should take into
-** account any potential expansion that could occur (e.g. via IN clauses
-** being expanded, etc).  This size returned is the total number of bytes
-** including the NUL terminator.
-*/
-static int estimateNormalizedSize(
-  const char *zSql, /* The original SQL string */
-  int nSql,         /* Length of original SQL string */
-  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
-){
-  int nOut = nSql + 4;
-  const char *z = zSql;
-  while( nOut<nSql*5 ){
-    while( z[0]!=0 && z[0]!='I' && z[0]!='i' ){ z++; }
-    if( z[0]==0 ) break;
-    z++;
-    if( z[0]!='N' && z[0]!='n' ) break;
-    z++;
-    while( sqlite3Isspace(z[0]) ){ z++; }
-    if( z[0]!='(' ) break;
-    z++;
-    nOut += 5; /* ?,?,? */
-  }
-  return nOut;
-}
-
-/*
-** Copy the current token into the output buffer while dealing with quoted
-** identifiers.  By default, all letters will be converted into lowercase.
-** If the bUpper flag is set, uppercase will be used.  The piOut argument
-** will be used to update the target index into the output string.
-*/
-static void copyNormalizedToken(
-  const char *zSql, /* The original SQL string */
-  int iIn,          /* Current index into the original SQL string */
-  int nToken,       /* Number of bytes in the current token */
-  int tokenFlags,   /* Flags returned by the tokenizer */
-  char *zOut,       /* The output string */
-  int *piOut        /* Pointer to target index into the output string */
-){
-  int bQuoted = tokenFlags & SQLITE_TOKEN_QUOTED;
-  int bKeyword = tokenFlags & SQLITE_TOKEN_KEYWORD;
-  int j = *piOut, k = 0;
-  for(; k<nToken; k++){
-    if( bQuoted ){
-      if( k==0 && iIn>0 ){
-        zOut[j++] = '"';
-        continue;
-      }else if( k==nToken-1 ){
-        zOut[j++] = '"';
-        continue;
-      }
-    }
-    if( bKeyword ){
-      zOut[j++] = sqlite3Toupper(zSql[iIn+k]);
-    }else{
-      zOut[j++] = sqlite3Tolower(zSql[iIn+k]);
-    }
-  }
-  *piOut = j;
-}
-
-/*
-** Perform normalization of the SQL contained in the prepared statement and
-** store the result in the zNormSql field.  The schema for the associated
-** databases are consulted while performing the normalization in order to
-** determine if a token appears to be an identifier.  All identifiers are
-** left intact in the normalized SQL and all literals are replaced with a
-** single '?'.
-*/
-void sqlite3Normalize(
-  Vdbe *pVdbe,      /* VM being reprepared */
-  const char *zSql, /* The original SQL string */
-  int nSql,         /* Size of the input string in bytes */
-  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
-){
-  sqlite3 *db;           /* Database handle. */
-  char *z;               /* The output string */
-  int nZ;                /* Size of the output string in bytes */
-  int i;                 /* Next character to read from zSql[] */
-  int j;                 /* Next character to fill in on z[] */
-  int tokenType = 0;     /* Type of the next token */
-  int prevTokenType = 0; /* Type of the previous token, except spaces */
-  int n;                 /* Size of the next token */
-  int nParen = 0;        /* Nesting level of parenthesis */
-  Hash inHash;           /* Table of parenthesis levels to output index. */
-
-  db = sqlite3VdbeDb(pVdbe);
-  assert( db!=0 );
-  assert( pVdbe->zNormSql==0 );
-  if( zSql==0 ) return;
-  nZ = estimateNormalizedSize(zSql, nSql, prepFlags);
-  z = sqlite3DbMallocRawNN(db, nZ);
-  if( z==0 ) return;
-  sqlite3HashInit(&inHash);
-  for(i=j=0; i<nSql && zSql[i]; i+=n){
-    int flags = 0;
-    if( tokenType!=TK_SPACE ) prevTokenType = tokenType;
-    n = sqlite3GetTokenNormalized((unsigned char*)zSql+i, &tokenType, &flags);
-    switch( tokenType ){
-      case TK_SPACE: {
-        break;
-      }
-      case TK_ILLEGAL: {
-        sqlite3DbFree(db, z);
-        sqlite3HashClear(&inHash);
-        return;
-      }
-      case TK_STRING:
-      case TK_INTEGER:
-      case TK_FLOAT:
-      case TK_VARIABLE:
-      case TK_BLOB: {
-        z[j++] = '?';
-        break;
-      }
-      case TK_LP:
-      case TK_RP: {
-        if( tokenType==TK_LP ){
-          nParen++;
-          if( prevTokenType==TK_IN ){
-            assert( nParen<nSql );
-            sqlite3HashInsert(&inHash, zSql+nParen, SQLITE_INT_TO_PTR(j));
-          }
-        }else{
-          int jj;
-          assert( nParen<nSql );
-          jj = SQLITE_PTR_TO_INT(sqlite3HashFind(&inHash, zSql+nParen));
-          if( jj>0 ){
-            sqlite3HashInsert(&inHash, zSql+nParen, 0);
-            assert( jj+6<nZ );
-            memcpy(z+jj+1, "?,?,?", 5);
-            j = jj+6;
-            assert( nZ-1-j>=0 );
-            assert( nZ-1-j<nZ );
-            memset(z+j, 0, nZ-1-j);
-          }
-          nParen--;
-        }
-        assert( nParen>=0 );
-        /* Fall through */
-      }
-      case TK_MINUS:
-      case TK_SEMI:
-      case TK_PLUS:
-      case TK_STAR:
-      case TK_SLASH:
-      case TK_REM:
-      case TK_EQ:
-      case TK_LE:
-      case TK_NE:
-      case TK_LSHIFT:
-      case TK_LT:
-      case TK_RSHIFT:
-      case TK_GT:
-      case TK_GE:
-      case TK_BITOR:
-      case TK_CONCAT:
-      case TK_COMMA:
-      case TK_BITAND:
-      case TK_BITNOT:
-      case TK_DOT:
-      case TK_IN:
-      case TK_IS:
-      case TK_NOT:
-      case TK_NULL:
-      case TK_ID: {
-        if( tokenType==TK_NULL ){
-          if( prevTokenType==TK_IS || prevTokenType==TK_NOT ){
-            /* NULL is a keyword in this case, not a literal value */
-          }else{
-            /* Here the NULL is a literal value */
-            z[j++] = '?';
-            break;
-          }
-        }
-        if( j>0 && sqlite3IsIdChar(z[j-1]) && sqlite3IsIdChar(zSql[i]) ){
-          z[j++] = ' ';
-        }
-        if( tokenType==TK_ID ){
-          int i2 = i, n2 = n, rc = SQLITE_OK;
-          if( nParen>0 ){
-            assert( nParen<nSql );
-            sqlite3HashInsert(&inHash, zSql+nParen, 0);
-          }
-          if( flags&SQLITE_TOKEN_QUOTED ){ i2++; n2-=2; }
-          if( shouldTreatAsIdentifier(db, zSql+i2, n2, &rc)==0 ){
-            if( rc!=SQLITE_OK ){
-              sqlite3DbFree(db, z);
-              sqlite3HashClear(&inHash);
-              return;
-            }
-            if( sqlite3_keyword_check(zSql+i2, n2)==0 ){
-              z[j++] = '?';
-              break;
-            }
-          }
-        }
-        copyNormalizedToken(zSql, i, n, flags, z, &j);
-        break;
-      }
-    }
-  }
-  assert( j<nZ && "one" );
-  while( j>0 && z[j-1]==' ' ){ j--; }
-  if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
-  z[j] = 0;
-  assert( j<nZ && "two" );
-  pVdbe->zNormSql = z;
-  sqlite3HashClear(&inHash);
-}
-#endif /* SQLITE_ENABLE_NORMALIZE */
 
 /*
 ** Rerun the compilation of a statement after a schema change.
 **
 ** If the statement is successfully recompiled, return SQLITE_OK. Otherwise,

Index: src/resolve.c
==================================================================
--- src/resolve.c
+++ src/resolve.c
@@ -78,11 +78,11 @@
   if( pDup!=0 ){
     if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery);
     if( pExpr->op==TK_COLLATE ){
       pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken);
     }
-    ExprSetProperty(pDup, EP_Alias);
+//    ExprSetProperty(pDup, EP_Alias);
 
     /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This 
     ** prevents ExprDelete() from deleting the Expr structure itself,
     ** allowing it to be repopulated by the memcpy() on the following line.
     ** The pExpr->u.zToken might point into memory that will be freed by the
@@ -472,10 +472,29 @@
   ** fields are not changed in any context.
   */
   if( cnt==0 && zTab==0 ){
     assert( pExpr->op==TK_ID );
     if( ExprHasProperty(pExpr,EP_DblQuoted) ){
+      /* If a double-quoted identifier does not match any known column name,
+      ** then treat it as a string.
+      **
+      ** This hack was added in the early days of SQLite in a misguided attempt
+      ** to be compatible with MySQL 3.x, which used double-quotes for strings.
+      ** I now sorely regret putting in this hack. The effect of this hack is
+      ** that misspelled identifier names are silently converted into strings
+      ** rather than causing an error, to the frustration of countless
+      ** programmers. To all those frustrated programmers, my apologies.
+      **
+      ** Someday, I hope to get rid of this hack. Unfortunately there is
+      ** a huge amount of legacy SQL that uses it. So for now, we just
+      ** issue a warning.
+      */
+      sqlite3_log(SQLITE_WARNING,
+        "double-quoted string literal: \"%w\"", zCol);
+#ifdef SQLITE_ENABLE_NORMALIZE
+      sqlite3VdbeAddDblquoteStr(db, pParse->pVdbe, zCol);
+#endif
       pExpr->op = TK_STRING;
       pExpr->y.pTab = 0;
       return WRC_Prune;
     }
     if( sqlite3ExprIdToTrueFalse(pExpr) ){

Index: src/select.c
==================================================================
--- src/select.c
+++ src/select.c
@@ -2075,14 +2075,14 @@
 ** the result set of that SELECT.
 */
 Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){
   Table *pTab;
   sqlite3 *db = pParse->db;
-  int savedFlags;
+  u64 savedFlags;
 
   savedFlags = db->flags;
-  db->flags &= ~SQLITE_FullColNames;
+  db->flags &= ~(u64)SQLITE_FullColNames;
   db->flags |= SQLITE_ShortColNames;
   sqlite3SelectPrep(pParse, pSelect, 0);
   if( pParse->nErr ) return 0;
   while( pSelect->pPrior ) pSelect = pSelect->pPrior;
   db->flags = savedFlags;
@@ -3459,10 +3459,11 @@
           ifNullRow.op = TK_IF_NULL_ROW;
           ifNullRow.pLeft = pCopy;
           ifNullRow.iTable = pSubst->iNewTable;
           pCopy = &ifNullRow;
         }
+        testcase( ExprHasProperty(pCopy, EP_Subquery) );
         pNew = sqlite3ExprDup(db, pCopy, 0);
         if( pNew && pSubst->isLeftJoin ){
           ExprSetProperty(pNew, EP_CanBeNull);
         }
         if( pNew && ExprHasProperty(pExpr,EP_FromJoin) ){
@@ -4023,11 +4024,12 @@
       }
       assert( pParent->pOrderBy==0 );
       pParent->pOrderBy = pOrderBy;
       pSub->pOrderBy = 0;
     }
-    pWhere = sqlite3ExprDup(db, pSub->pWhere, 0);
+    pWhere = pSub->pWhere;
+    pSub->pWhere = 0;
     if( isLeftJoin>0 ){
       setJoinExpr(pWhere, iNewParent);
     }
     pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere);
     if( db->mallocFailed==0 ){

Index: src/shell.c.in
==================================================================
--- src/shell.c.in
+++ src/shell.c.in
@@ -1005,10 +1005,11 @@
   u8 statsOn;            /* True to display memory stats before each finalize */
   u8 scanstatsOn;        /* True to display scan stats before each finalize */
   u8 openMode;           /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
   u8 doXdgOpen;          /* Invoke start/open/xdg-open in output_reset() */
   u8 nEqpLevel;          /* Depth of the EQP output graph */
+  u8 eTraceType;         /* SHELL_TRACE_* value for type of trace */
   unsigned mEqpLines;    /* Mask of veritical lines in the EQP output graph */
   int outCount;          /* Revert to stdout when reaching zero */
   int cnt;               /* Number of records displayed so far */
   FILE *out;             /* Write results here */
   FILE *traceOut;        /* Output for sqlite3_trace() */
@@ -1064,10 +1065,16 @@
 #define SHELL_OPEN_APPENDVFS   2      /* Use appendvfs */
 #define SHELL_OPEN_ZIPFILE     3      /* Use the zipfile virtual table */
 #define SHELL_OPEN_READONLY    4      /* Open a normal database read-only */
 #define SHELL_OPEN_DESERIALIZE 5      /* Open using sqlite3_deserialize() */
 
+/* Allowed values for ShellState.eTraceType
+*/
+#define SHELL_TRACE_PLAIN      0      /* Show input SQL text */
+#define SHELL_TRACE_EXPANDED   1      /* Show expanded SQL text */
+#define SHELL_TRACE_NORMALIZED 2      /* Show normalized SQL text */
+
 /*
 ** These are the allowed shellFlgs values
 */
 #define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
 #define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
@@ -3368,10 +3375,11 @@
 #ifndef SQLITE_OMIT_AUTHORIZATION
   ".auth ON|OFF             Show authorizer callbacks",
 #endif
   ".backup ?DB? FILE        Backup DB (default \"main\") to FILE",
   "       --append            Use the appendvfs",
+  "       --async             Write to FILE without a journal and without fsync()",
   ".bail on|off             Stop after hitting an error.  Default OFF",
   ".binary on|off           Turn binary output on or off.  Default OFF",
   ".cd DIRECTORY            Change the working directory to DIRECTORY",
   ".changes on|off          Show number of rows changed by SQL",
   ".check GLOB              Fail if output since .testcase does not match",
@@ -3490,11 +3498,26 @@
 #endif
   ".tables ?TABLE?          List names of tables matching LIKE pattern TABLE",
   ".testcase NAME           Begin redirecting output to 'testcase-out.txt'",
   ".timeout MS              Try opening locked tables for MS milliseconds",
   ".timer on|off            Turn SQL timer on or off",
-  ".trace FILE|off          Output each SQL statement as it is run",
+#ifndef SQLITE_OMIT_TRACE
+  ".trace ?OPTIONS?         Output each SQL statement as it is run",
+  "    FILE                    Send output to FILE",
+  "    stdout                  Send output to stdout",
+  "    stderr                  Send output to stderr",
+  "    off                     Disable tracing",
+  "    --expanded              Expand query parameters",
+#ifdef SQLITE_ENABLE_NORMALIZE
+  "    --normalized            Normal the SQL statements",
+#endif
+  "    --plain                 Show SQL as it is input",
+  "    --stmt                  Trace statement execution (SQLITE_TRACE_STMT)",
+  "    --profile               Profile statements (SQLITE_TRACE_PROFILE)",
+  "    --row                   Trace each row (SQLITE_TRACE_ROW)",
+  "    --close                 Trace connection close (SQLITE_TRACE_CLOSE)",
+#endif /* SQLITE_OMIT_TRACE */
   ".vfsinfo ?AUX?           Information about the top-level VFS",
   ".vfslist                 List all available VFSes",
   ".vfsname ?AUX?           Print the name of the VFS stack",
   ".width NUM1 NUM2 ...     Set column widths for \"column\" mode",
   "     Negative values right-justify",
@@ -3997,28 +4020,64 @@
     }
   }
   return f;
 }
 
-#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
+#ifndef SQLITE_OMIT_TRACE
 /*
 ** A routine for handling output from sqlite3_trace().
 */
 static int sql_trace_callback(
-  unsigned mType,
-  void *pArg,
-  void *pP,
-  void *pX
-){
-  FILE *f = (FILE*)pArg;
-  UNUSED_PARAMETER(mType);
-  UNUSED_PARAMETER(pP);
-  if( f ){
-    const char *z = (const char*)pX;
-    int i = strlen30(z);
-    while( i>0 && z[i-1]==';' ){ i--; }
-    utf8_printf(f, "%.*s;\n", i, z);
+  unsigned mType,         /* The trace type */
+  void *pArg,             /* The ShellState pointer */
+  void *pP,               /* Usually a pointer to sqlite_stmt */
+  void *pX                /* Auxiliary output */
+){
+  ShellState *p = (ShellState*)pArg;
+  sqlite3_stmt *pStmt;
+  const char *zSql;
+  int nSql;
+  if( p->traceOut==0 ) return 0;
+  if( mType==SQLITE_TRACE_CLOSE ){
+    utf8_printf(p->traceOut, "-- closing database connection\n");
+    return 0;
+  }
+  if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){
+    zSql = (const char*)pX;
+  }else{
+    pStmt = (sqlite3_stmt*)pP;
+    switch( p->eTraceType ){
+      case SHELL_TRACE_EXPANDED: {
+        zSql = sqlite3_expanded_sql(pStmt);
+        break;
+      }
+#ifdef SQLITE_ENABLE_NORMALIZE
+      case SHELL_TRACE_NORMALIZED: {
+        zSql = sqlite3_normalized_sql(pStmt);
+        break;
+      }
+#endif
+      default: {
+        zSql = sqlite3_sql(pStmt);
+        break;
+      }
+    }
+  }
+  if( zSql==0 ) return 0;
+  nSql = strlen30(zSql);
+  while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; }
+  switch( mType ){
+    case SQLITE_TRACE_ROW:
+    case SQLITE_TRACE_STMT: {
+      utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql);
+      break;
+    }
+    case SQLITE_TRACE_PROFILE: {
+      sqlite3_int64 nNanosec = *(sqlite3_int64*)pX;
+      utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec);
+      break;
+    }
   }
   return 0;
 }
 #endif
 
@@ -5780,17 +5839,21 @@
     const char *zDestFile = 0;
     const char *zDb = 0;
     sqlite3 *pDest;
     sqlite3_backup *pBackup;
     int j;
+    int bAsync = 0;
     const char *zVfs = 0;
     for(j=1; j<nArg; j++){
       const char *z = azArg[j];
       if( z[0]=='-' ){
         if( z[1]=='-' ) z++;
         if( strcmp(z, "-append")==0 ){
           zVfs = "apndvfs";
+        }else
+        if( strcmp(z, "-async")==0 ){
+          bAsync = 1;
         }else
         {
           utf8_printf(stderr, "unknown option: %s\n", azArg[j]);
           return 1;
         }
@@ -5798,11 +5861,11 @@
         zDestFile = azArg[j];
       }else if( zDb==0 ){
         zDb = zDestFile;
         zDestFile = azArg[j];
       }else{
-        raw_printf(stderr, "Usage: .backup ?DB? ?--append? FILENAME\n");
+        raw_printf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n");
         return 1;
       }
     }
     if( zDestFile==0 ){
       raw_printf(stderr, "missing FILENAME argument on .backup\n");
@@ -5814,10 +5877,14 @@
     if( rc!=SQLITE_OK ){
       utf8_printf(stderr, "Error: cannot open \"%s\"\n", zDestFile);
       close_db(pDest);
       return 1;
     }
+    if( bAsync ){
+      sqlite3_exec(pDest, "PRAGMA synchronous=OFF; PRAGMA journal_mode=OFF;",
+                   0, 0, 0);
+    }
     open_db(p, 0);
     pBackup = sqlite3_backup_init(pDest, "main", p->db, zDb);
     if( pBackup==0 ){
       utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest));
       close_db(pDest);
@@ -7836,27 +7903,59 @@
       raw_printf(stderr, "Usage: .timer on|off\n");
       rc = 1;
     }
   }else
 
+#ifndef SQLITE_OMIT_TRACE
   if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){
+    int mType = 0;
+    int jj;
     open_db(p, 0);
-    if( nArg!=2 ){
-      raw_printf(stderr, "Usage: .trace FILE|off\n");
-      rc = 1;
-      goto meta_command_exit;
+    for(jj=1; jj<nArg; jj++){
+      const char *z = azArg[jj];
+      if( z[0]=='-' ){
+        if( optionMatch(z, "expanded") ){
+          p->eTraceType = SHELL_TRACE_EXPANDED;
+        }
+#ifdef SQLITE_ENABLE_NORMALIZE
+        else if( optionMatch(z, "normalized") ){
+          p->eTraceType = SHELL_TRACE_NORMALIZED;
+        }
+#endif
+        else if( optionMatch(z, "plain") ){
+          p->eTraceType = SHELL_TRACE_PLAIN;
+        }
+        else if( optionMatch(z, "profile") ){
+          mType |= SQLITE_TRACE_PROFILE;
+        }
+        else if( optionMatch(z, "row") ){
+          mType |= SQLITE_TRACE_ROW;
+        }
+        else if( optionMatch(z, "stmt") ){
+          mType |= SQLITE_TRACE_STMT;
+        }
+        else if( optionMatch(z, "close") ){
+          mType |= SQLITE_TRACE_CLOSE;
+        }
+        else {
+          raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z);
+          rc = 1;
+          goto meta_command_exit;
+        }
+      }else{
+        output_file_close(p->traceOut);
+        p->traceOut = output_file_open(azArg[1], 0);
+      }
     }
-    output_file_close(p->traceOut);
-    p->traceOut = output_file_open(azArg[1], 0);
-#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
     if( p->traceOut==0 ){
       sqlite3_trace_v2(p->db, 0, 0, 0);
     }else{
-      sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut);
+      if( mType==0 ) mType = SQLITE_TRACE_STMT;
+      sqlite3_trace_v2(p->db, mType, sql_trace_callback, p);
     }
-#endif
   }else
+#endif /* !defined(SQLITE_OMIT_TRACE) */
 
 #if SQLITE_USER_AUTHENTICATION
   if( c=='u' && strncmp(azArg[0], "user", n)==0 ){
     if( nArg<2 ){
       raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n");

Index: src/sqlite.h.in
==================================================================
--- src/sqlite.h.in
+++ src/sqlite.h.in
@@ -2989,13 +2989,13 @@
 ** the original statement text and an estimate of wall-clock time
 ** of how long that statement took to run.  ^The profile callback
 ** time is in units of nanoseconds, however the current implementation
 ** is only capable of millisecond resolution so the six least significant
 ** digits in the time are meaningless.  Future versions of SQLite
-** might provide greater resolution on the profiler callback.  The
-** sqlite3_profile() function is considered experimental and is
-** subject to change in future versions of SQLite.
+** might provide greater resolution on the profiler callback.  Invoking
+** either [sqlite3_trace()] or [sqlite3_trace_v2()] will cancel the
+** profile callback.
 */
 SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*,
    void(*xTrace)(void*,const char*), void*);
 SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*,
    void(*xProfile)(void*,const char*,sqlite3_uint64), void*);
@@ -3627,18 +3627,17 @@
 ** [sqlite3_finalize()] relatively soon. The current implementation acts
 ** on this hint by avoiding the use of [lookaside memory] so as not to
 ** deplete the limited store of lookaside memory. Future versions of
 ** SQLite may act on this hint differently.
 **
-** [[SQLITE_PREPARE_NORMALIZE]] ^(<dt>SQLITE_PREPARE_NORMALIZE</dt>
-** <dd>The SQLITE_PREPARE_NORMALIZE flag indicates that a normalized
-** representation of the SQL statement should be calculated and then
-** associated with the prepared statement, which can be obtained via
-** the [sqlite3_normalized_sql()] interface.)^  The semantics used to
-** normalize a SQL statement are unspecified and subject to change.
-** At a minimum, literal values will be replaced with suitable
-** placeholders.
+** [[SQLITE_PREPARE_NORMALIZE]] <dt>SQLITE_PREPARE_NORMALIZE</dt>
+** <dd>The SQLITE_PREPARE_NORMALIZE flag is a no-op. This flag used
+** to be required for any prepared statement that wanted to use the
+** [sqlite3_normalized_sql()] interface.  However, the
+** [sqlite3_normalized_sql()] interface is now available to all
+** prepared statements, regardless of whether or not they use this
+** flag.
 ** </dl>
 */
 #define SQLITE_PREPARE_PERSISTENT              0x01
 #define SQLITE_PREPARE_NORMALIZE               0x02
 

Index: src/sqliteInt.h
==================================================================
--- src/sqliteInt.h
+++ src/sqliteInt.h
@@ -1354,14 +1354,17 @@
 
 #ifndef SQLITE_OMIT_DEPRECATED
 /* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing
 ** in the style of sqlite3_trace()
 */
-#define SQLITE_TRACE_LEGACY  0x80
+#define SQLITE_TRACE_LEGACY          0x40     /* Use the legacy xTrace */
+#define SQLITE_TRACE_XPROFILE        0x80     /* Use the legacy xProfile */
 #else
-#define SQLITE_TRACE_LEGACY  0
+#define SQLITE_TRACE_LEGACY          0
+#define SQLITE_TRACE_XPROFILE        0
 #endif /* SQLITE_OMIT_DEPRECATED */
+#define SQLITE_TRACE_NONLEGACY_MASK  0x0f     /* Normal flags */
 
 
 /*
 ** Each database connection is an instance of the following structure.
 */
@@ -1417,12 +1420,14 @@
   int nVDestroy;                /* Number of active OP_VDestroy operations */
   int nExtension;               /* Number of loaded extensions */
   void **aExtension;            /* Array of shared library handles */
   int (*xTrace)(u32,void*,void*,void*);     /* Trace function */
   void *pTraceArg;                          /* Argument to the trace function */
+#ifndef SQLITE_OMIT_DEPRECATED
   void (*xProfile)(void*,const char*,u64);  /* Profiling function */
   void *pProfileArg;                        /* Argument to profile function */
+#endif
   void *pCommitArg;                 /* Argument to xCommitCallback() */
   int (*xCommitCallback)(void*);    /* Invoked at every commit. */
   void *pRollbackArg;               /* Argument to xRollbackCallback() */
   void (*xRollbackCallback)(void*); /* Invoked at every commit. */
   void *pUpdateArg;
@@ -1951,13 +1956,10 @@
 ** by an instance of the following structure.
 */
 struct Table {
   char *zName;         /* Name of the table or view */
   Column *aCol;        /* Information about each column */
-#ifdef SQLITE_ENABLE_NORMALIZE
-  Hash *pColHash;      /* All columns indexed by name */
-#endif
   Index *pIndex;       /* List of SQL indexes on this table. */
   Select *pSelect;     /* NULL for tables.  Points to definition if a view. */
   FKey *pFKey;         /* Linked list of all foreign keys in this table */
   char *zColAff;       /* String defining the affinity of each column */
   ExprList *pCheck;    /* All CHECK constraints */
@@ -3982,12 +3984,12 @@
 Table *sqlite3LocateTable(Parse*,u32 flags,const char*, const char*);
 Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *);
 Index *sqlite3FindIndex(sqlite3*,const char*, const char*);
 void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*);
 void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*);
-void sqlite3Vacuum(Parse*,Token*);
-int sqlite3RunVacuum(char**, sqlite3*, int);
+void sqlite3Vacuum(Parse*,Token*,Expr*);
+int sqlite3RunVacuum(char**, sqlite3*, int, sqlite3_value*);
 char *sqlite3NameFromToken(sqlite3*, Token*);
 int sqlite3ExprCompare(Parse*,Expr*, Expr*, int);
 int sqlite3ExprCompareSkip(Expr*, Expr*, int);
 int sqlite3ExprListCompare(ExprList*, ExprList*, int);
 int sqlite3ExprImpliesExpr(Parse*,Expr*, Expr*, int);
@@ -4021,13 +4023,10 @@
 #endif
 int sqlite3ExprIsInteger(Expr*, int*);
 int sqlite3ExprCanBeNull(const Expr*);
 int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
 int sqlite3IsRowid(const char*);
-#ifdef SQLITE_ENABLE_NORMALIZE
-int sqlite3IsRowidN(const char*, int);
-#endif
 void sqlite3GenerateRowDelete(
     Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int);
 void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int);
 int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int);
 void sqlite3ResolvePartIdxLabel(Parse*,int);
@@ -4050,13 +4049,11 @@
 Expr *sqlite3ExprDup(sqlite3*,Expr*,int);
 ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int);
 SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int);
 IdList *sqlite3IdListDup(sqlite3*,IdList*);
 Select *sqlite3SelectDup(sqlite3*,Select*,int);
-#ifdef SQLITE_ENABLE_NORMALIZE
-FuncDef *sqlite3FunctionSearchN(int,const char*,int);
-#endif
+FuncDef *sqlite3FunctionSearch(int,const char*);
 void sqlite3InsertBuiltinFuncs(FuncDef*,int);
 FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8);
 void sqlite3RegisterBuiltinFunctions(void);
 void sqlite3RegisterDateTimeFunctions(void);
 void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*);
@@ -4257,13 +4254,10 @@
 void sqlite3Reindex(Parse*, Token*, Token*);
 void sqlite3AlterFunctions(void);
 void sqlite3AlterRenameTable(Parse*, SrcList*, Token*);
 void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*);
 int sqlite3GetToken(const unsigned char *, int *);
-#ifdef SQLITE_ENABLE_NORMALIZE
-int sqlite3GetTokenNormalized(const unsigned char *, int *, int *);
-#endif
 void sqlite3NestedParse(Parse*, const char*, ...);
 void sqlite3ExpirePreparedStatements(sqlite3*, int);
 int sqlite3CodeSubselect(Parse*, Expr *, int, int);
 void sqlite3SelectPrep(Parse*, Select*, NameContext*);
 void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p);
@@ -4418,11 +4412,11 @@
 sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*);
 int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
 int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
 void sqlite3ParserReset(Parse*);
 #ifdef SQLITE_ENABLE_NORMALIZE
-void sqlite3Normalize(Vdbe*, const char*, int, u8);
+char *sqlite3Normalize(Vdbe*, const char*);
 #endif
 int sqlite3Reprepare(Vdbe*);
 void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*);
 CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *);
 int sqlite3TempInMemory(const sqlite3*);

Index: src/test_hexio.c
==================================================================
--- src/test_hexio.c
+++ src/test_hexio.c
@@ -188,11 +188,11 @@
   Tcl_SetObjResult(interp, Tcl_NewIntObj(written));
   return TCL_OK;
 }
 
 /*
-** USAGE:   hexio_get_int   HEXDATA
+** USAGE:   hexio_get_int [-littleendian] HEXDATA
 **
 ** Interpret the HEXDATA argument as a big-endian integer.  Return
 ** the value of that integer.  HEXDATA can contain between 2 and 8
 ** hexadecimal digits.
 */
@@ -205,16 +205,24 @@
   int val;
   int nIn, nOut;
   const unsigned char *zIn;
   unsigned char *aOut;
   unsigned char aNum[4];
+  int bLittle = 0;
 
-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "HEXDATA");
+  if( objc==3 ){
+    int n;
+    char *z = Tcl_GetStringFromObj(objv[1], &n);
+    if( n>=2 && n<=13 && memcmp(z, "-littleendian", n)==0 ){
+      bLittle = 1;
+    }
+  }
+  if( (objc-bLittle)!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "[-littleendian] HEXDATA");
     return TCL_ERROR;
   }
-  zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1], &nIn);
+  zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1+bLittle], &nIn);
   aOut = sqlite3_malloc( nIn/2 );
   if( aOut==0 ){
     return TCL_ERROR;
   }
   nOut = sqlite3TestHexToBin(zIn, nIn, aOut);
@@ -223,11 +231,15 @@
   }else{
     memset(aNum, 0, sizeof(aNum));
     memcpy(&aNum[4-nOut], aOut, nOut);
   }
   sqlite3_free(aOut);
-  val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3];
+  if( bLittle ){
+    val = (aNum[3]<<24) | (aNum[2]<<16) | (aNum[1]<<8) | aNum[0];
+  }else{
+    val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3];
+  }
   Tcl_SetObjResult(interp, Tcl_NewIntObj(val));
   return TCL_OK;
 }
 
 

Index: src/tokenize.c
==================================================================
--- src/tokenize.c
+++ src/tokenize.c
@@ -543,77 +543,10 @@
   while( IdChar(z[i]) ){ i++; }
   *tokenType = TK_ID;
   return i;
 }
 
-#ifdef SQLITE_ENABLE_NORMALIZE
-/*
-** Return the length (in bytes) of the token that begins at z[0].
-** Store the token type in *tokenType before returning.  If flags has
-** SQLITE_TOKEN_NORMALIZE flag enabled, use the identifier token type
-** for keywords.  Add SQLITE_TOKEN_QUOTED to flags if the token was
-** actually a quoted identifier.  Add SQLITE_TOKEN_KEYWORD to flags
-** if the token was recognized as a keyword; this is useful when the
-** SQLITE_TOKEN_NORMALIZE flag is used, because it enables the caller
-** to differentiate between a keyword being treated as an identifier
-** (for normalization purposes) and an actual identifier.
-*/
-int sqlite3GetTokenNormalized(
-  const unsigned char *z,
-  int *tokenType,
-  int *flags
-){
-  int n;
-  unsigned char iClass = aiClass[*z];
-  if( iClass==CC_KYWD ){
-    int i;
-    for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
-    if( IdChar(z[i]) ){
-      /* This token started out using characters that can appear in keywords,
-      ** but z[i] is a character not allowed within keywords, so this must
-      ** be an identifier instead */
-      i++;
-      while( IdChar(z[i]) ){ i++; }
-      *tokenType = TK_ID;
-      return i;
-    }
-    *tokenType = TK_ID;
-    n = keywordCode((char*)z, i, tokenType);
-    /* If the token is no longer considered to be an identifier, then it is a
-    ** keyword of some kind.  Make the token back into an identifier and then
-    ** set the SQLITE_TOKEN_KEYWORD flag.  Several non-identifier tokens are
-    ** used verbatim, including IN, IS, NOT, and NULL. */
-    switch( *tokenType ){
-      case TK_ID: {
-        /* do nothing, handled by caller */
-        break;
-      }
-      case TK_IN:
-      case TK_IS:
-      case TK_NOT:
-      case TK_NULL: {
-        *flags |= SQLITE_TOKEN_KEYWORD;
-        break;
-      }
-      default: {
-        *tokenType = TK_ID;
-        *flags |= SQLITE_TOKEN_KEYWORD;
-        break;
-      }
-    }
-  }else{
-    n = sqlite3GetToken(z, tokenType);
-    /* If the token is considered to be an identifier and the character class
-    ** of the first character is a quote, set the SQLITE_TOKEN_QUOTED flag. */
-    if( *tokenType==TK_ID && (iClass==CC_QUOTE || iClass==CC_QUOTE2) ){
-      *flags |= SQLITE_TOKEN_QUOTED;
-    }
-  }
-  return n;
-}
-#endif /* SQLITE_ENABLE_NORMALIZE */
-
 /*
 ** Run the parser on the given SQL string.  The parser structure is
 ** passed in.  An SQLITE_ status code is returned.  If an error occurs
 ** then an and attempt is made to write an error message into 
 ** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that
@@ -779,5 +712,140 @@
     sqlite3DeleteTable(db, p);
   }
   assert( nErr==0 || pParse->rc!=SQLITE_OK );
   return nErr;
 }
+
+
+#ifdef SQLITE_ENABLE_NORMALIZE
+/*
+** Insert a single space character into pStr if the current string
+** ends with an identifier
+*/
+static void addSpaceSeparator(sqlite3_str *pStr){
+  if( pStr->nChar && sqlite3IsIdChar(pStr->zText[pStr->nChar-1]) ){
+    sqlite3_str_append(pStr, " ", 1);
+  }
+}
+
+/*
+** Compute a normalization of the SQL given by zSql[0..nSql-1].  Return
+** the normalization in space obtained from sqlite3DbMalloc().  Or return
+** NULL if anything goes wrong or if zSql is NULL.
+*/
+char *sqlite3Normalize(
+  Vdbe *pVdbe,       /* VM being reprepared */
+  const char *zSql   /* The original SQL string */
+){
+  sqlite3 *db;       /* The database connection */
+  int i;             /* Next unread byte of zSql[] */
+  int n;             /* length of current token */
+  int tokenType;     /* type of current token */
+  int prevType;      /* Previous non-whitespace token */
+  int nParen;        /* Number of nested levels of parentheses */
+  int iStartIN;      /* Start of RHS of IN operator in z[] */
+  int nParenAtIN;    /* Value of nParent at start of RHS of IN operator */
+  int j;             /* Bytes of normalized SQL generated so far */
+  sqlite3_str *pStr; /* The normalized SQL string under construction */
+
+  db = sqlite3VdbeDb(pVdbe);
+  tokenType = -1;
+  nParen = iStartIN = nParenAtIN = 0;
+  pStr = sqlite3_str_new(db);
+  assert( pStr!=0 );  /* sqlite3_str_new() never returns NULL */
+  for(i=0; zSql[i] && pStr->accError==0; i+=n){
+    if( tokenType!=TK_SPACE ){
+      prevType = tokenType;
+    }
+    n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType);
+    if( NEVER(n<=0) ) break;
+    switch( tokenType ){
+      case TK_SPACE: {
+        break;
+      }
+      case TK_NULL: {
+        if( prevType==TK_IS || prevType==TK_NOT ){
+          sqlite3_str_append(pStr, " NULL", 5);
+          break;
+        }
+        /* Fall through */
+      }
+      case TK_STRING:
+      case TK_INTEGER:
+      case TK_FLOAT:
+      case TK_VARIABLE:
+      case TK_BLOB: {
+        sqlite3_str_append(pStr, "?", 1);
+        break;
+      }
+      case TK_LP: {
+        nParen++;
+        if( prevType==TK_IN ){
+          iStartIN = pStr->nChar;
+          nParenAtIN = nParen;
+        }
+        sqlite3_str_append(pStr, "(", 1);
+        break;
+      }
+      case TK_RP: {
+        if( iStartIN>0 && nParen==nParenAtIN ){
+          assert( pStr->nChar>=iStartIN );
+          pStr->nChar = iStartIN+1;
+          sqlite3_str_append(pStr, "?,?,?", 5);
+          iStartIN = 0;
+        }
+        nParen--;
+        sqlite3_str_append(pStr, ")", 1);
+        break;
+      }
+      case TK_ID: {
+        iStartIN = 0;
+        j = pStr->nChar;
+        if( sqlite3Isquote(zSql[i]) ){
+          char *zId = sqlite3DbStrNDup(db, zSql+i, n);
+          int nId;
+          int eType = 0;
+          if( zId==0 ) break;
+          sqlite3Dequote(zId);
+          if( zSql[i]=='"' && sqlite3VdbeUsesDoubleQuotedString(pVdbe, zId) ){
+            sqlite3_str_append(pStr, "?", 1);
+            sqlite3DbFree(db, zId);
+            break;
+          }
+          nId = sqlite3Strlen30(zId);
+          if( sqlite3GetToken((u8*)zId, &eType)==nId && eType==TK_ID ){
+            addSpaceSeparator(pStr);
+            sqlite3_str_append(pStr, zId, nId);
+          }else{
+            sqlite3_str_appendf(pStr, "\"%w\"", zId);
+          }
+          sqlite3DbFree(db, zId);
+        }else{
+          addSpaceSeparator(pStr);
+          sqlite3_str_append(pStr, zSql+i, n);
+        }
+        while( j<pStr->nChar ){
+          pStr->zText[j] = sqlite3Tolower(pStr->zText[j]);
+          j++;
+        }
+        break;
+      }
+      case TK_SELECT: {
+        iStartIN = 0;
+        /* fall through */
+      }
+      default: {
+        if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr);
+        j = pStr->nChar;
+        sqlite3_str_append(pStr, zSql+i, n);
+        while( j<pStr->nChar ){
+          pStr->zText[j] = sqlite3Toupper(pStr->zText[j]);
+          j++;
+        }
+        break;
+      }
+    }
+  }
+  if( tokenType!=TK_SEMI ) sqlite3_str_append(pStr, ";", 1);
+  return sqlite3_str_finish(pStr);
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */

Index: src/vacuum.c
==================================================================
--- src/vacuum.c
+++ src/vacuum.c
@@ -100,20 +100,20 @@
 ** not work if other processes are attached to the original database.
 ** And a power loss in between deleting the original and renaming the
 ** transient would cause the database file to appear to be deleted
 ** following reboot.
 */
-void sqlite3Vacuum(Parse *pParse, Token *pNm){
+void sqlite3Vacuum(Parse *pParse, Token *pNm, Expr *pInto){
   Vdbe *v = sqlite3GetVdbe(pParse);
   int iDb = 0;
-  if( v==0 ) return;
+  if( v==0 ) goto build_vacuum_end;
   if( pNm ){
 #ifndef SQLITE_BUG_COMPATIBLE_20160819
     /* Default behavior:  Report an error if the argument to VACUUM is
     ** not recognized */
     iDb = sqlite3TwoPartName(pParse, pNm, pNm, &pNm);
-    if( iDb<0 ) return;
+    if( iDb<0 ) goto build_vacuum_end;
 #else
     /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments
     ** to VACUUM are silently ignored.  This is a back-out of a bug fix that
     ** occurred on 2016-08-19 (https://www.sqlite.org/src/info/083f9e6270).
     ** The buggy behavior is required for binary compatibility with some
@@ -121,42 +121,64 @@
     iDb = sqlite3FindDb(pParse->db, pNm);
     if( iDb<0 ) iDb = 0;
 #endif
   }
   if( iDb!=1 ){
-    sqlite3VdbeAddOp1(v, OP_Vacuum, iDb);
+    int iIntoReg = 0;
+    if( pInto ){
+      iIntoReg = ++pParse->nMem;
+      sqlite3ExprCode(pParse, pInto, iIntoReg);
+    }
+    sqlite3VdbeAddOp2(v, OP_Vacuum, iDb, iIntoReg);
     sqlite3VdbeUsesBtree(v, iDb);
   }
+build_vacuum_end:
+  sqlite3ExprDelete(pParse->db, pInto);
   return;
 }
 
 /*
 ** This routine implements the OP_Vacuum opcode of the VDBE.
 */
-int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){
+int sqlite3RunVacuum(
+  char **pzErrMsg,        /* Write error message here */
+  sqlite3 *db,            /* Database connection */
+  int iDb,                /* Which attached DB to vacuum */
+  sqlite3_value *pOut     /* Write results here, if not NULL */
+){
   int rc = SQLITE_OK;     /* Return code from service routines */
   Btree *pMain;           /* The database being vacuumed */
   Btree *pTemp;           /* The temporary database we vacuum into */
-  u16 saved_mDbFlags;     /* Saved value of db->mDbFlags */
-  u32 saved_flags;        /* Saved value of db->flags */
+  u32 saved_mDbFlags;     /* Saved value of db->mDbFlags */
+  u64 saved_flags;        /* Saved value of db->flags */
   int saved_nChange;      /* Saved value of db->nChange */
   int saved_nTotalChange; /* Saved value of db->nTotalChange */
   u8 saved_mTrace;        /* Saved trace settings */
   Db *pDb = 0;            /* Database to detach at end of vacuum */
   int isMemDb;            /* True if vacuuming a :memory: database */
   int nRes;               /* Bytes of reserved space at the end of each page */
   int nDb;                /* Number of attached databases */
   const char *zDbMain;    /* Schema name of database to vacuum */
+  const char *zOut;       /* Name of output file */
 
   if( !db->autoCommit ){
     sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction");
     return SQLITE_ERROR;
   }
   if( db->nVdbeActive>1 ){
     sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress");
     return SQLITE_ERROR;
   }
+  if( pOut ){
+    if( sqlite3_value_type(pOut)!=SQLITE_TEXT ){
+      sqlite3SetString(pzErrMsg, db, "non-text filename");
+      return SQLITE_ERROR;
+    }
+    zOut = (const char*)sqlite3_value_text(pOut);
+  }else{
+    zOut = "";
+  }
 
   /* Save the current value of the database flags so that it can be 
   ** restored before returning. Then set the writable-schema flag, and
   ** disable CHECK and foreign key constraints.  */
   saved_flags = db->flags;
@@ -164,11 +186,11 @@
   saved_nChange = db->nChange;
   saved_nTotalChange = db->nTotalChange;
   saved_mTrace = db->mTrace;
   db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks;
   db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum;
-  db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder
+  db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder
                    | SQLITE_Defensive | SQLITE_CountRows);
   db->mTrace = 0;
 
   zDbMain = db->aDb[iDb].zDbSName;
   pMain = db->aDb[iDb].pBt;
@@ -187,23 +209,25 @@
   ** empty.  Only the journal header is written.  Apparently it takes more
   ** time to parse and run the PRAGMA to turn journalling off than it does
   ** to write the journal header file.
   */
   nDb = db->nDb;
-  rc = execSql(db, pzErrMsg, "ATTACH''AS vacuum_db");
+  rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS vacuum_db", zOut);
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
   assert( (db->nDb-1)==nDb );
   pDb = &db->aDb[nDb];
   assert( strcmp(pDb->zDbSName,"vacuum_db")==0 );
   pTemp = pDb->pBt;
-
-  /* The call to execSql() to attach the temp database has left the file
-  ** locked (as there was more than one active statement when the transaction
-  ** to read the schema was concluded. Unlock it here so that this doesn't
-  ** cause problems for the call to BtreeSetPageSize() below.  */
-  sqlite3BtreeCommit(pTemp);
-
+  if( pOut ){
+    sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp));
+    i64 sz = 0;
+    if( id->pMethods!=0 && (sqlite3OsFileSize(id, &sz)!=SQLITE_OK || sz>0) ){
+      rc = SQLITE_ERROR;
+      sqlite3SetString(pzErrMsg, db, "output file already exists");
+      goto end_of_vacuum;
+    }
+  }
   nRes = sqlite3BtreeGetOptimalReserve(pMain);
 
   /* A VACUUM cannot change the pagesize of an encrypted database. */
 #ifdef SQLITE_HAS_CODEC
   if( db->nextPagesize ){
@@ -223,11 +247,11 @@
   ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below,
   ** to ensure that we do not try to change the page-size on a WAL database.
   */
   rc = execSql(db, pzErrMsg, "BEGIN");
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
-  rc = sqlite3BtreeBeginTrans(pMain, 2, 0);
+  rc = sqlite3BtreeBeginTrans(pMain, pOut==0 ? 2 : 0, 0);
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
 
   /* Do not attempt to change the page size for a WAL database */
   if( sqlite3PagerGetJournalMode(sqlite3BtreePager(pMain))
                                                ==PAGER_JOURNALMODE_WAL ){
@@ -318,11 +342,11 @@
        BTREE_USER_VERSION,       0,  /* Preserve the user version */
        BTREE_APPLICATION_ID,     0,  /* Preserve the application id */
     };
 
     assert( 1==sqlite3BtreeIsInTrans(pTemp) );
-    assert( 1==sqlite3BtreeIsInTrans(pMain) );
+    assert( pOut!=0 || 1==sqlite3BtreeIsInTrans(pMain) );
 
     /* Copy Btree meta values */
     for(i=0; i<ArraySize(aCopy); i+=2){
       /* GetMeta() and UpdateMeta() cannot fail in this context because
       ** we already have page 1 loaded into cache and marked dirty. */
@@ -329,21 +353,27 @@
       sqlite3BtreeGetMeta(pMain, aCopy[i], &meta);
       rc = sqlite3BtreeUpdateMeta(pTemp, aCopy[i], meta+aCopy[i+1]);
       if( NEVER(rc!=SQLITE_OK) ) goto end_of_vacuum;
     }
 
-    rc = sqlite3BtreeCopyFile(pMain, pTemp);
+    if( pOut==0 ){
+      rc = sqlite3BtreeCopyFile(pMain, pTemp);
+    }
     if( rc!=SQLITE_OK ) goto end_of_vacuum;
     rc = sqlite3BtreeCommit(pTemp);
     if( rc!=SQLITE_OK ) goto end_of_vacuum;
 #ifndef SQLITE_OMIT_AUTOVACUUM
-    sqlite3BtreeSetAutoVacuum(pMain, sqlite3BtreeGetAutoVacuum(pTemp));
+    if( pOut==0 ){
+      sqlite3BtreeSetAutoVacuum(pMain, sqlite3BtreeGetAutoVacuum(pTemp));
+    }
 #endif
   }
 
   assert( rc==SQLITE_OK );
-  rc = sqlite3BtreeSetPageSize(pMain, sqlite3BtreeGetPageSize(pTemp), nRes,1);
+  if( pOut==0 ){
+    rc = sqlite3BtreeSetPageSize(pMain, sqlite3BtreeGetPageSize(pTemp), nRes,1);
+  }
 
 end_of_vacuum:
   /* Restore the original value of db->flags */
   db->init.iDb = 0;
   db->mDbFlags = saved_mDbFlags;

Index: src/vdbe.c
==================================================================
--- src/vdbe.c
+++ src/vdbe.c
@@ -6725,18 +6725,23 @@
   break;
 };
 #endif /* SQLITE_OMIT_PRAGMA */
 
 #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
-/* Opcode: Vacuum P1 * * * *
+/* Opcode: Vacuum P1 P2 * * *
 **
 ** Vacuum the entire database P1.  P1 is 0 for "main", and 2 or more
 ** for an attached database.  The "temp" database may not be vacuumed.
+**
+** If P2 is not zero, then it is a register holding a string which is
+** the file into which the result of vacuum should be written.  When
+** P2 is zero, the vacuum overwrites the original database.
 */
 case OP_Vacuum: {
   assert( p->readOnly==0 );
-  rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1);
+  rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1,
+                        pOp->p2 ? &aMem[pOp->p2] : 0);
   if( rc ) goto abort_due_to_error;
   break;
 }
 #endif
 
@@ -7135,11 +7140,11 @@
   testcase( pName->enc==SQLITE_UTF16BE );
   testcase( pName->enc==SQLITE_UTF16LE );
   rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8);
   if( rc ) goto abort_due_to_error;
   rc = pVtab->pModule->xRename(pVtab, pName->z);
-  if( isLegacy==0 ) db->flags &= ~SQLITE_LegacyAlter;
+  if( isLegacy==0 ) db->flags &= ~(u64)SQLITE_LegacyAlter;
   sqlite3VtabImportErrmsg(p, pVtab);
   p->expired = 0;
   if( rc ) goto abort_due_to_error;
   break;
 }

Index: src/vdbe.h
==================================================================
--- src/vdbe.h
+++ src/vdbe.h
@@ -249,10 +249,14 @@
 int sqlite3VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*));
 void sqlite3VdbeCountChanges(Vdbe*);
 sqlite3 *sqlite3VdbeDb(Vdbe*);
 u8 sqlite3VdbePrepareFlags(Vdbe*);
 void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, u8);
+#ifdef SQLITE_ENABLE_NORMALIZE
+void sqlite3VdbeAddDblquoteStr(sqlite3*,Vdbe*,const char*);
+int sqlite3VdbeUsesDoubleQuotedString(Vdbe*,const char*);
+#endif
 void sqlite3VdbeSwap(Vdbe*,Vdbe*);
 VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*);
 sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8);
 void sqlite3VdbeSetVarmask(Vdbe*, int);
 #ifndef SQLITE_OMIT_TRACE

Index: src/vdbeInt.h
==================================================================
--- src/vdbeInt.h
+++ src/vdbeInt.h
@@ -333,19 +333,35 @@
 /* A bitfield type for use inside of structures.  Always follow with :N where
 ** N is the number of bits.
 */
 typedef unsigned bft;  /* Bit Field Type */
 
+/* The ScanStatus object holds a single value for the
+** sqlite3_stmt_scanstatus() interface.
+*/
 typedef struct ScanStatus ScanStatus;
 struct ScanStatus {
   int addrExplain;                /* OP_Explain for loop */
   int addrLoop;                   /* Address of "loops" counter */
   int addrVisit;                  /* Address of "rows visited" counter */
   int iSelectID;                  /* The "Select-ID" for this loop */
   LogEst nEst;                    /* Estimated output rows per loop */
   char *zName;                    /* Name of table or index */
 };
+
+/* The DblquoteStr object holds the text of a double-quoted
+** string for a prepared statement.  A linked list of these objects
+** is constructed during statement parsing and is held on Vdbe.pDblStr.
+** When computing a normalized SQL statement for an SQL statement, that
+** list is consulted for each double-quoted identifier to see if the
+** identifier should really be a string literal.
+*/
+typedef struct DblquoteStr DblquoteStr;
+struct DblquoteStr {
+  DblquoteStr *pNextStr;   /* Next string literal in the list */
+  char z[8];               /* Dequoted value for the string */
+};
 
 /*
 ** An instance of the virtual machine.  This structure contains the complete
 ** state of the virtual machine.
 **
@@ -362,11 +378,11 @@
   int nCursor;            /* Number of slots in apCsr[] */
   u32 cacheCtr;           /* VdbeCursor row cache generation counter */
   int pc;                 /* The program counter */
   int rc;                 /* Value to return */
   int nChange;            /* Number of db changes made since last reset */
-  int iStatement;         /* Statement number (or 0 if has not opened stmt) */
+  int iStatement;         /* Statement number (or 0 if has no opened stmt) */
   i64 iCurrentTime;       /* Value of julianday('now') for this statement */
   i64 nFkConstraint;      /* Number of imm. FK constraints this VM */
   i64 nStmtDefCons;       /* Number of def. constraints when stmt started */
   i64 nStmtDefImmCons;    /* Number of def. imm constraints when stmt started */
 
@@ -406,10 +422,11 @@
   yDbMask lockMask;       /* Subset of btreeMask that requires a lock */
   u32 aCounter[7];        /* Counters used by sqlite3_stmt_status() */
   char *zSql;             /* Text of the SQL statement that generated this */
 #ifdef SQLITE_ENABLE_NORMALIZE
   char *zNormSql;         /* Normalization of the associated SQL statement */
+  DblquoteStr *pDblStr;   /* List of double-quoted string literals */
 #endif
   void *pFree;            /* Free this when deleting the vdbe */
   VdbeFrame *pFrame;      /* Parent frame */
   VdbeFrame *pDelFrame;   /* List of frame objects to free on VM reset */
   int nFrame;             /* Number of frames in pFrame list */

Index: src/vdbeapi.c
==================================================================
--- src/vdbeapi.c
+++ src/vdbeapi.c
@@ -60,18 +60,20 @@
 */
 static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){
   sqlite3_int64 iNow;
   sqlite3_int64 iElapse;
   assert( p->startTime>0 );
-  assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 );
+  assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 );
   assert( db->init.busy==0 );
   assert( p->zSql!=0 );
   sqlite3OsCurrentTimeInt64(db->pVfs, &iNow);
   iElapse = (iNow - p->startTime)*1000000;
+#ifndef SQLITE_OMIT_DEPRECATED  	
   if( db->xProfile ){
     db->xProfile(db->pProfileArg, p->zSql, iElapse);
   }
+#endif
   if( db->mTrace & SQLITE_TRACE_PROFILE ){
     db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse);
   }
   p->startTime = 0;
 }
@@ -581,11 +583,11 @@
   if( db->mallocFailed ){
     p->rc = SQLITE_NOMEM;
     return SQLITE_NOMEM_BKPT;
   }
 
-  if( p->pc<=0 && p->expired ){
+  if( p->pc<0 && p->expired ){
     p->rc = SQLITE_SCHEMA;
     rc = SQLITE_ERROR;
     goto end_of_step;
   }
   if( p->pc<0 ){
@@ -600,11 +602,11 @@
     assert( db->nVdbeWrite>0 || db->autoCommit==0 
         || (db->nDeferredCons==0 && db->nDeferredImmCons==0)
     );
 
 #ifndef SQLITE_OMIT_TRACE
-    if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0)
+    if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0
         && !db->init.busy && p->zSql ){
       sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime);
     }else{
       assert( p->startTime==0 );
     }
@@ -627,20 +629,22 @@
     db->nVdbeExec++;
     rc = sqlite3VdbeExec(p);
     db->nVdbeExec--;
   }
 
+  if( rc!=SQLITE_ROW ){
 #ifndef SQLITE_OMIT_TRACE
-  /* If the statement completed successfully, invoke the profile callback */
-  if( rc!=SQLITE_ROW ) checkProfileCallback(db, p);
+    /* If the statement completed successfully, invoke the profile callback */
+    checkProfileCallback(db, p);
 #endif
 
-  if( rc==SQLITE_DONE && db->autoCommit ){
-    assert( p->rc==SQLITE_OK );
-    p->rc = doWalCallbacks(db);
-    if( p->rc!=SQLITE_OK ){
-      rc = SQLITE_ERROR;
+    if( rc==SQLITE_DONE && db->autoCommit ){
+      assert( p->rc==SQLITE_OK );
+      p->rc = doWalCallbacks(db);
+      if( p->rc!=SQLITE_OK ){
+        rc = SQLITE_ERROR;
+      }
     }
   }
 
   db->errCode = rc;
   if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){
@@ -656,13 +660,13 @@
   */
   assert( rc==SQLITE_ROW  || rc==SQLITE_DONE   || rc==SQLITE_ERROR 
        || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE
   );
   assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp );
-  if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 
-   && rc!=SQLITE_ROW 
-   && rc!=SQLITE_DONE 
+  if( rc!=SQLITE_ROW 
+   && rc!=SQLITE_DONE
+   && (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0
   ){
     /* If this statement was prepared using saved SQL and an 
     ** error has occurred, then return the error code in p->rc to the
     ** caller. Set the error code in the database handle to the same value.
     */ 
@@ -1280,11 +1284,11 @@
   }
   i--;
   pVar = &p->aVar[i];
   sqlite3VdbeMemRelease(pVar);
   pVar->flags = MEM_Null;
-  sqlite3Error(p->db, SQLITE_OK);
+  p->db->errCode = SQLITE_OK;
 
   /* If the bit corresponding to this variable in Vdbe.expmask is set, then 
   ** binding a new value to this variable invalidates the current query plan.
   **
   ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host
@@ -1706,11 +1710,17 @@
 /*
 ** Return the normalized SQL associated with a prepared statement.
 */
 const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){
   Vdbe *p = (Vdbe *)pStmt;
-  return p ? p->zNormSql : 0;
+  if( p==0 ) return 0;
+  if( p->zNormSql==0 && ALWAYS(p->zSql!=0) ){
+    sqlite3_mutex_enter(p->db->mutex);
+    p->zNormSql = sqlite3Normalize(p, p->zSql);
+    sqlite3_mutex_leave(p->db->mutex);
+  }
+  return p->zNormSql;
 }
 #endif /* SQLITE_ENABLE_NORMALIZE */
 
 #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
 /*

Index: src/vdbeaux.c
==================================================================
--- src/vdbeaux.c
+++ src/vdbeaux.c
@@ -62,18 +62,48 @@
   if( (prepFlags & SQLITE_PREPARE_SAVESQL)==0 ){
     p->expmask = 0;
   }
   assert( p->zSql==0 );
   p->zSql = sqlite3DbStrNDup(p->db, z, n);
+}
+
+#ifdef SQLITE_ENABLE_NORMALIZE
+/*
+** Add a new element to the Vdbe->pDblStr list.
+*/
+void sqlite3VdbeAddDblquoteStr(sqlite3 *db, Vdbe *p, const char *z){
+  if( p ){
+    int n = sqlite3Strlen30(z);
+    DblquoteStr *pStr = sqlite3DbMallocRawNN(db,
+                            sizeof(*pStr)+n+1-sizeof(pStr->z));
+    if( pStr ){
+      pStr->pNextStr = p->pDblStr;
+      p->pDblStr = pStr;
+      memcpy(pStr->z, z, n+1);
+    }
+  }
+}
+#endif
+
 #ifdef SQLITE_ENABLE_NORMALIZE
-  assert( p->zNormSql==0 );
-  if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){
-    sqlite3Normalize(p, p->zSql, n, prepFlags);
-    assert( p->zNormSql!=0 || p->db->mallocFailed );
+/*
+** zId of length nId is a double-quoted identifier.  Check to see if
+** that identifier is really used as a string literal.
+*/
+int sqlite3VdbeUsesDoubleQuotedString(
+  Vdbe *pVdbe,            /* The prepared statement */
+  const char *zId         /* The double-quoted identifier, already dequoted */
+){
+  DblquoteStr *pStr;
+  assert( zId!=0 );
+  if( pVdbe->pDblStr==0 ) return 0;
+  for(pStr=pVdbe->pDblStr; pStr; pStr=pStr->pNextStr){
+    if( strcmp(zId, pStr->z)==0 ) return 1;
   }
+  return 0;
+}
 #endif
-}
 
 /*
 ** Swap all content between two VDBE structures.
 */
 void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){
@@ -90,11 +120,11 @@
   pA->pPrev = pB->pPrev;
   pB->pPrev = pTmp;
   zTmp = pA->zSql;
   pA->zSql = pB->zSql;
   pB->zSql = zTmp;
-#ifdef SQLITE_ENABLE_NORMALIZE
+#if 0
   zTmp = pA->zNormSql;
   pA->zNormSql = pB->zNormSql;
   pB->zNormSql = zTmp;
 #endif
   pB->expmask = pA->expmask;
@@ -2871,11 +2901,11 @@
           sqlite3RollbackAll(db, SQLITE_OK);
           p->nChange = 0;
         }else{
           db->nDeferredCons = 0;
           db->nDeferredImmCons = 0;
-          db->flags &= ~SQLITE_DeferFKs;
+          db->flags &= ~(u64)SQLITE_DeferFKs;
           sqlite3CommitInternalChanges(db);
         }
       }else{
         sqlite3RollbackAll(db, SQLITE_OK);
         p->nChange = 0;
@@ -3188,10 +3218,17 @@
   vdbeFreeOpArray(db, p->aOp, p->nOp);
   sqlite3DbFree(db, p->aColName);
   sqlite3DbFree(db, p->zSql);
 #ifdef SQLITE_ENABLE_NORMALIZE
   sqlite3DbFree(db, p->zNormSql);
+  {
+    DblquoteStr *pThis, *pNext;
+    for(pThis=p->pDblStr; pThis; pThis=pNext){
+      pNext = pThis->pNextStr;
+      sqlite3DbFree(db, pThis);
+    }
+  }
 #endif
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
   {
     int i;
     for(i=0; i<p->nScan; i++){

Index: src/wal.c
==================================================================
--- src/wal.c
+++ src/wal.c
@@ -360,11 +360,10 @@
 **
 ** There is still a single WRITER and a single CHECKPOINTER lock. The
 ** recovery procedure still takes the same exclusive lock on the entire
 ** range of SQLITE_SHM_NLOCK shm-locks. This works because the read-locks
 ** above use four of the six read-locking slots used by legacy wal mode.
-** See the header comment for function walLockReader() for details.
 **
 ** STARTUP/RECOVERY
 **
 ** The read and write version fields of the database header in a wal2
 ** database are set to 0x03, instead of 0x02 as in legacy wal mode.
@@ -471,20 +470,24 @@
 **
 ** In wal mode, the Wal.readLock member is set to -1 when no read-lock
 ** is held, or else is the index of the read-mark on which a lock is
 ** held.
 **
-** In wal2 mode, Wal.readLock must be set to one of the following values.
-** A value of -1 still indicates that no read-lock is held, but the other
-** values are symbolic. See the implementation of walLockReader() for
-** details of how the symbols map to OS level locks.
+** In wal2 mode, a value of -1 still indicates that no read-lock is held.
+** And a non-zero value still represents the index of the read-mark on
+** which a lock is held. There are two differences:
+**
+**   1. wal2 mode never uses read-mark 0.
+**
+**   2. locks on each read-mark have a different interpretation, as 
+**      indicated by the symbolic names below.
 */
 #define WAL_LOCK_NONE        -1
 #define WAL_LOCK_PART1        1
 #define WAL_LOCK_PART1_FULL2  2
-#define WAL_LOCK_PART2        3
-#define WAL_LOCK_PART2_FULL1  4
+#define WAL_LOCK_PART2_FULL1  3
+#define WAL_LOCK_PART2        4
 
 /* 
 ** This constant is used in wal2 mode only.
 **
 ** In wal2 mode, when committing a transaction, if the current wal file 
@@ -1110,40 +1113,10 @@
                          SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE);
   WALTRACE(("WAL%p: release EXCLUSIVE-%s cnt=%d\n", pWal,
              walLockName(lockIdx), n));
 }
 
-/*
-** This function is used to take and release read-locks in wal2 mode.
-**
-** Use of WAL_READ_LOCK(x) slots for (1<=x<=4).
-**
-** 1) Partial read of *-wal-1   (blocks checkpointer from checkpointing)
-** 2) Full read of *-wal-2      (blocks writer from writing)
-** 3) Partial read of *-wal-2   (blocks checkpointer from checkpointing)
-** 4) Full read of *-wal-1      (blocks writer from writing)
-*/
-static int walLockReader(Wal *pWal, int eLock, int bLock){
-  int i;                          /* Index of first readmark to lock */
-  int n;                          /* Number of readmarks to lock */
-
-  assert( pWal->hdr.iVersion==WAL_VERSION2 );
-  if( pWal->exclusiveMode ) return SQLITE_OK;
-
-  switch( eLock ){
-    case WAL_LOCK_PART1      : i = 1; n = 1; break; 
-    case WAL_LOCK_PART1_FULL2: i = 1; n = 2; break; 
-    case WAL_LOCK_PART2      : i = 3; n = 1; break; 
-    case WAL_LOCK_PART2_FULL1: i = 3; n = 2; break; 
-    default: assert( !"cannot happen" );
-  }
-
-  return sqlite3OsShmLock(pWal->pDbFd, WAL_READ_LOCK(i), n,
-      SQLITE_SHM_SHARED | (bLock ? SQLITE_SHM_LOCK : SQLITE_SHM_UNLOCK) 
-  );
-}
-
 /*
 ** Compute a hash on a page number.  The resulting hash value must land
 ** between 0 and (HASHTABLE_NSLOT-1).  The walHashNext() function advances
 ** the hash to the next value in the event of a collision.
 */
@@ -1230,18 +1203,13 @@
     /* A frame in wal file 0 */
     *piRead = (iExternal <= HASHTABLE_NPAGE_ONE) ? iExternal :
       iExternal - (iHash/2) * HASHTABLE_NPAGE;
     return 0;
   }
-  if( iHash==0 ){
-    *piRead = iExternal;
-    return 0;
-  }else{
-    *piRead = iExternal - HASHTABLE_NPAGE_ONE - ((iHash-1)/2) * HASHTABLE_NPAGE;
-  }
 
-  return (iHash % 2);
+  *piRead = iExternal - HASHTABLE_NPAGE_ONE - ((iHash-1)/2) * HASHTABLE_NPAGE;
+  return 1;
 }
 
 /*
 ** Return the number of the wal-index page that contains the hash-table
 ** and page-number array that contain entries corresponding to WAL frame
@@ -1563,11 +1531,11 @@
           aFrameCksum[1] = pWal->hdr.aFrameCksum[1];
         }
       }
   
       sqlite3_free(aFrame);
-    }else if( pbZero && nSize==0 ){
+    }else if( pbZero ){
       *pbZero = 1;
     }
   }
 
   pWal->hdr.aFrameCksum[0] = aFrameCksum[0];
@@ -1687,12 +1655,12 @@
 
     if( isOpen(pWal->apWalFd[1]) ){
       /* The case where *-wal2 may follow *-wal */
       if( nCkpt2<=0x0F && nCkpt2==nCkpt1+1 ){
         if( sqlite3Get4byte((u8*)(&pWal->hdr.aSalt[0]))==hdr.aFrameCksum[0]
-            && sqlite3Get4byte((u8*)(&pWal->hdr.aSalt[1]))==hdr.aFrameCksum[1]
-          ){
+         && sqlite3Get4byte((u8*)(&pWal->hdr.aSalt[1]))==hdr.aFrameCksum[1]
+        ){
           walidxSetFile(&pWal->hdr, 1);
           walidxSetMxFrame(&pWal->hdr, 1, pWal->hdr.mxFrame);
           walidxSetMxFrame(&pWal->hdr, 0, hdr.mxFrame);
         }else{
           pWal->hdr = hdr;
@@ -1746,13 +1714,13 @@
     ** problems caused by applications routinely shutting down without
     ** checkpointing the log file.  */
     if( pWal->hdr.nPage ){
       if( isWalMode2(pWal) ){
         sqlite3_log(SQLITE_NOTICE_RECOVER_WAL,
-            "recovered (%d,%d) frames from WAL files %s[2] (%s mode)",
+            "recovered (%d,%d) frames from WAL files %s[2] (wal2 mode)",
             walidxGetMxFrame(&pWal->hdr, 0), walidxGetMxFrame(&pWal->hdr, 1), 
-            pWal->zWalName, isWalMode2(pWal) ? "wal2" : "wal"
+            pWal->zWalName
         );
       }else{
         sqlite3_log(SQLITE_NOTICE_RECOVER_WAL,
             "recovered %d frames from WAL file %s",
             pWal->hdr.mxFrame, pWal->zWalName
@@ -2249,10 +2217,72 @@
   pInfo->nBackfillAttempted = 0;
   pInfo->aReadMark[1] = 0;
   for(i=2; i<WAL_NREADER; i++) pInfo->aReadMark[i] = READMARK_NOT_USED;
   assert( pInfo->aReadMark[0]==0 );
 }
+
+/*
+** This function is used in wal2 mode.
+**
+** This function is called when writer pWal is just about to start 
+** writing out frames. Parameter iApp is the current wal file. The "other" wal
+** file (wal file !iApp) has been fully checkpointed. This function returns
+** SQLITE_OK if there are no readers preventing the writer from switching to
+** the other wal file. Or SQLITE_BUSY if there are.
+*/
+static int wal2RestartOk(Wal *pWal, int iApp){
+  /* The other wal file (wal file !iApp) can be overwritten if there
+  ** are no readers reading from it - no "full" or "partial" locks.
+  ** Technically speaking it is not possible for any reader to hold
+  ** a "part" lock, as this would have prevented the file from being
+  ** checkpointed. But checking anyway doesn't hurt. The following
+  ** is equivalent to:
+  **
+  **   if( iApp==0 ) eLock = WAL_LOCK_PART1_FULL2;
+  **   if( iApp==1 ) eLock = WAL_LOCK_PART1;
+  */
+  int eLock = 1 + (iApp==0);
+
+  assert( WAL_LOCK_PART1==1 );
+  assert( WAL_LOCK_PART1_FULL2==2 );
+  assert( WAL_LOCK_PART2_FULL1==3 );
+  assert( WAL_LOCK_PART2==4 );
+
+  assert( iApp!=0 || eLock==WAL_LOCK_PART1_FULL2 );
+  assert( iApp!=1 || eLock==WAL_LOCK_PART1 );
+
+  return walLockExclusive(pWal, WAL_READ_LOCK(eLock), 3);
+}
+static void wal2RestartFinished(Wal *pWal, int iApp){
+  walUnlockExclusive(pWal, WAL_READ_LOCK(1 + (iApp==0)), 3);
+}
+
+/*
+** This function is used in wal2 mode.
+**
+** This function is called when a checkpointer wishes to checkpoint wal
+** file iCkpt. It takes the required lock and, if successful, returns
+** SQLITE_OK. Otherwise, an SQLite error code (e.g. SQLITE_BUSY). If this
+** function returns SQLITE_OK, it is the responsibility of the caller
+** to invoke wal2CheckpointFinished() to release the lock.
+*/
+static int wal2CheckpointOk(Wal *pWal, int iCkpt){
+  int eLock = 1 + (iCkpt*2);
+
+  assert( WAL_LOCK_PART1==1 );
+  assert( WAL_LOCK_PART1_FULL2==2 );
+  assert( WAL_LOCK_PART2_FULL1==3 );
+  assert( WAL_LOCK_PART2==4 );
+
+  assert( iCkpt!=0 || eLock==WAL_LOCK_PART1 );
+  assert( iCkpt!=1 || eLock==WAL_LOCK_PART2_FULL1 );
+
+  return walLockExclusive(pWal, WAL_READ_LOCK(eLock), 2);
+}
+static void wal2CheckpointFinished(Wal *pWal, int iCkpt){
+  walUnlockExclusive(pWal, WAL_READ_LOCK(1 + (iCkpt*2)), 2);
+}
 
 /*
 ** Copy as much content as we can from the WAL back into the database file
 ** in response to an sqlite3_wal_checkpoint() request or the equivalent.
 **
@@ -2317,11 +2347,11 @@
 
     /* If this is a wal2 system, check for a reader holding a lock 
     ** preventing this checkpoint operation. If one is found, return
     ** early.  */
     if( bWal2 ){
-      rc = walLockExclusive(pWal, WAL_READ_LOCK(1 + iCkpt*2), 1);
+      rc = wal2CheckpointOk(pWal, iCkpt);
       if( rc!=SQLITE_OK ) return rc;
     }
 
     /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked
     ** in the SQLITE_CHECKPOINT_PASSIVE mode. */
@@ -2366,13 +2396,13 @@
       assert( bWal2==0 || pInfo->nBackfill==0 );
       rc = walIteratorInit(pWal, iCkpt, pInfo->nBackfill, &pIter);
       assert( rc==SQLITE_OK || pIter==0 );
     }
 
-    if( pIter
-     && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0),1))==SQLITE_OK
-    ){
+    if( pIter && (bWal2 
+     || (rc = walBusyLock(pWal, xBusy, pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK
+    )){
       u32 nBackfill = pInfo->nBackfill;
 
       assert( bWal2==0 || nBackfill==0 );
       pInfo->nBackfillAttempted = mxSafeFrame;
 
@@ -2433,21 +2463,21 @@
       if( rc==SQLITE_OK ){
         pInfo->nBackfill = bWal2 ? 1 : mxSafeFrame;
       }
 
       /* Release the reader lock held while backfilling */
-      if( bWal2 ){
-        walUnlockExclusive(pWal, WAL_READ_LOCK(1 + iCkpt*2), 1);
+      if( bWal2==0 ){
+        walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1);
       }
-      walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1);
     }
 
     if( rc==SQLITE_BUSY ){
       /* Reset the return code so as not to report a checkpoint failure
       ** just because there are active readers.  */
       rc = SQLITE_OK;
     }
+    if( bWal2 ) wal2CheckpointFinished(pWal, iCkpt);
   }
 
   /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the
   ** entire wal file has been copied into the database file, then block 
   ** until all readers have finished using the wal file. This ensures that 
@@ -3078,22 +3108,31 @@
 
   assert( pWal->nWiData>0 );
   assert( pWal->apWiData[0]!=0 );
   pInfo = walCkptInfo(pWal);
   if( isWalMode2(pWal) ){
-    int eLock = 1 + (walidxGetFile(&pWal->hdr)*2);
-    if( pInfo->nBackfill==0 ){
-      eLock += walidxGetMxFrame(&pWal->hdr, !walidxGetFile(&pWal->hdr))>0;
-    }
-    rc = walLockReader(pWal, eLock, 1);
+    /* This connection needs a "part" lock on the current wal file and, 
+    ** unless pInfo->nBackfill is set to indicate that it has already been
+    ** checkpointed, a "full" lock on the other wal file.  */
+    int iWal = walidxGetFile(&pWal->hdr);
+    int nBackfill = pInfo->nBackfill || walidxGetMxFrame(&pWal->hdr, !iWal)==0;
+    int eLock = 1 + (iWal*2) + (nBackfill==iWal);
+
+    assert( nBackfill==0 || nBackfill==1 );
+    assert( iWal==0 || iWal==1 );
+    assert( iWal!=0 || nBackfill!=1 || eLock==WAL_LOCK_PART1 );
+    assert( iWal!=0 || nBackfill!=0 || eLock==WAL_LOCK_PART1_FULL2 );
+    assert( iWal!=1 || nBackfill!=1 || eLock==WAL_LOCK_PART2 );
+    assert( iWal!=1 || nBackfill!=0 || eLock==WAL_LOCK_PART2_FULL1 );
+
+    rc = walLockShared(pWal, WAL_READ_LOCK(eLock));
     if( rc!=SQLITE_OK ){
-      return rc;
+      return (rc==SQLITE_BUSY ? WAL_RETRY : rc);
     }
-
     walShmBarrier(pWal);
     if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){
-      walLockReader(pWal, eLock, 0);
+      walUnlockShared(pWal, WAL_READ_LOCK(eLock));
       return WAL_RETRY;
     }else{
       pWal->readLock = eLock;
     }
     assert( pWal->minFrame==0 && walFramePage(pWal->minFrame)==0 );
@@ -3422,15 +3461,11 @@
 ** read-lock.
 */
 void sqlite3WalEndReadTransaction(Wal *pWal){
   sqlite3WalEndWriteTransaction(pWal);
   if( pWal->readLock!=WAL_LOCK_NONE ){
-    if( isWalMode2(pWal) ){
-      (void)walLockReader(pWal, pWal->readLock, 0);
-    }else{
-      walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
-    }
+    walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
     pWal->readLock = WAL_LOCK_NONE;
   }
 }
 
 /* Search hash table iHash for an entry matching page number
@@ -4061,37 +4096,10 @@
   }
 
   return rc;
 }
 
-/*
-** This function is used in wal2 mode.
-**
-** This function is called when writer pWal is just about to start 
-** writing out frames. The "other" wal file (wal file !pWal->hdr.iAppend)
-** has been fully checkpointed. This function returns SQLITE_OK if there
-** are no readers preventing the writer from switching to the other wal
-** file. Or SQLITE_BUSY if there are.
-*/
-static int walRestartOk(Wal *pWal){
-  int rc;                                        /* Return code */
-  int iApp = walidxGetFile(&pWal->hdr);          /* Current WAL file */
-
-  /* No reader can be doing a "partial" read of wal file !iApp - in that
-  ** case it would not have been possible to checkpoint the file. So
-  ** it is only necessary to test for "full" readers. See the comment
-  ** above walLockReader() function for exactly what this means in terms
-  ** of locks.  */
-  int i = (iApp==0) ? 2 : 4;
-
-  rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1);
-  if( rc==SQLITE_OK ){
-    walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
-  }
-  return rc;
-}
-
 /*
 ** This function is called just before writing a set of frames to the log
 ** file (see sqlite3WalFrames()). It checks to see if, instead of appending
 ** to the current log file, it is possible and desirable to switch to the
 ** other log file and write the new transaction to the start of it.
@@ -4115,23 +4123,24 @@
     }
 
     if( walidxGetMxFrame(&pWal->hdr, iApp)>=nWalSize ){
       volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
       if( walidxGetMxFrame(&pWal->hdr, !iApp)==0 || pInfo->nBackfill ){
-        rc = walRestartOk(pWal);
+        rc = wal2RestartOk(pWal, iApp);
         if( rc==SQLITE_OK ){
-          iApp = !iApp;
+          int iNew = !iApp;
           pWal->nCkpt++;
-          walidxSetFile(&pWal->hdr, iApp);
-          walidxSetMxFrame(&pWal->hdr, iApp, 0);
+          walidxSetFile(&pWal->hdr, iNew);
+          walidxSetMxFrame(&pWal->hdr, iNew, 0);
           sqlite3Put4byte((u8*)&pWal->hdr.aSalt[0], pWal->hdr.aFrameCksum[0]);
           sqlite3Put4byte((u8*)&pWal->hdr.aSalt[1], pWal->hdr.aFrameCksum[1]);
           walIndexWriteHdr(pWal);
           pInfo->nBackfill = 0;
-          walLockReader(pWal, pWal->readLock, 0);
-          pWal->readLock = iApp ? WAL_LOCK_PART2_FULL1 : WAL_LOCK_PART1_FULL2;
-          rc = walLockReader(pWal, pWal->readLock, 1);
+          wal2RestartFinished(pWal, iApp);
+          walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
+          pWal->readLock = iNew ? WAL_LOCK_PART2_FULL1 : WAL_LOCK_PART1_FULL2;
+          rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock));
         }else if( rc==SQLITE_BUSY ){
           rc = SQLITE_OK;
         }
       }
     }
@@ -4756,15 +4765,11 @@
   assert( pWal->readLock!=WAL_LOCK_NONE || (op<=0 && pWal->exclusiveMode==0) );
 
   if( op==0 ){
     if( pWal->exclusiveMode ){
       pWal->exclusiveMode = WAL_NORMAL_MODE;
-      if( isWalMode2(pWal) ){
-        rc = walLockReader(pWal, pWal->readLock, 1);
-      }else{
-        rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock));
-      }
+      rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock));
       if( rc!=SQLITE_OK ){
         pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
       }
       rc = pWal->exclusiveMode==WAL_NORMAL_MODE;
     }else{
@@ -4772,15 +4777,11 @@
       rc = 0;
     }
   }else if( op>0 ){
     assert( pWal->exclusiveMode==WAL_NORMAL_MODE );
     assert( pWal->readLock>=0 );
-    if( isWalMode2(pWal) ){
-      walLockReader(pWal, pWal->readLock, 0);
-    }else{
-      walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
-    }
+    walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
     pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
     rc = 1;
   }else{
     rc = pWal->exclusiveMode==WAL_NORMAL_MODE;
   }

Index: src/window.c
==================================================================
--- src/window.c
+++ src/window.c
@@ -2131,10 +2131,11 @@
   if( ALWAYS(p) ){
     pNew = sqlite3DbMallocZero(db, sizeof(Window));
     if( pNew ){
       pNew->zName = sqlite3DbStrDup(db, p->zName);
       pNew->pFilter = sqlite3ExprDup(db, p->pFilter, 0);
+      pNew->pFunc = p->pFunc;
       pNew->pPartition = sqlite3ExprListDup(db, p->pPartition, 0);
       pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, 0);
       pNew->eType = p->eType;
       pNew->eEnd = p->eEnd;
       pNew->eStart = p->eStart;

Index: test/altertab2.test
==================================================================
--- test/altertab2.test
+++ test/altertab2.test
@@ -10,11 +10,11 @@
 #*************************************************************************
 #
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-set testprefix altertab
+set testprefix altertab2
 
 # If SQLITE_OMIT_ALTERTABLE is defined, omit this file.
 ifcapable !altertable {
   finish_test
   return
@@ -83,7 +83,63 @@
   {CREATE TABLE c1(x REFERENCES "p3")}
   {CREATE TABLE c2(x, FOREIGN KEY (x) REFERENCES "p3")}
   {CREATE TABLE c3(x, FOREIGN KEY (x) REFERENCES "p3"(a))}
 }
 
+#-------------------------------------------------------------------------
+# Table name in WITH clauses that are part of views or triggers.
+#
+foreach {tn schema} {
+  1 {
+    CREATE TABLE log_entry(col1, y);
+    CREATE INDEX i1 ON log_entry(col1);
+  }
+
+  2 {
+    CREATE TABLE t1(a, b, c);
+    CREATE TABLE t2(x);
+    CREATE TABLE log_entry(col1);
+    CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN
+      INSERT INTO t2 SELECT col1 FROM log_entry;
+    END;
+  }
+
+  3 {
+    CREATE TABLE t1(a, b, c);
+    CREATE TABLE t2(x);
+    CREATE TABLE log_entry(col1);
+    CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN
+      INSERT INTO t2
+        WITH xyz(x) AS (SELECT col1 FROM log_entry)
+        SELECT x FROM xyz;
+    END;
+  }
+
+  4 {
+    CREATE TABLE log_entry(col1);
+    CREATE VIEW ttt AS
+        WITH xyz(x) AS (SELECT col1 FROM log_entry)
+        SELECT x FROM xyz;
+  }
+} {
+  reset_db
+  do_execsql_test 3.$tn.1 $schema
+  set expect [db eval "SELECT sql FROM sqlite_master"]
+  set expect [string map {log_entry {"newname"}} $expect]
+
+  do_execsql_test 3.$tn.2 {
+    ALTER TABLE log_entry RENAME TO newname;
+    SELECT sql FROM sqlite_master;
+  } $expect
+
+  reset_db
+  do_execsql_test 3.$tn.3 $schema
+  set expect [db eval "SELECT sql FROM sqlite_master"]
+  set expect [string map {col1 newname} $expect]
+
+  do_execsql_test 3.$tn.4 {
+    ALTER TABLE log_entry RENAME col1 TO newname;
+    SELECT sql FROM sqlite_master;
+  } $expect
+}
 
 finish_test

Index: test/dbfuzz2.c
==================================================================
--- test/dbfuzz2.c
+++ test/dbfuzz2.c
@@ -76,10 +76,11 @@
 
   if( eVerbosity>=1 ){
     printf("************** nByte=%d ***************\n", (int)nByte);
     fflush(stdout);
   }
+  if( sqlite3_initialize() ) return 0;
   rc = sqlite3_open(0, &db);
   if( rc ) return 1;
   a = sqlite3_malloc64(nByte+1);
   if( a==0 ) return 1;
   memcpy(a, aData, nByte);

ADDED   test/fts4umlaut.test
Index: test/fts4umlaut.test
==================================================================
--- test/fts4umlaut.test
+++ test/fts4umlaut.test
@@ -0,0 +1,65 @@
+# 2018 December 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts4umlaut
+
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  CREATE VIRTUAL TABLE t2 USING fts4(
+      x, 
+      tokenize=unicode61 "remove_diacritics=2"
+  );
+}
+
+foreach {tn q res1 res2} {
+  1 "Hà Nội"                  0 1
+  2 "Hà Noi"                  1 1
+  3 "Ha Noi"                  1 1
+  4 "Ha N\u1ed9i"             0 1
+  5 "Ha N\u006fi"             1 1
+  6 "Ha N\u006f\u0302i"       1 1
+  7 "Ha N\u006f\u0323\u0302i" 1 1
+} {
+  do_execsql_test 1.$tn.1 {
+    DELETE FROM t1;
+    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
+    SELECT count(*) FROM t1 WHERE t1 MATCH $q
+  } $res1
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t1;
+    INSERT INTO t1(rowid, x) VALUES (1, $q);
+    SELECT count(*) FROM t1 WHERE t1 MATCH 'Ha Noi'
+  } $res1
+
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t2;
+    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
+    SELECT count(*) FROM t2 WHERE t2 MATCH $q
+  } $res2
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t2;
+    INSERT INTO t2(rowid, x) VALUES (1, $q);
+    SELECT count(*) FROM t2 WHERE t2 MATCH 'Ha Noi'
+  } $res2
+}
+
+finish_test
+

Index: test/index6.test
==================================================================
--- test/index6.test
+++ test/index6.test
@@ -387,8 +387,27 @@
   EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99;
 } {/USING INDEX t11x/}
 do_execsql_test index6-11.2 {
   EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99 AND c<>98;
 } {/USING INDEX t11x/}
-  
 
+# 2018-12-08
+# Ticket https://www.sqlite.org/src/info/1d958d90596593a7
+# NOT IN operator fails when using a partial index.
+#
+do_execsql_test index6-12.1 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t1(a,b);
+  INSERT INTO t1 VALUES(1,1);
+  INSERT INTO t1 VALUES(2,2);
+  CREATE TABLE t2(x);
+  INSERT INTO t2 VALUES(1);
+  INSERT INTO t2 VALUES(2);
+  SELECT 'one', * FROM t2 WHERE x NOT IN (SELECT a FROM t1);
+  CREATE INDEX t1a ON t1(a) WHERE b=1;
+  SELECT 'two', * FROM t2 WHERE x NOT IN (SELECT a FROM t1);
+} {}
+do_execsql_test index6-12.2 {
+  SELECT x FROM t2 WHERE x IN (SELECT a FROM t1) ORDER BY +x;
+} {1 2}
 finish_test

Index: test/json101.test
==================================================================
--- test/json101.test
+++ test/json101.test
@@ -811,8 +811,25 @@
 } {$}
 do_execsql_test json-14.170 {
   SELECT fullkey FROM json_tree('null');
 } {$}
 
-
+# 2018-12-03
+# Make sure the table-valued functions contained within parentheses
+# work correctly.
+#
+# Bug reported via private email. See TH3 for more information.
+#
+do_execsql_test json-15.100 {
+  SELECT * FROM JSON_EACH('{"a":1, "b":2}');
+} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+do_execsql_test json-15.110 {
+  SELECT xyz.* FROM JSON_EACH('{"a":1, "b":2}') AS xyz;
+} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+do_execsql_test json-15.120 {
+  SELECT * FROM (JSON_EACH('{"a":1, "b":2}'));
+} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+do_execsql_test json-15.130 {
+  SELECT xyz.* FROM (JSON_EACH('{"a":1, "b":2}')) AS xyz;
+} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
 
 finish_test

Index: test/normalize.test
==================================================================
--- test/normalize.test
+++ test/normalize.test
@@ -84,11 +84,11 @@
 
   sqlite3_bind_null $STMT 1
 } {}
 do_test 202 {
   sqlite3_normalized_sql $STMT
-} {}
+} {SELECT a,b FROM t1 WHERE b=?ORDER BY a;}
 do_test 203 {
   sqlite3_finalize $STMT
 } {SQLITE_OK}
 
 do_test 210 {
@@ -205,21 +205,21 @@
   {0 {SELECT a FROM t1 WHERE x IN(?,?,?)AND hex8(?);}}
 
   430
   {SELECT "a" FROM t1 WHERE "x" IN ("1","2",'3');}
   0x2
-  {0 {SELECT"a"FROM t1 WHERE"x"IN(?,?,?);}}
+  {0 {SELECT a FROM t1 WHERE x IN(?,?,?);}}
 
   440
   {SELECT 'a' FROM t1 WHERE 'x';}
   0x2
   {0 {SELECT?FROM t1 WHERE?;}}
 
   450
   {SELECT [a] FROM t1 WHERE [x];}
   0x2
-  {0 {SELECT"a"FROM t1 WHERE"x";}}
+  {0 {SELECT a FROM t1 WHERE x;}}
 
   460
   {SELECT * FROM t1 WHERE x IN (x);}
   0x2
   {0 {SELECT*FROM t1 WHERE x IN(x);}}
@@ -230,31 +230,31 @@
   {0 {SELECT*FROM t1 WHERE x IN(x,a);}}
 
   480
   {SELECT * FROM t1 WHERE x IN ([x],"a");}
   0x2
-  {0 {SELECT*FROM t1 WHERE x IN("x","a");}}
+  {0 {SELECT*FROM t1 WHERE x IN(x,a);}}
 
   500
   {SELECT * FROM t1 WHERE x IN ([x],"a",'b',sqlite_version());}
   0x2
-  {0 {SELECT*FROM t1 WHERE x IN("x","a",?,sqlite_version());}}
+  {0 {SELECT*FROM t1 WHERE x IN(x,a,?,sqlite_version());}}
 
   520
   {SELECT * FROM t1 WHERE x IN (SELECT x FROM t1);}
   0x2
   {0 {SELECT*FROM t1 WHERE x IN(SELECT x FROM t1);}}
 
   540
   {SELECT * FROM t1 WHERE x IN ((SELECT x FROM t1));}
   0x2
-  {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}
+  {0 {SELECT*FROM t1 WHERE x IN((SELECT x FROM t1));}}
 
   550
   {SELECT a, a+1, a||'b', a+"b" FROM t1;}
   0x2
-  {0 {SELECT a,a+?,a||?,a+"b"FROM t1;}}
+  {0 {SELECT a,a+?,a||?,a+b FROM t1;}}
 
   570
   {SELECT * FROM t1 WHERE x IN (1);}
   0x2
   {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}
@@ -314,11 +314,11 @@
   {0 {SELECT"col f","col f"FROM t1;}}
 
   680
   {SELECT a, "col f" FROM t1 LEFT OUTER JOIN t2 ON [t1].[col f] == [t2].[col y];}
   0x2
-  {0 {SELECT a,"col f"FROM t1 LEFT OUTER JOIN t2 ON"t1"."col f"=="t2"."col y";}}
+  {0 {SELECT a,"col f"FROM t1 LEFT OUTER JOIN t2 ON t1."col f"==t2."col y";}}
 
   690
   {SELECT * FROM ( WITH x AS ( SELECT * FROM t1 WHERE x IN ( 1)) SELECT 10);}
   0x2
   {0 {SELECT*FROM(WITH x AS(SELECT*FROM t1 WHERE x IN(?,?,?))SELECT?);}}
@@ -344,11 +344,11 @@
   {0 {SELECT x FROM t1 WHERE x=?;}}
 
   760
   {SELECT x FROM t1 WHERE x IN ([x] IS NOT NULL, NULL, 1, 'a', "b", x'00');}
   0x2
-  {0 {SELECT x FROM t1 WHERE x IN("x"IS NOT NULL,?,?,?,"b",?);}}
+  {0 {SELECT x FROM t1 WHERE x IN(x IS NOT NULL,?,?,?,b,?);}}
 } {
   do_test $tnum {
     set code [catch {
       set STMT [sqlite3_prepare_v3 $DB $sql -1 $flags TAIL]
       sqlite3_normalized_sql $STMT

Index: test/permutations.test
==================================================================
--- test/permutations.test
+++ test/permutations.test
@@ -428,14 +428,20 @@
 #   coverage-wal
 #
 test_suite "coverage-wal" -description {
   Coverage tests for file wal.c.
 } -files {
-  wal.test       wal2.test     wal3.test       walmode.test    
-  walbak.test    walhook.test  walcrash2.test  walcksum.test
-  walfault.test  walbig.test   walnoshm.test
-  wal5.test
+wal2big.test wal2recover.test wal2rewrite.test
+wal2simple.test wal2snapshot.test wal2.test
+wal3.test wal4.test wal5.test
+wal64k.test wal6.test wal7.test wal8.test wal9.test
+walbak.test walbig.test walblock.test walcksum.test 
+walfault.test walhook.test walmode.test walnoshm.test
+waloverwrite.test walpersist.test walprotocol2.test
+walprotocol.test walro2.test walrofault.test walro.test
+walshared.test walslow.test wal.test
+wal2savepoint.test wal2lock.test wal2recover2.test
 } 
 
 test_suite "coverage-pager" -description {
   Coverage tests for file pager.c.
 } -files {

Index: test/shell1.test
==================================================================
--- test/shell1.test
+++ test/shell1.test
@@ -254,11 +254,11 @@
   catchcmd "test.db" ".backup FOO BAR"
 } {1 {Error: unknown database FOO}}
 do_test shell1-3.1.4 {
   # too many arguments
   catchcmd "test.db" ".backup FOO BAR BAD"
-} {1 {Usage: .backup ?DB? ?--append? FILENAME}}
+} {1 {Usage: .backup ?DB? ?OPTIONS? FILENAME}}
 
 # .bail ON|OFF           Stop after hitting an error.  Default OFF
 do_test shell1-3.2.1 {
   catchcmd "test.db" ".bail"
 } {1 {Usage: .bail on|off}}

Index: test/shell4.test
==================================================================
--- test/shell4.test
+++ test/shell4.test
@@ -105,18 +105,18 @@
        [regexp {Heap Usage} $res] \
        [regexp {Autoindex Inserts} $res]
 } {1 1 1}
 
 do_test shell4-2.1 {
-  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace"
-} {1 {Usage: .trace FILE|off}}
+  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace --unknown"
+} {1 {Unknown option "--unknown" on ".trace"}}
 do_test shell4-2.2 {
   catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace off\n.trace off\n"
 } {0 {}}
 do_test shell4-2.3 {
-  catchcmd ":memory:" ".trace stdout\n.trace\n.trace off\n.dump\n"
-} {/^1 {PRAGMA.*Usage:.*}$/}
+  catchcmd ":memory:" ".trace stdout\n.dump\n.trace off\n"
+} {/^0 {PRAGMA.*}$/}
 ifcapable trace {
 do_test shell4-2.4 {
   catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;"
 } {0 {CREATE TABLE t1(x);
 SELECT * FROM t1;}}

Index: test/tabfunc01.test
==================================================================
--- test/tabfunc01.test
+++ test/tabfunc01.test
@@ -122,10 +122,15 @@
 do_execsql_test tabfunc01-4.3 {
   ATTACH ':memory:' AS aux1;
   CREATE TABLE aux1.t1(a,b,c);
   SELECT * FROM aux1.generate_series(1,4)
 } {1 2 3 4}
+
+# 2018-12-03: Fix bug reported by by private email.
+do_execsql_test tabfunc01-4.4 {
+  SELECT * FROM (generate_series(1,5,2)) AS x LIMIT 10;
+} {1 3 5}
 
 # The next series of tests is verifying that virtual table are able
 # to optimize the IN operator, even on terms that are not marked "omit".
 # When the generate_series virtual table is compiled for the testfixture,
 # the special -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 option is used, which

ADDED   test/vacuum-into.test
Index: test/vacuum-into.test
==================================================================
--- test/vacuum-into.test
+++ test/vacuum-into.test
@@ -0,0 +1,69 @@
+# 2018-12-07
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the VACUUM INTO statement.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# If the VACUUM statement is disabled in the current build, skip all
+# the tests in this file.
+#
+ifcapable {!vacuum} {
+  omit_test vacuum.test {Compiled with SQLITE_OMIT_VACUUM}
+  finish_test
+  return
+}
+
+forcedelete out.db
+do_execsql_test vacuum-into-100 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
+  INSERT INTO t1(a,b) SELECT x, randomblob(600) FROM c;
+  CREATE INDEX t1b ON t1(b);
+  DELETE FROM t1 WHERE a%2;
+  SELECT count(*), sum(a), sum(length(b)) FROM t1;
+} {50 2550 30000}
+do_execsql_test vacuum-into-110 {
+  VACUUM main INTO 'out.db';
+} {}
+sqlite3 db2 out.db
+do_test vacuum-into-120 {
+  db2 eval {SELECT count(*), sum(a), sum(length(b)) FROM t1}
+} {50 2550 30000}
+do_catchsql_test vacuum-into-130 {
+  VACUUM INTO 'out.db';
+} {1 {output file already exists}}
+forcedelete out2.db
+do_catchsql_test vacuum-into-140 {
+  VACUUM INTO 'out2.db';
+} {0 {}}
+do_catchsql_test vacuum-into-150 {
+  VACUUM INTO 'out2.db';
+} {1 {output file already exists}}
+
+do_catchsql_test vacuum-into-200 {
+  VACUUM main INTO ':memory:';
+} {0 {}}
+
+# The INTO argument can be an arbitrary expression.
+#
+do_execsql_test vacuum-into-300 {
+  CREATE TABLE t2(name TEXT);
+  INSERT INTO t2 VALUES(':memory:');
+  VACUUM main INTO (SELECT name FROM t2);
+} {}
+do_catchsql_test vacuum-into-310 {
+  VACUUM INTO null;
+} {1 {non-text filename}}
+
+finish_test

ADDED   test/wal2big.test
Index: test/wal2big.test
==================================================================
--- test/wal2big.test
+++ test/wal2big.test
@@ -0,0 +1,71 @@
+# 2017 September 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+
+set testprefix wal2big
+ifcapable !wal {finish_test ; return }
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1b ON t1(b);
+  CREATE INDEX t1c ON t1(c);
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_size_limit = 10000000;
+
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200000
+  )
+  INSERT INTO t1 SELECT random(), random(), random() FROM s;
+} {wal2 10000000}
+
+do_execsql_test 1.1 {
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200000
+  )
+  INSERT INTO t1 SELECT random(), random(), random() FROM s;
+}
+
+do_test 1.2 {
+  list [expr [file size test.db-wal]>10000000] \
+       [expr [file size test.db-wal2]>10000000]
+} {1 1}
+
+do_test 1.3 {
+  sqlite3 db2 test.db
+  execsql {
+    SELECT count(*) FROM t1;
+    PRAGMA integrity_check;
+  } db2
+} {400000 ok}
+
+do_test 1.4 {
+  db2 close
+  forcecopy test.db test.db2
+  forcecopy test.db-wal test.db2-wal
+  forcecopy test.db-wal2 test.db2-wal2
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT count(*) FROM t1;
+    PRAGMA integrity_check;
+  }
+} {400000 ok}
+
+finish_test

ADDED   test/wal2lock.test
Index: test/wal2lock.test
==================================================================
--- test/wal2lock.test
+++ test/wal2lock.test
@@ -0,0 +1,106 @@
+# 2018 December 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+
+set testprefix wal2lock
+ifcapable !wal {finish_test ; return }
+
+db close
+testvfs tvfs 
+sqlite3 db test.db -vfs tvfs
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal2;
+  CREATE TABLE y1(y, yy);
+  CREATE INDEX y1y ON y1(y);
+  CREATE INDEX y1yy ON y1(yy);
+  INSERT INTO y1 VALUES(1, 2), (3, 4), (5, 6);
+} {wal2}
+
+tvfs script vfs_callback
+tvfs filter xShmLock
+
+set ::lock [list]
+proc vfs_callback {func file name lock} {
+  lappend ::lock $lock
+  return SQLITE_OK
+}
+
+do_execsql_test 1.1.1 {
+  SELECT * FROM y1
+} {1 2 3 4 5 6}
+do_test 1.1.2 {
+  set ::lock
+} {{4 1 lock shared} {4 1 unlock shared}}
+
+set ::bFirst 1
+proc vfs_callback {func file name lock} {
+  if {$::bFirst} {
+    set ::bFirst 0
+    return SQLITE_BUSY
+  }
+  return SQLITE_OK
+}
+do_execsql_test 1.2 {
+  SELECT * FROM y1
+} {1 2 3 4 5 6}
+
+set ::bFirst 1
+proc vfs_callback {func file name lock} {
+  if {$::bFirst} {
+    set ::bFirst 0
+    return SQLITE_IOERR
+  }
+  return SQLITE_OK
+}
+do_catchsql_test 1.3 {
+  SELECT * FROM y1
+} {1 {disk I/O error}}
+
+puts "# Warning: This next test case causes SQLite to call xSleep(1) 100 times."
+puts "# Normally this equates to a delay of roughly 10 seconds, but if SQLite"
+puts "# is built on unix without HAVE_USLEEP defined, it may be much longer."
+proc vfs_callback {func file name lock} { return SQLITE_BUSY }
+do_catchsql_test 1.4 {
+  SELECT * FROM y1
+} {1 {locking protocol}}
+proc vfs_callback {func file name lock} { return SQLITE_OK }
+
+sqlite3 db2 test.db -vfs tvfs
+set ::bFirst 1
+
+proc vfs_callback {func file name lock} {
+  if {$::bFirst} {
+    set ::bFirst 0
+    db2 eval { INSERT INTO y1 VALUES(7, 8) }
+  }
+}
+
+do_execsql_test 1.5.1 {
+  SELECT * FROM y1
+} {1 2 3 4 5 6 7 8}
+do_execsql_test 1.5.2 {
+  SELECT * FROM y1
+} {1 2 3 4 5 6 7 8}
+
+db close
+db2 close
+tvfs delete
+finish_test

ADDED   test/wal2recover.test
Index: test/wal2recover.test
==================================================================
--- test/wal2recover.test
+++ test/wal2recover.test
@@ -0,0 +1,232 @@
+# 2018 December 13
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+
+set testprefix wal2recover
+ifcapable !wal {finish_test ; return }
+
+proc db_copy {from to} {
+  forcecopy $from $to
+  forcecopy ${from}-wal ${to}-wal
+  forcecopy ${from}-wal2 ${to}-wal2
+}
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1b ON t1(b);
+  CREATE INDEX t1c ON t1(c);
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_size_limit = 15000;
+  PRAGMA wal_autocheckpoint = 0;
+} {wal2 15000 0}
+
+do_test 1.1 {
+  for {set i 1} {$i <= 1000} {incr i} {
+    execsql { INSERT INTO t1 VALUES(random(), random(), random()) }
+    db_copy test.db test.db2
+    sqlite3 db2 test.db
+    set res [execsql {
+      SELECT count(*) FROM t1;
+      PRAGMA integrity_check;
+    } db2]
+    db2 close
+    if {$res != [list $i ok]} {
+      error "failure on iteration $i"
+    }
+  }
+  set {} {}
+} {}
+
+#--------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE TABLE t1(x UNIQUE);
+  CREATE TABLE t2(x UNIQUE);
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_size_limit = 10000;
+  PRAGMA wal_autocheckpoint = 0;
+  BEGIN;
+    INSERT INTO t1 VALUES(randomblob(4000));
+    INSERT INTO t1 VALUES(randomblob(4000));
+    INSERT INTO t1 VALUES(randomblob(4000));
+  COMMIT;
+  BEGIN;
+    INSERT INTO t2 VALUES(randomblob(4000));
+    INSERT INTO t2 VALUES(randomblob(4000));
+    INSERT INTO t2 VALUES(randomblob(4000));
+  COMMIT;
+} {wal2 10000 0}
+do_test 2.0.1 {
+  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
+} {5120 28328 28328}
+
+# Test recovery with both wal files intact.
+#
+do_test 2.1 {
+  db_copy test.db test.db2
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT count(*) FROM t1;
+    SELECT count(*) FROM t2;
+    PRAGMA integrity_check;
+  } db2
+} {3 3 ok}
+
+do_test 2.2 {
+  db2 close
+  db_copy test.db test.db2
+  hexio_write test.db2-wal 16 12345678
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT count(*) FROM t1;
+    SELECT count(*) FROM t2;
+  } db2
+} {0 3}
+
+do_test 2.3 {
+  db2 close
+  db_copy test.db test.db2
+  hexio_write test.db2-wal2 16 12345678
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT count(*) FROM t1;
+    SELECT count(*) FROM t2;
+    PRAGMA integrity_check;
+  } db2
+} {3 0 ok}
+
+do_test 2.4 {
+  db2 close
+  db_copy test.db test.db2
+  forcecopy test.db-wal test.db2-wal2
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT count(*) FROM t1;
+    SELECT count(*) FROM t2;
+    PRAGMA integrity_check;
+  } db2
+} {3 0 ok}
+
+do_test 2.5 {
+  db2 close
+  db_copy test.db test.db2
+  forcecopy test.db-wal  test.db2-wal2
+  forcecopy test.db-wal2  test.db2-wal
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT count(*) FROM t1;
+    SELECT count(*) FROM t2;
+    PRAGMA integrity_check;
+  } db2
+} {3 3 ok}
+
+do_test 2.6 {
+  db2 close
+  db_copy test.db test.db2
+  forcecopy test.db-wal test.db2-wal2
+  close [open test.db-wal w]
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT count(*) FROM t1;
+    SELECT count(*) FROM t2;
+    PRAGMA integrity_check;
+  } db2
+} {3 0 ok}
+
+do_test 2.7 {
+  db2 close
+  db_copy test.db test.db2
+  forcedelete test.db2-wal
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT count(*) FROM t1;
+    SELECT count(*) FROM t2;
+    PRAGMA integrity_check;
+  } db2
+} {0 0 ok}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE t1(a TEXT, b TEXT, c TEXT);
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1b ON t1(b);
+  CREATE INDEX t1c ON t1(c);
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_size_limit = 10000;
+  PRAGMA wal_autocheckpoint = 0;
+  PRAGMA cache_size = 5;
+} {wal2 10000 0}
+
+do_execsql_test 3.1 {
+  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200)
+  INSERT INTO t1 SELECT i, i, i FROM s;
+
+  INSERT INTO t1 VALUES(201, 201, 201);
+} {}
+
+do_test 3.2 {
+  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
+} {5120 15752 4224}
+
+do_test 3.3 {
+  forcecopy test.db test.db2
+  forcecopy test.db-wal test.db2-wal
+  forcecopy test.db-wal2 test.db2-wal2
+  sqlite3 db2 test.db2
+  execsql {
+    PRAGMA journal_size_limit = 10000;
+    PRAGMA wal_autocheckpoint = 0;
+    PRAGMA cache_size = 5;
+    BEGIN;
+      WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200)
+      INSERT INTO t1 SELECT i, i, i FROM s;
+  } db2
+  list [file size test.db2] [file size test.db2-wal] [file size test.db2-wal2]
+} {5120 15752 23088}
+
+do_test 3.4 {
+  set fd [open test.db2-shm]
+  fconfigure $fd -encoding binary -translation binary
+  set data [read $fd]
+  close $fd
+
+  set fd [open test.db-shm w]
+  fconfigure $fd -encoding binary -translation binary
+  puts -nonewline $fd $data
+  close $fd
+
+  execsql {
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 10)
+      INSERT INTO t1 SELECT i, i, i FROM s;
+    SELECT count(*) FROM t1;
+    PRAGMA integrity_check;
+  }
+} {211 ok}
+
+do_test 3.5 {
+  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
+} {5120 15752 18896}
+
+
+finish_test
+

ADDED   test/wal2recover2.test
Index: test/wal2recover2.test
==================================================================
--- test/wal2recover2.test
+++ test/wal2recover2.test
@@ -0,0 +1,227 @@
+# 2018 December 13
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+
+set testprefix wal2recover2
+ifcapable !wal {finish_test ; return }
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(x);
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 )
+    INSERT INTO t1 SELECT i FROM s;
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 )
+    INSERT INTO t2 SELECT i FROM s;
+
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_size_limit = 10000;
+} {wal2 10000}
+
+set ::L 1125750
+set ::M 1126500
+set ::H 1127250
+
+do_execsql_test 1.1 {
+  UPDATE t1 SET x=x+1;
+  UPDATE t2 SET x=x+1 WHERE rowid<=750;
+
+  SELECT sum(x) FROM t1;
+  SELECT sum(x) FROM t2;
+} [list $H $M]
+
+do_test 1.2 {
+  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
+} {31744 14704 7368}
+
+proc cksum {zIn data} {
+  if {[string length $zIn]==0} {
+    set s0 0
+    set s1 0
+  } else {
+    set s0 [hexio_get_int [string range $zIn 0 7]]
+    set s1 [hexio_get_int [string range $zIn 8 15]]
+  }
+  set n [expr [string length $data] / 8]
+
+  for {set i 0} {$i < $n} {incr i 2} {
+    set x0 [hexio_get_int -l [string range $data [expr $i*8]   [expr $i*8+7]]]
+    set x1 [hexio_get_int -l [string range $data [expr $i*8+8] [expr $i*8+8+7]]]
+
+    set s0 [expr ($s0 + $x0 + $s1) & 0xFFFFFFFF]
+    set s1 [expr ($s1 + $x1 + $s0) & 0xFFFFFFFF]
+  }
+
+  return "[hexio_render_int32 $s0][hexio_render_int32 $s1]"
+}
+
+proc fix_wal_cksums {file} {
+  # Fix the checksum on the wal header.
+  set data [hexio_read $file 0 32]
+  set cksum [cksum {} [string range $data 0 47]]
+  set salt [hexio_read $file 16 8]
+  hexio_write $file 24 $cksum
+
+  # Fix the checksums for all pages in the wal file.
+  set pgsz [hexio_get_int [hexio_read $file 8 4]]
+  set sz [file size $file]
+  for {set off 32} {$off < $sz} {incr off [expr $pgsz+24]} {
+    set e [hexio_read $file $off 8]
+    set cksum [cksum $cksum $e]
+
+    set p [hexio_read $file [expr $off+24] $pgsz]
+    set cksum [cksum $cksum $p]
+
+    hexio_write $file [expr $off+8] $salt
+    hexio_write $file [expr $off+16] $cksum
+  }
+}
+
+proc wal_incr_hdrfield {file field} {
+  switch -- $field {
+    nCkpt { set offset 12 }
+    salt0 { set offset 16 }
+    salt1 { set offset 20 }
+    default {
+      error "unknown field $field - should be \"nCkpt\", \"salt0\" or \"salt1\""
+    }
+  }
+
+  # Increment the value in the wal header.
+  set v [hexio_get_int [hexio_read $file $offset 4]]
+  incr v
+  hexio_write $file $offset [hexio_render_int32 $v]
+  
+  # Fix various checksums
+  fix_wal_cksums $file
+}
+
+proc wal_set_nckpt {file val} {
+  # Increment the value in the wal header.
+  hexio_write $file 12 [hexio_render_int32 $val]
+  
+  # Fix various checksums
+  fix_wal_cksums $file
+}
+
+proc wal_set_follow {file prevfile} {
+  set pgsz [hexio_get_int [hexio_read $prevfile 8 4]]
+  set sz [file size $prevfile]
+  set cksum [hexio_read $prevfile [expr $sz-$pgsz-8] 8]
+
+  hexio_write $file 16 $cksum
+  fix_wal_cksums $file
+}
+
+foreach {tn file field} {
+  1 test.db2-wal    salt0
+  2 test.db2-wal    salt1
+  3 test.db2-wal    nCkpt
+  4 test.db2-wal2   salt0
+  5 test.db2-wal2   salt1
+  6 test.db2-wal2   nCkpt
+} {
+  do_test 1.3.$tn {
+    forcecopy test.db test.db2
+    forcecopy test.db-wal test.db2-wal
+    forcecopy test.db-wal2 test.db2-wal2
+    wal_incr_hdrfield $file $field
+    sqlite3 db2 test.db2
+    execsql {
+      SELECT sum(x) FROM t1;
+      SELECT sum(x) FROM t2;
+    } db2
+  } [list $H $L]
+  db2 close
+}
+
+do_test 1.4 {
+  forcecopy test.db test.db2
+  forcecopy test.db-wal2 test.db2-wal
+  forcedelete test.db2-wal2
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT sum(x) FROM t1;
+    SELECT sum(x) FROM t2;
+  } db2
+} [list $L $M]
+
+do_test 1.5 {
+  forcecopy test.db test.db2
+  forcecopy test.db-wal2 test.db2-wal
+  forcecopy test.db-wal test.db2-wal2
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT sum(x) FROM t1;
+    SELECT sum(x) FROM t2;
+  } db2
+} [list $H $M]
+
+foreach {tn file field} {
+  1 test.db2-wal    salt0
+  2 test.db2-wal    salt1
+  3 test.db2-wal2   salt0
+  4 test.db2-wal2   salt1
+} {
+  do_test 1.6.$tn {
+    forcecopy test.db test.db2
+    forcecopy test.db-wal2 test.db2-wal
+    forcecopy test.db-wal test.db2-wal2
+    wal_incr_hdrfield $file $field
+    breakpoint
+    sqlite3 db2 test.db2
+    execsql {
+      SELECT sum(x) FROM t1;
+      SELECT sum(x) FROM t2;
+    } db2
+  } [list $H $L]
+  db2 close
+}
+
+foreach {tn nCkpt1 nCkpt2 res} [list \
+  1   2 1   "$H $M"                  \
+  2   2 2   "$L $M"                  \
+  3   3 1   "$H $L"                  \
+  4   15 14 "$H $M"                  \
+  5   0 15  "$H $M"                  \
+  6   1 15  "$L $M"                  \
+] {
+  do_test 1.7.$tn {
+    forcecopy test.db test.db2
+    forcecopy test.db-wal2 test.db2-wal
+    forcecopy test.db-wal test.db2-wal2
+
+    wal_set_nckpt test.db2-wal2 $nCkpt2
+    wal_set_nckpt test.db2-wal  $nCkpt1
+    wal_set_follow test.db2-wal test.db2-wal2
+
+    if {$tn==1} breakpoint
+
+    sqlite3 db2 test.db2
+    execsql {
+      SELECT sum(x) FROM t1;
+      SELECT sum(x) FROM t2;
+    } db2
+  } $res
+  db2 close
+}
+
+
+finish_test
+

ADDED   test/wal2savepoint.test
Index: test/wal2savepoint.test
==================================================================
--- test/wal2savepoint.test
+++ test/wal2savepoint.test
@@ -0,0 +1,60 @@
+# 2018 December 13
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+
+set testprefix wal2savepoint
+ifcapable !wal {finish_test ; return }
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1b ON t1(b);
+  CREATE INDEX t1c ON t1(c);
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_size_limit = 15000;
+  PRAGMA wal_autocheckpoint = 0;
+  PRAGMA cache_size = 5;
+} {wal2 15000 0}
+
+do_execsql_test 1.1 {
+  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200)
+  INSERT INTO t1 SELECT random(), random(), random() FROM s;
+} {}
+
+do_test 1.2 {
+  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
+} {5120 23088 0}
+
+do_execsql_test 1.3 {
+  BEGIN;
+    SAVEPOINT abc;
+      WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 100)
+      INSERT INTO t1 SELECT random(), random(), random() FROM s;
+    ROLLBACK TO abc;
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 10)
+    INSERT INTO t1 SELECT random(), random(), random() FROM s;
+  COMMIT;
+  SELECT count(*) FROM t1;
+  PRAGMA integrity_check;
+} {210 ok}
+
+
+finish_test
+

Index: test/wal2simple.test
==================================================================
--- test/wal2simple.test
+++ test/wal2simple.test
@@ -112,11 +112,10 @@
   CREATE INDEX i1 ON t1(x);
   PRAGMA cache_size = 5;
   PRAGMA journal_mode = wal2;
 } {wal2}
 
-breakpoint
 do_test 3.1 {
   execsql BEGIN
   for {set i 1} {$i < 1000} {incr i} {
     execsql { INSERT INTO t1 VALUES(randomblob(800), $i) }
   }
@@ -241,12 +240,10 @@
   for {set i 0} {$i < 10} {incr i} {
     execsql "CREATE TABLE t$i (x);"
   }
 } {}
 
-puts "[file size test.db-wal] [file size test.db-wal2]"
-
 do_test 6.2.1 {
   foreach f [glob -nocomplain test.db2*] { forcedelete $f }
   forcecopy test.db-wal2 test.db2-wal2
   sqlite3 db2 test.db2
   db2 eval { SELECT * FROM sqlite_master }
@@ -286,7 +283,192 @@
     PRAGMA journal_mode = wal2;
     SELECT * FROM sqlite_master;
   }
 } {wal2}
 db2 close
+
+#-------------------------------------------------------------------------
+reset_db
+sqlite3 db2 test.db
+do_execsql_test 7.0 {
+  PRAGMA journal_size_limit = 10000;
+  PRAGMA journal_mode = wal2;
+  PRAGMA wal_autocheckpoint = 0;
+  BEGIN;
+    CREATE TABLE t1(a);
+    INSERT INTO t1 VALUES( randomblob(8000) );
+  COMMIT;
+} {10000 wal2 0}
+
+do_test 7.1 {
+  list [file size test.db-wal] [file size test.db-wal2]
+} {9464 0}
+
+# Connection db2 is holding a PART1 lock. 
+#
+#   7.2.2: Test that the PART1 does not prevent db from switching to the
+#          other wal file.
+#
+#   7.2.3: Test that the PART1 does prevent a checkpoint of test.db-wal.
+#
+#   7.2.4: Test that after the PART1 is released the checkpoint is possible.
+#
+do_test 7.2.1 {
+  execsql {
+    BEGIN;
+      SELECT count(*) FROM t1;
+  } db2
+} {1}
+do_test 7.2.2 {
+  execsql {
+    INSERT INTO t1 VALUES( randomblob(800) );
+    INSERT INTO t1 VALUES( randomblob(800) );
+  }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {13656 3176 1024}
+do_test 7.2.3 {
+  execsql { PRAGMA wal_checkpoint }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {13656 3176 1024}
+do_test 7.2.4 {
+  execsql { END } db2
+  execsql { PRAGMA wal_checkpoint }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {13656 3176 11264}
+
+# Connection db2 is holding a PART2_FULL1 lock. 
+#
+#   7.3.2: Test that the lock does not prevent checkpointing.
+#
+#   7.3.3: Test that the lock does prevent the writer from overwriting 
+#          test.db-wal.
+#
+#   7.3.4: Test that after the PART2_FULL1 is released the writer can
+#          switch wal files and overwrite test.db-wal
+#
+db close
+db2 close
+sqlite3 db test.db
+sqlite3 db2 test.db
+do_test 7.3.1 {
+  execsql {
+    PRAGMA wal_autocheckpoint = 0;
+    PRAGMA journal_size_limit = 10000;
+    INSERT INTO t1 VALUES(randomblob(10000));
+    INSERT INTO t1 VALUES(randomblob(500));
+  }
+  execsql {
+    BEGIN;
+      SELECT count(*) FROM t1;
+  } db2
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 3176 11264}
+do_test 7.3.2 {
+  execsql { PRAGMA wal_checkpoint }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 3176 21504}
+do_test 7.3.3 {
+  execsql { 
+    INSERT INTO t1 VALUES(randomblob(10000));
+    INSERT INTO t1 VALUES(randomblob(500));
+  }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 18896 21504}
+do_test 7.3.4 {
+  execsql END db2
+  execsql { INSERT INTO t1 VALUES(randomblob(5000)); }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 18896 21504}
+
+# Connection db2 is holding a PART2 lock. 
+#
+#   7.4.2: Test that the lock does not prevent writer switching to test.db-wal.
+#
+#   7.3.3: Test that the lock does prevent checkpointing of test.db-wal2.
+#
+#   7.3.4: Test that after the PART2 is released test.db-wal2 can be
+#          checkpointed.
+#
+db close
+db2 close
+sqlite3 db test.db
+sqlite3 db2 test.db
+do_test 7.4.1 {
+  execsql {
+    PRAGMA wal_autocheckpoint = 0;
+    PRAGMA journal_size_limit = 10000;
+    INSERT INTO t1 VALUES(randomblob(10000));
+    INSERT INTO t1 VALUES(randomblob(10000));
+    PRAGMA wal_checkpoint;
+  }
+  execsql {
+    BEGIN;
+      SELECT count(*) FROM t1;
+  } db2
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 12608 44032}
+do_test 7.4.2 {
+  execsql { 
+    INSERT INTO t1 VALUES(randomblob(5000));
+  }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 12608 44032}
+do_test 7.4.3 {
+  execsql { PRAGMA wal_checkpoint }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 12608 44032}
+do_test 7.4.4 {
+  execsql END db2
+  execsql { PRAGMA wal_checkpoint }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 12608 54272}
+
+# Connection db2 is holding a PART1_FULL2 lock. 
+#
+#   7.5.2: Test that the lock does not prevent a checkpoint of test.db-wal2.
+#
+#   7.5.3: Test that the lock does prevent the writer from overwriting
+#          test.db-wal2.
+#
+#   7.5.4: Test that after the PART1_FULL2 lock is released, the writer
+#          can switch to test.db-wal2.
+#
+db close
+db2 close
+sqlite3 db test.db
+sqlite3 db2 test.db
+do_test 7.5.1 {
+  execsql {
+    PRAGMA wal_autocheckpoint = 0;
+    PRAGMA journal_size_limit = 10000;
+    INSERT INTO t1 VALUES(randomblob(10000));
+    INSERT INTO t1 VALUES(randomblob(10000));
+    PRAGMA wal_checkpoint;
+    INSERT INTO t1 VALUES(randomblob(5000));
+  }
+  execsql {
+    BEGIN;
+      SELECT count(*) FROM t1;
+  } db2
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 12608 64512}
+do_test 7.5.2 {
+  execsql { PRAGMA wal_checkpoint }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 12608 75776}
+do_test 7.5.3.1 {
+  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {14704 12608 75776}
+do_test 7.5.3.2 {
+  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {22040 12608 75776}
+do_test 7.5.4 {
+  execsql END db2
+  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {22040 12608 75776}
+
 
 finish_test
+

ADDED   test/wal2snapshot.test
Index: test/wal2snapshot.test
==================================================================
--- test/wal2snapshot.test
+++ test/wal2snapshot.test
@@ -0,0 +1,84 @@
+# 2018 December 5
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+set testprefix wal2snapshot
+ifcapable !wal {finish_test ; return }
+ifcapable !snapshot {finish_test; return}
+
+foreach {tn mode} {1 wal 2 wal2} {
+  reset_db
+  do_execsql_test $tn.1 "PRAGMA journal_mode = $mode" $mode
+
+  do_execsql_test $tn.2 {
+    CREATE TABLE t1(a, b);
+    INSERT INTO t1 VALUES(1, 2);
+    INSERT INTO t1 VALUES(3, 4);
+    BEGIN;
+  }
+
+  # Check that sqlite3_snapshot_get() is an error for a wal2 db.
+  #
+  if {$tn==1} {
+    do_test 1.3 {
+      set S [sqlite3_snapshot_get db main]
+      sqlite3_snapshot_free $S
+    } {}
+  } else {
+    do_test 2.3 {
+      list [catch { sqlite3_snapshot_get db main } msg] $msg
+    } {1 SQLITE_ERROR}
+  }
+  
+  # Check that sqlite3_snapshot_recover() is an error for a wal2 db.
+  #
+  do_execsql_test $tn.4 COMMIT
+  if {$tn==1} {
+    do_test 1.5 {
+      sqlite3_snapshot_recover db main
+    } {}
+  } else {
+    do_test 2.5 {
+      list [catch { sqlite3_snapshot_recover db main } msg] $msg
+    } {1 SQLITE_ERROR}
+  }
+ 
+  # Check that sqlite3_snapshot_open() is an error for a wal2 db.
+  #
+  if {$tn==1} {
+    do_test 1.6 {
+      execsql BEGIN
+      set SNAPSHOT [sqlite3_snapshot_get_blob db main]
+      sqlite3_snapshot_open_blob db main $SNAPSHOT
+      execsql COMMIT
+    } {}
+  } else {
+    do_test 2.6 {
+      execsql BEGIN
+      set res [
+        list [catch { sqlite3_snapshot_open_blob db main $SNAPSHOT } msg] $msg
+      ]
+      execsql COMMIT
+      set res
+    } {1 SQLITE_ERROR}
+  }
+}
+
+
+finish_test
+
+

Index: test/window1.test
==================================================================
--- test/window1.test
+++ test/window1.test
@@ -591,7 +591,34 @@
   SELECT a, rank() OVER(ORDER BY b) FROM t1
     INTERSECT 
   SELECT a, rank() OVER(ORDER BY b DESC) FROM t1;
 } {
 }
+
+# 2018-12-06
+# https://www.sqlite.org/src/info/f09fcd17810f65f7
+# Assertion fault when window functions are used.
+#
+# Root cause is the query flattener invoking sqlite3ExprDup() on
+# expressions that contain subqueries with window functions.  The
+# sqlite3ExprDup() routine is not making correctly initializing
+# Select.pWin field of the subqueries.
+#
+sqlite3 db :memory:
+do_execsql_test 14.0 {
+  SELECT * FROM(
+    SELECT * FROM (SELECT 1 AS c) WHERE c IN (
+        SELECT (row_number() OVER()) FROM (VALUES (0))
+    )
+  );
+} {1}
+do_execsql_test 14.1 {
+  CREATE TABLE t1(x); INSERT INTO t1(x) VALUES(12345);
+  CREATE TABLE t2(c); INSERT INTO t2(c) VALUES(1);
+  SELECT y, y+1, y+2 FROM (
+    SELECT c IN (
+      SELECT (row_number() OVER()) FROM t1
+    ) AS y FROM t2
+  );
+} {1 2 3}
 
 finish_test

ADDED   tool/index_usage.c
Index: tool/index_usage.c
==================================================================
--- tool/index_usage.c
+++ tool/index_usage.c
@@ -0,0 +1,164 @@
+/*
+** 2018-12-04
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** 
+** This file implements a utility program used to help determine which
+** indexes in a database schema are used and unused, and how often specific
+** indexes are used.
+*/
+#include "sqlite3.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+
+static void usage(const char *argv0){
+  printf("Usage: %s DATABASE LOG\n\n", argv0);
+  printf(
+    "DATABASE is an SQLite database against which various statements\n"
+    "have been run.  The SQL text is stored in LOG.  LOG is an SQLite\n"
+    "database with this schema:\n"
+    "\n"
+    "    CREATE TABLE sqllog(sql TEXT);\n"
+    "\n"
+    "This utility program analyzes statements contained in LOG and prints\n"
+    "a report showing how many times each index in DATABASE is used by the\n"
+    "statements in LOG.\n"
+    "\n"
+    "DATABASE only needs to contain the schema used by the statements in\n"
+    "LOG. The content can be removed from DATABASE.\n"
+  );
+  printf("\nAnalysis will be done by SQLite version %s dated %.20s\n"
+         "checkin number %.40s. Different versions\n"
+         "of SQLite might use different indexes.\n",
+         sqlite3_libversion(), sqlite3_sourceid(), sqlite3_sourceid()+21);
+  exit(1);
+}
+
+int main(int argc, char **argv){
+  sqlite3 *db = 0;          /* The main database */
+  sqlite3_stmt *pStmt = 0;  /* a query */
+  char *zSql;
+  int nErr = 0;
+  int rc;
+
+  if( argc!=3 ) usage(argv[0]);
+  rc = sqlite3_open_v2(argv[1], &db, SQLITE_OPEN_READONLY, 0);
+  if( rc ){
+    printf("Cannot open \"%s\" for reading: %s\n", argv[1], sqlite3_errmsg(db));
+    goto errorOut;
+  }
+  rc = sqlite3_prepare_v2(db, "SELECT * FROM sqlite_master", -1, &pStmt, 0);
+  if( rc ){
+    printf("Cannot read the schema from \"%s\" - %s\n", argv[1],
+           sqlite3_errmsg(db));
+    goto errorOut;
+  }
+  sqlite3_finalize(pStmt);
+  pStmt = 0;
+  rc = sqlite3_exec(db, 
+     "CREATE TABLE temp.idxu(\n"
+     "  tbl TEXT,\n"
+     "  idx TEXT,\n"
+     "  cnt INT,\n"
+     "  PRIMARY KEY(idx)\n"
+     ") WITHOUT ROWID;", 0, 0, 0);
+  if( rc ){
+    printf("Cannot create the result table - %s\n",
+           sqlite3_errmsg(db));
+    goto errorOut;
+  }
+  rc = sqlite3_exec(db,
+     "INSERT INTO temp.idxu(tbl,idx,cnt)"
+     " SELECT tbl_name, name, 0 FROM sqlite_master"
+     " WHERE type='index' AND sql IS NOT NULL", 0, 0, 0);
+
+  /* Open the LOG database */
+  zSql = sqlite3_mprintf("ATTACH %Q AS log", argv[2]);
+  rc = sqlite3_exec(db, zSql, 0, 0, 0);
+  sqlite3_free(zSql);
+  if( rc ){
+    printf("Cannot open the LOG database \"%s\" - %s\n",
+           argv[2], sqlite3_errmsg(db));
+    goto errorOut;
+  }
+  rc = sqlite3_prepare_v2(db, "SELECT sql, rowid FROM log.sqllog",
+                          -1, &pStmt, 0);
+  if( rc ){
+    printf("Cannot read the SQLLOG table in the LOG database \"%s\" - %s\n",
+           argv[2], sqlite3_errmsg(db));
+    goto errorOut;
+  }
+
+  /* Update the counts based on LOG */
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    const char *zLog = (const char*)sqlite3_column_text(pStmt, 0);
+    sqlite3_stmt *pS2;
+    if( zLog==0 ) continue;
+    zSql = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zLog);
+    rc = sqlite3_prepare_v2(db, zSql, -1, &pS2, 0);
+    sqlite3_free(zSql);
+    if( rc ){
+      printf("Cannot compile LOG entry %d (%s): %s\n",
+             sqlite3_column_int(pStmt, 1), zLog, sqlite3_errmsg(db));
+      nErr++;
+    }else{
+      while( sqlite3_step(pS2)==SQLITE_ROW ){
+        const char *zExplain = (const char*)sqlite3_column_text(pS2,3);
+        const char *z1, *z2;
+        int n;
+        /* printf("EXPLAIN: %s\n", zExplain); */
+        z1 = strstr(zExplain, " USING INDEX ");
+        if( z1==0 ) continue;
+        z1 += 13;
+        for(z2=z1+1; z2[1] && z2[1]!='('; z2++){}
+        n = z2 - z1;
+        zSql = sqlite3_mprintf(
+          "UPDATE temp.idxu SET cnt=cnt+1 WHERE idx='%.*q'", n, z1
+        );
+        /* printf("sql: %s\n", zSql); */
+        sqlite3_exec(db, zSql, 0, 0, 0);
+        sqlite3_free(zSql);
+      }
+    }
+    sqlite3_finalize(pS2);
+  }
+  sqlite3_finalize(pStmt);
+
+  /* Generate the report */
+  rc = sqlite3_prepare_v2(db,
+     "SELECT tbl, idx, cnt, "
+     "   (SELECT group_concat(name,',') FROM pragma_index_info(idx))"
+     " FROM temp.idxu, main.sqlite_master"
+     " WHERE temp.idxu.tbl=main.sqlite_master.tbl_name"
+     "   AND temp.idxu.idx=main.sqlite_master.name"
+     " ORDER BY cnt DESC, tbl, idx",
+     -1, &pStmt, 0);
+  if( rc ){
+    printf("Cannot query the result table - %s\n",
+           sqlite3_errmsg(db));
+    goto errorOut;
+  }
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    printf("%10d %s on %s(%s)\n", 
+       sqlite3_column_int(pStmt, 2),
+       sqlite3_column_text(pStmt, 1),
+       sqlite3_column_text(pStmt, 0),
+       sqlite3_column_text(pStmt, 3));
+  }
+  sqlite3_finalize(pStmt);
+  pStmt = 0;
+
+errorOut:
+  sqlite3_finalize(pStmt);
+  sqlite3_close(db);
+  return nErr;
+}

Index: tool/lemon.c
==================================================================
--- tool/lemon.c
+++ tool/lemon.c
@@ -4588,17 +4588,24 @@
 
   /* Generate code which executes whenever the parser stack overflows */
   tplt_print(out,lemp,lemp->overflow,&lineno);
   tplt_xfer(lemp->name,in,out,&lineno);
 
-  /* Generate the table of rule information
+  /* Generate the tables of rule information.  yyRuleInfoLhs[] and
+  ** yyRuleInfoNRhs[].
   **
   ** Note: This code depends on the fact that rules are number
   ** sequentually beginning with 0.
   */
   for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
-    fprintf(out,"  { %4d, %4d }, /* (%d) ",rp->lhs->index,-rp->nrhs,i);
+    fprintf(out,"  %4d,  /* (%d) ", rp->lhs->index, i);
+     rule_print(out, rp);
+    fprintf(out," */\n"); lineno++;
+  }
+  tplt_xfer(lemp->name,in,out,&lineno);
+  for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
+    fprintf(out,"  %3d,  /* (%d) ", -rp->nrhs, i);
     rule_print(out, rp);
     fprintf(out," */\n"); lineno++;
   }
   tplt_xfer(lemp->name,in,out,&lineno);
 

Index: tool/lempar.c
==================================================================
--- tool/lempar.c
+++ tool/lempar.c
@@ -684,17 +684,19 @@
   yytos->major = yyMajor;
   yytos->minor.yy0 = yyMinor;
   yyTraceShift(yypParser, yyNewState, "Shift");
 }
 
-/* The following table contains information about every rule that
-** is used during the reduce.
-*/
-static const struct {
-  YYCODETYPE lhs;       /* Symbol on the left-hand side of the rule */
-  signed char nrhs;     /* Negative of the number of RHS symbols in the rule */
-} yyRuleInfo[] = {
+/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side
+** of that rule */
+static const YYCODETYPE yyRuleInfoLhs[] = {
+%%
+};
+
+/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number
+** of symbols on the right-hand side of that rule. */
+static const signed char yyRuleInfoNRhs[] = {
 %%
 };
 
 static void yy_accept(yyParser*);  /* Forward Declaration */
 
@@ -723,11 +725,11 @@
   (void)yyLookahead;
   (void)yyLookaheadToken;
   yymsp = yypParser->yytos;
 #ifndef NDEBUG
   if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
-    yysize = yyRuleInfo[yyruleno].nrhs;
+    yysize = yyRuleInfoNRhs[yyruleno];
     if( yysize ){
       fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n",
         yyTracePrompt,
         yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno);
     }else{
@@ -738,11 +740,11 @@
 #endif /* NDEBUG */
 
   /* Check that the stack is large enough to grow by a single entry
   ** if the RHS of the rule is empty.  This ensures that there is room
   ** enough on the stack to push the LHS value */
-  if( yyRuleInfo[yyruleno].nrhs==0 ){
+  if( yyRuleInfoNRhs[yyruleno]==0 ){
 #ifdef YYTRACKMAXSTACKDEPTH
     if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
       yypParser->yyhwm++;
       assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack));
     }
@@ -780,13 +782,13 @@
   */
 /********** Begin reduce actions **********************************************/
 %%
 /********** End reduce actions ************************************************/
   };
-  assert( yyruleno<sizeof(yyRuleInfo)/sizeof(yyRuleInfo[0]) );
-  yygoto = yyRuleInfo[yyruleno].lhs;
-  yysize = yyRuleInfo[yyruleno].nrhs;
+  assert( yyruleno<sizeof(yyRuleInfoLhs)/sizeof(yyRuleInfoLhs[0]) );
+  yygoto = yyRuleInfoLhs[yyruleno];
+  yysize = yyRuleInfoNRhs[yyruleno];
   yyact = yy_find_reduce_action(yymsp[yysize].stateno,(YYCODETYPE)yygoto);
 
   /* There are no SHIFTREDUCE actions on nonterminals because the table
   ** generator has simplified them to pure REDUCE actions. */
   assert( !(yyact>YY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) );