Index: Makefile.in
==================================================================
--- Makefile.in
+++ Makefile.in
@@ -1317,10 +1317,13 @@
 	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)
 
 showshm$(TEXE):	$(TOP)/tool/showshm.c
 	$(LTLINK) -o $@ $(TOP)/tool/showshm.c
 
+index_usage$(TEXE): $(TOP)/tool/index_usage.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/tool/index_usage.c sqlite3.lo $(TLIBS)
+
 changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
 
 changesetfuzz$(TEXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/ext/session/changesetfuzz.c sqlite3.lo $(TLIBS)

Index: Makefile.msc
==================================================================
--- Makefile.msc
+++ Makefile.msc
@@ -2442,10 +2442,14 @@
 		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
 showshm.exe:	$(TOP)\tool\showshm.c
 	$(LTLINK) $(NO_WARN)	$(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS)
 
+index_usage.exe: $(TOP)\tool\index_usage.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
+		$(TOP)\tool\index_usage.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
 changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
 		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 

Index: autoconf/Makefile.msc
==================================================================
--- autoconf/Makefile.msc
+++ autoconf/Makefile.msc
@@ -281,10 +281,11 @@
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF
 
 # Should the session extension be enabled?  If so, add compilation options
@@ -935,10 +936,11 @@
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF
 
 
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.

Index: ext/fts3/fts3_unicode.c
==================================================================
--- ext/fts3/fts3_unicode.c
+++ ext/fts3/fts3_unicode.c
@@ -80,11 +80,11 @@
 typedef struct unicode_tokenizer unicode_tokenizer;
 typedef struct unicode_cursor unicode_cursor;
 
 struct unicode_tokenizer {
   sqlite3_tokenizer base;
-  int bRemoveDiacritic;
+  int eRemoveDiacritic;
   int nException;
   int *aiException;
 };
 
 struct unicode_cursor {
@@ -225,21 +225,24 @@
   int rc = SQLITE_OK;
 
   pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
   if( pNew==NULL ) return SQLITE_NOMEM;
   memset(pNew, 0, sizeof(unicode_tokenizer));
-  pNew->bRemoveDiacritic = 1;
+  pNew->eRemoveDiacritic = 1;
 
   for(i=0; rc==SQLITE_OK && i<nArg; i++){
     const char *z = azArg[i];
     int n = (int)strlen(z);
 
     if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
-      pNew->bRemoveDiacritic = 1;
+      pNew->eRemoveDiacritic = 1;
     }
     else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
-      pNew->bRemoveDiacritic = 0;
+      pNew->eRemoveDiacritic = 0;
+    }
+    else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){
+      pNew->eRemoveDiacritic = 2;
     }
     else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){
       rc = unicodeAddExceptions(pNew, 1, &z[11], n-11);
     }
     else if( n>=11 && memcmp("separators=", z, 11)==0 ){
@@ -348,11 +351,11 @@
       pCsr->nAlloc += 64;
     }
 
     /* Write the folded case of the last character read to the output */
     zEnd = z;
-    iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
+    iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic);
     if( iOut ){
       WRITE_UTF8(zOut, iOut);
     }
 
     /* If the cursor is not at EOF, read the next character */

Index: ext/fts3/fts3_unicode2.c
==================================================================
--- ext/fts3/fts3_unicode2.c
+++ ext/fts3/fts3_unicode2.c
@@ -157,36 +157,51 @@
 ** of the ASCII letter only. For example, if passed 235 - "LATIN
 ** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
 ** E"). The resuls of passing a codepoint that corresponds to an
 ** uppercase letter are undefined.
 */
-static int remove_diacritic(int c){
+static int remove_diacritic(int c, int bComplex){
   unsigned short aDia[] = {
         0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
      2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
      2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
      2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
-     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
-     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
-     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
-     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
-    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
-    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
-    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
-    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
-    62924, 63050, 63082, 63274, 63390, 
+     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
+     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
+     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
+     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
+     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
+    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
+    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
+    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
+    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
+    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
+    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
+    63182, 63242, 63274, 63310, 63368, 63390, 
   };
   char aChar[] = {
-    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
-    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
-    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
-    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
-    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
-    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
-    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
-    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
-    'e',  'i',  'o',  'u',  'y',  
+    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
+    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
+    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
+    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
+    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
+    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
+    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
+    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
+    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
+    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
+    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
+    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
+    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
+    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
+    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
+    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
+    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
+    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
+    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
+    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
+    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
   };
 
   unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
   int iRes = 0;
   int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
@@ -199,11 +214,12 @@
     }else{
       iHi = iTest-1;
     }
   }
   assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
 }
 
 
 /*
 ** Return true if the argument interpreted as a unicode codepoint
@@ -226,11 +242,11 @@
 ** Otherwise, return a copy of the argument.
 **
 ** The results are undefined if the value passed to this function
 ** is less than zero.
 */
-int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
+int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){
   /* Each entry in the following array defines a rule for folding a range
   ** of codepoints to lower case. The rule applies to a range of nRange
   ** codepoints starting at codepoint iCode.
   **
   ** If the least significant bit in flags is clear, then the rule applies
@@ -349,11 +365,13 @@
     if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
       ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
       assert( ret>0 );
     }
 
-    if( bRemoveDiacritic ) ret = remove_diacritic(ret);
+    if( eRemoveDiacritic ){
+      ret = remove_diacritic(ret, eRemoveDiacritic==2);
+    }
   }
   
   else if( c>=66560 && c<66600 ){
     ret = c + 40;
   }

Index: ext/fts3/unicode/mkunicode.tcl
==================================================================
--- ext/fts3/unicode/mkunicode.tcl
+++ ext/fts3/unicode/mkunicode.tcl
@@ -7,15 +7,16 @@
   set lRange [list]
 
   set nRange 1
   set iFirst  [lindex $map 0 0]
   set cPrev   [lindex $map 0 1]
+  set fPrev   [lindex $map 0 2]
 
   foreach m [lrange $map 1 end] {
-    foreach {i c} $m {}
+    foreach {i c f} $m {}
 
-    if {$cPrev == $c} {
+    if {$cPrev == $c && $fPrev==$f} {
       for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} {
         if {[info exists tl_lookup_table($j)]==0} break
       }
 
       if {$j==$i} {
@@ -27,27 +28,30 @@
       }
     }
 
     lappend lRange [list $iFirst $nRange]
     lappend aChar  $cPrev
+    lappend aFlag  $fPrev
 
     set iFirst $i
     set cPrev  $c
+    set fPrev  $f
     set nRange 1
   }
   lappend lRange [list $iFirst $nRange]
   lappend aChar $cPrev
+  lappend aFlag $fPrev
 
   puts "/*"
   puts "** If the argument is a codepoint corresponding to a lowercase letter"
   puts "** in the ASCII range with a diacritic added, return the codepoint"
   puts "** of the ASCII letter only. For example, if passed 235 - \"LATIN"
   puts "** SMALL LETTER E WITH DIAERESIS\" - return 65 (\"LATIN SMALL LETTER"
   puts "** E\"). The resuls of passing a codepoint that corresponds to an"
   puts "** uppercase letter are undefined."
   puts "*/"
-  puts "static int ${::remove_diacritic}(int c)\{"
+  puts "static int ${::remove_diacritic}(int c, int bComplex)\{"
   puts "  unsigned short aDia\[\] = \{"
   puts -nonewline "        0, "
   set i 1
   foreach r $lRange {
     foreach {iCode nRange} $r {}
@@ -58,17 +62,21 @@
     puts -nonewline ", "
   }
   puts ""
   puts "  \};"
   puts "  char aChar\[\] = \{"
-  puts -nonewline "    '\\0', "
+  puts -nonewline "    '\\0',      "
   set i 1
-  foreach c $aChar {
-    set str "'$c',  "
-    if {$c == ""} { set str "'\\0', " }
+  foreach c $aChar f $aFlag {
+    if { $f } {
+      set str "'$c'|0x80,  "
+    } else {
+      set str "'$c'|0x00,  "
+    }
+    if {$c == ""} { set str "'\\0',      " }
 
-    if {($i % 12)==0} {puts "" ; puts -nonewline "    " }
+    if {($i % 6)==0} {puts "" ; puts -nonewline "    " }
     incr i
     puts -nonewline "$str"
   }
   puts ""
   puts "  \};"
@@ -85,19 +93,21 @@
     }else{
       iHi = iTest-1;
     }
   }
   assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);}
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);}
   puts "\}"
 }
 
 proc print_isdiacritic {zFunc map} {
 
   set lCode [list]
   foreach m $map {
-    foreach {code char} $m {}
+    foreach {code char flag} $m {}
+    if {$flag} continue
     if {$code && $char == ""} { lappend lCode $code }
   }
   set lCode [lsort -integer $lCode]
   set iFirst [lindex $lCode 0]
   set iLast [lindex $lCode end]
@@ -470,11 +480,11 @@
   puts "** Otherwise, return a copy of the argument."
   puts "**"
   puts "** The results are undefined if the value passed to this function"
   puts "** is less than zero."
   puts "*/"
-  puts "int ${zFunc}\(int c, int bRemoveDiacritic)\{"
+  puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{"
 
   set liOff [tl_generate_ioff_table $lRecord]
   tl_print_table_header
   foreach entry $lRecord { 
     if {[tl_print_table_entry toggle $entry $liOff]} { 
@@ -514,11 +524,13 @@
     if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
       ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
       assert( ret>0 );
     }
 
-    if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);
+    if( eRemoveDiacritic ){
+      ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2);
+    }
   }
   }]
 
   foreach entry $lHigh {
     tl_print_if_entry $entry
@@ -603,14 +615,10 @@
   set caseS [categories_switch C S {c k m o}]
   set caseZ [categories_switch C Z {l p s}]
 
   set nCat [expr [llength [array names C]] + 1]
   puts [code {
-    int sqlite3Fts5UnicodeNCat(void) { 
-      return $nCat;
-    }
-
     int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
       aArray[0] = 1;
       switch( zCat[0] ){
         $caseC
         $caseL

Index: ext/fts3/unicode/parseunicode.tcl
==================================================================
--- ext/fts3/unicode/parseunicode.tcl
+++ ext/fts3/unicode/parseunicode.tcl
@@ -5,15 +5,27 @@
 # diacritical marks from a unicode string. Each mapping is itself a list
 # consisting of two elements - the unicode codepoint and the single ASCII
 # character that it should be replaced with, or an empty string if the 
 # codepoint should simply be removed from the input. Examples:
 #
-#   { 224 a  }     (replace codepoint 224 to "a")
-#   { 769 "" }     (remove codepoint 769 from input)
+#   { 224 a  0 }     (replace codepoint 224 to "a")
+#   { 769 "" 0 }     (remove codepoint 769 from input)
 #
 # Mappings are only returned for non-upper case codepoints. It is assumed
 # that the input has already been folded to lower case.
+#
+# The third value in the list is always either 0 or 1. 0 if the 
+# UnicodeData.txt file maps the codepoint to a single ASCII character and
+# a diacritic, or 1 if the mapping is indirect. For example, consider the 
+# two entries:
+#
+# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC
+# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8
+#
+# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a 
+# diacritic). The second is an indirect mapping, as it maps to the
+# first codepoint plus 0302 (a diacritic).
 #
 proc rd_load_unicodedata_text {zName} {
   global tl_lookup_table
 
   set fd [open $zName]
@@ -51,22 +63,33 @@
 
     set iCode  [expr "0x$code"]
     set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
     set iDia   [expr "0x[lindex $character_decomposition_mapping 1]"]
 
+    # Filter out upper-case characters, as they will be mapped to their
+    # lower-case equivalents before this data is used.
     if {[info exists tl_lookup_table($iCode)]} continue
+
+    # Check if this is an indirect mapping. If so, set bIndirect to true
+    # and change $iAscii to the indirectly mappped ASCII character.
+    set bIndirect 0
+    if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} {
+      set iAscii $mapping($iAscii)
+      set bIndirect 1
+    }
 
     if { ($iAscii >= 97 && $iAscii <= 122)
       || ($iAscii >= 65 && $iAscii <= 90)
     } {
-      lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
+      lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect]
+      set mapping($iCode) $iAscii
       set dia($iDia) 1
     }
   }
 
   foreach d [array names dia] {
-    lappend lRet [list $d ""]
+    lappend lRet [list $d "" 0]
   }
   set lRet [lsort -integer -index 0 $lRet]
 
   close $fd
   set lRet

Index: ext/fts5/fts5_tokenize.c
==================================================================
--- ext/fts5/fts5_tokenize.c
+++ ext/fts5/fts5_tokenize.c
@@ -232,16 +232,21 @@
 typedef struct Unicode61Tokenizer Unicode61Tokenizer;
 struct Unicode61Tokenizer {
   unsigned char aTokenChar[128];  /* ASCII range token characters */
   char *aFold;                    /* Buffer to fold text into */
   int nFold;                      /* Size of aFold[] in bytes */
-  int bRemoveDiacritic;           /* True if remove_diacritics=1 is set */
+  int eRemoveDiacritic;           /* True if remove_diacritics=1 is set */
   int nException;
   int *aiException;
 
   unsigned char aCategory[32];    /* True for token char categories */
 };
+
+/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */
+#define FTS5_REMOVE_DIACRITICS_NONE    0
+#define FTS5_REMOVE_DIACRITICS_SIMPLE  1
+#define FTS5_REMOVE_DIACRITICS_COMPLEX 2
 
 static int fts5UnicodeAddExceptions(
   Unicode61Tokenizer *p,          /* Tokenizer object */
   const char *z,                  /* Characters to treat as exceptions */
   int bTokenChars                 /* 1 for 'tokenchars', 0 for 'separators' */
@@ -359,11 +364,11 @@
     if( p ){
       const char *zCat = "L* N* Co";
       int i;
       memset(p, 0, sizeof(Unicode61Tokenizer));
 
-      p->bRemoveDiacritic = 1;
+      p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE;
       p->nFold = 64;
       p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
       if( p->aFold==0 ){
         rc = SQLITE_NOMEM;
       }
@@ -380,14 +385,19 @@
       }
 
       for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
         const char *zArg = azArg[i+1];
         if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
-          if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
+          if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
             rc = SQLITE_ERROR;
+          }else{
+            p->eRemoveDiacritic = (zArg[0] - '0');
+            assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE
+                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE
+                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX
+            );
           }
-          p->bRemoveDiacritic = (zArg[0]=='1');
         }else
         if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
           rc = fts5UnicodeAddExceptions(p, zArg, 1);
         }else
         if( 0==sqlite3_stricmp(azArg[i], "separators") ){
@@ -497,11 +507,11 @@
         /* An non-ascii-range character. Fold it into the output buffer if
         ** it is a token character, or break out of the loop if it is not. */
         READ_UTF8(zCsr, zTerm, iCode);
         if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){
  non_ascii_tokenchar:
-          iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic);
+          iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic);
           if( iCode ) WRITE_UTF8(zOut, iCode);
         }else{
           break;
         }
       }else if( a[*zCsr]==0 ){

Index: ext/fts5/fts5_unicode2.c
==================================================================
--- ext/fts5/fts5_unicode2.c
+++ ext/fts5/fts5_unicode2.c
@@ -26,36 +26,51 @@
 ** of the ASCII letter only. For example, if passed 235 - "LATIN
 ** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
 ** E"). The resuls of passing a codepoint that corresponds to an
 ** uppercase letter are undefined.
 */
-static int fts5_remove_diacritic(int c){
+static int fts5_remove_diacritic(int c, int bComplex){
   unsigned short aDia[] = {
         0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
      2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
      2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
      2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
-     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
-     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
-     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
-     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
-    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
-    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
-    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
-    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
-    62924, 63050, 63082, 63274, 63390, 
+     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
+     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
+     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
+     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
+     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
+    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
+    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
+    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
+    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
+    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
+    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
+    63182, 63242, 63274, 63310, 63368, 63390, 
   };
   char aChar[] = {
-    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
-    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
-    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
-    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
-    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
-    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
-    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
-    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
-    'e',  'i',  'o',  'u',  'y',  
+    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
+    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
+    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
+    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
+    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
+    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
+    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
+    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
+    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
+    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
+    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
+    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
+    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
+    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
+    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
+    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
+    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
+    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
+    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
+    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
+    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
   };
 
   unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
   int iRes = 0;
   int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
@@ -68,11 +83,12 @@
     }else{
       iHi = iTest-1;
     }
   }
   assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
 }
 
 
 /*
 ** Return true if the argument interpreted as a unicode codepoint
@@ -95,11 +111,11 @@
 ** Otherwise, return a copy of the argument.
 **
 ** The results are undefined if the value passed to this function
 ** is less than zero.
 */
-int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
+int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){
   /* Each entry in the following array defines a rule for folding a range
   ** of codepoints to lower case. The rule applies to a range of nRange
   ** codepoints starting at codepoint iCode.
   **
   ** If the least significant bit in flags is clear, then the rule applies
@@ -218,27 +234,22 @@
     if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
       ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
       assert( ret>0 );
     }
 
-    if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret);
+    if( eRemoveDiacritic ){
+      ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2);
+    }
   }
   
   else if( c>=66560 && c<66600 ){
     ret = c + 40;
   }
 
   return ret;
 }
 
-
-#if 0
-int sqlite3Fts5UnicodeNCat(void) { 
-  return 32;
-}
-#endif
-
 int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
   aArray[0] = 1;
   switch( zCat[0] ){
     case 'C':
           switch( zCat[1] ){
@@ -754,11 +765,10 @@
   int iTbl = 0;
   while( i<128 ){
     int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
     int n = (aFts5UnicodeData[iTbl] >> 5) + i;
     for(; i<128 && i<n; i++){
-      aAscii[i] = (u8)bToken;
+      aAscii[i] = bToken;
     }
     iTbl++;
   }
 }
-

Index: ext/fts5/test/fts5tokenizer.test
==================================================================
--- ext/fts5/test/fts5tokenizer.test
+++ ext/fts5/test/fts5tokenizer.test
@@ -187,11 +187,11 @@
 do_catchsql_test 6.2 {
   CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'unicode61 a b');
 } {1 {error in tokenizer constructor}}
 do_catchsql_test 6.3 {
   CREATE VIRTUAL TABLE a3 USING fts5(
-    x, y, tokenize = 'unicode61 remove_diacritics 2'
+    x, y, tokenize = 'unicode61 remove_diacritics 3'
   );
 } {1 {error in tokenizer constructor}}
 do_catchsql_test 6.4 {
   CREATE VIRTUAL TABLE a3 USING fts5(
     x, y, tokenize = 'unicode61 remove_diacritics 10'

ADDED   ext/fts5/test/fts5umlaut.test
Index: ext/fts5/test/fts5umlaut.test
==================================================================
--- /dev/null
+++ ext/fts5/test/fts5umlaut.test
@@ -0,0 +1,65 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5umlaut
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  CREATE VIRTUAL TABLE t2 USING fts5(
+      x, 
+      tokenize="unicode61 remove_diacritics 2"
+  );
+}
+
+foreach {tn q res1 res2} {
+  1 "Hà Nội"                  0 1
+  2 "Hà Noi"                  1 1
+  3 "Ha Noi"                  1 1
+  4 "Ha N\u1ed9i"             0 1
+  5 "Ha N\u006fi"             1 1
+  6 "Ha N\u006f\u0302i"       1 1
+  7 "Ha N\u006f\u0323\u0302i" 1 1
+} {
+  do_execsql_test 1.$tn.1 {
+    DELETE FROM t1;
+    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
+    SELECT count(*) FROM t1($q)
+  } $res1
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t1;
+    INSERT INTO t1(rowid, x) VALUES (1, $q);
+    SELECT count(*) FROM t1('Ha Noi')
+  } $res1
+
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t2;
+    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
+    SELECT count(*) FROM t2($q)
+  } $res2
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t2;
+    INSERT INTO t2(rowid, x) VALUES (1, $q);
+    SELECT count(*) FROM t2('Ha Noi')
+  } $res2
+}
+
+finish_test
+

Index: ext/fts5/test/fts5unicode3.test
==================================================================
--- ext/fts5/test/fts5unicode3.test
+++ ext/fts5/test/fts5unicode3.test
@@ -34,28 +34,30 @@
 foreach x [an_load_unicodedata_text $UD] {
   set aNotAlnum($x) 1
 }
 
 foreach {y} [rd_load_unicodedata_text $UD] {
-  foreach {code ascii} $y {}
+  foreach {code ascii f} $y {}
   if {$ascii==""} {
     set int 0
   } else {
     binary scan $ascii c int
   }
-  set aDiacritic($code) $int
+  set aDiacritic($code,$f) $int
+  if {$f==0} { set aDiacritic($code,1) $int }
 }
 
 proc tcl_fold {i {bRemoveDiacritic 0}} {
   global tl_lookup_table
   global aDiacritic
+  set f [expr $bRemoveDiacritic==2]
 
   if {[info exists tl_lookup_table($i)]} {
     set i $tl_lookup_table($i)
   }
-  if {$bRemoveDiacritic && [info exists aDiacritic($i)]} {
-    set i $aDiacritic($i)
+  if {$bRemoveDiacritic && [info exists aDiacritic($i,$f)]} {
+    set i $aDiacritic($i,$f)
   }
   expr $i
 }
 db func tcl_fold tcl_fold
 
@@ -83,19 +85,29 @@
     SELECT i+1 FROM ii WHERE i<100000
   )
   SELECT count(*), min(i) FROM ii WHERE fts5_fold(i)!=CAST(tcl_fold(i) AS int);
 } {0 {}}
 
-do_execsql_test 1.2 {
+do_execsql_test 1.2.1 {
   WITH ii(i) AS (
     SELECT -1
     UNION ALL
     SELECT i+1 FROM ii WHERE i<100000
   )
   SELECT count(*), min(i) FROM ii 
   WHERE fts5_fold(i,1)!=CAST(tcl_fold(i,1) AS int);
 } {0 {}}
+
+do_execsql_test 1.2.2 {
+  WITH ii(i) AS (
+    SELECT -1
+    UNION ALL
+    SELECT i+1 FROM ii WHERE i<100000
+  )
+  SELECT count(*), min(i) FROM ii 
+  WHERE fts5_fold(i,2)!=CAST(tcl_fold(i,2) AS int);
+} {0 {}}
 
 do_execsql_test 1.3 {
   WITH ii(i) AS (
     SELECT -1
     UNION ALL

Index: ext/rtree/sqlite3rtree.h
==================================================================
--- ext/rtree/sqlite3rtree.h
+++ ext/rtree/sqlite3rtree.h
@@ -94,11 +94,11 @@
   int iLevel;                       /* Level of current node or entry */
   int mxLevel;                      /* The largest iLevel value in the tree */
   sqlite3_int64 iRowid;             /* Rowid for current entry */
   sqlite3_rtree_dbl rParentScore;   /* Score of parent node */
   int eParentWithin;                /* Visibility of parent node */
-  int eWithin;                      /* OUT: Visiblity */
+  int eWithin;                      /* OUT: Visibility */
   sqlite3_rtree_dbl rScore;         /* OUT: Write the score here */
   /* The following fields are only available in 3.8.11 and later */
   sqlite3_value **apSqlParam;       /* Original SQL values of parameters */
 };
 

Index: main.mk
==================================================================
--- main.mk
+++ main.mk
@@ -997,10 +997,14 @@
 	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \
 		$(TOP)/tool/showwal.c sqlite3.o $(THREADLIB)
 
 showshm$(EXE):	$(TOP)/tool/showshm.c
 	$(TCC) -o showshm$(EXE) $(TOP)/tool/showshm.c
+
+index_usage$(EXE): $(TOP)/tool/index_usage.c sqlite3.o
+	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_DEPRECATED -o index_usage$(EXE) \
+		$(TOP)/tool/index_usage.c sqlite3.o $(THREADLIB)
 
 changeset$(EXE):	$(TOP)/ext/session/changeset.c sqlite3.o
 	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \
 		$(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB)
 

Index: src/alter.c
==================================================================
--- src/alter.c
+++ src/alter.c
@@ -777,19 +777,36 @@
       pCtx->nList++;
       break;
     }
   }
 }
+
+/*
+** Iterate through the Select objects that are part of WITH clauses attached
+** to select statement pSelect.
+*/
+static void renameWalkWith(Walker *pWalker, Select *pSelect){
+  if( pSelect->pWith ){
+    int i;
+    for(i=0; i<pSelect->pWith->nCte; i++){
+      Select *p = pSelect->pWith->a[i].pSelect;
+      NameContext sNC;
+      memset(&sNC, 0, sizeof(sNC));
+      sNC.pParse = pWalker->pParse;
+      sqlite3SelectPrep(sNC.pParse, p, &sNC);
+      sqlite3WalkSelect(pWalker, p);
+    }
+  }
+}
 
 /*
 ** This is a Walker select callback. It does nothing. It is only required
 ** because without a dummy callback, sqlite3WalkExpr() and similar do not
 ** descend into sub-select statements.
 */
 static int renameColumnSelectCb(Walker *pWalker, Select *p){
-  UNUSED_PARAMETER(pWalker);
-  UNUSED_PARAMETER(p);
+  renameWalkWith(pWalker, p);
   return WRC_Continue;
 }
 
 /*
 ** This is a Walker expression callback.
@@ -1362,10 +1379,11 @@
     struct SrcList_item *pItem = &pSrc->a[i];
     if( pItem->pTab==p->pTab ){
       renameTokenFind(pWalker->pParse, p, pItem->zName);
     }
   }
+  renameWalkWith(pWalker, pSelect);
 
   return WRC_Continue;
 }
 
 

Index: src/build.c
==================================================================
--- src/build.c
+++ src/build.c
@@ -225,11 +225,11 @@
   /* Get the VDBE program ready for execution
   */
   if( v && pParse->nErr==0 && !db->mallocFailed ){
     /* A minimum of one cursor is required if autoincrement is used
     *  See ticket [a696379c1f08866] */
-    if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1;
+    assert( pParse->pAinc==0 || pParse->nTab>0 );
     sqlite3VdbeMakeReady(v, pParse);
     pParse->rc = SQLITE_DONE;
   }else{
     pParse->rc = SQLITE_ERROR;
   }

Index: src/callback.c
==================================================================
--- src/callback.c
+++ src/callback.c
@@ -281,11 +281,11 @@
 
 /*
 ** Search a FuncDefHash for a function with the given name.  Return
 ** a pointer to the matching FuncDef if found, or 0 if there is no match.
 */
-static FuncDef *functionSearch(
+FuncDef *sqlite3FunctionSearch(
   int h,               /* Hash of the name */
   const char *zFunc    /* Name of function */
 ){
   FuncDef *p;
   for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){
@@ -293,25 +293,10 @@
       return p;
     }
   }
   return 0;
 }
-#ifdef SQLITE_ENABLE_NORMALIZE
-FuncDef *sqlite3FunctionSearchN(
-  int h,               /* Hash of the name */
-  const char *zFunc,   /* Name of function */
-  int nFunc            /* Length of the name */
-){
-  FuncDef *p;
-  for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){
-    if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 ){
-      return p;
-    }
-  }
-  return 0;
-}
-#endif /* SQLITE_ENABLE_NORMALIZE */
 
 /*
 ** Insert a new FuncDef into a FuncDefHash hash table.
 */
 void sqlite3InsertBuiltinFuncs(
@@ -323,11 +308,11 @@
     FuncDef *pOther;
     const char *zName = aDef[i].zName;
     int nName = sqlite3Strlen30(zName);
     int h = SQLITE_FUNC_HASH(zName[0], nName);
     assert( zName[0]>='a' && zName[0]<='z' );
-    pOther = functionSearch(h, zName);
+    pOther = sqlite3FunctionSearch(h, zName);
     if( pOther ){
       assert( pOther!=&aDef[i] && pOther->pNext!=&aDef[i] );
       aDef[i].pNext = pOther->pNext;
       pOther->pNext = &aDef[i];
     }else{
@@ -401,11 +386,11 @@
   ** So we must not search for built-ins when creating a new function.
   */ 
   if( !createFlag && (pBest==0 || (db->mDbFlags & DBFLAG_PreferBuiltin)!=0) ){
     bestScore = 0;
     h = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zName[0]], nName);
-    p = functionSearch(h, zName);
+    p = sqlite3FunctionSearch(h, zName);
     while( p ){
       int score = matchQuality(p, nArg, enc);
       if( score>bestScore ){
         pBest = p;
         bestScore = score;

Index: src/hash.c
==================================================================
--- src/hash.c
+++ src/hash.c
@@ -62,24 +62,10 @@
     h += sqlite3UpperToLower[c];
     h *= 0x9e3779b1;
   }
   return h;
 }
-#ifdef SQLITE_ENABLE_NORMALIZE
-static unsigned int strHashN(const char *z, int n){
-  unsigned int h = 0;
-  int i;
-  for(i=0; i<n; i++){
-    /* Knuth multiplicative hashing.  (Sorting & Searching, p. 510).
-    ** 0x9e3779b1 is 2654435761 which is the closest prime number to
-    ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */
-    h += sqlite3UpperToLower[z[i]];
-    h *= 0x9e3779b1;
-  }
-  return h;
-}
-#endif /* SQLITE_ENABLE_NORMALIZE */
 
 
 /* Link pNew element into the hash table pH.  If pEntry!=0 then also
 ** insert pNew into the pEntry hash bucket.
 */
@@ -187,44 +173,10 @@
     }
     elem = elem->next;
   }
   return &nullElement;
 }
-#ifdef SQLITE_ENABLE_NORMALIZE
-static HashElem *findElementWithHashN(
-  const Hash *pH,     /* The pH to be searched */
-  const char *pKey,   /* The key we are searching for */
-  int nKey,           /* Number of key bytes to use */
-  unsigned int *pHash /* Write the hash value here */
-){
-  HashElem *elem;                /* Used to loop thru the element list */
-  int count;                     /* Number of elements left to test */
-  unsigned int h;                /* The computed hash */
-  static HashElem nullElement = { 0, 0, 0, 0 };
-
-  if( pH->ht ){   /*OPTIMIZATION-IF-TRUE*/
-    struct _ht *pEntry;
-    h = strHashN(pKey, nKey) % pH->htsize;
-    pEntry = &pH->ht[h];
-    elem = pEntry->chain;
-    count = pEntry->count;
-  }else{
-    h = 0;
-    elem = pH->first;
-    count = pH->count;
-  }
-  if( pHash ) *pHash = h;
-  while( count-- ){
-    assert( elem!=0 );
-    if( sqlite3StrNICmp(elem->pKey,pKey,nKey)==0 ){ 
-      return elem;
-    }
-    elem = elem->next;
-  }
-  return &nullElement;
-}
-#endif /* SQLITE_ENABLE_NORMALIZE */
 
 /* Remove a single entry from the hash table given a pointer to that
 ** element and a hash on the element's key.
 */
 static void removeElementGivenHash(
@@ -265,18 +217,10 @@
 void *sqlite3HashFind(const Hash *pH, const char *pKey){
   assert( pH!=0 );
   assert( pKey!=0 );
   return findElementWithHash(pH, pKey, 0)->data;
 }
-#ifdef SQLITE_ENABLE_NORMALIZE
-void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey){
-  assert( pH!=0 );
-  assert( pKey!=0 );
-  assert( nKey>=0 );
-  return findElementWithHashN(pH, pKey, nKey, 0)->data;
-}
-#endif /* SQLITE_ENABLE_NORMALIZE */
 
 /* Insert an element into the hash table pH.  The key is pKey
 ** and the data is "data".
 **
 ** If no element exists with a matching key, then a new

Index: src/insert.c
==================================================================
--- src/insert.c
+++ src/insert.c
@@ -317,10 +317,11 @@
     aOp[5].p3 = memId;
     aOp[6].p1 = memId;
     aOp[7].p2 = memId+2;
     aOp[7].p1 = memId;
     aOp[10].p2 = memId;
+    if( pParse->nTab==0 ) pParse->nTab = 1;
   }
 }
 
 /*
 ** Update the maximum rowid for an autoincrement calculation.

Index: src/main.c
==================================================================
--- src/main.c
+++ src/main.c
@@ -2002,10 +2002,11 @@
   }
 #endif
   sqlite3_mutex_enter(db->mutex);
   pOld = db->pTraceArg;
   db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0;
+  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
   db->xTrace = (int(*)(u32,void*,void*,void*))xTrace;
   db->pTraceArg = pArg;
   sqlite3_mutex_leave(db->mutex);
   return pOld;
 }
@@ -2026,10 +2027,13 @@
 #endif
   sqlite3_mutex_enter(db->mutex);
   if( mTrace==0 ) xTrace = 0;
   if( xTrace==0 ) mTrace = 0;
   db->mTrace = mTrace;
+#ifndef SQLITE_OMIT_DEPRECATED
+  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
+#endif
   db->xTrace = xTrace;
   db->pTraceArg = pArg;
   sqlite3_mutex_leave(db->mutex);
   return SQLITE_OK;
 }
@@ -2058,10 +2062,12 @@
 #endif
   sqlite3_mutex_enter(db->mutex);
   pOld = db->pProfileArg;
   db->xProfile = xProfile;
   db->pProfileArg = pArg;
+  db->mTrace &= SQLITE_TRACE_NONLEGACY_MASK;
+  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
   sqlite3_mutex_leave(db->mutex);
   return pOld;
 }
 #endif /* SQLITE_OMIT_DEPRECATED */
 #endif /* SQLITE_OMIT_TRACE */

Index: src/parse.y
==================================================================
--- src/parse.y
+++ src/parse.y
@@ -662,10 +662,16 @@
         struct SrcList_item *pNew = &A->a[A->nSrc-1];
         struct SrcList_item *pOld = F->a;
         pNew->zName = pOld->zName;
         pNew->zDatabase = pOld->zDatabase;
         pNew->pSelect = pOld->pSelect;
+        if( pOld->fg.isTabFunc ){
+          pNew->u1.pFuncArg = pOld->u1.pFuncArg;
+          pOld->u1.pFuncArg = 0;
+          pOld->fg.isTabFunc = 0;
+          pNew->fg.isTabFunc = 1;
+        }
         pOld->zName = pOld->zDatabase = 0;
         pOld->pSelect = 0;
       }
       sqlite3SrcListDelete(pParse->db, F);
     }else{

Index: src/pcache1.c
==================================================================
--- src/pcache1.c
+++ src/pcache1.c
@@ -100,10 +100,11 @@
   u8 isAnchor;                   /* This is the PGroup.lru element */
   PgHdr1 *pNext;                 /* Next in hash table chain */
   PCache1 *pCache;               /* Cache that currently owns this page */
   PgHdr1 *pLruNext;              /* Next in LRU list of unpinned pages */
   PgHdr1 *pLruPrev;              /* Previous in LRU list of unpinned pages */
+                                 /* NB: pLruPrev is only valid if pLruNext!=0 */
 };
 
 /*
 ** A page is pinned if it is not on the LRU list.  To be "pinned" means
 ** that the page is in active use and must not be deallocated.
@@ -567,11 +568,12 @@
   assert( pPage->pLruPrev );
   assert( sqlite3_mutex_held(pPage->pCache->pGroup->mutex) );
   pPage->pLruPrev->pLruNext = pPage->pLruNext;
   pPage->pLruNext->pLruPrev = pPage->pLruPrev;
   pPage->pLruNext = 0;
-  pPage->pLruPrev = 0;
+  /* pPage->pLruPrev = 0;
+  ** No need to clear pLruPrev as it is never accessed if pLruNext is 0 */
   assert( pPage->isAnchor==0 );
   assert( pPage->pCache->pGroup->lru.isAnchor==1 );
   pPage->pCache->nRecyclable--;
   return pPage;
 }
@@ -905,12 +907,13 @@
     unsigned int h = iKey % pCache->nHash;
     pCache->nPage++;
     pPage->iKey = iKey;
     pPage->pNext = pCache->apHash[h];
     pPage->pCache = pCache;
-    pPage->pLruPrev = 0;
     pPage->pLruNext = 0;
+    /* pPage->pLruPrev = 0;
+    ** No need to clear pLruPrev since it is not accessed when pLruNext==0 */
     *(void **)pPage->page.pExtra = 0;
     pCache->apHash[h] = pPage;
     if( iKey>pCache->iMaxKey ){
       pCache->iMaxKey = iKey;
     }
@@ -1066,11 +1069,11 @@
   pcache1EnterMutex(pGroup);
 
   /* It is an error to call this function if the page is already 
   ** part of the PGroup LRU list.
   */
-  assert( pPage->pLruPrev==0 && pPage->pLruNext==0 );
+  assert( pPage->pLruNext==0 );
   assert( PAGE_IS_PINNED(pPage) );
 
   if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){
     pcache1RemoveFromHash(pPage, 1);
   }else{

Index: src/prepare.c
==================================================================
--- src/prepare.c
+++ src/prepare.c
@@ -729,31 +729,45 @@
   int i, j;           /* Database and column loop indexes. */
   Schema *pSchema;    /* Schema for current database. */
   Hash *pHash;        /* Hash table of tables for current database. */
   HashElem *e;        /* Hash element for hash table iteration. */
   Table *pTab;        /* Database table for columns being checked. */
+  char *zId;          /* Zero terminated name of the identifier */
+  char zSpace[50];    /* Static space for the zero-terminated name */
 
-  if( sqlite3IsRowidN(zToken, nToken) ){
-    return 1;
+  if( nToken<sizeof(zSpace) ){
+    memcpy(zSpace, zToken, nToken);
+    zSpace[nToken] = 0;
+    zId = zSpace;
+  }else{
+    zId = sqlite3DbStrNDup(db, zToken, nToken);
+    if( zId==0 ) return 0;
+  }
+  if( sqlite3IsRowid(zId) ){
+    bFound = 1;
+    goto done;
   }
   if( nToken>0 ){
     int hash = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zToken[0]], nToken);
-    if( sqlite3FunctionSearchN(hash, zToken, nToken) ) return 1;
+    if( sqlite3FunctionSearch(hash, zId) ){
+      bFound = 1;
+      goto done;
+    }
   }
   assert( db!=0 );
   sqlite3_mutex_enter(db->mutex);
   sqlite3BtreeEnterAll(db);
   for(i=0; i<db->nDb; i++){
     pHash = &db->aFunc;
-    if( sqlite3HashFindN(pHash, zToken, nToken) ){
+    if( sqlite3HashFind(pHash, zId) ){
       bFound = 1;
       break;
     }
     pSchema = db->aDb[i].pSchema;
     if( pSchema==0 ) continue;
     pHash = &pSchema->tblHash;
-    if( sqlite3HashFindN(pHash, zToken, nToken) ){
+    if( sqlite3HashFind(pHash, zId) ){
       bFound = 1;
       break;
     }
     for(e=sqliteHashFirst(pHash); e; e=sqliteHashNext(e)){
       pTab = sqliteHashData(e);
@@ -771,19 +785,20 @@
           *pRc = SQLITE_NOMEM_BKPT;
           bFound = 0;
           goto done;
         }
       }
-      if( pHash && sqlite3HashFindN(pHash, zToken, nToken) ){
+      if( pHash && sqlite3HashFind(pHash, zId) ){
         bFound = 1;
         goto done;
       }
     }
   }
 done:
   sqlite3BtreeLeaveAll(db);
   sqlite3_mutex_leave(db->mutex);
+  if( zId!=zSpace ) sqlite3DbFree(db, zId);
   return bFound;
 }
 
 /*
 ** Attempt to estimate the final output buffer size needed for the fully
@@ -792,12 +807,11 @@
 ** being expanded, etc).  This size returned is the total number of bytes
 ** including the NUL terminator.
 */
 static int estimateNormalizedSize(
   const char *zSql, /* The original SQL string */
-  int nSql,         /* Length of original SQL string */
-  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
+  int nSql          /* Length of original SQL string */
 ){
   int nOut = nSql + 4;
   const char *z = zSql;
   while( nOut<nSql*5 ){
     while( z[0]!=0 && z[0]!='I' && z[0]!='i' ){ z++; }
@@ -848,22 +862,18 @@
   }
   *piOut = j;
 }
 
 /*
-** Perform normalization of the SQL contained in the prepared statement and
-** store the result in the zNormSql field.  The schema for the associated
-** databases are consulted while performing the normalization in order to
-** determine if a token appears to be an identifier.  All identifiers are
-** left intact in the normalized SQL and all literals are replaced with a
-** single '?'.
+** Compute a normalization of the SQL given by zSql[0..nSql-1].  Return
+** the normalization in space obtained from sqlite3DbMalloc().  Or return
+** NULL if anything goes wrong or if zSql is NULL.
 */
-void sqlite3Normalize(
+char *sqlite3Normalize(
   Vdbe *pVdbe,      /* VM being reprepared */
   const char *zSql, /* The original SQL string */
-  int nSql,         /* Size of the input string in bytes */
-  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
+  int nSql          /* Size of the input string in bytes */
 ){
   sqlite3 *db;           /* Database handle. */
   char *z;               /* The output string */
   int nZ;                /* Size of the output string in bytes */
   int i;                 /* Next character to read from zSql[] */
@@ -874,15 +884,14 @@
   int nParen = 0;        /* Nesting level of parenthesis */
   Hash inHash;           /* Table of parenthesis levels to output index. */
 
   db = sqlite3VdbeDb(pVdbe);
   assert( db!=0 );
-  assert( pVdbe->zNormSql==0 );
-  if( zSql==0 ) return;
-  nZ = estimateNormalizedSize(zSql, nSql, prepFlags);
+  if( zSql==0 ) return 0;
+  nZ = estimateNormalizedSize(zSql, nSql);
   z = sqlite3DbMallocRawNN(db, nZ);
-  if( z==0 ) return;
+  if( z==0 ) goto normalizeError;
   sqlite3HashInit(&inHash);
   for(i=j=0; i<nSql && zSql[i]; i+=n){
     int flags = 0;
     if( tokenType!=TK_SPACE ) prevTokenType = tokenType;
     n = sqlite3GetTokenNormalized((unsigned char*)zSql+i, &tokenType, &flags);
@@ -889,13 +898,11 @@
     switch( tokenType ){
       case TK_SPACE: {
         break;
       }
       case TK_ILLEGAL: {
-        sqlite3DbFree(db, z);
-        sqlite3HashClear(&inHash);
-        return;
+        goto normalizeError;
       }
       case TK_STRING:
       case TK_INTEGER:
       case TK_FLOAT:
       case TK_VARIABLE:
@@ -972,15 +979,11 @@
             assert( nParen<nSql );
             sqlite3HashInsert(&inHash, zSql+nParen, 0);
           }
           if( flags&SQLITE_TOKEN_QUOTED ){ i2++; n2-=2; }
           if( shouldTreatAsIdentifier(db, zSql+i2, n2, &rc)==0 ){
-            if( rc!=SQLITE_OK ){
-              sqlite3DbFree(db, z);
-              sqlite3HashClear(&inHash);
-              return;
-            }
+            if( rc!=SQLITE_OK ) goto normalizeError;
             if( sqlite3_keyword_check(zSql+i2, n2)==0 ){
               z[j++] = '?';
               break;
             }
           }
@@ -993,12 +996,17 @@
   assert( j<nZ && "one" );
   while( j>0 && z[j-1]==' ' ){ j--; }
   if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
   z[j] = 0;
   assert( j<nZ && "two" );
-  pVdbe->zNormSql = z;
+  sqlite3HashClear(&inHash);
+  return z;
+
+normalizeError:
+  sqlite3DbFree(db, z);
   sqlite3HashClear(&inHash);
+  return 0;
 }
 #endif /* SQLITE_ENABLE_NORMALIZE */
 
 /*
 ** Rerun the compilation of a statement after a schema change.

Index: src/resolve.c
==================================================================
--- src/resolve.c
+++ src/resolve.c
@@ -78,11 +78,11 @@
   if( pDup!=0 ){
     if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery);
     if( pExpr->op==TK_COLLATE ){
       pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken);
     }
-    ExprSetProperty(pDup, EP_Alias);
+//    ExprSetProperty(pDup, EP_Alias);
 
     /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This 
     ** prevents ExprDelete() from deleting the Expr structure itself,
     ** allowing it to be repopulated by the memcpy() on the following line.
     ** The pExpr->u.zToken might point into memory that will be freed by the

Index: src/shell.c.in
==================================================================
--- src/shell.c.in
+++ src/shell.c.in
@@ -1005,10 +1005,11 @@
   u8 statsOn;            /* True to display memory stats before each finalize */
   u8 scanstatsOn;        /* True to display scan stats before each finalize */
   u8 openMode;           /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
   u8 doXdgOpen;          /* Invoke start/open/xdg-open in output_reset() */
   u8 nEqpLevel;          /* Depth of the EQP output graph */
+  u8 eTraceType;         /* SHELL_TRACE_* value for type of trace */
   unsigned mEqpLines;    /* Mask of veritical lines in the EQP output graph */
   int outCount;          /* Revert to stdout when reaching zero */
   int cnt;               /* Number of records displayed so far */
   FILE *out;             /* Write results here */
   FILE *traceOut;        /* Output for sqlite3_trace() */
@@ -1064,10 +1065,16 @@
 #define SHELL_OPEN_APPENDVFS   2      /* Use appendvfs */
 #define SHELL_OPEN_ZIPFILE     3      /* Use the zipfile virtual table */
 #define SHELL_OPEN_READONLY    4      /* Open a normal database read-only */
 #define SHELL_OPEN_DESERIALIZE 5      /* Open using sqlite3_deserialize() */
 
+/* Allowed values for ShellState.eTraceType
+*/
+#define SHELL_TRACE_PLAIN      0      /* Show input SQL text */
+#define SHELL_TRACE_EXPANDED   1      /* Show expanded SQL text */
+#define SHELL_TRACE_NORMALIZED 2      /* Show normalized SQL text */
+
 /*
 ** These are the allowed shellFlgs values
 */
 #define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
 #define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
@@ -3490,11 +3497,26 @@
 #endif
   ".tables ?TABLE?          List names of tables matching LIKE pattern TABLE",
   ".testcase NAME           Begin redirecting output to 'testcase-out.txt'",
   ".timeout MS              Try opening locked tables for MS milliseconds",
   ".timer on|off            Turn SQL timer on or off",
-  ".trace FILE|off          Output each SQL statement as it is run",
+#ifndef SQLITE_OMIT_TRACE
+  ".trace ?OPTIONS?         Output each SQL statement as it is run",
+  "    FILE                    Send output to FILE",
+  "    stdout                  Send output to stdout",
+  "    stderr                  Send output to stderr",
+  "    off                     Disable tracing",
+  "    --expanded              Expand query parameters",
+#ifdef SQLITE_ENABLE_NORMALIZE
+  "    --normalized            Normal the SQL statements",
+#endif
+  "    --plain                 Show SQL as it is input",
+  "    --stmt                  Trace statement execution (SQLITE_TRACE_STMT)",
+  "    --profile               Profile statements (SQLITE_TRACE_PROFILE)",
+  "    --row                   Trace each row (SQLITE_TRACE_ROW)",
+  "    --close                 Trace connection close (SQLITE_TRACE_CLOSE)",
+#endif /* SQLITE_OMIT_TRACE */
   ".vfsinfo ?AUX?           Information about the top-level VFS",
   ".vfslist                 List all available VFSes",
   ".vfsname ?AUX?           Print the name of the VFS stack",
   ".width NUM1 NUM2 ...     Set column widths for \"column\" mode",
   "     Negative values right-justify",
@@ -3997,28 +4019,64 @@
     }
   }
   return f;
 }
 
-#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
+#ifndef SQLITE_OMIT_TRACE
 /*
 ** A routine for handling output from sqlite3_trace().
 */
 static int sql_trace_callback(
-  unsigned mType,
-  void *pArg,
-  void *pP,
-  void *pX
+  unsigned mType,         /* The trace type */
+  void *pArg,             /* The ShellState pointer */
+  void *pP,               /* Usually a pointer to sqlite_stmt */
+  void *pX                /* Auxiliary output */
 ){
-  FILE *f = (FILE*)pArg;
-  UNUSED_PARAMETER(mType);
-  UNUSED_PARAMETER(pP);
-  if( f ){
-    const char *z = (const char*)pX;
-    int i = strlen30(z);
-    while( i>0 && z[i-1]==';' ){ i--; }
-    utf8_printf(f, "%.*s;\n", i, z);
+  ShellState *p = (ShellState*)pArg;
+  sqlite3_stmt *pStmt;
+  const char *zSql;
+  int nSql;
+  if( p->traceOut==0 ) return 0;
+  if( mType==SQLITE_TRACE_CLOSE ){
+    utf8_printf(p->traceOut, "-- closing database connection\n");
+    return 0;
+  }
+  if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){
+    zSql = (const char*)pX;
+  }else{
+    pStmt = (sqlite3_stmt*)pP;
+    switch( p->eTraceType ){
+      case SHELL_TRACE_EXPANDED: {
+        zSql = sqlite3_expanded_sql(pStmt);
+        break;
+      }
+#ifdef SQLITE_ENABLE_NORMALIZE
+      case SHELL_TRACE_NORMALIZED: {
+        zSql = sqlite3_normalized_sql(pStmt);
+        break;
+      }
+#endif
+      default: {
+        zSql = sqlite3_sql(pStmt);
+        break;
+      }
+    }
+  }
+  if( zSql==0 ) return 0;
+  nSql = strlen30(zSql);
+  while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; }
+  switch( mType ){
+    case SQLITE_TRACE_ROW:
+    case SQLITE_TRACE_STMT: {
+      utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql);
+      break;
+    }
+    case SQLITE_TRACE_PROFILE: {
+      sqlite3_int64 nNanosec = *(sqlite3_int64*)pX;
+      utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec);
+      break;
+    }
   }
   return 0;
 }
 #endif
 
@@ -7836,27 +7894,59 @@
       raw_printf(stderr, "Usage: .timer on|off\n");
       rc = 1;
     }
   }else
 
+#ifndef SQLITE_OMIT_TRACE
   if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){
+    int mType = 0;
+    int jj;
     open_db(p, 0);
-    if( nArg!=2 ){
-      raw_printf(stderr, "Usage: .trace FILE|off\n");
-      rc = 1;
-      goto meta_command_exit;
-    }
-    output_file_close(p->traceOut);
-    p->traceOut = output_file_open(azArg[1], 0);
-#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
+    for(jj=1; jj<nArg; jj++){
+      const char *z = azArg[jj];
+      if( z[0]=='-' ){
+        if( optionMatch(z, "expanded") ){
+          p->eTraceType = SHELL_TRACE_EXPANDED;
+        }
+#ifdef SQLITE_ENABLE_NORMALIZE
+        else if( optionMatch(z, "normalized") ){
+          p->eTraceType = SHELL_TRACE_NORMALIZED;
+        }
+#endif
+        else if( optionMatch(z, "plain") ){
+          p->eTraceType = SHELL_TRACE_PLAIN;
+        }
+        else if( optionMatch(z, "profile") ){
+          mType |= SQLITE_TRACE_PROFILE;
+        }
+        else if( optionMatch(z, "row") ){
+          mType |= SQLITE_TRACE_ROW;
+        }
+        else if( optionMatch(z, "stmt") ){
+          mType |= SQLITE_TRACE_STMT;
+        }
+        else if( optionMatch(z, "close") ){
+          mType |= SQLITE_TRACE_CLOSE;
+        }
+        else {
+          raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z);
+          rc = 1;
+          goto meta_command_exit;
+        }
+      }else{
+        output_file_close(p->traceOut);
+        p->traceOut = output_file_open(azArg[1], 0);
+      }
+    }
     if( p->traceOut==0 ){
       sqlite3_trace_v2(p->db, 0, 0, 0);
     }else{
-      sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut);
+      if( mType==0 ) mType = SQLITE_TRACE_STMT;
+      sqlite3_trace_v2(p->db, mType, sql_trace_callback, p);
     }
-#endif
   }else
+#endif /* !defined(SQLITE_OMIT_TRACE) */
 
 #if SQLITE_USER_AUTHENTICATION
   if( c=='u' && strncmp(azArg[0], "user", n)==0 ){
     if( nArg<2 ){
       raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n");

Index: src/sqliteInt.h
==================================================================
--- src/sqliteInt.h
+++ src/sqliteInt.h
@@ -1370,14 +1370,17 @@
 
 #ifndef SQLITE_OMIT_DEPRECATED
 /* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing
 ** in the style of sqlite3_trace()
 */
-#define SQLITE_TRACE_LEGACY  0x80
+#define SQLITE_TRACE_LEGACY          0x40     /* Use the legacy xTrace */
+#define SQLITE_TRACE_XPROFILE        0x80     /* Use the legacy xProfile */
 #else
-#define SQLITE_TRACE_LEGACY  0
+#define SQLITE_TRACE_LEGACY          0
+#define SQLITE_TRACE_XPROFILE        0
 #endif /* SQLITE_OMIT_DEPRECATED */
+#define SQLITE_TRACE_NONLEGACY_MASK  0x0f     /* Normal flags */
 
 
 /*
 ** Each database connection is an instance of the following structure.
 */
@@ -1432,12 +1435,14 @@
   int nVDestroy;                /* Number of active OP_VDestroy operations */
   int nExtension;               /* Number of loaded extensions */
   void **aExtension;            /* Array of shared library handles */
   int (*xTrace)(u32,void*,void*,void*);     /* Trace function */
   void *pTraceArg;                          /* Argument to the trace function */
+#ifndef SQLITE_OMIT_DEPRECATED
   void (*xProfile)(void*,const char*,u64);  /* Profiling function */
   void *pProfileArg;                        /* Argument to profile function */
+#endif
   void *pCommitArg;                 /* Argument to xCommitCallback() */
   int (*xCommitCallback)(void*);    /* Invoked at every commit. */
   void *pRollbackArg;               /* Argument to xRollbackCallback() */
   void (*xRollbackCallback)(void*); /* Invoked at every commit. */
   void *pUpdateArg;
@@ -4065,13 +4070,11 @@
 Expr *sqlite3ExprDup(sqlite3*,Expr*,int);
 ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int);
 SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int);
 IdList *sqlite3IdListDup(sqlite3*,IdList*);
 Select *sqlite3SelectDup(sqlite3*,Select*,int);
-#ifdef SQLITE_ENABLE_NORMALIZE
-FuncDef *sqlite3FunctionSearchN(int,const char*,int);
-#endif
+FuncDef *sqlite3FunctionSearch(int,const char*);
 void sqlite3InsertBuiltinFuncs(FuncDef*,int);
 FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8);
 void sqlite3RegisterBuiltinFunctions(void);
 void sqlite3RegisterDateTimeFunctions(void);
 void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*);
@@ -4433,11 +4436,11 @@
 sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*);
 int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
 int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
 void sqlite3ParserReset(Parse*);
 #ifdef SQLITE_ENABLE_NORMALIZE
-void sqlite3Normalize(Vdbe*, const char*, int, u8);
+char *sqlite3Normalize(Vdbe*, const char*, int);
 #endif
 int sqlite3Reprepare(Vdbe*);
 void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*);
 CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *);
 int sqlite3TempInMemory(const sqlite3*);

Index: src/vdbeapi.c
==================================================================
--- src/vdbeapi.c
+++ src/vdbeapi.c
@@ -63,18 +63,20 @@
 */
 static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){
   sqlite3_int64 iNow;
   sqlite3_int64 iElapse;
   assert( p->startTime>0 );
-  assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 );
+  assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 );
   assert( db->init.busy==0 );
   assert( p->zSql!=0 );
   sqlite3OsCurrentTimeInt64(db->pVfs, &iNow);
   iElapse = (iNow - p->startTime)*1000000;
+#ifndef SQLITE_OMIT_DEPRECATED  	
   if( db->xProfile ){
     db->xProfile(db->pProfileArg, p->zSql, iElapse);
   }
+#endif
   if( db->mTrace & SQLITE_TRACE_PROFILE ){
     db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse);
   }
   p->startTime = 0;
 }
@@ -616,11 +618,11 @@
     assert( db->nVdbeWrite>0 || db->autoCommit==0 
         || (db->nDeferredCons==0 && db->nDeferredImmCons==0)
     );
 
 #ifndef SQLITE_OMIT_TRACE
-    if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0)
+    if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0
         && !db->init.busy && p->zSql ){
       sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime);
     }else{
       assert( p->startTime==0 );
     }
@@ -643,20 +645,22 @@
     db->nVdbeExec++;
     rc = sqlite3VdbeExec(p);
     db->nVdbeExec--;
   }
 
+  if( rc!=SQLITE_ROW ){
 #ifndef SQLITE_OMIT_TRACE
-  /* If the statement completed successfully, invoke the profile callback */
-  if( rc!=SQLITE_ROW ) checkProfileCallback(db, p);
+    /* If the statement completed successfully, invoke the profile callback */
+    checkProfileCallback(db, p);
 #endif
 
-  if( rc==SQLITE_DONE && db->autoCommit ){
-    assert( p->rc==SQLITE_OK );
-    p->rc = doWalCallbacks(db);
-    if( p->rc!=SQLITE_OK ){
-      rc = SQLITE_ERROR;
+    if( rc==SQLITE_DONE && db->autoCommit ){
+      assert( p->rc==SQLITE_OK );
+      p->rc = doWalCallbacks(db);
+      if( p->rc!=SQLITE_OK ){
+        rc = SQLITE_ERROR;
+      }
     }
   }
 
   db->errCode = rc;
   if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){
@@ -1749,11 +1753,15 @@
 /*
 ** Return the normalized SQL associated with a prepared statement.
 */
 const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){
   Vdbe *p = (Vdbe *)pStmt;
-  return p ? p->zNormSql : 0;
+  if( p==0 ) return 0;
+  if( p->zNormSql==0 && p->zSql!=0 ){
+    p->zNormSql = sqlite3Normalize(p, p->zSql, sqlite3Strlen30(p->zSql));
+  }
+  return p->zNormSql;
 }
 #endif /* SQLITE_ENABLE_NORMALIZE */
 
 #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
 /*

Index: src/vdbeaux.c
==================================================================
--- src/vdbeaux.c
+++ src/vdbeaux.c
@@ -65,11 +65,11 @@
   assert( p->zSql==0 );
   p->zSql = sqlite3DbStrNDup(p->db, z, n);
 #ifdef SQLITE_ENABLE_NORMALIZE
   assert( p->zNormSql==0 );
   if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){
-    sqlite3Normalize(p, p->zSql, n, prepFlags);
+    p->zNormSql = sqlite3Normalize(p, p->zSql, n);
     assert( p->zNormSql!=0 || p->db->mallocFailed );
   }
 #endif
 }
 

Index: test/altertab2.test
==================================================================
--- test/altertab2.test
+++ test/altertab2.test
@@ -10,11 +10,11 @@
 #*************************************************************************
 #
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-set testprefix altertab
+set testprefix altertab2
 
 # If SQLITE_OMIT_ALTERTABLE is defined, omit this file.
 ifcapable !altertable {
   finish_test
   return
@@ -83,7 +83,63 @@
   {CREATE TABLE c1(x REFERENCES "p3")}
   {CREATE TABLE c2(x, FOREIGN KEY (x) REFERENCES "p3")}
   {CREATE TABLE c3(x, FOREIGN KEY (x) REFERENCES "p3"(a))}
 }
 
+#-------------------------------------------------------------------------
+# Table name in WITH clauses that are part of views or triggers.
+#
+foreach {tn schema} {
+  1 {
+    CREATE TABLE log_entry(col1, y);
+    CREATE INDEX i1 ON log_entry(col1);
+  }
+
+  2 {
+    CREATE TABLE t1(a, b, c);
+    CREATE TABLE t2(x);
+    CREATE TABLE log_entry(col1);
+    CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN
+      INSERT INTO t2 SELECT col1 FROM log_entry;
+    END;
+  }
+
+  3 {
+    CREATE TABLE t1(a, b, c);
+    CREATE TABLE t2(x);
+    CREATE TABLE log_entry(col1);
+    CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN
+      INSERT INTO t2
+        WITH xyz(x) AS (SELECT col1 FROM log_entry)
+        SELECT x FROM xyz;
+    END;
+  }
+
+  4 {
+    CREATE TABLE log_entry(col1);
+    CREATE VIEW ttt AS
+        WITH xyz(x) AS (SELECT col1 FROM log_entry)
+        SELECT x FROM xyz;
+  }
+} {
+  reset_db
+  do_execsql_test 3.$tn.1 $schema
+  set expect [db eval "SELECT sql FROM sqlite_master"]
+  set expect [string map {log_entry {"newname"}} $expect]
+
+  do_execsql_test 3.$tn.2 {
+    ALTER TABLE log_entry RENAME TO newname;
+    SELECT sql FROM sqlite_master;
+  } $expect
+
+  reset_db
+  do_execsql_test 3.$tn.3 $schema
+  set expect [db eval "SELECT sql FROM sqlite_master"]
+  set expect [string map {col1 newname} $expect]
+
+  do_execsql_test 3.$tn.4 {
+    ALTER TABLE log_entry RENAME col1 TO newname;
+    SELECT sql FROM sqlite_master;
+  } $expect
+}
 
 finish_test

ADDED   test/fts4umlaut.test
Index: test/fts4umlaut.test
==================================================================
--- /dev/null
+++ test/fts4umlaut.test
@@ -0,0 +1,65 @@
+# 2018 December 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts4umlaut
+
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  CREATE VIRTUAL TABLE t2 USING fts4(
+      x, 
+      tokenize=unicode61 "remove_diacritics=2"
+  );
+}
+
+foreach {tn q res1 res2} {
+  1 "Hà Nội"                  0 1
+  2 "Hà Noi"                  1 1
+  3 "Ha Noi"                  1 1
+  4 "Ha N\u1ed9i"             0 1
+  5 "Ha N\u006fi"             1 1
+  6 "Ha N\u006f\u0302i"       1 1
+  7 "Ha N\u006f\u0323\u0302i" 1 1
+} {
+  do_execsql_test 1.$tn.1 {
+    DELETE FROM t1;
+    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
+    SELECT count(*) FROM t1 WHERE t1 MATCH $q
+  } $res1
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t1;
+    INSERT INTO t1(rowid, x) VALUES (1, $q);
+    SELECT count(*) FROM t1 WHERE t1 MATCH 'Ha Noi'
+  } $res1
+
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t2;
+    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
+    SELECT count(*) FROM t2 WHERE t2 MATCH $q
+  } $res2
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t2;
+    INSERT INTO t2(rowid, x) VALUES (1, $q);
+    SELECT count(*) FROM t2 WHERE t2 MATCH 'Ha Noi'
+  } $res2
+}
+
+finish_test
+

Index: test/json101.test
==================================================================
--- test/json101.test
+++ test/json101.test
@@ -811,8 +811,25 @@
 } {$}
 do_execsql_test json-14.170 {
   SELECT fullkey FROM json_tree('null');
 } {$}
 
-
+# 2018-12-03
+# Make sure the table-valued functions contained within parentheses
+# work correctly.
+#
+# Bug reported via private email. See TH3 for more information.
+#
+do_execsql_test json-15.100 {
+  SELECT * FROM JSON_EACH('{"a":1, "b":2}');
+} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+do_execsql_test json-15.110 {
+  SELECT xyz.* FROM JSON_EACH('{"a":1, "b":2}') AS xyz;
+} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+do_execsql_test json-15.120 {
+  SELECT * FROM (JSON_EACH('{"a":1, "b":2}'));
+} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+do_execsql_test json-15.130 {
+  SELECT xyz.* FROM (JSON_EACH('{"a":1, "b":2}')) AS xyz;
+} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
 
 finish_test

Index: test/shell4.test
==================================================================
--- test/shell4.test
+++ test/shell4.test
@@ -105,18 +105,18 @@
        [regexp {Heap Usage} $res] \
        [regexp {Autoindex Inserts} $res]
 } {1 1 1}
 
 do_test shell4-2.1 {
-  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace"
-} {1 {Usage: .trace FILE|off}}
+  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace --unknown"
+} {1 {Unknown option "--unknown" on ".trace"}}
 do_test shell4-2.2 {
   catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace off\n.trace off\n"
 } {0 {}}
 do_test shell4-2.3 {
-  catchcmd ":memory:" ".trace stdout\n.trace\n.trace off\n.dump\n"
-} {/^1 {PRAGMA.*Usage:.*}$/}
+  catchcmd ":memory:" ".trace stdout\n.dump\n.trace off\n"
+} {/^0 {PRAGMA.*}$/}
 ifcapable trace {
 do_test shell4-2.4 {
   catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;"
 } {0 {CREATE TABLE t1(x);
 SELECT * FROM t1;}}

Index: test/tabfunc01.test
==================================================================
--- test/tabfunc01.test
+++ test/tabfunc01.test
@@ -122,10 +122,15 @@
 do_execsql_test tabfunc01-4.3 {
   ATTACH ':memory:' AS aux1;
   CREATE TABLE aux1.t1(a,b,c);
   SELECT * FROM aux1.generate_series(1,4)
 } {1 2 3 4}
+
+# 2018-12-03: Fix bug reported by by private email.
+do_execsql_test tabfunc01-4.4 {
+  SELECT * FROM (generate_series(1,5,2)) AS x LIMIT 10;
+} {1 3 5}
 
 # The next series of tests is verifying that virtual table are able
 # to optimize the IN operator, even on terms that are not marked "omit".
 # When the generate_series virtual table is compiled for the testfixture,
 # the special -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 option is used, which

ADDED   tool/index_usage.c
Index: tool/index_usage.c
==================================================================
--- /dev/null
+++ tool/index_usage.c
@@ -0,0 +1,164 @@
+/*
+** 2018-12-04
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** 
+** This file implements a utility program used to help determine which
+** indexes in a database schema are used and unused, and how often specific
+** indexes are used.
+*/
+#include "sqlite3.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+
+static void usage(const char *argv0){
+  printf("Usage: %s DATABASE LOG\n\n", argv0);
+  printf(
+    "DATABASE is an SQLite database against which various statements\n"
+    "have been run.  The SQL text is stored in LOG.  LOG is an SQLite\n"
+    "database with this schema:\n"
+    "\n"
+    "    CREATE TABLE sqllog(sql TEXT);\n"
+    "\n"
+    "This utility program analyzes statements contained in LOG and prints\n"
+    "a report showing how many times each index in DATABASE is used by the\n"
+    "statements in LOG.\n"
+    "\n"
+    "DATABASE only needs to contain the schema used by the statements in\n"
+    "LOG. The content can be removed from DATABASE.\n"
+  );
+  printf("\nAnalysis will be done by SQLite version %s dated %.20s\n"
+         "checkin number %.40s. Different versions\n"
+         "of SQLite might use different indexes.\n",
+         sqlite3_libversion(), sqlite3_sourceid(), sqlite3_sourceid()+21);
+  exit(1);
+}
+
+int main(int argc, char **argv){
+  sqlite3 *db = 0;          /* The main database */
+  sqlite3_stmt *pStmt = 0;  /* a query */
+  char *zSql;
+  int nErr = 0;
+  int rc;
+
+  if( argc!=3 ) usage(argv[0]);
+  rc = sqlite3_open_v2(argv[1], &db, SQLITE_OPEN_READONLY, 0);
+  if( rc ){
+    printf("Cannot open \"%s\" for reading: %s\n", argv[1], sqlite3_errmsg(db));
+    goto errorOut;
+  }
+  rc = sqlite3_prepare_v2(db, "SELECT * FROM sqlite_master", -1, &pStmt, 0);
+  if( rc ){
+    printf("Cannot read the schema from \"%s\" - %s\n", argv[1],
+           sqlite3_errmsg(db));
+    goto errorOut;
+  }
+  sqlite3_finalize(pStmt);
+  pStmt = 0;
+  rc = sqlite3_exec(db, 
+     "CREATE TABLE temp.idxu(\n"
+     "  tbl TEXT,\n"
+     "  idx TEXT,\n"
+     "  cnt INT,\n"
+     "  PRIMARY KEY(idx)\n"
+     ") WITHOUT ROWID;", 0, 0, 0);
+  if( rc ){
+    printf("Cannot create the result table - %s\n",
+           sqlite3_errmsg(db));
+    goto errorOut;
+  }
+  rc = sqlite3_exec(db,
+     "INSERT INTO temp.idxu(tbl,idx,cnt)"
+     " SELECT tbl_name, name, 0 FROM sqlite_master"
+     " WHERE type='index' AND sql IS NOT NULL", 0, 0, 0);
+
+  /* Open the LOG database */
+  zSql = sqlite3_mprintf("ATTACH %Q AS log", argv[2]);
+  rc = sqlite3_exec(db, zSql, 0, 0, 0);
+  sqlite3_free(zSql);
+  if( rc ){
+    printf("Cannot open the LOG database \"%s\" - %s\n",
+           argv[2], sqlite3_errmsg(db));
+    goto errorOut;
+  }
+  rc = sqlite3_prepare_v2(db, "SELECT sql, rowid FROM log.sqllog",
+                          -1, &pStmt, 0);
+  if( rc ){
+    printf("Cannot read the SQLLOG table in the LOG database \"%s\" - %s\n",
+           argv[2], sqlite3_errmsg(db));
+    goto errorOut;
+  }
+
+  /* Update the counts based on LOG */
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    const char *zLog = (const char*)sqlite3_column_text(pStmt, 0);
+    sqlite3_stmt *pS2;
+    if( zLog==0 ) continue;
+    zSql = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zLog);
+    rc = sqlite3_prepare_v2(db, zSql, -1, &pS2, 0);
+    sqlite3_free(zSql);
+    if( rc ){
+      printf("Cannot compile LOG entry %d (%s): %s\n",
+             sqlite3_column_int(pStmt, 1), zLog, sqlite3_errmsg(db));
+      nErr++;
+    }else{
+      while( sqlite3_step(pS2)==SQLITE_ROW ){
+        const char *zExplain = (const char*)sqlite3_column_text(pS2,3);
+        const char *z1, *z2;
+        int n;
+        /* printf("EXPLAIN: %s\n", zExplain); */
+        z1 = strstr(zExplain, " USING INDEX ");
+        if( z1==0 ) continue;
+        z1 += 13;
+        for(z2=z1+1; z2[1] && z2[1]!='('; z2++){}
+        n = z2 - z1;
+        zSql = sqlite3_mprintf(
+          "UPDATE temp.idxu SET cnt=cnt+1 WHERE idx='%.*q'", n, z1
+        );
+        /* printf("sql: %s\n", zSql); */
+        sqlite3_exec(db, zSql, 0, 0, 0);
+        sqlite3_free(zSql);
+      }
+    }
+    sqlite3_finalize(pS2);
+  }
+  sqlite3_finalize(pStmt);
+
+  /* Generate the report */
+  rc = sqlite3_prepare_v2(db,
+     "SELECT tbl, idx, cnt, "
+     "   (SELECT group_concat(name,',') FROM pragma_index_info(idx))"
+     " FROM temp.idxu, main.sqlite_master"
+     " WHERE temp.idxu.tbl=main.sqlite_master.tbl_name"
+     "   AND temp.idxu.idx=main.sqlite_master.name"
+     " ORDER BY cnt DESC, tbl, idx",
+     -1, &pStmt, 0);
+  if( rc ){
+    printf("Cannot query the result table - %s\n",
+           sqlite3_errmsg(db));
+    goto errorOut;
+  }
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    printf("%10d %s on %s(%s)\n", 
+       sqlite3_column_int(pStmt, 2),
+       sqlite3_column_text(pStmt, 1),
+       sqlite3_column_text(pStmt, 0),
+       sqlite3_column_text(pStmt, 3));
+  }
+  sqlite3_finalize(pStmt);
+  pStmt = 0;
+
+errorOut:
+  sqlite3_finalize(pStmt);
+  sqlite3_close(db);
+  return nErr;
+}

Index: tool/lemon.c
==================================================================
--- tool/lemon.c
+++ tool/lemon.c
@@ -4588,17 +4588,24 @@
 
   /* Generate code which executes whenever the parser stack overflows */
   tplt_print(out,lemp,lemp->overflow,&lineno);
   tplt_xfer(lemp->name,in,out,&lineno);
 
-  /* Generate the table of rule information
+  /* Generate the tables of rule information.  yyRuleInfoLhs[] and
+  ** yyRuleInfoNRhs[].
   **
   ** Note: This code depends on the fact that rules are number
   ** sequentually beginning with 0.
   */
   for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
-    fprintf(out,"  { %4d, %4d }, /* (%d) ",rp->lhs->index,-rp->nrhs,i);
+    fprintf(out,"  %4d,  /* (%d) ", rp->lhs->index, i);
+     rule_print(out, rp);
+    fprintf(out," */\n"); lineno++;
+  }
+  tplt_xfer(lemp->name,in,out,&lineno);
+  for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
+    fprintf(out,"  %3d,  /* (%d) ", -rp->nrhs, i);
     rule_print(out, rp);
     fprintf(out," */\n"); lineno++;
   }
   tplt_xfer(lemp->name,in,out,&lineno);
 

Index: tool/lempar.c
==================================================================
--- tool/lempar.c
+++ tool/lempar.c
@@ -684,17 +684,19 @@
   yytos->major = yyMajor;
   yytos->minor.yy0 = yyMinor;
   yyTraceShift(yypParser, yyNewState, "Shift");
 }
 
-/* The following table contains information about every rule that
-** is used during the reduce.
-*/
-static const struct {
-  YYCODETYPE lhs;       /* Symbol on the left-hand side of the rule */
-  signed char nrhs;     /* Negative of the number of RHS symbols in the rule */
-} yyRuleInfo[] = {
+/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side
+** of that rule */
+static const YYCODETYPE yyRuleInfoLhs[] = {
+%%
+};
+
+/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number
+** of symbols on the right-hand side of that rule. */
+static const signed char yyRuleInfoNRhs[] = {
 %%
 };
 
 static void yy_accept(yyParser*);  /* Forward Declaration */
 
@@ -723,11 +725,11 @@
   (void)yyLookahead;
   (void)yyLookaheadToken;
   yymsp = yypParser->yytos;
 #ifndef NDEBUG
   if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
-    yysize = yyRuleInfo[yyruleno].nrhs;
+    yysize = yyRuleInfoNRhs[yyruleno];
     if( yysize ){
       fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n",
         yyTracePrompt,
         yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno);
     }else{
@@ -738,11 +740,11 @@
 #endif /* NDEBUG */
 
   /* Check that the stack is large enough to grow by a single entry
   ** if the RHS of the rule is empty.  This ensures that there is room
   ** enough on the stack to push the LHS value */
-  if( yyRuleInfo[yyruleno].nrhs==0 ){
+  if( yyRuleInfoNRhs[yyruleno]==0 ){
 #ifdef YYTRACKMAXSTACKDEPTH
     if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
       yypParser->yyhwm++;
       assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack));
     }
@@ -780,13 +782,13 @@
   */
 /********** Begin reduce actions **********************************************/
 %%
 /********** End reduce actions ************************************************/
   };
-  assert( yyruleno<sizeof(yyRuleInfo)/sizeof(yyRuleInfo[0]) );
-  yygoto = yyRuleInfo[yyruleno].lhs;
-  yysize = yyRuleInfo[yyruleno].nrhs;
+  assert( yyruleno<sizeof(yyRuleInfoLhs)/sizeof(yyRuleInfoLhs[0]) );
+  yygoto = yyRuleInfoLhs[yyruleno];
+  yysize = yyRuleInfoNRhs[yyruleno];
   yyact = yy_find_reduce_action(yymsp[yysize].stateno,(YYCODETYPE)yygoto);
 
   /* There are no SHIFTREDUCE actions on nonterminals because the table
   ** generator has simplified them to pure REDUCE actions. */
   assert( !(yyact>YY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) );