/ Check-in [342c9538]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge enhancements from trunk, especially the sqlite3_normalized_sql() enhancement.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | begin-concurrent-pnu
Files: files | file ages | folders
SHA3-256: 342c9538d9c6a993ac0acaa6f74ad58886bcef7bb53783d053f9b24c131aec5d
User & Date: drh 2018-12-05 13:49:23
Context
2018-12-06
00:08
Merge ALTER TABLE and sqlite3_normalized_sql() bug fixes from trunk. check-in: 3793e5d5 user: drh tags: begin-concurrent-pnu
2018-12-05
13:49
Merge enhancements from trunk, especially the sqlite3_normalized_sql() enhancement. check-in: 342c9538 user: drh tags: begin-concurrent-pnu
13:44
Merge enhancements from trunk, especially the enhanced sqlite3_normalized_sql() interface. check-in: 47b73f6b user: drh tags: begin-concurrent
2018-12-03
18:24
Merge in all changes for release 3.26.0. check-in: 85fd92c7 user: drh tags: begin-concurrent-pnu
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

  1302   1302   
  1303   1303   showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
  1304   1304   	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)
  1305   1305   
  1306   1306   showshm$(TEXE):	$(TOP)/tool/showshm.c
  1307   1307   	$(LTLINK) -o $@ $(TOP)/tool/showshm.c
  1308   1308   
         1309  +index_usage$(TEXE): $(TOP)/tool/index_usage.c sqlite3.lo
         1310  +	$(LTLINK) -o $@ $(TOP)/tool/index_usage.c sqlite3.lo $(TLIBS)
         1311  +
  1309   1312   changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
  1310   1313   	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
  1311   1314   
  1312   1315   changesetfuzz$(TEXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.lo
  1313   1316   	$(LTLINK) -o $@ $(TOP)/ext/session/changesetfuzz.c sqlite3.lo $(TLIBS)
  1314   1317   
  1315   1318   rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo

Changes to Makefile.msc.

  2440   2440   showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
  2441   2441   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2442   2442   		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2443   2443   
  2444   2444   showshm.exe:	$(TOP)\tool\showshm.c
  2445   2445   	$(LTLINK) $(NO_WARN)	$(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS)
  2446   2446   
         2447  +index_usage.exe: $(TOP)\tool\index_usage.c $(SQLITE3C) $(SQLITE3H)
         2448  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
         2449  +		$(TOP)\tool\index_usage.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         2450  +
  2447   2451   changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
  2448   2452   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2449   2453   		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
  2450   2454   		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2451   2455   
  2452   2456   changesetfuzz.exe:	$(TOP)\ext\session\changesetfuzz.c $(SQLITE3C) $(SQLITE3H)
  2453   2457   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \

Changes to autoconf/Makefile.msc.

   279    279   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
   280    280   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
   281    281   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
   282    282   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
   283    283   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
   284    284   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
   285    285   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
          286  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
   286    287   !ENDIF
   287    288   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
   288    289   !ENDIF
   289    290   
   290    291   # Should the session extension be enabled?  If so, add compilation options
   291    292   # to enable it.
   292    293   #
................................................................................
   933    934   # Additional compiler options for the shell.  These are only effective
   934    935   # when the shell is not being dynamically linked.
   935    936   #
   936    937   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
   937    938   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
   938    939   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
   939    940   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
          941  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
   940    942   !ENDIF
   941    943   
   942    944   
   943    945   # This is the default Makefile target.  The objects listed here
   944    946   # are what get build when you type just "make" with no arguments.
   945    947   #
   946    948   core:	dll shell

Changes to ext/fts3/fts3_unicode.c.

    78     78   #endif /* ifndef SQLITE_AMALGAMATION */
    79     79   
    80     80   typedef struct unicode_tokenizer unicode_tokenizer;
    81     81   typedef struct unicode_cursor unicode_cursor;
    82     82   
    83     83   struct unicode_tokenizer {
    84     84     sqlite3_tokenizer base;
    85         -  int bRemoveDiacritic;
           85  +  int eRemoveDiacritic;
    86     86     int nException;
    87     87     int *aiException;
    88     88   };
    89     89   
    90     90   struct unicode_cursor {
    91     91     sqlite3_tokenizer_cursor base;
    92     92     const unsigned char *aInput;    /* Input text being tokenized */
................................................................................
   223    223     unicode_tokenizer *pNew;        /* New tokenizer object */
   224    224     int i;
   225    225     int rc = SQLITE_OK;
   226    226   
   227    227     pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
   228    228     if( pNew==NULL ) return SQLITE_NOMEM;
   229    229     memset(pNew, 0, sizeof(unicode_tokenizer));
   230         -  pNew->bRemoveDiacritic = 1;
          230  +  pNew->eRemoveDiacritic = 1;
   231    231   
   232    232     for(i=0; rc==SQLITE_OK && i<nArg; i++){
   233    233       const char *z = azArg[i];
   234    234       int n = (int)strlen(z);
   235    235   
   236    236       if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
   237         -      pNew->bRemoveDiacritic = 1;
          237  +      pNew->eRemoveDiacritic = 1;
   238    238       }
   239    239       else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
   240         -      pNew->bRemoveDiacritic = 0;
          240  +      pNew->eRemoveDiacritic = 0;
          241  +    }
          242  +    else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){
          243  +      pNew->eRemoveDiacritic = 2;
   241    244       }
   242    245       else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){
   243    246         rc = unicodeAddExceptions(pNew, 1, &z[11], n-11);
   244    247       }
   245    248       else if( n>=11 && memcmp("separators=", z, 11)==0 ){
   246    249         rc = unicodeAddExceptions(pNew, 0, &z[11], n-11);
   247    250       }
................................................................................
   346    349         zOut = &zNew[zOut - pCsr->zToken];
   347    350         pCsr->zToken = zNew;
   348    351         pCsr->nAlloc += 64;
   349    352       }
   350    353   
   351    354       /* Write the folded case of the last character read to the output */
   352    355       zEnd = z;
   353         -    iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
          356  +    iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic);
   354    357       if( iOut ){
   355    358         WRITE_UTF8(zOut, iOut);
   356    359       }
   357    360   
   358    361       /* If the cursor is not at EOF, read the next character */
   359    362       if( z>=zTerm ) break;
   360    363       READ_UTF8(z, zTerm, iCode);

Changes to ext/fts3/fts3_unicode2.c.

   155    155   ** If the argument is a codepoint corresponding to a lowercase letter
   156    156   ** in the ASCII range with a diacritic added, return the codepoint
   157    157   ** of the ASCII letter only. For example, if passed 235 - "LATIN
   158    158   ** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
   159    159   ** E"). The resuls of passing a codepoint that corresponds to an
   160    160   ** uppercase letter are undefined.
   161    161   */
   162         -static int remove_diacritic(int c){
          162  +static int remove_diacritic(int c, int bComplex){
   163    163     unsigned short aDia[] = {
   164    164           0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
   165    165        2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
   166    166        2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
   167    167        2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
   168         -     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
   169         -     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
   170         -     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
   171         -     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
   172         -    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
   173         -    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
   174         -    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
   175         -    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
   176         -    62924, 63050, 63082, 63274, 63390, 
          168  +     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
          169  +     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
          170  +     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
          171  +     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
          172  +     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
          173  +    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
          174  +    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
          175  +    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
          176  +    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
          177  +    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
          178  +    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
          179  +    63182, 63242, 63274, 63310, 63368, 63390, 
   177    180     };
   178    181     char aChar[] = {
   179         -    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
   180         -    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
   181         -    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
   182         -    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
   183         -    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
   184         -    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
   185         -    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
   186         -    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
   187         -    'e',  'i',  'o',  'u',  'y',  
          182  +    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
          183  +    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
          184  +    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
          185  +    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
          186  +    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
          187  +    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
          188  +    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
          189  +    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
          190  +    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
          191  +    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
          192  +    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
          193  +    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
          194  +    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
          195  +    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
          196  +    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
          197  +    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
          198  +    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
          199  +    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
          200  +    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
          201  +    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
          202  +    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
   188    203     };
   189    204   
   190    205     unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
   191    206     int iRes = 0;
   192    207     int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
   193    208     int iLo = 0;
   194    209     while( iHi>=iLo ){
................................................................................
   197    212         iRes = iTest;
   198    213         iLo = iTest+1;
   199    214       }else{
   200    215         iHi = iTest-1;
   201    216       }
   202    217     }
   203    218     assert( key>=aDia[iRes] );
   204         -  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
          219  +  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
          220  +  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
   205    221   }
   206    222   
   207    223   
   208    224   /*
   209    225   ** Return true if the argument interpreted as a unicode codepoint
   210    226   ** is a diacritical modifier character.
   211    227   */
................................................................................
   224    240   ** is an upper case character that has a lower case equivalent,
   225    241   ** return the codepoint corresponding to the lower case version.
   226    242   ** Otherwise, return a copy of the argument.
   227    243   **
   228    244   ** The results are undefined if the value passed to this function
   229    245   ** is less than zero.
   230    246   */
   231         -int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
          247  +int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){
   232    248     /* Each entry in the following array defines a rule for folding a range
   233    249     ** of codepoints to lower case. The rule applies to a range of nRange
   234    250     ** codepoints starting at codepoint iCode.
   235    251     **
   236    252     ** If the least significant bit in flags is clear, then the rule applies
   237    253     ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
   238    254     ** need to be folded). Or, if it is set, then the rule only applies to
................................................................................
   347    363       assert( iRes>=0 && c>=aEntry[iRes].iCode );
   348    364       p = &aEntry[iRes];
   349    365       if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
   350    366         ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
   351    367         assert( ret>0 );
   352    368       }
   353    369   
   354         -    if( bRemoveDiacritic ) ret = remove_diacritic(ret);
          370  +    if( eRemoveDiacritic ){
          371  +      ret = remove_diacritic(ret, eRemoveDiacritic==2);
          372  +    }
   355    373     }
   356    374     
   357    375     else if( c>=66560 && c<66600 ){
   358    376       ret = c + 40;
   359    377     }
   360    378   
   361    379     return ret;
   362    380   }
   363    381   #endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */
   364    382   #endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */

Changes to ext/fts3/unicode/mkunicode.tcl.

     5      5     global tl_lookup_table
     6      6     set aChar [list]
     7      7     set lRange [list]
     8      8   
     9      9     set nRange 1
    10     10     set iFirst  [lindex $map 0 0]
    11     11     set cPrev   [lindex $map 0 1]
           12  +  set fPrev   [lindex $map 0 2]
    12     13   
    13     14     foreach m [lrange $map 1 end] {
    14         -    foreach {i c} $m {}
           15  +    foreach {i c f} $m {}
    15     16   
    16         -    if {$cPrev == $c} {
           17  +    if {$cPrev == $c && $fPrev==$f} {
    17     18         for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} {
    18     19           if {[info exists tl_lookup_table($j)]==0} break
    19     20         }
    20     21   
    21     22         if {$j==$i} {
    22     23           set nNew [expr {(1 + $i - $iFirst)}]
    23     24           if {$nNew<=8} {
................................................................................
    25     26             continue
    26     27           }
    27     28         }
    28     29       }
    29     30   
    30     31       lappend lRange [list $iFirst $nRange]
    31     32       lappend aChar  $cPrev
           33  +    lappend aFlag  $fPrev
    32     34   
    33     35       set iFirst $i
    34     36       set cPrev  $c
           37  +    set fPrev  $f
    35     38       set nRange 1
    36     39     }
    37     40     lappend lRange [list $iFirst $nRange]
    38     41     lappend aChar $cPrev
           42  +  lappend aFlag $fPrev
    39     43   
    40     44     puts "/*"
    41     45     puts "** If the argument is a codepoint corresponding to a lowercase letter"
    42     46     puts "** in the ASCII range with a diacritic added, return the codepoint"
    43     47     puts "** of the ASCII letter only. For example, if passed 235 - \"LATIN"
    44     48     puts "** SMALL LETTER E WITH DIAERESIS\" - return 65 (\"LATIN SMALL LETTER"
    45     49     puts "** E\"). The resuls of passing a codepoint that corresponds to an"
    46     50     puts "** uppercase letter are undefined."
    47     51     puts "*/"
    48         -  puts "static int ${::remove_diacritic}(int c)\{"
           52  +  puts "static int ${::remove_diacritic}(int c, int bComplex)\{"
    49     53     puts "  unsigned short aDia\[\] = \{"
    50     54     puts -nonewline "        0, "
    51     55     set i 1
    52     56     foreach r $lRange {
    53     57       foreach {iCode nRange} $r {}
    54     58       if {($i % 8)==0} {puts "" ; puts -nonewline "    " }
    55     59       incr i
................................................................................
    56     60   
    57     61       puts -nonewline [format "%5d" [expr ($iCode<<3) + $nRange-1]]
    58     62       puts -nonewline ", "
    59     63     }
    60     64     puts ""
    61     65     puts "  \};"
    62     66     puts "  char aChar\[\] = \{"
    63         -  puts -nonewline "    '\\0', "
           67  +  puts -nonewline "    '\\0',      "
    64     68     set i 1
    65         -  foreach c $aChar {
    66         -    set str "'$c',  "
    67         -    if {$c == ""} { set str "'\\0', " }
           69  +  foreach c $aChar f $aFlag {
           70  +    if { $f } {
           71  +      set str "'$c'|0x80,  "
           72  +    } else {
           73  +      set str "'$c'|0x00,  "
           74  +    }
           75  +    if {$c == ""} { set str "'\\0',      " }
    68     76   
    69         -    if {($i % 12)==0} {puts "" ; puts -nonewline "    " }
           77  +    if {($i % 6)==0} {puts "" ; puts -nonewline "    " }
    70     78       incr i
    71     79       puts -nonewline "$str"
    72     80     }
    73     81     puts ""
    74     82     puts "  \};"
    75     83     puts {
    76     84     unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
................................................................................
    83     91         iRes = iTest;
    84     92         iLo = iTest+1;
    85     93       }else{
    86     94         iHi = iTest-1;
    87     95       }
    88     96     }
    89     97     assert( key>=aDia[iRes] );
    90         -  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);}
           98  +  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
           99  +  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);}
    91    100     puts "\}"
    92    101   }
    93    102   
    94    103   proc print_isdiacritic {zFunc map} {
    95    104   
    96    105     set lCode [list]
    97    106     foreach m $map {
    98         -    foreach {code char} $m {}
          107  +    foreach {code char flag} $m {}
          108  +    if {$flag} continue
    99    109       if {$code && $char == ""} { lappend lCode $code }
   100    110     }
   101    111     set lCode [lsort -integer $lCode]
   102    112     set iFirst [lindex $lCode 0]
   103    113     set iLast [lindex $lCode end]
   104    114   
   105    115     set i1 0
................................................................................
   468    478     puts "** is an upper case character that has a lower case equivalent,"
   469    479     puts "** return the codepoint corresponding to the lower case version."
   470    480     puts "** Otherwise, return a copy of the argument."
   471    481     puts "**"
   472    482     puts "** The results are undefined if the value passed to this function"
   473    483     puts "** is less than zero."
   474    484     puts "*/"
   475         -  puts "int ${zFunc}\(int c, int bRemoveDiacritic)\{"
          485  +  puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{"
   476    486   
   477    487     set liOff [tl_generate_ioff_table $lRecord]
   478    488     tl_print_table_header
   479    489     foreach entry $lRecord { 
   480    490       if {[tl_print_table_entry toggle $entry $liOff]} { 
   481    491         lappend lHigh $entry 
   482    492       } 
................................................................................
   512    522       assert( iRes>=0 && c>=aEntry[iRes].iCode );
   513    523       p = &aEntry[iRes];
   514    524       if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
   515    525         ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
   516    526         assert( ret>0 );
   517    527       }
   518    528   
   519         -    if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);
          529  +    if( eRemoveDiacritic ){
          530  +      ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2);
          531  +    }
   520    532     }
   521    533     }]
   522    534   
   523    535     foreach entry $lHigh {
   524    536       tl_print_if_entry $entry
   525    537     }
   526    538   
................................................................................
   601    613     set caseN [categories_switch C N {d l o}]
   602    614     set caseP [categories_switch C P {c d e f i o s}]
   603    615     set caseS [categories_switch C S {c k m o}]
   604    616     set caseZ [categories_switch C Z {l p s}]
   605    617   
   606    618     set nCat [expr [llength [array names C]] + 1]
   607    619     puts [code {
   608         -    int sqlite3Fts5UnicodeNCat(void) { 
   609         -      return $nCat;
   610         -    }
   611         -
   612    620       int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
   613    621         aArray[0] = 1;
   614    622         switch( zCat[0] ){
   615    623           $caseC
   616    624           $caseL
   617    625           $caseM
   618    626           $caseN

Changes to ext/fts3/unicode/parseunicode.tcl.

     3      3   # Parameter $zName must be a path to the file UnicodeData.txt. This command
     4      4   # reads the file and returns a list of mappings required to remove all
     5      5   # diacritical marks from a unicode string. Each mapping is itself a list
     6      6   # consisting of two elements - the unicode codepoint and the single ASCII
     7      7   # character that it should be replaced with, or an empty string if the 
     8      8   # codepoint should simply be removed from the input. Examples:
     9      9   #
    10         -#   { 224 a  }     (replace codepoint 224 to "a")
    11         -#   { 769 "" }     (remove codepoint 769 from input)
           10  +#   { 224 a  0 }     (replace codepoint 224 to "a")
           11  +#   { 769 "" 0 }     (remove codepoint 769 from input)
    12     12   #
    13     13   # Mappings are only returned for non-upper case codepoints. It is assumed
    14     14   # that the input has already been folded to lower case.
           15  +#
           16  +# The third value in the list is always either 0 or 1. 0 if the 
           17  +# UnicodeData.txt file maps the codepoint to a single ASCII character and
           18  +# a diacritic, or 1 if the mapping is indirect. For example, consider the 
           19  +# two entries:
           20  +#
           21  +# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC
           22  +# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8
           23  +#
           24  +# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a 
           25  +# diacritic). The second is an indirect mapping, as it maps to the
           26  +# first codepoint plus 0302 (a diacritic).
    15     27   #
    16     28   proc rd_load_unicodedata_text {zName} {
    17     29     global tl_lookup_table
    18     30   
    19     31     set fd [open $zName]
    20     32     set lField {
    21     33       code
................................................................................
    49     61         continue
    50     62       }
    51     63   
    52     64       set iCode  [expr "0x$code"]
    53     65       set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
    54     66       set iDia   [expr "0x[lindex $character_decomposition_mapping 1]"]
    55     67   
           68  +    # Filter out upper-case characters, as they will be mapped to their
           69  +    # lower-case equivalents before this data is used.
    56     70       if {[info exists tl_lookup_table($iCode)]} continue
           71  +
           72  +    # Check if this is an indirect mapping. If so, set bIndirect to true
           73  +    # and change $iAscii to the indirectly mappped ASCII character.
           74  +    set bIndirect 0
           75  +    if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} {
           76  +      set iAscii $mapping($iAscii)
           77  +      set bIndirect 1
           78  +    }
    57     79   
    58     80       if { ($iAscii >= 97 && $iAscii <= 122)
    59     81         || ($iAscii >= 65 && $iAscii <= 90)
    60     82       } {
    61         -      lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
           83  +      lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect]
           84  +      set mapping($iCode) $iAscii
    62     85         set dia($iDia) 1
    63     86       }
    64     87     }
    65     88   
    66     89     foreach d [array names dia] {
    67         -    lappend lRet [list $d ""]
           90  +    lappend lRet [list $d "" 0]
    68     91     }
    69     92     set lRet [lsort -integer -index 0 $lRet]
    70     93   
    71     94     close $fd
    72     95     set lRet
    73     96   }
    74     97   

Changes to ext/fts5/fts5_tokenize.c.

   230    230   #endif /* ifndef SQLITE_AMALGAMATION */
   231    231   
   232    232   typedef struct Unicode61Tokenizer Unicode61Tokenizer;
   233    233   struct Unicode61Tokenizer {
   234    234     unsigned char aTokenChar[128];  /* ASCII range token characters */
   235    235     char *aFold;                    /* Buffer to fold text into */
   236    236     int nFold;                      /* Size of aFold[] in bytes */
   237         -  int bRemoveDiacritic;           /* True if remove_diacritics=1 is set */
          237  +  int eRemoveDiacritic;           /* True if remove_diacritics=1 is set */
   238    238     int nException;
   239    239     int *aiException;
   240    240   
   241    241     unsigned char aCategory[32];    /* True for token char categories */
   242    242   };
          243  +
          244  +/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */
          245  +#define FTS5_REMOVE_DIACRITICS_NONE    0
          246  +#define FTS5_REMOVE_DIACRITICS_SIMPLE  1
          247  +#define FTS5_REMOVE_DIACRITICS_COMPLEX 2
   243    248   
   244    249   static int fts5UnicodeAddExceptions(
   245    250     Unicode61Tokenizer *p,          /* Tokenizer object */
   246    251     const char *z,                  /* Characters to treat as exceptions */
   247    252     int bTokenChars                 /* 1 for 'tokenchars', 0 for 'separators' */
   248    253   ){
   249    254     int rc = SQLITE_OK;
................................................................................
   357    362     }else{
   358    363       p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
   359    364       if( p ){
   360    365         const char *zCat = "L* N* Co";
   361    366         int i;
   362    367         memset(p, 0, sizeof(Unicode61Tokenizer));
   363    368   
   364         -      p->bRemoveDiacritic = 1;
          369  +      p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE;
   365    370         p->nFold = 64;
   366    371         p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
   367    372         if( p->aFold==0 ){
   368    373           rc = SQLITE_NOMEM;
   369    374         }
   370    375   
   371    376         /* Search for a "categories" argument */
................................................................................
   378    383         if( rc==SQLITE_OK ){
   379    384           rc = unicodeSetCategories(p, zCat);
   380    385         }
   381    386   
   382    387         for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
   383    388           const char *zArg = azArg[i+1];
   384    389           if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
   385         -          if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
          390  +          if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
   386    391               rc = SQLITE_ERROR;
          392  +          }else{
          393  +            p->eRemoveDiacritic = (zArg[0] - '0');
          394  +            assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE
          395  +                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE
          396  +                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX
          397  +            );
   387    398             }
   388         -          p->bRemoveDiacritic = (zArg[0]=='1');
   389    399           }else
   390    400           if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
   391    401             rc = fts5UnicodeAddExceptions(p, zArg, 1);
   392    402           }else
   393    403           if( 0==sqlite3_stricmp(azArg[i], "separators") ){
   394    404             rc = fts5UnicodeAddExceptions(p, zArg, 0);
   395    405           }else
................................................................................
   495    505   
   496    506         if( *zCsr & 0x80 ){
   497    507           /* An non-ascii-range character. Fold it into the output buffer if
   498    508           ** it is a token character, or break out of the loop if it is not. */
   499    509           READ_UTF8(zCsr, zTerm, iCode);
   500    510           if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){
   501    511    non_ascii_tokenchar:
   502         -          iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic);
          512  +          iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic);
   503    513             if( iCode ) WRITE_UTF8(zOut, iCode);
   504    514           }else{
   505    515             break;
   506    516           }
   507    517         }else if( a[*zCsr]==0 ){
   508    518           /* An ascii-range separator character. End of token. */
   509    519           break; 

Changes to ext/fts5/fts5_unicode2.c.

    24     24   ** If the argument is a codepoint corresponding to a lowercase letter
    25     25   ** in the ASCII range with a diacritic added, return the codepoint
    26     26   ** of the ASCII letter only. For example, if passed 235 - "LATIN
    27     27   ** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
    28     28   ** E"). The resuls of passing a codepoint that corresponds to an
    29     29   ** uppercase letter are undefined.
    30     30   */
    31         -static int fts5_remove_diacritic(int c){
           31  +static int fts5_remove_diacritic(int c, int bComplex){
    32     32     unsigned short aDia[] = {
    33     33           0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
    34     34        2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
    35     35        2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
    36     36        2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
    37         -     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
    38         -     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
    39         -     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
    40         -     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
    41         -    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
    42         -    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
    43         -    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
    44         -    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
    45         -    62924, 63050, 63082, 63274, 63390, 
           37  +     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
           38  +     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
           39  +     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
           40  +     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
           41  +     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
           42  +    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
           43  +    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
           44  +    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
           45  +    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
           46  +    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
           47  +    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
           48  +    63182, 63242, 63274, 63310, 63368, 63390, 
    46     49     };
    47     50     char aChar[] = {
    48         -    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
    49         -    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
    50         -    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
    51         -    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
    52         -    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
    53         -    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
    54         -    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
    55         -    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
    56         -    'e',  'i',  'o',  'u',  'y',  
           51  +    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
           52  +    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
           53  +    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
           54  +    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
           55  +    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
           56  +    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
           57  +    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
           58  +    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
           59  +    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
           60  +    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
           61  +    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
           62  +    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
           63  +    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
           64  +    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
           65  +    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
           66  +    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
           67  +    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
           68  +    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
           69  +    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
           70  +    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
           71  +    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
    57     72     };
    58     73   
    59     74     unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
    60     75     int iRes = 0;
    61     76     int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
    62     77     int iLo = 0;
    63     78     while( iHi>=iLo ){
................................................................................
    66     81         iRes = iTest;
    67     82         iLo = iTest+1;
    68     83       }else{
    69     84         iHi = iTest-1;
    70     85       }
    71     86     }
    72     87     assert( key>=aDia[iRes] );
    73         -  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
           88  +  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
           89  +  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
    74     90   }
    75     91   
    76     92   
    77     93   /*
    78     94   ** Return true if the argument interpreted as a unicode codepoint
    79     95   ** is a diacritical modifier character.
    80     96   */
................................................................................
    93    109   ** is an upper case character that has a lower case equivalent,
    94    110   ** return the codepoint corresponding to the lower case version.
    95    111   ** Otherwise, return a copy of the argument.
    96    112   **
    97    113   ** The results are undefined if the value passed to this function
    98    114   ** is less than zero.
    99    115   */
   100         -int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
          116  +int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){
   101    117     /* Each entry in the following array defines a rule for folding a range
   102    118     ** of codepoints to lower case. The rule applies to a range of nRange
   103    119     ** codepoints starting at codepoint iCode.
   104    120     **
   105    121     ** If the least significant bit in flags is clear, then the rule applies
   106    122     ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
   107    123     ** need to be folded). Or, if it is set, then the rule only applies to
................................................................................
   216    232       assert( iRes>=0 && c>=aEntry[iRes].iCode );
   217    233       p = &aEntry[iRes];
   218    234       if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
   219    235         ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
   220    236         assert( ret>0 );
   221    237       }
   222    238   
   223         -    if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret);
          239  +    if( eRemoveDiacritic ){
          240  +      ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2);
          241  +    }
   224    242     }
   225    243     
   226    244     else if( c>=66560 && c<66600 ){
   227    245       ret = c + 40;
   228    246     }
   229    247   
   230    248     return ret;
   231    249   }
   232    250   
   233         -
   234         -#if 0
   235         -int sqlite3Fts5UnicodeNCat(void) { 
   236         -  return 32;
   237         -}
   238         -#endif
   239         -
   240    251   int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
   241    252     aArray[0] = 1;
   242    253     switch( zCat[0] ){
   243    254       case 'C':
   244    255             switch( zCat[1] ){
   245    256               case 'c': aArray[1] = 1; break;
   246    257               case 'f': aArray[2] = 1; break;
................................................................................
   752    763   void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
   753    764     int i = 0;
   754    765     int iTbl = 0;
   755    766     while( i<128 ){
   756    767       int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
   757    768       int n = (aFts5UnicodeData[iTbl] >> 5) + i;
   758    769       for(; i<128 && i<n; i++){
   759         -      aAscii[i] = (u8)bToken;
          770  +      aAscii[i] = bToken;
   760    771       }
   761    772       iTbl++;
   762    773     }
   763    774   }
   764         -

Changes to ext/fts5/test/fts5tokenizer.test.

   185    185     CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'unicode61 tokenchars');
   186    186   } {1 {error in tokenizer constructor}}
   187    187   do_catchsql_test 6.2 {
   188    188     CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'unicode61 a b');
   189    189   } {1 {error in tokenizer constructor}}
   190    190   do_catchsql_test 6.3 {
   191    191     CREATE VIRTUAL TABLE a3 USING fts5(
   192         -    x, y, tokenize = 'unicode61 remove_diacritics 2'
          192  +    x, y, tokenize = 'unicode61 remove_diacritics 3'
   193    193     );
   194    194   } {1 {error in tokenizer constructor}}
   195    195   do_catchsql_test 6.4 {
   196    196     CREATE VIRTUAL TABLE a3 USING fts5(
   197    197       x, y, tokenize = 'unicode61 remove_diacritics 10'
   198    198     );
   199    199   } {1 {error in tokenizer constructor}}

Added ext/fts5/test/fts5umlaut.test.

            1  +# 2014 June 17
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5umlaut
           17  +
           18  +# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
           19  +ifcapable !fts5 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +do_execsql_test 1.0 {
           25  +  CREATE VIRTUAL TABLE t1 USING fts5(x);
           26  +  CREATE VIRTUAL TABLE t2 USING fts5(
           27  +      x, 
           28  +      tokenize="unicode61 remove_diacritics 2"
           29  +  );
           30  +}
           31  +
           32  +foreach {tn q res1 res2} {
           33  +  1 "Hà Nội"                  0 1
           34  +  2 "Hà Noi"                  1 1
           35  +  3 "Ha Noi"                  1 1
           36  +  4 "Ha N\u1ed9i"             0 1
           37  +  5 "Ha N\u006fi"             1 1
           38  +  6 "Ha N\u006f\u0302i"       1 1
           39  +  7 "Ha N\u006f\u0323\u0302i" 1 1
           40  +} {
           41  +  do_execsql_test 1.$tn.1 {
           42  +    DELETE FROM t1;
           43  +    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
           44  +    SELECT count(*) FROM t1($q)
           45  +  } $res1
           46  +  do_execsql_test 1.$tn.2 {
           47  +    DELETE FROM t1;
           48  +    INSERT INTO t1(rowid, x) VALUES (1, $q);
           49  +    SELECT count(*) FROM t1('Ha Noi')
           50  +  } $res1
           51  +
           52  +  do_execsql_test 1.$tn.2 {
           53  +    DELETE FROM t2;
           54  +    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
           55  +    SELECT count(*) FROM t2($q)
           56  +  } $res2
           57  +  do_execsql_test 1.$tn.2 {
           58  +    DELETE FROM t2;
           59  +    INSERT INTO t2(rowid, x) VALUES (1, $q);
           60  +    SELECT count(*) FROM t2('Ha Noi')
           61  +  } $res2
           62  +}
           63  +
           64  +finish_test
           65  +

Changes to ext/fts5/test/fts5unicode3.test.

    32     32   
    33     33   tl_load_casefolding_txt $CF
    34     34   foreach x [an_load_unicodedata_text $UD] {
    35     35     set aNotAlnum($x) 1
    36     36   }
    37     37   
    38     38   foreach {y} [rd_load_unicodedata_text $UD] {
    39         -  foreach {code ascii} $y {}
           39  +  foreach {code ascii f} $y {}
    40     40     if {$ascii==""} {
    41     41       set int 0
    42     42     } else {
    43     43       binary scan $ascii c int
    44     44     }
    45         -  set aDiacritic($code) $int
           45  +  set aDiacritic($code,$f) $int
           46  +  if {$f==0} { set aDiacritic($code,1) $int }
    46     47   }
    47     48   
    48     49   proc tcl_fold {i {bRemoveDiacritic 0}} {
    49     50     global tl_lookup_table
    50     51     global aDiacritic
           52  +  set f [expr $bRemoveDiacritic==2]
    51     53   
    52     54     if {[info exists tl_lookup_table($i)]} {
    53     55       set i $tl_lookup_table($i)
    54     56     }
    55         -  if {$bRemoveDiacritic && [info exists aDiacritic($i)]} {
    56         -    set i $aDiacritic($i)
           57  +  if {$bRemoveDiacritic && [info exists aDiacritic($i,$f)]} {
           58  +    set i $aDiacritic($i,$f)
    57     59     }
    58     60     expr $i
    59     61   }
    60     62   db func tcl_fold tcl_fold
    61     63   
    62     64   proc tcl_isalnum {i} {
    63     65     global aNotAlnum
................................................................................
    81     83       SELECT -1
    82     84       UNION ALL
    83     85       SELECT i+1 FROM ii WHERE i<100000
    84     86     )
    85     87     SELECT count(*), min(i) FROM ii WHERE fts5_fold(i)!=CAST(tcl_fold(i) AS int);
    86     88   } {0 {}}
    87     89   
    88         -do_execsql_test 1.2 {
           90  +do_execsql_test 1.2.1 {
    89     91     WITH ii(i) AS (
    90     92       SELECT -1
    91     93       UNION ALL
    92     94       SELECT i+1 FROM ii WHERE i<100000
    93     95     )
    94     96     SELECT count(*), min(i) FROM ii 
    95     97     WHERE fts5_fold(i,1)!=CAST(tcl_fold(i,1) AS int);
    96     98   } {0 {}}
           99  +
          100  +do_execsql_test 1.2.2 {
          101  +  WITH ii(i) AS (
          102  +    SELECT -1
          103  +    UNION ALL
          104  +    SELECT i+1 FROM ii WHERE i<100000
          105  +  )
          106  +  SELECT count(*), min(i) FROM ii 
          107  +  WHERE fts5_fold(i,2)!=CAST(tcl_fold(i,2) AS int);
          108  +} {0 {}}
    97    109   
    98    110   do_execsql_test 1.3 {
    99    111     WITH ii(i) AS (
   100    112       SELECT -1
   101    113       UNION ALL
   102    114       SELECT i+1 FROM ii WHERE i<100000
   103    115     )

Changes to main.mk.

   994    994   
   995    995   showwal$(EXE):	$(TOP)/tool/showwal.c sqlite3.o
   996    996   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \
   997    997   		$(TOP)/tool/showwal.c sqlite3.o $(THREADLIB)
   998    998   
   999    999   showshm$(EXE):	$(TOP)/tool/showshm.c
  1000   1000   	$(TCC) -o showshm$(EXE) $(TOP)/tool/showshm.c
         1001  +
         1002  +index_usage$(EXE): $(TOP)/tool/index_usage.c sqlite3.o
         1003  +	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_DEPRECATED -o index_usage$(EXE) \
         1004  +		$(TOP)/tool/index_usage.c sqlite3.o $(THREADLIB)
  1001   1005   
  1002   1006   changeset$(EXE):	$(TOP)/ext/session/changeset.c sqlite3.o
  1003   1007   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \
  1004   1008   		$(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB)
  1005   1009   
  1006   1010   changesetfuzz$(EXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.o
  1007   1011   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changesetfuzz$(EXE) \

Changes to src/build.c.

   225    225   
   226    226   
   227    227     /* Get the VDBE program ready for execution
   228    228     */
   229    229     if( v && pParse->nErr==0 && !db->mallocFailed ){
   230    230       /* A minimum of one cursor is required if autoincrement is used
   231    231       *  See ticket [a696379c1f08866] */
   232         -    if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1;
          232  +    assert( pParse->pAinc==0 || pParse->nTab>0 );
   233    233       sqlite3VdbeMakeReady(v, pParse);
   234    234       pParse->rc = SQLITE_DONE;
   235    235     }else{
   236    236       pParse->rc = SQLITE_ERROR;
   237    237     }
   238    238   }
   239    239   

Changes to src/hash.c.

    68     68   static unsigned int strHashN(const char *z, int n){
    69     69     unsigned int h = 0;
    70     70     int i;
    71     71     for(i=0; i<n; i++){
    72     72       /* Knuth multiplicative hashing.  (Sorting & Searching, p. 510).
    73     73       ** 0x9e3779b1 is 2654435761 which is the closest prime number to
    74     74       ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */
    75         -    h += sqlite3UpperToLower[z[i]];
           75  +    h += sqlite3UpperToLower[(unsigned char)z[i]];
    76     76       h *= 0x9e3779b1;
    77     77     }
    78     78     return h;
    79     79   }
    80     80   #endif /* SQLITE_ENABLE_NORMALIZE */
    81     81   
    82     82   

Changes to src/hash.h.

    64     64   
    65     65   /*
    66     66   ** Access routines.  To delete, insert a NULL pointer.
    67     67   */
    68     68   void sqlite3HashInit(Hash*);
    69     69   void *sqlite3HashInsert(Hash*, const char *pKey, void *pData);
    70     70   void *sqlite3HashFind(const Hash*, const char *pKey);
           71  +#ifdef SQLITE_ENABLE_NORMALIZE
           72  +void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey);
           73  +#endif
    71     74   void sqlite3HashClear(Hash*);
    72     75   
    73     76   /*
    74     77   ** Macros for looping over all elements of a hash table.  The idiom is
    75     78   ** like this:
    76     79   **
    77     80   **   Hash h;

Changes to src/insert.c.

   315    315       aOp[3].p5 = SQLITE_JUMPIFNULL;
   316    316       aOp[4].p2 = memId+1;
   317    317       aOp[5].p3 = memId;
   318    318       aOp[6].p1 = memId;
   319    319       aOp[7].p2 = memId+2;
   320    320       aOp[7].p1 = memId;
   321    321       aOp[10].p2 = memId;
          322  +    if( pParse->nTab==0 ) pParse->nTab = 1;
   322    323     }
   323    324   }
   324    325   
   325    326   /*
   326    327   ** Update the maximum rowid for an autoincrement calculation.
   327    328   **
   328    329   ** This routine should be called when the regRowid register holds a

Changes to src/main.c.

  1992   1992       (void)SQLITE_MISUSE_BKPT;
  1993   1993       return 0;
  1994   1994     }
  1995   1995   #endif
  1996   1996     sqlite3_mutex_enter(db->mutex);
  1997   1997     pOld = db->pTraceArg;
  1998   1998     db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0;
         1999  +  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
  1999   2000     db->xTrace = (int(*)(u32,void*,void*,void*))xTrace;
  2000   2001     db->pTraceArg = pArg;
  2001   2002     sqlite3_mutex_leave(db->mutex);
  2002   2003     return pOld;
  2003   2004   }
  2004   2005   #endif /* SQLITE_OMIT_DEPRECATED */
  2005   2006   
................................................................................
  2016   2017       return SQLITE_MISUSE_BKPT;
  2017   2018     }
  2018   2019   #endif
  2019   2020     sqlite3_mutex_enter(db->mutex);
  2020   2021     if( mTrace==0 ) xTrace = 0;
  2021   2022     if( xTrace==0 ) mTrace = 0;
  2022   2023     db->mTrace = mTrace;
         2024  +#ifndef SQLITE_OMIT_DEPRECATED
         2025  +  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
         2026  +#endif
  2023   2027     db->xTrace = xTrace;
  2024   2028     db->pTraceArg = pArg;
  2025   2029     sqlite3_mutex_leave(db->mutex);
  2026   2030     return SQLITE_OK;
  2027   2031   }
  2028   2032   
  2029   2033   #ifndef SQLITE_OMIT_DEPRECATED
................................................................................
  2048   2052       return 0;
  2049   2053     }
  2050   2054   #endif
  2051   2055     sqlite3_mutex_enter(db->mutex);
  2052   2056     pOld = db->pProfileArg;
  2053   2057     db->xProfile = xProfile;
  2054   2058     db->pProfileArg = pArg;
         2059  +  db->mTrace &= SQLITE_TRACE_NONLEGACY_MASK;
         2060  +  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
  2055   2061     sqlite3_mutex_leave(db->mutex);
  2056   2062     return pOld;
  2057   2063   }
  2058   2064   #endif /* SQLITE_OMIT_DEPRECATED */
  2059   2065   #endif /* SQLITE_OMIT_TRACE */
  2060   2066   
  2061   2067   /*

Changes to src/parse.y.

   675    675         A = sqlite3SrcListAppendFromTerm(pParse,A,0,0,&Z,0,N,U);
   676    676         if( A ){
   677    677           struct SrcList_item *pNew = &A->a[A->nSrc-1];
   678    678           struct SrcList_item *pOld = F->a;
   679    679           pNew->zName = pOld->zName;
   680    680           pNew->zDatabase = pOld->zDatabase;
   681    681           pNew->pSelect = pOld->pSelect;
          682  +        if( pOld->fg.isTabFunc ){
          683  +          pNew->u1.pFuncArg = pOld->u1.pFuncArg;
          684  +          pOld->u1.pFuncArg = 0;
          685  +          pOld->fg.isTabFunc = 0;
          686  +          pNew->fg.isTabFunc = 1;
          687  +        }
   682    688           pOld->zName = pOld->zDatabase = 0;
   683    689           pOld->pSelect = 0;
   684    690         }
   685    691         sqlite3SrcListDelete(pParse->db, F);
   686    692       }else{
   687    693         Select *pSubquery;
   688    694         sqlite3SrcListShiftJoinType(F);

Changes to src/pcache1.c.

    98     98     unsigned int iKey;             /* Key value (page number) */
    99     99     u8 isBulkLocal;                /* This page from bulk local storage */
   100    100     u8 isAnchor;                   /* This is the PGroup.lru element */
   101    101     PgHdr1 *pNext;                 /* Next in hash table chain */
   102    102     PCache1 *pCache;               /* Cache that currently owns this page */
   103    103     PgHdr1 *pLruNext;              /* Next in LRU list of unpinned pages */
   104    104     PgHdr1 *pLruPrev;              /* Previous in LRU list of unpinned pages */
          105  +                                 /* NB: pLruPrev is only valid if pLruNext!=0 */
   105    106   };
   106    107   
   107    108   /*
   108    109   ** A page is pinned if it is not on the LRU list.  To be "pinned" means
   109    110   ** that the page is in active use and must not be deallocated.
   110    111   */
   111    112   #define PAGE_IS_PINNED(p)    ((p)->pLruNext==0)
................................................................................
   566    567     assert( PAGE_IS_UNPINNED(pPage) );
   567    568     assert( pPage->pLruNext );
   568    569     assert( pPage->pLruPrev );
   569    570     assert( sqlite3_mutex_held(pPage->pCache->pGroup->mutex) );
   570    571     pPage->pLruPrev->pLruNext = pPage->pLruNext;
   571    572     pPage->pLruNext->pLruPrev = pPage->pLruPrev;
   572    573     pPage->pLruNext = 0;
   573         -  pPage->pLruPrev = 0;
          574  +  /* pPage->pLruPrev = 0;
          575  +  ** No need to clear pLruPrev as it is never accessed if pLruNext is 0 */
   574    576     assert( pPage->isAnchor==0 );
   575    577     assert( pPage->pCache->pGroup->lru.isAnchor==1 );
   576    578     pPage->pCache->nRecyclable--;
   577    579     return pPage;
   578    580   }
   579    581   
   580    582   
................................................................................
   904    906   
   905    907     if( pPage ){
   906    908       unsigned int h = iKey % pCache->nHash;
   907    909       pCache->nPage++;
   908    910       pPage->iKey = iKey;
   909    911       pPage->pNext = pCache->apHash[h];
   910    912       pPage->pCache = pCache;
   911         -    pPage->pLruPrev = 0;
   912    913       pPage->pLruNext = 0;
          914  +    /* pPage->pLruPrev = 0;
          915  +    ** No need to clear pLruPrev since it is not accessed when pLruNext==0 */
   913    916       *(void **)pPage->page.pExtra = 0;
   914    917       pCache->apHash[h] = pPage;
   915    918       if( iKey>pCache->iMaxKey ){
   916    919         pCache->iMaxKey = iKey;
   917    920       }
   918    921     }
   919    922     return pPage;
................................................................................
  1065   1068    
  1066   1069     assert( pPage->pCache==pCache );
  1067   1070     pcache1EnterMutex(pGroup);
  1068   1071   
  1069   1072     /* It is an error to call this function if the page is already 
  1070   1073     ** part of the PGroup LRU list.
  1071   1074     */
  1072         -  assert( pPage->pLruPrev==0 && pPage->pLruNext==0 );
         1075  +  assert( pPage->pLruNext==0 );
  1073   1076     assert( PAGE_IS_PINNED(pPage) );
  1074   1077   
  1075   1078     if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){
  1076   1079       pcache1RemoveFromHash(pPage, 1);
  1077   1080     }else{
  1078   1081       /* Add the page to the PGroup LRU list. */
  1079   1082       PgHdr1 **ppFirst = &pGroup->lru.pLruNext;

Changes to src/prepare.c.

   787    787   ** normalized version of the specified SQL string.  This should take into
   788    788   ** account any potential expansion that could occur (e.g. via IN clauses
   789    789   ** being expanded, etc).  This size returned is the total number of bytes
   790    790   ** including the NUL terminator.
   791    791   */
   792    792   static int estimateNormalizedSize(
   793    793     const char *zSql, /* The original SQL string */
   794         -  int nSql,         /* Length of original SQL string */
   795         -  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
          794  +  int nSql          /* Length of original SQL string */
   796    795   ){
   797    796     int nOut = nSql + 4;
   798    797     const char *z = zSql;
   799    798     while( nOut<nSql*5 ){
   800    799       while( z[0]!=0 && z[0]!='I' && z[0]!='i' ){ z++; }
   801    800       if( z[0]==0 ) break;
   802    801       z++;
................................................................................
   843    842         zOut[j++] = sqlite3Tolower(zSql[iIn+k]);
   844    843       }
   845    844     }
   846    845     *piOut = j;
   847    846   }
   848    847   
   849    848   /*
   850         -** Perform normalization of the SQL contained in the prepared statement and
   851         -** store the result in the zNormSql field.  The schema for the associated
   852         -** databases are consulted while performing the normalization in order to
   853         -** determine if a token appears to be an identifier.  All identifiers are
   854         -** left intact in the normalized SQL and all literals are replaced with a
   855         -** single '?'.
          849  +** Compute a normalization of the SQL given by zSql[0..nSql-1].  Return
          850  +** the normalization in space obtained from sqlite3DbMalloc().  Or return
          851  +** NULL if anything goes wrong or if zSql is NULL.
   856    852   */
   857         -void sqlite3Normalize(
          853  +char *sqlite3Normalize(
   858    854     Vdbe *pVdbe,      /* VM being reprepared */
   859    855     const char *zSql, /* The original SQL string */
   860         -  int nSql,         /* Size of the input string in bytes */
   861         -  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
          856  +  int nSql          /* Size of the input string in bytes */
   862    857   ){
   863    858     sqlite3 *db;           /* Database handle. */
   864    859     char *z;               /* The output string */
   865    860     int nZ;                /* Size of the output string in bytes */
   866    861     int i;                 /* Next character to read from zSql[] */
   867    862     int j;                 /* Next character to fill in on z[] */
   868    863     int tokenType = 0;     /* Type of the next token */
................................................................................
   869    864     int prevTokenType = 0; /* Type of the previous token, except spaces */
   870    865     int n;                 /* Size of the next token */
   871    866     int nParen = 0;        /* Nesting level of parenthesis */
   872    867     Hash inHash;           /* Table of parenthesis levels to output index. */
   873    868   
   874    869     db = sqlite3VdbeDb(pVdbe);
   875    870     assert( db!=0 );
   876         -  assert( pVdbe->zNormSql==0 );
   877         -  if( zSql==0 ) return;
   878         -  nZ = estimateNormalizedSize(zSql, nSql, prepFlags);
          871  +  if( zSql==0 ) return 0;
          872  +  nZ = estimateNormalizedSize(zSql, nSql);
   879    873     z = sqlite3DbMallocRawNN(db, nZ);
   880         -  if( z==0 ) return;
          874  +  if( z==0 ) goto normalizeError;
   881    875     sqlite3HashInit(&inHash);
   882    876     for(i=j=0; i<nSql && zSql[i]; i+=n){
   883    877       int flags = 0;
   884    878       if( tokenType!=TK_SPACE ) prevTokenType = tokenType;
   885    879       n = sqlite3GetTokenNormalized((unsigned char*)zSql+i, &tokenType, &flags);
   886    880       switch( tokenType ){
   887    881         case TK_SPACE: {
   888    882           break;
   889    883         }
   890    884         case TK_ILLEGAL: {
   891         -        sqlite3DbFree(db, z);
   892         -        sqlite3HashClear(&inHash);
   893         -        return;
          885  +        goto normalizeError;
   894    886         }
   895    887         case TK_STRING:
   896    888         case TK_INTEGER:
   897    889         case TK_FLOAT:
   898    890         case TK_VARIABLE:
   899    891         case TK_BLOB: {
   900    892           z[j++] = '?';
................................................................................
   967    959             int i2 = i, n2 = n, rc = SQLITE_OK;
   968    960             if( nParen>0 ){
   969    961               assert( nParen<nSql );
   970    962               sqlite3HashInsert(&inHash, zSql+nParen, 0);
   971    963             }
   972    964             if( flags&SQLITE_TOKEN_QUOTED ){ i2++; n2-=2; }
   973    965             if( shouldTreatAsIdentifier(db, zSql+i2, n2, &rc)==0 ){
   974         -            if( rc!=SQLITE_OK ){
   975         -              sqlite3DbFree(db, z);
   976         -              sqlite3HashClear(&inHash);
   977         -              return;
   978         -            }
          966  +            if( rc!=SQLITE_OK ) goto normalizeError;
   979    967               if( sqlite3_keyword_check(zSql+i2, n2)==0 ){
   980    968                 z[j++] = '?';
   981    969                 break;
   982    970               }
   983    971             }
   984    972           }
   985    973           copyNormalizedToken(zSql, i, n, flags, z, &j);
................................................................................
   988    976       }
   989    977     }
   990    978     assert( j<nZ && "one" );
   991    979     while( j>0 && z[j-1]==' ' ){ j--; }
   992    980     if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
   993    981     z[j] = 0;
   994    982     assert( j<nZ && "two" );
   995         -  pVdbe->zNormSql = z;
          983  +  sqlite3HashClear(&inHash);
          984  +  return z;
          985  +
          986  +normalizeError:
          987  +  sqlite3DbFree(db, z);
   996    988     sqlite3HashClear(&inHash);
          989  +  return 0;
   997    990   }
   998    991   #endif /* SQLITE_ENABLE_NORMALIZE */
   999    992   
  1000    993   /*
  1001    994   ** Rerun the compilation of a statement after a schema change.
  1002    995   **
  1003    996   ** If the statement is successfully recompiled, return SQLITE_OK. Otherwise,

Changes to src/resolve.c.

    76     76     db = pParse->db;
    77     77     pDup = sqlite3ExprDup(db, pOrig, 0);
    78     78     if( pDup!=0 ){
    79     79       if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery);
    80     80       if( pExpr->op==TK_COLLATE ){
    81     81         pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken);
    82     82       }
    83         -    ExprSetProperty(pDup, EP_Alias);
           83  +//    ExprSetProperty(pDup, EP_Alias);
    84     84   
    85     85       /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This 
    86     86       ** prevents ExprDelete() from deleting the Expr structure itself,
    87     87       ** allowing it to be repopulated by the memcpy() on the following line.
    88     88       ** The pExpr->u.zToken might point into memory that will be freed by the
    89     89       ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to
    90     90       ** make a copy of the token before doing the sqlite3DbFree().

Changes to src/shell.c.in.

  1003   1003     u8 autoEQP;            /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
  1004   1004     u8 autoEQPtest;        /* autoEQP is in test mode */
  1005   1005     u8 statsOn;            /* True to display memory stats before each finalize */
  1006   1006     u8 scanstatsOn;        /* True to display scan stats before each finalize */
  1007   1007     u8 openMode;           /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
  1008   1008     u8 doXdgOpen;          /* Invoke start/open/xdg-open in output_reset() */
  1009   1009     u8 nEqpLevel;          /* Depth of the EQP output graph */
         1010  +  u8 eTraceType;         /* SHELL_TRACE_* value for type of trace */
  1010   1011     unsigned mEqpLines;    /* Mask of veritical lines in the EQP output graph */
  1011   1012     int outCount;          /* Revert to stdout when reaching zero */
  1012   1013     int cnt;               /* Number of records displayed so far */
  1013   1014     FILE *out;             /* Write results here */
  1014   1015     FILE *traceOut;        /* Output for sqlite3_trace() */
  1015   1016     int nErr;              /* Number of errors seen */
  1016   1017     int mode;              /* An output mode setting */
................................................................................
  1062   1063   #define SHELL_OPEN_UNSPEC      0      /* No open-mode specified */
  1063   1064   #define SHELL_OPEN_NORMAL      1      /* Normal database file */
  1064   1065   #define SHELL_OPEN_APPENDVFS   2      /* Use appendvfs */
  1065   1066   #define SHELL_OPEN_ZIPFILE     3      /* Use the zipfile virtual table */
  1066   1067   #define SHELL_OPEN_READONLY    4      /* Open a normal database read-only */
  1067   1068   #define SHELL_OPEN_DESERIALIZE 5      /* Open using sqlite3_deserialize() */
  1068   1069   
         1070  +/* Allowed values for ShellState.eTraceType
         1071  +*/
         1072  +#define SHELL_TRACE_PLAIN      0      /* Show input SQL text */
         1073  +#define SHELL_TRACE_EXPANDED   1      /* Show expanded SQL text */
         1074  +#define SHELL_TRACE_NORMALIZED 2      /* Show normalized SQL text */
         1075  +
  1069   1076   /*
  1070   1077   ** These are the allowed shellFlgs values
  1071   1078   */
  1072   1079   #define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
  1073   1080   #define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
  1074   1081   #define SHFLG_Backslash      0x00000004 /* The --backslash option is used */
  1075   1082   #define SHFLG_PreserveRowid  0x00000008 /* .dump preserves rowid values */
................................................................................
  3488   3495   #ifndef SQLITE_NOHAVE_SYSTEM
  3489   3496     ".system CMD ARGS...      Run CMD ARGS... in a system shell",
  3490   3497   #endif
  3491   3498     ".tables ?TABLE?          List names of tables matching LIKE pattern TABLE",
  3492   3499     ".testcase NAME           Begin redirecting output to 'testcase-out.txt'",
  3493   3500     ".timeout MS              Try opening locked tables for MS milliseconds",
  3494   3501     ".timer on|off            Turn SQL timer on or off",
  3495         -  ".trace FILE|off          Output each SQL statement as it is run",
         3502  +#ifndef SQLITE_OMIT_TRACE
         3503  +  ".trace ?OPTIONS?         Output each SQL statement as it is run",
         3504  +  "    FILE                    Send output to FILE",
         3505  +  "    stdout                  Send output to stdout",
         3506  +  "    stderr                  Send output to stderr",
         3507  +  "    off                     Disable tracing",
         3508  +  "    --expanded              Expand query parameters",
         3509  +#ifdef SQLITE_ENABLE_NORMALIZE
         3510  +  "    --normalized            Normal the SQL statements",
         3511  +#endif
         3512  +  "    --plain                 Show SQL as it is input",
         3513  +  "    --stmt                  Trace statement execution (SQLITE_TRACE_STMT)",
         3514  +  "    --profile               Profile statements (SQLITE_TRACE_PROFILE)",
         3515  +  "    --row                   Trace each row (SQLITE_TRACE_ROW)",
         3516  +  "    --close                 Trace connection close (SQLITE_TRACE_CLOSE)",
         3517  +#endif /* SQLITE_OMIT_TRACE */
  3496   3518     ".vfsinfo ?AUX?           Information about the top-level VFS",
  3497   3519     ".vfslist                 List all available VFSes",
  3498   3520     ".vfsname ?AUX?           Print the name of the VFS stack",
  3499   3521     ".width NUM1 NUM2 ...     Set column widths for \"column\" mode",
  3500   3522     "     Negative values right-justify",
  3501   3523   };
  3502   3524   
................................................................................
  3995   4017       if( f==0 ){
  3996   4018         utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile);
  3997   4019       }
  3998   4020     }
  3999   4021     return f;
  4000   4022   }
  4001   4023   
  4002         -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
         4024  +#ifndef SQLITE_OMIT_TRACE
  4003   4025   /*
  4004   4026   ** A routine for handling output from sqlite3_trace().
  4005   4027   */
  4006   4028   static int sql_trace_callback(
  4007         -  unsigned mType,
  4008         -  void *pArg,
  4009         -  void *pP,
  4010         -  void *pX
  4011         -){
  4012         -  FILE *f = (FILE*)pArg;
  4013         -  UNUSED_PARAMETER(mType);
  4014         -  UNUSED_PARAMETER(pP);
  4015         -  if( f ){
  4016         -    const char *z = (const char*)pX;
  4017         -    int i = strlen30(z);
  4018         -    while( i>0 && z[i-1]==';' ){ i--; }
  4019         -    utf8_printf(f, "%.*s;\n", i, z);
         4029  +  unsigned mType,         /* The trace type */
         4030  +  void *pArg,             /* The ShellState pointer */
         4031  +  void *pP,               /* Usually a pointer to sqlite_stmt */
         4032  +  void *pX                /* Auxiliary output */
         4033  +){
         4034  +  ShellState *p = (ShellState*)pArg;
         4035  +  sqlite3_stmt *pStmt;
         4036  +  const char *zSql;
         4037  +  int nSql;
         4038  +  if( p->traceOut==0 ) return 0;
         4039  +  if( mType==SQLITE_TRACE_CLOSE ){
         4040  +    utf8_printf(p->traceOut, "-- closing database connection\n");
         4041  +    return 0;
         4042  +  }
         4043  +  if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){
         4044  +    zSql = (const char*)pX;
         4045  +  }else{
         4046  +    pStmt = (sqlite3_stmt*)pP;
         4047  +    switch( p->eTraceType ){
         4048  +      case SHELL_TRACE_EXPANDED: {
         4049  +        zSql = sqlite3_expanded_sql(pStmt);
         4050  +        break;
         4051  +      }
         4052  +#ifdef SQLITE_ENABLE_NORMALIZE
         4053  +      case SHELL_TRACE_NORMALIZED: {
         4054  +        zSql = sqlite3_normalized_sql(pStmt);
         4055  +        break;
         4056  +      }
         4057  +#endif
         4058  +      default: {
         4059  +        zSql = sqlite3_sql(pStmt);
         4060  +        break;
         4061  +      }
         4062  +    }
         4063  +  }
         4064  +  if( zSql==0 ) return 0;
         4065  +  nSql = strlen30(zSql);
         4066  +  while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; }
         4067  +  switch( mType ){
         4068  +    case SQLITE_TRACE_ROW:
         4069  +    case SQLITE_TRACE_STMT: {
         4070  +      utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql);
         4071  +      break;
         4072  +    }
         4073  +    case SQLITE_TRACE_PROFILE: {
         4074  +      sqlite3_int64 nNanosec = *(sqlite3_int64*)pX;
         4075  +      utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec);
         4076  +      break;
         4077  +    }
  4020   4078     }
  4021   4079     return 0;
  4022   4080   }
  4023   4081   #endif
  4024   4082   
  4025   4083   /*
  4026   4084   ** A no-op routine that runs with the ".breakpoint" doc-command.  This is
................................................................................
  7834   7892         }
  7835   7893       }else{
  7836   7894         raw_printf(stderr, "Usage: .timer on|off\n");
  7837   7895         rc = 1;
  7838   7896       }
  7839   7897     }else
  7840   7898   
         7899  +#ifndef SQLITE_OMIT_TRACE
  7841   7900     if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){
         7901  +    int mType = 0;
         7902  +    int jj;
  7842   7903       open_db(p, 0);
  7843         -    if( nArg!=2 ){
  7844         -      raw_printf(stderr, "Usage: .trace FILE|off\n");
  7845         -      rc = 1;
  7846         -      goto meta_command_exit;
         7904  +    for(jj=1; jj<nArg; jj++){
         7905  +      const char *z = azArg[jj];
         7906  +      if( z[0]=='-' ){
         7907  +        if( optionMatch(z, "expanded") ){
         7908  +          p->eTraceType = SHELL_TRACE_EXPANDED;
         7909  +        }
         7910  +#ifdef SQLITE_ENABLE_NORMALIZE
         7911  +        else if( optionMatch(z, "normalized") ){
         7912  +          p->eTraceType = SHELL_TRACE_NORMALIZED;
         7913  +        }
         7914  +#endif
         7915  +        else if( optionMatch(z, "plain") ){
         7916  +          p->eTraceType = SHELL_TRACE_PLAIN;
         7917  +        }
         7918  +        else if( optionMatch(z, "profile") ){
         7919  +          mType |= SQLITE_TRACE_PROFILE;
         7920  +        }
         7921  +        else if( optionMatch(z, "row") ){
         7922  +          mType |= SQLITE_TRACE_ROW;
         7923  +        }
         7924  +        else if( optionMatch(z, "stmt") ){
         7925  +          mType |= SQLITE_TRACE_STMT;
         7926  +        }
         7927  +        else if( optionMatch(z, "close") ){
         7928  +          mType |= SQLITE_TRACE_CLOSE;
         7929  +        }
         7930  +        else {
         7931  +          raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z);
         7932  +          rc = 1;
         7933  +          goto meta_command_exit;
         7934  +        }
         7935  +      }else{
         7936  +        output_file_close(p->traceOut);
         7937  +        p->traceOut = output_file_open(azArg[1], 0);
         7938  +      }
  7847   7939       }
  7848         -    output_file_close(p->traceOut);
  7849         -    p->traceOut = output_file_open(azArg[1], 0);
  7850         -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
  7851   7940       if( p->traceOut==0 ){
  7852   7941         sqlite3_trace_v2(p->db, 0, 0, 0);
  7853   7942       }else{
  7854         -      sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut);
         7943  +      if( mType==0 ) mType = SQLITE_TRACE_STMT;
         7944  +      sqlite3_trace_v2(p->db, mType, sql_trace_callback, p);
  7855   7945       }
  7856         -#endif
  7857   7946     }else
         7947  +#endif /* !defined(SQLITE_OMIT_TRACE) */
  7858   7948   
  7859   7949   #if SQLITE_USER_AUTHENTICATION
  7860   7950     if( c=='u' && strncmp(azArg[0], "user", n)==0 ){
  7861   7951       if( nArg<2 ){
  7862   7952         raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n");
  7863   7953         rc = 1;
  7864   7954         goto meta_command_exit;

Changes to src/sqliteInt.h.

  1361   1361                                  const char*);
  1362   1362   #endif
  1363   1363   
  1364   1364   #ifndef SQLITE_OMIT_DEPRECATED
  1365   1365   /* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing
  1366   1366   ** in the style of sqlite3_trace()
  1367   1367   */
  1368         -#define SQLITE_TRACE_LEGACY  0x80
         1368  +#define SQLITE_TRACE_LEGACY          0x40     /* Use the legacy xTrace */
         1369  +#define SQLITE_TRACE_XPROFILE        0x80     /* Use the legacy xProfile */
  1369   1370   #else
  1370         -#define SQLITE_TRACE_LEGACY  0
         1371  +#define SQLITE_TRACE_LEGACY          0
         1372  +#define SQLITE_TRACE_XPROFILE        0
  1371   1373   #endif /* SQLITE_OMIT_DEPRECATED */
         1374  +#define SQLITE_TRACE_NONLEGACY_MASK  0x0f     /* Normal flags */
  1372   1375   
  1373   1376   
  1374   1377   /*
  1375   1378   ** Each database connection is an instance of the following structure.
  1376   1379   */
  1377   1380   struct sqlite3 {
  1378   1381     sqlite3_vfs *pVfs;            /* OS Interface */
................................................................................
  1425   1428     int nVdbeWrite;               /* Number of active VDBEs that read and write */
  1426   1429     int nVdbeExec;                /* Number of nested calls to VdbeExec() */
  1427   1430     int nVDestroy;                /* Number of active OP_VDestroy operations */
  1428   1431     int nExtension;               /* Number of loaded extensions */
  1429   1432     void **aExtension;            /* Array of shared library handles */
  1430   1433     int (*xTrace)(u32,void*,void*,void*);     /* Trace function */
  1431   1434     void *pTraceArg;                          /* Argument to the trace function */
         1435  +#ifndef SQLITE_OMIT_DEPRECATED
  1432   1436     void (*xProfile)(void*,const char*,u64);  /* Profiling function */
  1433   1437     void *pProfileArg;                        /* Argument to profile function */
         1438  +#endif
  1434   1439     void *pCommitArg;                 /* Argument to xCommitCallback() */
  1435   1440     int (*xCommitCallback)(void*);    /* Invoked at every commit. */
  1436   1441     void *pRollbackArg;               /* Argument to xRollbackCallback() */
  1437   1442     void (*xRollbackCallback)(void*); /* Invoked at every commit. */
  1438   1443     void *pUpdateArg;
  1439   1444     void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64);
  1440   1445   #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
................................................................................
  4429   4434   int sqlite3VtabBegin(sqlite3 *, VTable *);
  4430   4435   FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*);
  4431   4436   sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*);
  4432   4437   int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
  4433   4438   int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
  4434   4439   void sqlite3ParserReset(Parse*);
  4435   4440   #ifdef SQLITE_ENABLE_NORMALIZE
  4436         -void sqlite3Normalize(Vdbe*, const char*, int, u8);
         4441  +char *sqlite3Normalize(Vdbe*, const char*, int);
  4437   4442   #endif
  4438   4443   int sqlite3Reprepare(Vdbe*);
  4439   4444   void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*);
  4440   4445   CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *);
  4441   4446   int sqlite3TempInMemory(const sqlite3*);
  4442   4447   const char *sqlite3JournalModename(int);
  4443   4448   #ifndef SQLITE_OMIT_WAL

Changes to src/vdbeapi.c.

    58     58   ** Invoke the profile callback.  This routine is only called if we already
    59     59   ** know that the profile callback is defined and needs to be invoked.
    60     60   */
    61     61   static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){
    62     62     sqlite3_int64 iNow;
    63     63     sqlite3_int64 iElapse;
    64     64     assert( p->startTime>0 );
    65         -  assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 );
           65  +  assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 );
    66     66     assert( db->init.busy==0 );
    67     67     assert( p->zSql!=0 );
    68     68     sqlite3OsCurrentTimeInt64(db->pVfs, &iNow);
    69     69     iElapse = (iNow - p->startTime)*1000000;
           70  +#ifndef SQLITE_OMIT_DEPRECATED  	
    70     71     if( db->xProfile ){
    71     72       db->xProfile(db->pProfileArg, p->zSql, iElapse);
    72     73     }
           74  +#endif
    73     75     if( db->mTrace & SQLITE_TRACE_PROFILE ){
    74     76       db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse);
    75     77     }
    76     78     p->startTime = 0;
    77     79   }
    78     80   /*
    79     81   ** The checkProfileCallback(DB,P) macro checks to see if a profile callback
................................................................................
   598    600       }
   599    601   
   600    602       assert( db->nVdbeWrite>0 || db->autoCommit==0 
   601    603           || (db->nDeferredCons==0 && db->nDeferredImmCons==0)
   602    604       );
   603    605   
   604    606   #ifndef SQLITE_OMIT_TRACE
   605         -    if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0)
          607  +    if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0
   606    608           && !db->init.busy && p->zSql ){
   607    609         sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime);
   608    610       }else{
   609    611         assert( p->startTime==0 );
   610    612       }
   611    613   #endif
   612    614   
................................................................................
   625    627   #endif /* SQLITE_OMIT_EXPLAIN */
   626    628     {
   627    629       db->nVdbeExec++;
   628    630       rc = sqlite3VdbeExec(p);
   629    631       db->nVdbeExec--;
   630    632     }
   631    633   
          634  +  if( rc!=SQLITE_ROW ){
   632    635   #ifndef SQLITE_OMIT_TRACE
   633         -  /* If the statement completed successfully, invoke the profile callback */
   634         -  if( rc!=SQLITE_ROW ) checkProfileCallback(db, p);
          636  +    /* If the statement completed successfully, invoke the profile callback */
          637  +    checkProfileCallback(db, p);
   635    638   #endif
   636    639   
   637         -  if( rc==SQLITE_DONE && db->autoCommit ){
   638         -    assert( p->rc==SQLITE_OK );
   639         -    p->rc = doWalCallbacks(db);
   640         -    if( p->rc!=SQLITE_OK ){
   641         -      rc = SQLITE_ERROR;
          640  +    if( rc==SQLITE_DONE && db->autoCommit ){
          641  +      assert( p->rc==SQLITE_OK );
          642  +      p->rc = doWalCallbacks(db);
          643  +      if( p->rc!=SQLITE_OK ){
          644  +        rc = SQLITE_ERROR;
          645  +      }
   642    646       }
   643    647     }
   644    648   
   645    649     db->errCode = rc;
   646    650     if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){
   647    651       p->rc = SQLITE_NOMEM_BKPT;
   648    652     }
................................................................................
  1704   1708   
  1705   1709   #ifdef SQLITE_ENABLE_NORMALIZE
  1706   1710   /*
  1707   1711   ** Return the normalized SQL associated with a prepared statement.
  1708   1712   */
  1709   1713   const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){
  1710   1714     Vdbe *p = (Vdbe *)pStmt;
  1711         -  return p ? p->zNormSql : 0;
         1715  +  if( p==0 ) return 0;
         1716  +  if( p->zNormSql==0 && p->zSql!=0 ){
         1717  +    p->zNormSql = sqlite3Normalize(p, p->zSql, sqlite3Strlen30(p->zSql));
         1718  +  }
         1719  +  return p->zNormSql;
  1712   1720   }
  1713   1721   #endif /* SQLITE_ENABLE_NORMALIZE */
  1714   1722   
  1715   1723   #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
  1716   1724   /*
  1717   1725   ** Allocate and populate an UnpackedRecord structure based on the serialized
  1718   1726   ** record in nKey/pKey. Return a pointer to the new UnpackedRecord structure

Changes to src/vdbeaux.c.

    63     63       p->expmask = 0;
    64     64     }
    65     65     assert( p->zSql==0 );
    66     66     p->zSql = sqlite3DbStrNDup(p->db, z, n);
    67     67   #ifdef SQLITE_ENABLE_NORMALIZE
    68     68     assert( p->zNormSql==0 );
    69     69     if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){
    70         -    sqlite3Normalize(p, p->zSql, n, prepFlags);
           70  +    p->zNormSql = sqlite3Normalize(p, p->zSql, n);
    71     71       assert( p->zNormSql!=0 || p->db->mallocFailed );
    72     72     }
    73     73   #endif
    74     74   }
    75     75   
    76     76   /*
    77     77   ** Swap all content between two VDBE structures.

Changes to src/wal.c.

  2852   2852     if( pWal->readLock>=0 ){
  2853   2853       walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
  2854   2854       pWal->readLock = -1;
  2855   2855     }
  2856   2856   }
  2857   2857   
  2858   2858   /*
  2859         -** Search the hash tables for an entry matching page number pgno. Ignore
  2860         -** any entries that lie after frame iLast within the wal file.
         2859  +** Search the wal file for page pgno. If found, set *piRead to the frame that
         2860  +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead
         2861  +** to zero.
         2862  +**
         2863  +** Return SQLITE_OK if successful, or an error code if an error occurs. If an
         2864  +** error does occur, the final value of *piRead is undefined.
  2861   2865   */
  2862         -static int walFindFrame(
  2863         -  Wal *pWal, 
  2864         -  Pgno pgno, 
  2865         -  u32 iLast, 
  2866         -  u32 *piRead
         2866  +int sqlite3WalFindFrame(
         2867  +  Wal *pWal,                      /* WAL handle */
         2868  +  Pgno pgno,                      /* Database page number to read data for */
         2869  +  u32 *piRead                     /* OUT: Frame number (or zero) */
  2867   2870   ){
         2871  +  u32 iRead = 0;                  /* If !=0, WAL frame to return data from */
         2872  +  u32 iLast = pWal->hdr.mxFrame;  /* Last page in WAL for this reader */
  2868   2873     int iHash;                      /* Used to loop through N hash tables */
  2869         -  u32 iRead = 0;
  2870   2874     int iMinHash;
         2875  +
         2876  +  /* This routine is only be called from within a read transaction. */
         2877  +  assert( pWal->readLock>=0 || pWal->lockError );
         2878  +
         2879  +  /* If the "last page" field of the wal-index header snapshot is 0, then
         2880  +  ** no data will be read from the wal under any circumstances. Return early
         2881  +  ** in this case as an optimization.  Likewise, if pWal->readLock==0, 
         2882  +  ** then the WAL is ignored by the reader so return early, as if the 
         2883  +  ** WAL were empty.
         2884  +  */
         2885  +  if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){
         2886  +    *piRead = 0;
         2887  +    return SQLITE_OK;
         2888  +  }
  2871   2889   
  2872   2890     /* Each iteration of the following for() loop searches one
  2873   2891     ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames).
  2874   2892     **
  2875   2893     ** This code might run concurrently to the code in walIndexAppend()
  2876   2894     ** that adds entries to the wal-index (and possibly to this hash 
  2877   2895     ** table). This means the value just read from the hash 
................................................................................
  2915   2933         if( (nCollide--)==0 ){
  2916   2934           return SQLITE_CORRUPT_BKPT;
  2917   2935         }
  2918   2936       }
  2919   2937       if( iRead ) break;
  2920   2938     }
  2921   2939   
  2922         -  *piRead = iRead;
  2923         -  return SQLITE_OK;
  2924         -}
  2925         -
  2926         -/*
  2927         -** Search the wal file for page pgno. If found, set *piRead to the frame that
  2928         -** contains the page. Otherwise, if pgno is not in the wal file, set *piRead
  2929         -** to zero.
  2930         -**
  2931         -** Return SQLITE_OK if successful, or an error code if an error occurs. If an
  2932         -** error does occur, the final value of *piRead is undefined.
  2933         -*/
  2934         -int sqlite3WalFindFrame(
  2935         -  Wal *pWal,                      /* WAL handle */
  2936         -  Pgno pgno,                      /* Database page number to read data for */
  2937         -  u32 *piRead                     /* OUT: Frame number (or zero) */
  2938         -){
  2939         -  u32 iRead = 0;                  /* If !=0, WAL frame to return data from */
  2940         -  u32 iLast = pWal->hdr.mxFrame;  /* Last page in WAL for this reader */
  2941         -  int rc;
  2942         -
  2943         -  /* This routine is only be called from within a read transaction. */
  2944         -  assert( pWal->readLock>=0 || pWal->lockError );
  2945         -
  2946         -  /* If the "last page" field of the wal-index header snapshot is 0, then
  2947         -  ** no data will be read from the wal under any circumstances. Return early
  2948         -  ** in this case as an optimization.  Likewise, if pWal->readLock==0, 
  2949         -  ** then the WAL is ignored by the reader so return early, as if the 
  2950         -  ** WAL were empty.
  2951         -  */
  2952         -  if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){
  2953         -    *piRead = 0;
  2954         -    return SQLITE_OK;
  2955         -  }
  2956         -
  2957         -  rc = walFindFrame(pWal, pgno, iLast, &iRead);
  2958         -
  2959   2940   #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT
  2960   2941     /* If expensive assert() statements are available, do a linear search
  2961   2942     ** of the wal-index file content. Make sure the results agree with the
  2962   2943     ** result obtained using the hash indexes above.  */
  2963         -  if( rc==SQLITE_OK ){
         2944  +  {
  2964   2945       u32 iRead2 = 0;
  2965   2946       u32 iTest;
  2966   2947       assert( pWal->bShmUnreliable || pWal->minFrame>0 );
  2967   2948       for(iTest=iLast; iTest>=pWal->minFrame && iTest>0; iTest--){
  2968   2949         if( walFramePgno(pWal, iTest)==pgno ){
  2969   2950           iRead2 = iTest;
  2970   2951           break;
................................................................................
  2971   2952         }
  2972   2953       }
  2973   2954       assert( iRead==iRead2 );
  2974   2955     }
  2975   2956   #endif
  2976   2957   
  2977   2958     *piRead = iRead;
  2978         -  return rc;
         2959  +  return SQLITE_OK;
  2979   2960   }
  2980   2961   
  2981   2962   /*
  2982   2963   ** Read the contents of frame iRead from the wal file into buffer pOut
  2983   2964   ** (which is nOut bytes in size). Return SQLITE_OK if successful, or an
  2984   2965   ** error code otherwise.
  2985   2966   */

Added test/fts4umlaut.test.

            1  +# 2018 December 3
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +set testprefix fts4umlaut
           18  +
           19  +ifcapable !fts3 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +do_execsql_test 1.0 {
           25  +  CREATE VIRTUAL TABLE t1 USING fts5(x);
           26  +  CREATE VIRTUAL TABLE t2 USING fts4(
           27  +      x, 
           28  +      tokenize=unicode61 "remove_diacritics=2"
           29  +  );
           30  +}
           31  +
           32  +foreach {tn q res1 res2} {
           33  +  1 "Hà Nội"                  0 1
           34  +  2 "Hà Noi"                  1 1
           35  +  3 "Ha Noi"                  1 1
           36  +  4 "Ha N\u1ed9i"             0 1
           37  +  5 "Ha N\u006fi"             1 1
           38  +  6 "Ha N\u006f\u0302i"       1 1
           39  +  7 "Ha N\u006f\u0323\u0302i" 1 1
           40  +} {
           41  +  do_execsql_test 1.$tn.1 {
           42  +    DELETE FROM t1;
           43  +    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
           44  +    SELECT count(*) FROM t1 WHERE t1 MATCH $q
           45  +  } $res1
           46  +  do_execsql_test 1.$tn.2 {
           47  +    DELETE FROM t1;
           48  +    INSERT INTO t1(rowid, x) VALUES (1, $q);
           49  +    SELECT count(*) FROM t1 WHERE t1 MATCH 'Ha Noi'
           50  +  } $res1
           51  +
           52  +  do_execsql_test 1.$tn.2 {
           53  +    DELETE FROM t2;
           54  +    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
           55  +    SELECT count(*) FROM t2 WHERE t2 MATCH $q
           56  +  } $res2
           57  +  do_execsql_test 1.$tn.2 {
           58  +    DELETE FROM t2;
           59  +    INSERT INTO t2(rowid, x) VALUES (1, $q);
           60  +    SELECT count(*) FROM t2 WHERE t2 MATCH 'Ha Noi'
           61  +  } $res2
           62  +}
           63  +
           64  +finish_test
           65  +

Changes to test/json101.test.

   809    809   do_execsql_test json-14.160 {
   810    810     SELECT fullkey FROM json_tree('"hello"');
   811    811   } {$}
   812    812   do_execsql_test json-14.170 {
   813    813     SELECT fullkey FROM json_tree('null');
   814    814   } {$}
   815    815   
   816         -
          816  +# 2018-12-03
          817  +# Make sure the table-valued functions contained within parentheses
          818  +# work correctly.
          819  +#
          820  +# Bug reported via private email. See TH3 for more information.
          821  +#
          822  +do_execsql_test json-15.100 {
          823  +  SELECT * FROM JSON_EACH('{"a":1, "b":2}');
          824  +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
          825  +do_execsql_test json-15.110 {
          826  +  SELECT xyz.* FROM JSON_EACH('{"a":1, "b":2}') AS xyz;
          827  +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
          828  +do_execsql_test json-15.120 {
          829  +  SELECT * FROM (JSON_EACH('{"a":1, "b":2}'));
          830  +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
          831  +do_execsql_test json-15.130 {
          832  +  SELECT xyz.* FROM (JSON_EACH('{"a":1, "b":2}')) AS xyz;
          833  +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
   817    834   
   818    835   finish_test

Changes to test/shell4.test.

   103    103   }]
   104    104     list [regexp {Memory Used} $res] \
   105    105          [regexp {Heap Usage} $res] \
   106    106          [regexp {Autoindex Inserts} $res]
   107    107   } {1 1 1}
   108    108   
   109    109   do_test shell4-2.1 {
   110         -  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace"
   111         -} {1 {Usage: .trace FILE|off}}
          110  +  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace --unknown"
          111  +} {1 {Unknown option "--unknown" on ".trace"}}
   112    112   do_test shell4-2.2 {
   113    113     catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace off\n.trace off\n"
   114    114   } {0 {}}
   115    115   do_test shell4-2.3 {
   116         -  catchcmd ":memory:" ".trace stdout\n.trace\n.trace off\n.dump\n"
   117         -} {/^1 {PRAGMA.*Usage:.*}$/}
          116  +  catchcmd ":memory:" ".trace stdout\n.dump\n.trace off\n"
          117  +} {/^0 {PRAGMA.*}$/}
   118    118   ifcapable trace {
   119    119   do_test shell4-2.4 {
   120    120     catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;"
   121    121   } {0 {CREATE TABLE t1(x);
   122    122   SELECT * FROM t1;}}
   123    123   do_test shell4-2.5 {
   124    124     catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace stdout\nSELECT * FROM t1;"

Changes to test/tabfunc01.test.

   120    120     SELECT * FROM temp.generate_series(1,4)
   121    121   } {1 2 3 4}
   122    122   do_execsql_test tabfunc01-4.3 {
   123    123     ATTACH ':memory:' AS aux1;
   124    124     CREATE TABLE aux1.t1(a,b,c);
   125    125     SELECT * FROM aux1.generate_series(1,4)
   126    126   } {1 2 3 4}
          127  +
          128  +# 2018-12-03: Fix bug reported by by private email.
          129  +do_execsql_test tabfunc01-4.4 {
          130  +  SELECT * FROM (generate_series(1,5,2)) AS x LIMIT 10;
          131  +} {1 3 5}
   127    132   
   128    133   # The next series of tests is verifying that virtual table are able
   129    134   # to optimize the IN operator, even on terms that are not marked "omit".
   130    135   # When the generate_series virtual table is compiled for the testfixture,
   131    136   # the special -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 option is used, which
   132    137   # causes the xBestIndex method of generate_series to leave the
   133    138   # sqlite3_index_constraint_usage.omit flag set to 0, which should cause

Added tool/index_usage.c.

            1  +/*
            2  +** 2018-12-04
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +** 
           13  +** This file implements a utility program used to help determine which
           14  +** indexes in a database schema are used and unused, and how often specific
           15  +** indexes are used.
           16  +*/
           17  +#include "sqlite3.h"
           18  +#include <stdio.h>
           19  +#include <stdlib.h>
           20  +#include <assert.h>
           21  +#include <string.h>
           22  +
           23  +static void usage(const char *argv0){
           24  +  printf("Usage: %s DATABASE LOG\n\n", argv0);
           25  +  printf(
           26  +    "DATABASE is an SQLite database against which various statements\n"
           27  +    "have been run.  The SQL text is stored in LOG.  LOG is an SQLite\n"
           28  +    "database with this schema:\n"
           29  +    "\n"
           30  +    "    CREATE TABLE sqllog(sql TEXT);\n"
           31  +    "\n"
           32  +    "This utility program analyzes statements contained in LOG and prints\n"
           33  +    "a report showing how many times each index in DATABASE is used by the\n"
           34  +    "statements in LOG.\n"
           35  +    "\n"
           36  +    "DATABASE only needs to contain the schema used by the statements in\n"
           37  +    "LOG. The content can be removed from DATABASE.\n"
           38  +  );
           39  +  printf("\nAnalysis will be done by SQLite version %s dated %.20s\n"
           40  +         "checkin number %.40s. Different versions\n"
           41  +         "of SQLite might use different indexes.\n",
           42  +         sqlite3_libversion(), sqlite3_sourceid(), sqlite3_sourceid()+21);
           43  +  exit(1);
           44  +}
           45  +
           46  +int main(int argc, char **argv){
           47  +  sqlite3 *db = 0;          /* The main database */
           48  +  sqlite3_stmt *pStmt = 0;  /* a query */
           49  +  char *zSql;
           50  +  int nErr = 0;
           51  +  int rc;
           52  +
           53  +  if( argc!=3 ) usage(argv[0]);
           54  +  rc = sqlite3_open_v2(argv[1], &db, SQLITE_OPEN_READONLY, 0);
           55  +  if( rc ){
           56  +    printf("Cannot open \"%s\" for reading: %s\n", argv[1], sqlite3_errmsg(db));
           57  +    goto errorOut;
           58  +  }
           59  +  rc = sqlite3_prepare_v2(db, "SELECT * FROM sqlite_master", -1, &pStmt, 0);
           60  +  if( rc ){
           61  +    printf("Cannot read the schema from \"%s\" - %s\n", argv[1],
           62  +           sqlite3_errmsg(db));
           63  +    goto errorOut;
           64  +  }
           65  +  sqlite3_finalize(pStmt);
           66  +  pStmt = 0;
           67  +  rc = sqlite3_exec(db, 
           68  +     "CREATE TABLE temp.idxu(\n"
           69  +     "  tbl TEXT,\n"
           70  +     "  idx TEXT,\n"
           71  +     "  cnt INT,\n"
           72  +     "  PRIMARY KEY(idx)\n"
           73  +     ") WITHOUT ROWID;", 0, 0, 0);
           74  +  if( rc ){
           75  +    printf("Cannot create the result table - %s\n",
           76  +           sqlite3_errmsg(db));
           77  +    goto errorOut;
           78  +  }
           79  +  rc = sqlite3_exec(db,
           80  +     "INSERT INTO temp.idxu(tbl,idx,cnt)"
           81  +     " SELECT tbl_name, name, 0 FROM sqlite_master"
           82  +     " WHERE type='index' AND sql IS NOT NULL", 0, 0, 0);
           83  +
           84  +  /* Open the LOG database */
           85  +  zSql = sqlite3_mprintf("ATTACH %Q AS log", argv[2]);
           86  +  rc = sqlite3_exec(db, zSql, 0, 0, 0);
           87  +  sqlite3_free(zSql);
           88  +  if( rc ){
           89  +    printf("Cannot open the LOG database \"%s\" - %s\n",
           90  +           argv[2], sqlite3_errmsg(db));
           91  +    goto errorOut;
           92  +  }
           93  +  rc = sqlite3_prepare_v2(db, "SELECT sql, rowid FROM log.sqllog",
           94  +                          -1, &pStmt, 0);
           95  +  if( rc ){
           96  +    printf("Cannot read the SQLLOG table in the LOG database \"%s\" - %s\n",
           97  +           argv[2], sqlite3_errmsg(db));
           98  +    goto errorOut;
           99  +  }
          100  +
          101  +  /* Update the counts based on LOG */
          102  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          103  +    const char *zLog = (const char*)sqlite3_column_text(pStmt, 0);
          104  +    sqlite3_stmt *pS2;
          105  +    if( zLog==0 ) continue;
          106  +    zSql = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zLog);
          107  +    rc = sqlite3_prepare_v2(db, zSql, -1, &pS2, 0);
          108  +    sqlite3_free(zSql);
          109  +    if( rc ){
          110  +      printf("Cannot compile LOG entry %d (%s): %s\n",
          111  +             sqlite3_column_int(pStmt, 1), zLog, sqlite3_errmsg(db));
          112  +      nErr++;
          113  +    }else{
          114  +      while( sqlite3_step(pS2)==SQLITE_ROW ){
          115  +        const char *zExplain = (const char*)sqlite3_column_text(pS2,3);
          116  +        const char *z1, *z2;
          117  +        int n;
          118  +        /* printf("EXPLAIN: %s\n", zExplain); */
          119  +        z1 = strstr(zExplain, " USING INDEX ");
          120  +        if( z1==0 ) continue;
          121  +        z1 += 13;
          122  +        for(z2=z1+1; z2[1] && z2[1]!='('; z2++){}
          123  +        n = z2 - z1;
          124  +        zSql = sqlite3_mprintf(
          125  +          "UPDATE temp.idxu SET cnt=cnt+1 WHERE idx='%.*q'", n, z1
          126  +        );
          127  +        /* printf("sql: %s\n", zSql); */
          128  +        sqlite3_exec(db, zSql, 0, 0, 0);
          129  +        sqlite3_free(zSql);
          130  +      }
          131  +    }
          132  +    sqlite3_finalize(pS2);
          133  +  }
          134  +  sqlite3_finalize(pStmt);
          135  +
          136  +  /* Generate the report */
          137  +  rc = sqlite3_prepare_v2(db,
          138  +     "SELECT tbl, idx, cnt, "
          139  +     "   (SELECT group_concat(name,',') FROM pragma_index_info(idx))"
          140  +     " FROM temp.idxu, main.sqlite_master"
          141  +     " WHERE temp.idxu.tbl=main.sqlite_master.tbl_name"
          142  +     "   AND temp.idxu.idx=main.sqlite_master.name"
          143  +     " ORDER BY cnt DESC, tbl, idx",
          144  +     -1, &pStmt, 0);
          145  +  if( rc ){
          146  +    printf("Cannot query the result table - %s\n",
          147  +           sqlite3_errmsg(db));
          148  +    goto errorOut;
          149  +  }
          150  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          151  +    printf("%10d %s on %s(%s)\n", 
          152  +       sqlite3_column_int(pStmt, 2),
          153  +       sqlite3_column_text(pStmt, 1),
          154  +       sqlite3_column_text(pStmt, 0),
          155  +       sqlite3_column_text(pStmt, 3));
          156  +  }
          157  +  sqlite3_finalize(pStmt);
          158  +  pStmt = 0;
          159  +
          160  +errorOut:
          161  +  sqlite3_finalize(pStmt);
          162  +  sqlite3_close(db);
          163  +  return nErr;
          164  +}

Changes to tool/lemon.c.

  4586   4586     }
  4587   4587     tplt_xfer(lemp->name,in,out,&lineno);
  4588   4588   
  4589   4589     /* Generate code which executes whenever the parser stack overflows */
  4590   4590     tplt_print(out,lemp,lemp->overflow,&lineno);
  4591   4591     tplt_xfer(lemp->name,in,out,&lineno);
  4592   4592   
  4593         -  /* Generate the table of rule information
         4593  +  /* Generate the tables of rule information.  yyRuleInfoLhs[] and
         4594  +  ** yyRuleInfoNRhs[].
  4594   4595     **
  4595   4596     ** Note: This code depends on the fact that rules are number
  4596   4597     ** sequentually beginning with 0.
  4597   4598     */
  4598   4599     for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
  4599         -    fprintf(out,"  { %4d, %4d }, /* (%d) ",rp->lhs->index,-rp->nrhs,i);
         4600  +    fprintf(out,"  %4d,  /* (%d) ", rp->lhs->index, i);
         4601  +     rule_print(out, rp);
         4602  +    fprintf(out," */\n"); lineno++;
         4603  +  }
         4604  +  tplt_xfer(lemp->name,in,out,&lineno);
         4605  +  for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
         4606  +    fprintf(out,"  %3d,  /* (%d) ", -rp->nrhs, i);
  4600   4607       rule_print(out, rp);
  4601   4608       fprintf(out," */\n"); lineno++;
  4602   4609     }
  4603   4610     tplt_xfer(lemp->name,in,out,&lineno);
  4604   4611   
  4605   4612     /* Generate code which execution during each REDUCE action */
  4606   4613     i = 0;

Changes to tool/lempar.c.

   682    682     yytos = yypParser->yytos;
   683    683     yytos->stateno = yyNewState;
   684    684     yytos->major = yyMajor;
   685    685     yytos->minor.yy0 = yyMinor;
   686    686     yyTraceShift(yypParser, yyNewState, "Shift");
   687    687   }
   688    688   
   689         -/* The following table contains information about every rule that
   690         -** is used during the reduce.
   691         -*/
   692         -static const struct {
   693         -  YYCODETYPE lhs;       /* Symbol on the left-hand side of the rule */
   694         -  signed char nrhs;     /* Negative of the number of RHS symbols in the rule */
   695         -} yyRuleInfo[] = {
          689  +/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side
          690  +** of that rule */
          691  +static const YYCODETYPE yyRuleInfoLhs[] = {
          692  +%%
          693  +};
          694  +
          695  +/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number
          696  +** of symbols on the right-hand side of that rule. */
          697  +static const signed char yyRuleInfoNRhs[] = {
   696    698   %%
   697    699   };
   698    700   
   699    701   static void yy_accept(yyParser*);  /* Forward Declaration */
   700    702   
   701    703   /*
   702    704   ** Perform a reduce action and the shift that must immediately
................................................................................
   721    723     int yysize;                     /* Amount to pop the stack */
   722    724     ParseARG_FETCH
   723    725     (void)yyLookahead;
   724    726     (void)yyLookaheadToken;
   725    727     yymsp = yypParser->yytos;
   726    728   #ifndef NDEBUG
   727    729     if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
   728         -    yysize = yyRuleInfo[yyruleno].nrhs;
          730  +    yysize = yyRuleInfoNRhs[yyruleno];
   729    731       if( yysize ){
   730    732         fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n",
   731    733           yyTracePrompt,
   732    734           yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno);
   733    735       }else{
   734    736         fprintf(yyTraceFILE, "%sReduce %d [%s].\n",
   735    737           yyTracePrompt, yyruleno, yyRuleName[yyruleno]);
................................................................................
   736    738       }
   737    739     }
   738    740   #endif /* NDEBUG */
   739    741   
   740    742     /* Check that the stack is large enough to grow by a single entry
   741    743     ** if the RHS of the rule is empty.  This ensures that there is room
   742    744     ** enough on the stack to push the LHS value */
   743         -  if( yyRuleInfo[yyruleno].nrhs==0 ){
          745  +  if( yyRuleInfoNRhs[yyruleno]==0 ){
   744    746   #ifdef YYTRACKMAXSTACKDEPTH
   745    747       if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
   746    748         yypParser->yyhwm++;
   747    749         assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack));
   748    750       }
   749    751   #endif
   750    752   #if YYSTACKDEPTH>0 
................................................................................
   778    780     **  #line <lineno> <thisfile>
   779    781     **     break;
   780    782     */
   781    783   /********** Begin reduce actions **********************************************/
   782    784   %%
   783    785   /********** End reduce actions ************************************************/
   784    786     };
   785         -  assert( yyruleno<sizeof(yyRuleInfo)/sizeof(yyRuleInfo[0]) );
   786         -  yygoto = yyRuleInfo[yyruleno].lhs;
   787         -  yysize = yyRuleInfo[yyruleno].nrhs;
          787  +  assert( yyruleno<sizeof(yyRuleInfoLhs)/sizeof(yyRuleInfoLhs[0]) );
          788  +  yygoto = yyRuleInfoLhs[yyruleno];
          789  +  yysize = yyRuleInfoNRhs[yyruleno];
   788    790     yyact = yy_find_reduce_action(yymsp[yysize].stateno,(YYCODETYPE)yygoto);
   789    791   
   790    792     /* There are no SHIFTREDUCE actions on nonterminals because the table
   791    793     ** generator has simplified them to pure REDUCE actions. */
   792    794     assert( !(yyact>YY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) );
   793    795   
   794    796     /* It is not possible for a REDUCE to be followed by an error */