/ Check-in [123cbb33]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge latest trunk changes into this branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | begin-concurrent
Files: files | file ages | folders
SHA3-256: 123cbb3312917ad5b3c32556547c5b7e8ba4e2d2def8651ff80f9fc1bdc1875c
User & Date: dan 2018-12-18 17:20:01
Wiki:begin-concurrent
Context
2018-12-29
20:42
Fix a problem causing a corrupt pager-cache if an OOM or IO error was encountered while committing a concurrent transacation. check-in: 48ca30f9 user: dan tags: begin-concurrent
2018-12-18
17:47
Merge latest begin-concurrent changes into this branch. check-in: a93ca38b user: dan tags: begin-concurrent-pnu
17:20
Merge latest trunk changes into this branch. check-in: 123cbb33 user: dan tags: begin-concurrent
2018-12-17
22:19
Move variable declaration to address compilation issue (C89). check-in: d64f248d user: mistachkin tags: trunk
2018-12-06
02:08
Merge bug fixes from trunk. check-in: 1e13aaa2 user: drh tags: begin-concurrent
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   665    665   	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
   666    666   
   667    667   ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
   668    668   	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
   669    669                $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
   670    670   
   671    671   sessionfuzz$(TEXE):	$(TOP)/test/sessionfuzz.c sqlite3.c sqlite3.h
   672         -	$(CC) $(CFLAGS) -I. -o $@ $(TOP)/test/sessionfuzz.c $(TLIBS)
          672  +	$(LTLINK) -o $@ $(TOP)/test/sessionfuzz.c $(TLIBS)
   673    673   
   674    674   dbfuzz$(TEXE):	$(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h
   675    675   	$(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS)
   676    676   
   677    677   DBFUZZ2_OPTS = \
   678    678     -DSQLITE_THREADSAFE=0 \
   679    679     -DSQLITE_OMIT_LOAD_EXTENSION \
................................................................................
   681    681     -DSQLITE_DEBUG \
   682    682     -DSQLITE_ENABLE_DBSTAT_VTAB \
   683    683     -DSQLITE_ENABLE_RTREE \
   684    684     -DSQLITE_ENABLE_FTS4 \
   685    685     -DSQLITE_EANBLE_FTS5
   686    686   
   687    687   dbfuzz2:	$(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h
   688         -	clang-6.0 -I. -g -O0 -fsanitize=fuzzer,undefined,address -o dbfuzz2 \
          688  +	clang-6.0 $(OPT_FEATURE_FLAGS) $(OPTS) -I. -g -O0 \
          689  +		-fsanitize=fuzzer,undefined,address -o dbfuzz2 \
   689    690   		$(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c
   690    691   	mkdir -p dbfuzz2-dir
   691    692   	cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir
   692    693   
   693    694   mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
   694    695   	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
   695    696   		$(TLIBS) -rpath "$(libdir)"
................................................................................
  1185   1186   TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit
  1186   1187   TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
  1187   1188   TESTFIXTURE_FLAGS += -DBUILD_sqlite
  1188   1189   TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
  1189   1190   TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
  1190   1191   TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
  1191   1192   TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB
         1193  +TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DESERIALIZE
  1192   1194   
  1193   1195   TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
  1194   1196   TESTFIXTURE_SRC1 = sqlite3.c
  1195   1197   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c
  1196   1198   TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))
  1197   1199   
  1198   1200   testfixture$(TEXE):	$(TESTFIXTURE_SRC)
................................................................................
  1286   1288   
  1287   1289   sqlite3_checker$(TEXE):	sqlite3_checker.c
  1288   1290   	$(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS)
  1289   1291   
  1290   1292   dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
  1291   1293   	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
  1292   1294              $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
         1295  +
         1296  +dbtotxt$(TEXE): $(TOP)/tool/dbtotxt.c
         1297  +	$(LTLINK)-o $@ $(TOP)/tool/dbtotxt.c
  1293   1298   
  1294   1299   showdb$(TEXE):	$(TOP)/tool/showdb.c sqlite3.lo
  1295   1300   	$(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS)
  1296   1301   
  1297   1302   showstat4$(TEXE):	$(TOP)/tool/showstat4.c sqlite3.lo
  1298   1303   	$(LTLINK) -o $@ $(TOP)/tool/showstat4.c sqlite3.lo $(TLIBS)
  1299   1304   

Changes to Makefile.msc.

  2294   2294   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
  2295   2295   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
  2296   2296   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
  2297   2297   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
  2298   2298   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
  2299   2299   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
  2300   2300   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
         2301  +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
  2301   2302   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
  2302   2303   
  2303   2304   TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
  2304   2305   TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C)
  2305   2306   !IF $(USE_AMALGAMATION)==0
  2306   2307   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0)
  2307   2308   !ELSE
................................................................................
  2421   2422   
  2422   2423   testloadext.lo:	$(TOP)\src\test_loadext.c $(SQLITE3H)
  2423   2424   	$(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c
  2424   2425   
  2425   2426   testloadext.dll:	testloadext.lo
  2426   2427   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo
  2427   2428   
         2429  +dbtotxt.exe:	$(TOP)\tool\dbtotxt.c
         2430  +	$(LTLINK) $(NO_WARN)	$(TOP)\tool\dbtotxt.c /link $(LDFLAGS) $(LTLINKOPTS)
         2431  +
  2428   2432   showdb.exe:	$(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H)
  2429   2433   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2430   2434   		$(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2431   2435   
  2432   2436   showstat4.exe:	$(TOP)\tool\showstat4.c $(SQLITE3C) $(SQLITE3H)
  2433   2437   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2434   2438   		$(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

Changes to ext/fts3/fts3_unicode2.c.

   174    174       61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
   175    175       61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
   176    176       62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
   177    177       62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
   178    178       62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
   179    179       63182, 63242, 63274, 63310, 63368, 63390, 
   180    180     };
          181  +#define HIBIT ((char)0x80)
   181    182     char aChar[] = {
   182         -    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
   183         -    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
   184         -    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
   185         -    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
   186         -    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
   187         -    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
   188         -    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
   189         -    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
   190         -    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
   191         -    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
   192         -    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
   193         -    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
   194         -    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
   195         -    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
   196         -    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
   197         -    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
   198         -    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
   199         -    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
   200         -    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
   201         -    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
   202         -    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
          183  +    '\0',      'a',       'c',       'e',       'i',       'n',       
          184  +    'o',       'u',       'y',       'y',       'a',       'c',       
          185  +    'd',       'e',       'e',       'g',       'h',       'i',       
          186  +    'j',       'k',       'l',       'n',       'o',       'r',       
          187  +    's',       't',       'u',       'u',       'w',       'y',       
          188  +    'z',       'o',       'u',       'a',       'i',       'o',       
          189  +    'u',       'u'|HIBIT, 'a'|HIBIT, 'g',       'k',       'o',       
          190  +    'o'|HIBIT, 'j',       'g',       'n',       'a'|HIBIT, 'a',       
          191  +    'e',       'i',       'o',       'r',       'u',       's',       
          192  +    't',       'h',       'a',       'e',       'o'|HIBIT, 'o',       
          193  +    'o'|HIBIT, 'y',       '\0',      '\0',      '\0',      '\0',      
          194  +    '\0',      '\0',      '\0',      '\0',      'a',       'b',       
          195  +    'c'|HIBIT, 'd',       'd',       'e'|HIBIT, 'e',       'e'|HIBIT, 
          196  +    'f',       'g',       'h',       'h',       'i',       'i'|HIBIT, 
          197  +    'k',       'l',       'l'|HIBIT, 'l',       'm',       'n',       
          198  +    'o'|HIBIT, 'p',       'r',       'r'|HIBIT, 'r',       's',       
          199  +    's'|HIBIT, 't',       'u',       'u'|HIBIT, 'v',       'w',       
          200  +    'w',       'x',       'y',       'z',       'h',       't',       
          201  +    'w',       'y',       'a',       'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, 
          202  +    'e',       'e'|HIBIT, 'e'|HIBIT, 'i',       'o',       'o'|HIBIT, 
          203  +    'o'|HIBIT, 'o'|HIBIT, 'u',       'u'|HIBIT, 'u'|HIBIT, 'y',  
   203    204     };
   204    205   
   205    206     unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
   206    207     int iRes = 0;
   207    208     int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
   208    209     int iLo = 0;
   209    210     while( iHi>=iLo ){

Changes to ext/fts5/fts5.h.

   410    410   **            same token for inputs "first" and "1st". Say that token is in
   411    411   **            fact "first", so that when the user inserts the document "I won
   412    412   **            1st place" entries are added to the index for tokens "i", "won",
   413    413   **            "first" and "place". If the user then queries for '1st + place',
   414    414   **            the tokenizer substitutes "first" for "1st" and the query works
   415    415   **            as expected.
   416    416   **
   417         -**       <li> By adding multiple synonyms for a single term to the FTS index.
   418         -**            In this case, when tokenizing query text, the tokenizer may 
   419         -**            provide multiple synonyms for a single term within the document.
   420         -**            FTS5 then queries the index for each synonym individually. For
   421         -**            example, faced with the query:
          417  +**       <li> By querying the index for all synonyms of each query term
          418  +**            separately. In this case, when tokenizing query text, the
          419  +**            tokenizer may provide multiple synonyms for a single term 
          420  +**            within the document. FTS5 then queries the index for each 
          421  +**            synonym individually. For example, faced with the query:
   422    422   **
   423    423   **   <codeblock>
   424    424   **     ... MATCH 'first place'</codeblock>
   425    425   **
   426    426   **            the tokenizer offers both "1st" and "first" as synonyms for the
   427    427   **            first token in the MATCH query and FTS5 effectively runs a query 
   428    428   **            similar to:
................................................................................
   438    438   **            Using this method, when tokenizing document text, the tokenizer
   439    439   **            provides multiple synonyms for each token. So that when a 
   440    440   **            document such as "I won first place" is tokenized, entries are
   441    441   **            added to the FTS index for "i", "won", "first", "1st" and
   442    442   **            "place".
   443    443   **
   444    444   **            This way, even if the tokenizer does not provide synonyms
   445         -**            when tokenizing query text (it should not - to do would be
          445  +**            when tokenizing query text (it should not - to do so would be
   446    446   **            inefficient), it doesn't matter if the user queries for 
   447    447   **            'first + place' or '1st + place', as there are entries in the
   448    448   **            FTS index corresponding to both forms of the first token.
   449    449   **   </ol>
   450    450   **
   451    451   **   Whether it is parsing document or query text, any call to xToken that
   452    452   **   specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit

Changes to ext/fts5/fts5_unicode2.c.

    43     43       61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
    44     44       61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
    45     45       62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
    46     46       62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
    47     47       62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
    48     48       63182, 63242, 63274, 63310, 63368, 63390, 
    49     49     };
           50  +#define HIBIT ((char)0x80)
    50     51     char aChar[] = {
    51         -    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
    52         -    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
    53         -    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
    54         -    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
    55         -    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
    56         -    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
    57         -    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
    58         -    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
    59         -    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
    60         -    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
    61         -    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
    62         -    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
    63         -    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
    64         -    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
    65         -    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
    66         -    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
    67         -    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
    68         -    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
    69         -    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
    70         -    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
    71         -    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
           52  +    '\0',      'a',       'c',       'e',       'i',       'n',       
           53  +    'o',       'u',       'y',       'y',       'a',       'c',       
           54  +    'd',       'e',       'e',       'g',       'h',       'i',       
           55  +    'j',       'k',       'l',       'n',       'o',       'r',       
           56  +    's',       't',       'u',       'u',       'w',       'y',       
           57  +    'z',       'o',       'u',       'a',       'i',       'o',       
           58  +    'u',       'u'|HIBIT, 'a'|HIBIT, 'g',       'k',       'o',       
           59  +    'o'|HIBIT, 'j',       'g',       'n',       'a'|HIBIT, 'a',       
           60  +    'e',       'i',       'o',       'r',       'u',       's',       
           61  +    't',       'h',       'a',       'e',       'o'|HIBIT, 'o',       
           62  +    'o'|HIBIT, 'y',       '\0',      '\0',      '\0',      '\0',      
           63  +    '\0',      '\0',      '\0',      '\0',      'a',       'b',       
           64  +    'c'|HIBIT, 'd',       'd',       'e'|HIBIT, 'e',       'e'|HIBIT, 
           65  +    'f',       'g',       'h',       'h',       'i',       'i'|HIBIT, 
           66  +    'k',       'l',       'l'|HIBIT, 'l',       'm',       'n',       
           67  +    'o'|HIBIT, 'p',       'r',       'r'|HIBIT, 'r',       's',       
           68  +    's'|HIBIT, 't',       'u',       'u'|HIBIT, 'v',       'w',       
           69  +    'w',       'x',       'y',       'z',       'h',       't',       
           70  +    'w',       'y',       'a',       'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, 
           71  +    'e',       'e'|HIBIT, 'e'|HIBIT, 'i',       'o',       'o'|HIBIT, 
           72  +    'o'|HIBIT, 'o'|HIBIT, 'u',       'u'|HIBIT, 'u'|HIBIT, 'y',  
    72     73     };
    73     74   
    74     75     unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
    75     76     int iRes = 0;
    76     77     int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
    77     78     int iLo = 0;
    78     79     while( iHi>=iLo ){

Changes to ext/rbu/rbu_common.tcl.

    82     82       sqlite3 tmpdb $rbu
    83     83       tmpdb eval { DELETE FROM rbu_state WHERE k==10 }
    84     84       tmpdb close
    85     85     }
    86     86     set rc
    87     87   }
    88     88   
    89         -proc do_rbu_vacuum_test {tn step} {
    90         -  forcedelete state.db
    91         -  uplevel [list do_test $tn.1 {
    92         -    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
           89  +proc do_rbu_vacuum_test {tn step {statedb state.db}} {
           90  +  forcedelete $statedb
           91  +  if {$statedb=="" && $step==1} breakpoint
           92  +  uplevel [list do_test $tn.1 [string map [list %state% $statedb] {
           93  +    if {$step==0} { sqlite3rbu_vacuum rbu test.db {%state%}}
    93     94       while 1 {
    94         -      if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
           95  +      if {$step==1} { sqlite3rbu_vacuum rbu test.db {%state%}}
    95     96         set state [rbu state]
    96     97         check_prestep_state test.db $state
    97     98         set rc [rbu step]
    98     99         check_poststep_state $rc test.db $state
    99    100         if {$rc!="SQLITE_OK"} break
   100    101         if {$step==1} { rbu close }
   101    102       }
   102    103       rbu close
   103         -  } {SQLITE_DONE}]
          104  +  }] {SQLITE_DONE}]
   104    105   
   105    106     uplevel [list do_execsql_test $tn.2 {
   106    107       PRAGMA integrity_check
   107    108     } ok]
   108    109   }
   109    110   

Changes to ext/rbu/rbuvacuum2.test.

    12     12   # This file contains tests for the RBU module. More specifically, it
    13     13   # contains tests to ensure that the sqlite3rbu_vacuum() API works as
    14     14   # expected.
    15     15   #
    16     16   
    17     17   source [file join [file dirname [info script]] rbu_common.tcl]
    18     18   
    19         -foreach step {0 1} {
    20         -  set ::testprefix rbuvacuum2-$step
           19  +foreach {step} {0 1} {
           20  +foreach {ttt state} {
           21  +  s state.db t test.db-vacuum n {}
           22  +} {
           23  +  set ::testprefix rbuvacuum2-$step$ttt
    21     24     
    22     25     #-------------------------------------------------------------------------
    23     26     # Test that a database that contains fts3 tables can be vacuumed.
    24     27     #
    25     28     ifcapable fts3 {
    26     29       reset_db
    27     30       do_execsql_test 1.1 {
    28     31         CREATE VIRTUAL TABLE t1 USING fts3(z, y);
    29     32         INSERT INTO t1 VALUES('fix this issue', 'at some point');
    30     33       }
    31     34     
    32         -    do_rbu_vacuum_test 1.2 $step
           35  +    do_rbu_vacuum_test 1.2 $step $state
    33     36     
    34     37       do_execsql_test 1.3 {
    35     38         SELECT * FROM t1;
    36     39       } {{fix this issue} {at some point}}
    37     40     
    38     41       do_execsql_test 1.4 {
    39     42         SELECT rowid FROM t1 WHERE t1 MATCH 'fix';
................................................................................
    42     45       do_execsql_test 1.5 {
    43     46         INSERT INTO t1 VALUES('a b c', 'd e f');
    44     47         INSERT INTO t1 VALUES('l h i', 'd e f');
    45     48         DELETE FROM t1 WHERE docid = 2;
    46     49         INSERT INTO t1 VALUES('a b c', 'x y z');
    47     50       }
    48     51   
    49         -    do_rbu_vacuum_test 1.6 $step
           52  +    do_rbu_vacuum_test 1.6 $step $state
    50     53       do_execsql_test 1.7 {
    51     54         INSERT INTO t1(t1) VALUES('integrity-check');
    52     55         SELECT * FROM t1;
    53     56       } {
    54     57         {fix this issue} {at some point}
    55     58         {l h i} {d e f}
    56     59         {a b c} {x y z}
................................................................................
    63     66     ifcapable fts5 {
    64     67       reset_db
    65     68       do_execsql_test 2.1 {
    66     69         CREATE VIRTUAL TABLE t1 USING fts5(z, y);
    67     70         INSERT INTO t1 VALUES('fix this issue', 'at some point');
    68     71       }
    69     72     
    70         -    do_rbu_vacuum_test 2.2 $step
           73  +    do_rbu_vacuum_test 2.2 $step $state
    71     74     
    72     75       do_execsql_test 2.3 {
    73     76         SELECT * FROM t1;
    74     77       } {{fix this issue} {at some point}}
    75     78     
    76     79       do_execsql_test 2.4 {
    77     80         SELECT rowid FROM t1 ('fix');
................................................................................
    80     83       do_execsql_test 2.5 {
    81     84         INSERT INTO t1 VALUES('a b c', 'd e f');
    82     85         INSERT INTO t1 VALUES('l h i', 'd e f');
    83     86         DELETE FROM t1 WHERE rowid = 2;
    84     87         INSERT INTO t1 VALUES('a b c', 'x y z');
    85     88       }
    86     89   
    87         -    do_rbu_vacuum_test 2.6 $step
           90  +    do_rbu_vacuum_test 2.6 $step $state
    88     91       do_execsql_test 2.7 {
    89     92         INSERT INTO t1(t1) VALUES('integrity-check');
    90     93         SELECT * FROM t1;
    91     94       } {
    92     95         {fix this issue} {at some point}
    93     96         {l h i} {d e f}
    94     97         {a b c} {x y z}
................................................................................
   103    106       do_execsql_test 3.1 {
   104    107         CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2);
   105    108         INSERT INTO rt VALUES(1, 45, 55);
   106    109         INSERT INTO rt VALUES(2, 50, 60);
   107    110         INSERT INTO rt VALUES(3, 55, 65);
   108    111       }
   109    112     
   110         -    do_rbu_vacuum_test 3.2 $step
          113  +    do_rbu_vacuum_test 3.2 $step $state
   111    114     
   112    115       do_execsql_test 3.3 {
   113    116         SELECT * FROM rt;
   114    117       } {1 45.0 55.0 2 50.0 60.0 3 55.0 65.0}
   115    118     
   116    119       do_execsql_test 3.4.1 {
   117    120         SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
   118    121       } {1 2}
   119    122       do_execsql_test 3.4.2 {
   120    123         SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
   121    124       } {2 3}
   122    125   
   123         -    do_rbu_vacuum_test 3.5 $step
          126  +    do_rbu_vacuum_test 3.5 $step $state
   124    127   
   125    128       do_execsql_test 3.6.1 {
   126    129         SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
   127    130       } {1 2}
   128    131       do_execsql_test 3.6.2 {
   129    132         SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
   130    133       } {2 3}
................................................................................
   143    146         SELECT * FROM sqlite_master;
   144    147       } {
   145    148       table t1 t1 2 {CREATE TABLE t1(a, b, c)}
   146    149       view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
   147    150       trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
   148    151       }
   149    152   
   150         -    do_rbu_vacuum_test 4.3 $step
          153  +    do_rbu_vacuum_test 4.3 $step $state
   151    154       do_execsql_test 4.4 {
   152    155         SELECT * FROM sqlite_master;
   153    156       } {
   154    157       table t1 t1 2 {CREATE TABLE t1(a, b, c)}
   155    158       view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
   156    159       trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
   157    160       }
   158    161     }
          162  +}
   159    163   }
   160    164     
   161    165   #-------------------------------------------------------------------------
   162    166   # Test that passing a NULL value as the second argument to 
   163    167   # sqlite3rbu_vacuum() causes it to:
   164    168   #
   165    169   #   * Use <database>-vacuum as the state db, and
................................................................................
   227    231   do_test 6.3 {
   228    232     sqlite3rbu_vacuum rbu test.db test.db2
   229    233     while {[rbu step]!="SQLITE_DONE"} { rbu step }
   230    234     rbu close
   231    235     execsql { PRAGMA integrity_check }
   232    236   } {ok}
   233    237   
          238  +do_test 6.4 {
          239  +  sqlite3rbu_vacuum rbu test.db test.db-vactmp
          240  +  list [catch { rbu close } msg] $msg
          241  +} {1 SQLITE_MISUSE}
          242  +
   234    243   finish_test

Changes to ext/rbu/sqlite3rbu.c.

  2473   2473           zExtra = &p->zRbu[5];
  2474   2474           while( *zExtra ){
  2475   2475             if( *zExtra++=='?' ) break;
  2476   2476           }
  2477   2477           if( *zExtra=='\0' ) zExtra = 0;
  2478   2478         }
  2479   2479   
  2480         -      zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", 
         2480  +      zTarget = sqlite3_mprintf("file:%s-vactmp?rbu_memory=1%s%s", 
  2481   2481             sqlite3_db_filename(p->dbRbu, "main"),
  2482   2482             (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra)
  2483   2483         );
  2484   2484   
  2485   2485         if( zTarget==0 ){
  2486   2486           p->rc = SQLITE_NOMEM;
  2487   2487           return;
................................................................................
  3739   3739   ** Open a handle to begin or resume an RBU VACUUM operation.
  3740   3740   */
  3741   3741   sqlite3rbu *sqlite3rbu_vacuum(
  3742   3742     const char *zTarget, 
  3743   3743     const char *zState
  3744   3744   ){
  3745   3745     if( zTarget==0 ){ return rbuMisuseError(); }
         3746  +  if( zState ){
         3747  +    int n = strlen(zState);
         3748  +    if( n>=7 && 0==memcmp("-vactmp", &zState[n-7], 7) ){
         3749  +      return rbuMisuseError();
         3750  +    }
         3751  +  }
  3746   3752     /* TODO: Check that both arguments are non-NULL */
  3747   3753     return openRbuHandle(0, zTarget, zState);
  3748   3754   }
  3749   3755   
  3750   3756   /*
  3751   3757   ** Return the database handle used by pRbu.
  3752   3758   */

Changes to ext/rbu/sqlite3rbu.h.

   329    329   ** The vacuum can be resumed by calling this function to open a new RBU
   330    330   ** handle specifying the same target and state databases.
   331    331   **
   332    332   ** If the second argument passed to this function is NULL, then the
   333    333   ** name of the state database is "<database>-vacuum", where <database>
   334    334   ** is the name of the target database file. In this case, on UNIX, if the
   335    335   ** state database is not already present in the file-system, it is created
   336         -** with the same permissions as the target db is made.
          336  +** with the same permissions as the target db is made. 
          337  +**
          338  +** With an RBU vacuum, it is an SQLITE_MISUSE error if the name of the 
          339  +** state database ends with "-vactmp". This name is reserved for internal 
          340  +** use.
   337    341   **
   338    342   ** This function does not delete the state database after an RBU vacuum
   339    343   ** is completed, even if it created it. However, if the call to
   340    344   ** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents
   341    345   ** of the state tables within the state database are zeroed. This way,
   342    346   ** the next call to sqlite3rbu_vacuum() opens a handle that starts a 
   343    347   ** new RBU vacuum operation.

Changes to ext/rbu/test_rbu.c.

   269    269     if( objc!=3 && objc!=4 ){
   270    270       Tcl_WrongNumArgs(interp, 1, objv, "NAME TARGET-DB ?STATE-DB?");
   271    271       return TCL_ERROR;
   272    272     }
   273    273     zCmd = Tcl_GetString(objv[1]);
   274    274     zTarget = Tcl_GetString(objv[2]);
   275    275     if( objc==4 ) zStateDb = Tcl_GetString(objv[3]);
          276  +  if( zStateDb && zStateDb[0]=='\0' ) zStateDb = 0;
   276    277   
   277    278     pRbu = sqlite3rbu_vacuum(zTarget, zStateDb);
   278    279     Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0);
   279    280     Tcl_SetObjResult(interp, objv[1]);
   280    281     return TCL_OK;
   281    282   }
   282    283   

Changes to ext/session/sqlite3session.c.

  1157   1157     int op,                         /* One of SQLITE_UPDATE, INSERT, DELETE */
  1158   1158     sqlite3_session *pSession,      /* Session object pTab is attached to */
  1159   1159     SessionTable *pTab              /* Table that change applies to */
  1160   1160   ){
  1161   1161     int iHash; 
  1162   1162     int bNull = 0; 
  1163   1163     int rc = SQLITE_OK;
  1164         -  SessionStat1Ctx stat1 = {0};
         1164  +  SessionStat1Ctx stat1 = {{0,0,0,0,0},0};
  1165   1165   
  1166   1166     if( pSession->rc ) return;
  1167   1167   
  1168   1168     /* Load table details if required */
  1169   1169     if( sessionInitTable(pSession, pTab) ) return;
  1170   1170   
  1171   1171     /* Check the number of columns in this xPreUpdate call matches the 

Changes to main.mk.

   976    976   TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO)
   977    977   $(TEST_EXTENSION): $(TOP)/src/test_loadext.c
   978    978   	$(MKSHLIB) $(TOP)/src/test_loadext.c -o $(TEST_EXTENSION)
   979    979   
   980    980   extensiontest: testfixture$(EXE) $(TEST_EXTENSION)
   981    981   	./testfixture$(EXE) $(TOP)/test/loadext.test
   982    982   
          983  +dbtotxt$(EXE):	$(TOP)/tool/dbtotxt.c
          984  +	$(TCC) -o dbtotxt$(EXE) $(TOP)/tool/dbtotxt.c
          985  +
   983    986   showdb$(EXE):	$(TOP)/tool/showdb.c sqlite3.o
   984    987   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showdb$(EXE) \
   985    988   		$(TOP)/tool/showdb.c sqlite3.o $(THREADLIB)
   986    989   
   987    990   showstat4$(EXE):	$(TOP)/tool/showstat4.c sqlite3.o
   988    991   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showstat4$(EXE) \
   989    992   		$(TOP)/tool/showstat4.c sqlite3.o $(THREADLIB)

Changes to src/btree.c.

   889    889     assert( 0==pCur->pKey );
   890    890     assert( cursorHoldsMutex(pCur) );
   891    891   
   892    892     if( pCur->curIntKey ){
   893    893       /* Only the rowid is required for a table btree */
   894    894       pCur->nKey = sqlite3BtreeIntegerKey(pCur);
   895    895     }else{
   896         -    /* For an index btree, save the complete key content */
          896  +    /* For an index btree, save the complete key content. It is possible
          897  +    ** that the current key is corrupt. In that case, it is possible that
          898  +    ** the sqlite3VdbeRecordUnpack() function may overread the buffer by
          899  +    ** up to the size of 1 varint plus 1 8-byte value when the cursor 
          900  +    ** position is restored. Hence the 17 bytes of padding allocated 
          901  +    ** below. */
   897    902       void *pKey;
   898    903       pCur->nKey = sqlite3BtreePayloadSize(pCur);
   899         -    pKey = sqlite3Malloc( pCur->nKey );
          904  +    pKey = sqlite3Malloc( pCur->nKey + 9 + 8 );
   900    905       if( pKey ){
   901    906         rc = sqlite3BtreePayload(pCur, 0, (int)pCur->nKey, pKey);
   902    907         if( rc==SQLITE_OK ){
   903    908           pCur->pKey = pKey;
   904    909         }else{
   905    910           sqlite3_free(pKey);
   906    911         }
................................................................................
  1227   1232       return;
  1228   1233     }
  1229   1234     iPtrmap = PTRMAP_PAGENO(pBt, key);
  1230   1235     rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage, 0);
  1231   1236     if( rc!=SQLITE_OK ){
  1232   1237       *pRC = rc;
  1233   1238       return;
         1239  +  }
         1240  +  if( ((char*)sqlite3PagerGetExtra(pDbPage))[0]!=0 ){
         1241  +    /* The first byte of the extra data is the MemPage.isInit byte.
         1242  +    ** If that byte is set, it means this page is also being used
         1243  +    ** as a btree page. */
         1244  +    *pRC = SQLITE_CORRUPT_BKPT;
         1245  +    goto ptrmap_exit;
  1234   1246     }
  1235   1247     offset = PTRMAP_PTROFFSET(iPtrmap, key);
  1236   1248     if( offset<0 ){
  1237   1249       *pRC = SQLITE_CORRUPT_BKPT;
  1238   1250       goto ptrmap_exit;
  1239   1251     }
  1240   1252     assert( offset <= (int)pBt->usableSize-5 );
................................................................................
  1593   1605   */
  1594   1606   static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){
  1595   1607     CellInfo info;
  1596   1608     if( *pRC ) return;
  1597   1609     assert( pCell!=0 );
  1598   1610     pPage->xParseCell(pPage, pCell, &info);
  1599   1611     if( info.nLocal<info.nPayload ){
  1600         -    Pgno ovfl = get4byte(&pCell[info.nSize-4]);
         1612  +    Pgno ovfl;
         1613  +    if( SQLITE_WITHIN(pPage->aDataEnd, pCell, pCell+info.nLocal) ){
         1614  +      *pRC = SQLITE_CORRUPT_BKPT;
         1615  +      return;
         1616  +    }
         1617  +    ovfl = get4byte(&pCell[info.nSize-4]);
  1601   1618       ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC);
  1602   1619     }
  1603   1620   }
  1604   1621   #endif
  1605   1622   
  1606   1623   
  1607   1624   /*
................................................................................
  1648   1665     /* This block handles pages with two or fewer free blocks and nMaxFrag
  1649   1666     ** or fewer fragmented bytes. In this case it is faster to move the
  1650   1667     ** two (or one) blocks of cells using memmove() and add the required
  1651   1668     ** offsets to each pointer in the cell-pointer array than it is to 
  1652   1669     ** reconstruct the entire page.  */
  1653   1670     if( (int)data[hdr+7]<=nMaxFrag ){
  1654   1671       int iFree = get2byte(&data[hdr+1]);
         1672  +
         1673  +    /* If the initial freeblock offset were out of bounds, that would
         1674  +    ** have been detected by btreeInitPage() when it was computing the
         1675  +    ** number of free bytes on the page. */
         1676  +    assert( iFree<=usableSize-4 );
  1655   1677       if( iFree ){
  1656   1678         int iFree2 = get2byte(&data[iFree]);
  1657         -
  1658         -      /* pageFindSlot() has already verified that free blocks are sorted
  1659         -      ** in order of offset within the page, and that no block extends
  1660         -      ** past the end of the page. Provided the two free slots do not 
  1661         -      ** overlap, this guarantees that the memmove() calls below will not
  1662         -      ** overwrite the usableSize byte buffer, even if the database page
  1663         -      ** is corrupt.  */
  1664         -      assert( iFree2==0 || iFree2>iFree );
  1665         -      assert( iFree+get2byte(&data[iFree+2]) <= usableSize );
  1666         -      assert( iFree2==0 || iFree2+get2byte(&data[iFree2+2]) <= usableSize );
  1667         -
         1679  +      if( iFree2>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage);
  1668   1680         if( 0==iFree2 || (data[iFree2]==0 && data[iFree2+1]==0) ){
  1669   1681           u8 *pEnd = &data[cellOffset + nCell*2];
  1670   1682           u8 *pAddr;
  1671   1683           int sz2 = 0;
  1672   1684           int sz = get2byte(&data[iFree+2]);
  1673   1685           int top = get2byte(&data[hdr+5]);
  1674   1686           if( top>=iFree ){
  1675   1687             return SQLITE_CORRUPT_PAGE(pPage);
  1676   1688           }
  1677   1689           if( iFree2 ){
  1678         -          assert( iFree+sz<=iFree2 ); /* Verified by pageFindSlot() */
         1690  +          if( iFree+sz>iFree2 ) return SQLITE_CORRUPT_PAGE(pPage);
  1679   1691             sz2 = get2byte(&data[iFree2+2]);
  1680         -          assert( iFree+sz+sz2+iFree2-(iFree+sz) <= usableSize );
         1692  +          if( iFree2+sz2 > usableSize ) return SQLITE_CORRUPT_PAGE(pPage);
  1681   1693             memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz));
  1682   1694             sz += sz2;
  1683   1695           }
  1684   1696           cbrk = top+sz;
  1685   1697           assert( cbrk+(iFree-top) <= usableSize );
  1686   1698           memmove(&data[cbrk], &data[top], iFree-top);
  1687   1699           for(pAddr=&data[cellOffset]; pAddr<pEnd; pAddr+=2){
................................................................................
  7212   7224     i = get2byte(&aData[hdr+5]);
  7213   7225     memcpy(&pTmp[i], &aData[i], usableSize - i);
  7214   7226   
  7215   7227     pData = pEnd;
  7216   7228     for(i=0; i<nCell; i++){
  7217   7229       u8 *pCell = apCell[i];
  7218   7230       if( SQLITE_WITHIN(pCell,aData,pEnd) ){
         7231  +      if( ((uptr)(pCell+szCell[i]))>(uptr)pEnd ) return SQLITE_CORRUPT_BKPT;
  7219   7232         pCell = &pTmp[pCell - aData];
  7220   7233       }
  7221   7234       pData -= szCell[i];
  7222   7235       put2byte(pCellptr, (pData - aData));
  7223   7236       pCellptr += 2;
  7224   7237       if( pData < pCellptr ) return SQLITE_CORRUPT_BKPT;
  7225   7238       memcpy(pData, pCell, szCell[i]);
................................................................................
  7506   7519     int rc;                              /* Return Code */
  7507   7520     Pgno pgnoNew;                        /* Page number of pNew */
  7508   7521   
  7509   7522     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  7510   7523     assert( sqlite3PagerIswriteable(pParent->pDbPage) );
  7511   7524     assert( pPage->nOverflow==1 );
  7512   7525   
  7513         -  /* This error condition is now caught prior to reaching this function */
  7514         -  if( NEVER(pPage->nCell==0) ) return SQLITE_CORRUPT_BKPT;
         7526  +  if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT;  /* dbfuzz001.test */
  7515   7527   
  7516   7528     /* Allocate a new page. This page will become the right-sibling of 
  7517   7529     ** pPage. Make the parent page writable, so that the new divider cell
  7518   7530     ** may be inserted. If both these operations are successful, proceed.
  7519   7531     */
  7520   7532     rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0);
  7521   7533   
................................................................................
  9088   9100     **
  9089   9101     ** Or, if the current delete will not cause a rebalance, then the cursor
  9090   9102     ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately
  9091   9103     ** before or after the deleted entry. In this case set bSkipnext to true.  */
  9092   9104     if( bPreserve ){
  9093   9105       if( !pPage->leaf 
  9094   9106        || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3)
         9107  +     || pPage->nCell==1  /* See dbfuzz001.test for a test case */
  9095   9108       ){
  9096   9109         /* A b-tree rebalance will be required after deleting this entry.
  9097   9110         ** Save the cursor key.  */
  9098   9111         rc = saveCursorKey(pCur);
  9099   9112         if( rc ) return rc;
  9100   9113       }else{
  9101   9114         bSkipnext = 1;
................................................................................
  9870   9883       N--;
  9871   9884       if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage, 0) ){
  9872   9885         checkAppendMsg(pCheck, "failed to get page %d", iPage);
  9873   9886         break;
  9874   9887       }
  9875   9888       pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage);
  9876   9889       if( isFreeList ){
  9877         -      int n = get4byte(&pOvflData[4]);
         9890  +      u32 n = (u32)get4byte(&pOvflData[4]);
  9878   9891   #ifndef SQLITE_OMIT_AUTOVACUUM
  9879   9892         if( pCheck->pBt->autoVacuum ){
  9880   9893           checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0);
  9881   9894         }
  9882   9895   #endif
  9883         -      if( n>(int)pCheck->pBt->usableSize/4-2 ){
         9896  +      if( n>pCheck->pBt->usableSize/4-2 ){
  9884   9897           checkAppendMsg(pCheck,
  9885   9898              "freelist leaf count too big on page %d", iPage);
  9886   9899           N--;
  9887   9900         }else{
  9888         -        for(i=0; i<n; i++){
         9901  +        for(i=0; i<(int)n; i++){
  9889   9902             Pgno iFreePage = get4byte(&pOvflData[8+i*4]);
  9890   9903   #ifndef SQLITE_OMIT_AUTOVACUUM
  9891   9904             if( pCheck->pBt->autoVacuum ){
  9892   9905               checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0);
  9893   9906             }
  9894   9907   #endif
  9895   9908             checkRef(pCheck, iFreePage);
................................................................................
 10325  10338     }else if( get4byte(&pBt->pPage1->aData[64])!=0 ){
 10326  10339       checkAppendMsg(&sCheck,
 10327  10340         "incremental_vacuum enabled with a max rootpage of zero"
 10328  10341       );
 10329  10342     }
 10330  10343   #endif
 10331  10344     testcase( pBt->db->flags & SQLITE_CellSizeCk );
 10332         -  pBt->db->flags &= ~SQLITE_CellSizeCk;
        10345  +  pBt->db->flags &= ~(u64)SQLITE_CellSizeCk;
 10333  10346     for(i=0; (int)i<nRoot && sCheck.mxErr; i++){
 10334  10347       i64 notUsed;
 10335  10348       if( aRoot[i]==0 ) continue;
 10336  10349   #ifndef SQLITE_OMIT_AUTOVACUUM
 10337  10350       if( pBt->autoVacuum && aRoot[i]>1 ){
 10338  10351         checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0);
 10339  10352       }

Changes to src/expr.c.

   137    137   CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){
   138    138     sqlite3 *db = pParse->db;
   139    139     CollSeq *pColl = 0;
   140    140     Expr *p = pExpr;
   141    141     while( p ){
   142    142       int op = p->op;
   143    143       if( p->flags & EP_Generic ) break;
   144         -    if( (op==TK_AGG_COLUMN || op==TK_COLUMN
   145         -          || op==TK_REGISTER || op==TK_TRIGGER)
          144  +    if( op==TK_REGISTER ) op = p->op2;
          145  +    if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER)
   146    146        && p->y.pTab!=0
   147    147       ){
   148    148         /* op==TK_REGISTER && p->y.pTab!=0 happens when pExpr was originally
   149    149         ** a TK_COLUMN but was previously evaluated and cached in a register */
   150    150         int j = p->iColumn;
   151    151         if( j>=0 ){
   152    152           const char *zColl = p->y.pTab->aCol[j].zColl;
................................................................................
   154    154         }
   155    155         break;
   156    156       }
   157    157       if( op==TK_CAST || op==TK_UPLUS ){
   158    158         p = p->pLeft;
   159    159         continue;
   160    160       }
   161         -    if( op==TK_COLLATE || (op==TK_REGISTER && p->op2==TK_COLLATE) ){
          161  +    if( op==TK_COLLATE ){
   162    162         pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken);
   163    163         break;
   164    164       }
   165    165       if( p->flags & EP_Collate ){
   166    166         if( p->pLeft && (p->pLeft->flags & EP_Collate)!=0 ){
   167    167           p = p->pLeft;
   168    168         }else{
................................................................................
  1325   1325       }
  1326   1326     }
  1327   1327     return pRet;
  1328   1328   }
  1329   1329   #else
  1330   1330   # define withDup(x,y) 0
  1331   1331   #endif
         1332  +
         1333  +#ifndef SQLITE_OMIT_WINDOWFUNC
         1334  +/*
         1335  +** The gatherSelectWindows() procedure and its helper routine
         1336  +** gatherSelectWindowsCallback() are used to scan all the expressions
         1337  +** an a newly duplicated SELECT statement and gather all of the Window
         1338  +** objects found there, assembling them onto the linked list at Select->pWin.
         1339  +*/
         1340  +static int gatherSelectWindowsCallback(Walker *pWalker, Expr *pExpr){
         1341  +  if( pExpr->op==TK_FUNCTION && pExpr->y.pWin!=0 ){
         1342  +    assert( ExprHasProperty(pExpr, EP_WinFunc) );
         1343  +    pExpr->y.pWin->pNextWin = pWalker->u.pSelect->pWin;
         1344  +    pWalker->u.pSelect->pWin = pExpr->y.pWin;
         1345  +  }
         1346  +  return WRC_Continue;
         1347  +}
         1348  +static int gatherSelectWindowsSelectCallback(Walker *pWalker, Select *p){
         1349  +  return p==pWalker->u.pSelect ? WRC_Continue : WRC_Prune;
         1350  +}
         1351  +static void gatherSelectWindows(Select *p){
         1352  +  Walker w;
         1353  +  w.xExprCallback = gatherSelectWindowsCallback;
         1354  +  w.xSelectCallback = gatherSelectWindowsSelectCallback;
         1355  +  w.xSelectCallback2 = 0;
         1356  +  w.u.pSelect = p;
         1357  +  sqlite3WalkSelect(&w, p);
         1358  +}
         1359  +#endif
         1360  +
  1332   1361   
  1333   1362   /*
  1334   1363   ** The following group of routines make deep copies of expressions,
  1335   1364   ** expression lists, ID lists, and select statements.  The copies can
  1336   1365   ** be deleted (by being passed to their respective ...Delete() routines)
  1337   1366   ** without effecting the originals.
  1338   1367   **
................................................................................
  1493   1522       pNew->addrOpenEphm[0] = -1;
  1494   1523       pNew->addrOpenEphm[1] = -1;
  1495   1524       pNew->nSelectRow = p->nSelectRow;
  1496   1525       pNew->pWith = withDup(db, p->pWith);
  1497   1526   #ifndef SQLITE_OMIT_WINDOWFUNC
  1498   1527       pNew->pWin = 0;
  1499   1528       pNew->pWinDefn = sqlite3WindowListDup(db, p->pWinDefn);
         1529  +    if( p->pWin ) gatherSelectWindows(pNew);
  1500   1530   #endif
  1501   1531       pNew->selId = p->selId;
  1502   1532       *pp = pNew;
  1503   1533       pp = &pNew->pPrior;
  1504   1534       pNext = pNew;
  1505   1535     }
  1506   1536   
................................................................................
  2409   2439   
  2410   2440         if( affinity_ok ){
  2411   2441           /* Search for an existing index that will work for this IN operator */
  2412   2442           for(pIdx=pTab->pIndex; pIdx && eType==0; pIdx=pIdx->pNext){
  2413   2443             Bitmask colUsed;      /* Columns of the index used */
  2414   2444             Bitmask mCol;         /* Mask for the current column */
  2415   2445             if( pIdx->nColumn<nExpr ) continue;
         2446  +          if( pIdx->pPartIdxWhere!=0 ) continue;
  2416   2447             /* Maximum nColumn is BMS-2, not BMS-1, so that we can compute
  2417   2448             ** BITMASK(nExpr) without overflowing */
  2418   2449             testcase( pIdx->nColumn==BMS-2 );
  2419   2450             testcase( pIdx->nColumn==BMS-1 );
  2420   2451             if( pIdx->nColumn>=BMS-1 ) continue;
  2421   2452             if( mustBeUnique ){
  2422   2453               if( pIdx->nKeyCol>nExpr
................................................................................
  4743   4774       }else if( pA->op==TK_COLLATE ){
  4744   4775         if( sqlite3_stricmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2;
  4745   4776       }else if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){
  4746   4777         return 2;
  4747   4778       }
  4748   4779     }
  4749   4780     if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2;
  4750         -  if( ALWAYS((combinedFlags & EP_TokenOnly)==0) ){
         4781  +  if( (combinedFlags & EP_TokenOnly)==0 ){
  4751   4782       if( combinedFlags & EP_xIsSelect ) return 2;
  4752   4783       if( (combinedFlags & EP_FixedCol)==0
  4753   4784        && sqlite3ExprCompare(pParse, pA->pLeft, pB->pLeft, iTab) ) return 2;
  4754   4785       if( sqlite3ExprCompare(pParse, pA->pRight, pB->pRight, iTab) ) return 2;
  4755   4786       if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList, iTab) ) return 2;
  4756         -    assert( (combinedFlags & EP_Reduced)==0 );
  4757         -    if( pA->op!=TK_STRING && pA->op!=TK_TRUEFALSE ){
         4787  +    if( pA->op!=TK_STRING
         4788  +     && pA->op!=TK_TRUEFALSE
         4789  +     && (combinedFlags & EP_Reduced)==0
         4790  +    ){
  4758   4791         if( pA->iColumn!=pB->iColumn ) return 2;
  4759   4792         if( pA->iTable!=pB->iTable 
  4760   4793          && (pA->iTable!=iTab || NEVER(pB->iTable>=0)) ) return 2;
  4761   4794       }
  4762   4795     }
  4763   4796     return 0;
  4764   4797   }

Changes to src/loadext.c.

   646    646   ** default so as not to open security holes in older applications.
   647    647   */
   648    648   int sqlite3_enable_load_extension(sqlite3 *db, int onoff){
   649    649     sqlite3_mutex_enter(db->mutex);
   650    650     if( onoff ){
   651    651       db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc;
   652    652     }else{
   653         -    db->flags &= ~(SQLITE_LoadExtension|SQLITE_LoadExtFunc);
          653  +    db->flags &= ~(u64)(SQLITE_LoadExtension|SQLITE_LoadExtFunc);
   654    654     }
   655    655     sqlite3_mutex_leave(db->mutex);
   656    656     return SQLITE_OK;
   657    657   }
   658    658   
   659    659   #endif /* !defined(SQLITE_OMIT_LOAD_EXTENSION) */
   660    660   

Changes to src/main.c.

   839    839         };
   840    840         unsigned int i;
   841    841         rc = SQLITE_ERROR; /* IMP: R-42790-23372 */
   842    842         for(i=0; i<ArraySize(aFlagOp); i++){
   843    843           if( aFlagOp[i].op==op ){
   844    844             int onoff = va_arg(ap, int);
   845    845             int *pRes = va_arg(ap, int*);
   846         -          u32 oldFlags = db->flags;
          846  +          u64 oldFlags = db->flags;
   847    847             if( onoff>0 ){
   848    848               db->flags |= aFlagOp[i].mask;
   849    849             }else if( onoff==0 ){
   850         -            db->flags &= ~aFlagOp[i].mask;
          850  +            db->flags &= ~(u64)aFlagOp[i].mask;
   851    851             }
   852    852             if( oldFlags!=db->flags ){
   853    853               sqlite3ExpirePreparedStatements(db, 0);
   854    854             }
   855    855             if( pRes ){
   856    856               *pRes = (db->flags & aFlagOp[i].mask)!=0;
   857    857             }
................................................................................
  1306   1306       sqlite3ResetAllSchemasOfConnection(db);
  1307   1307     }
  1308   1308     sqlite3BtreeLeaveAll(db);
  1309   1309   
  1310   1310     /* Any deferred constraint violations have now been resolved. */
  1311   1311     db->nDeferredCons = 0;
  1312   1312     db->nDeferredImmCons = 0;
  1313         -  db->flags &= ~SQLITE_DeferFKs;
         1313  +  db->flags &= ~(u64)SQLITE_DeferFKs;
  1314   1314   
  1315   1315     /* If one has been configured, invoke the rollback-hook callback */
  1316   1316     if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){
  1317   1317       db->xRollbackCallback(db->pRollbackArg);
  1318   1318     }
  1319   1319   }
  1320   1320   
................................................................................
  1992   1992       (void)SQLITE_MISUSE_BKPT;
  1993   1993       return 0;
  1994   1994     }
  1995   1995   #endif
  1996   1996     sqlite3_mutex_enter(db->mutex);
  1997   1997     pOld = db->pTraceArg;
  1998   1998     db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0;
  1999         -  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
  2000   1999     db->xTrace = (int(*)(u32,void*,void*,void*))xTrace;
  2001   2000     db->pTraceArg = pArg;
  2002   2001     sqlite3_mutex_leave(db->mutex);
  2003   2002     return pOld;
  2004   2003   }
  2005   2004   #endif /* SQLITE_OMIT_DEPRECATED */
  2006   2005   
................................................................................
  2017   2016       return SQLITE_MISUSE_BKPT;
  2018   2017     }
  2019   2018   #endif
  2020   2019     sqlite3_mutex_enter(db->mutex);
  2021   2020     if( mTrace==0 ) xTrace = 0;
  2022   2021     if( xTrace==0 ) mTrace = 0;
  2023   2022     db->mTrace = mTrace;
  2024         -#ifndef SQLITE_OMIT_DEPRECATED
  2025         -  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
  2026         -#endif
  2027   2023     db->xTrace = xTrace;
  2028   2024     db->pTraceArg = pArg;
  2029   2025     sqlite3_mutex_leave(db->mutex);
  2030   2026     return SQLITE_OK;
  2031   2027   }
  2032   2028   
  2033   2029   #ifndef SQLITE_OMIT_DEPRECATED
................................................................................
  2405   2401       return sqlite3ErrStr(SQLITE_MISUSE_BKPT);
  2406   2402     }
  2407   2403     sqlite3_mutex_enter(db->mutex);
  2408   2404     if( db->mallocFailed ){
  2409   2405       z = sqlite3ErrStr(SQLITE_NOMEM_BKPT);
  2410   2406     }else{
  2411   2407       testcase( db->pErr==0 );
  2412         -    z = (char*)sqlite3_value_text(db->pErr);
         2408  +    z = db->errCode ? (char*)sqlite3_value_text(db->pErr) : 0;
  2413   2409       assert( !db->mallocFailed );
  2414   2410       if( z==0 ){
  2415   2411         z = sqlite3ErrStr(db->errCode);
  2416   2412       }
  2417   2413     }
  2418   2414     sqlite3_mutex_leave(db->mutex);
  2419   2415     return z;

Changes to src/parse.y.

  1378   1378   //
  1379   1379   cmd ::= DROP INDEX ifexists(E) fullname(X).   {sqlite3DropIndex(pParse, X, E);}
  1380   1380   
  1381   1381   ///////////////////////////// The VACUUM command /////////////////////////////
  1382   1382   //
  1383   1383   %ifndef SQLITE_OMIT_VACUUM
  1384   1384   %ifndef SQLITE_OMIT_ATTACH
  1385         -cmd ::= VACUUM.                {sqlite3Vacuum(pParse,0);}
  1386         -cmd ::= VACUUM nm(X).          {sqlite3Vacuum(pParse,&X);}
         1385  +%type vinto {Expr*}
         1386  +%destructor vinto {sqlite3ExprDelete(pParse->db, $$);}
         1387  +cmd ::= VACUUM vinto(Y).                {sqlite3Vacuum(pParse,0,Y);}
         1388  +cmd ::= VACUUM nm(X) vinto(Y).          {sqlite3Vacuum(pParse,&X,Y);}
         1389  +vinto(A) ::= INTO expr(X).              {A = X;}
         1390  +vinto(A) ::= .                          {A = 0;}
  1387   1391   %endif  SQLITE_OMIT_ATTACH
  1388   1392   %endif  SQLITE_OMIT_VACUUM
  1389   1393   
  1390   1394   ///////////////////////////// The PRAGMA command /////////////////////////////
  1391   1395   //
  1392   1396   %ifndef SQLITE_OMIT_PRAGMA
  1393   1397   cmd ::= PRAGMA nm(X) dbnm(Z).                {sqlite3Pragma(pParse,&X,&Z,0,0);}

Changes to src/pcache1.c.

   473    473   
   474    474   /*
   475    475   ** Malloc function used by SQLite to obtain space from the buffer configured
   476    476   ** using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no such buffer
   477    477   ** exists, this function falls back to sqlite3Malloc().
   478    478   */
   479    479   void *sqlite3PageMalloc(int sz){
          480  +  /* During rebalance operations on a corrupt database file, it is sometimes
          481  +  ** (rarely) possible to overread the temporary page buffer by a few bytes.
          482  +  ** Enlarge the allocation slightly so that this does not cause problems. */
   480    483     return pcache1Alloc(sz);
   481    484   }
   482    485   
   483    486   /*
   484    487   ** Free an allocated buffer obtained from sqlite3PageMalloc().
   485    488   */
   486    489   void sqlite3PageFree(void *p){

Changes to src/pragma.c.

   812    812         int size = 1;
   813    813         if( sqlite3GetInt32(zRight, &size) ){
   814    814           sqlite3BtreeSetSpillSize(pDb->pBt, size);
   815    815         }
   816    816         if( sqlite3GetBoolean(zRight, size!=0) ){
   817    817           db->flags |= SQLITE_CacheSpill;
   818    818         }else{
   819         -        db->flags &= ~SQLITE_CacheSpill;
          819  +        db->flags &= ~(u64)SQLITE_CacheSpill;
   820    820         }
   821    821         setAllPagerFlags(db);
   822    822       }
   823    823       break;
   824    824     }
   825    825   
   826    826     /*

Changes to src/prepare.c.

   289    289   
   290    290     /* Ticket #2804:  When we open a database in the newer file format,
   291    291     ** clear the legacy_file_format pragma flag so that a VACUUM will
   292    292     ** not downgrade the database and thus invalidate any descending
   293    293     ** indices that the user might have created.
   294    294     */
   295    295     if( iDb==0 && meta[BTREE_FILE_FORMAT-1]>=4 ){
   296         -    db->flags &= ~SQLITE_LegacyFileFmt;
          296  +    db->flags &= ~(u64)SQLITE_LegacyFileFmt;
   297    297     }
   298    298   
   299    299     /* Read the schema information out of the schema tables
   300    300     */
   301    301     assert( db->init.busy );
   302    302     {
   303    303       char *zSql;
................................................................................
   705    705     sqlite3BtreeLeaveAll(db);
   706    706     rc = sqlite3ApiExit(db, rc);
   707    707     assert( (rc&db->errMask)==rc );
   708    708     sqlite3_mutex_leave(db->mutex);
   709    709     return rc;
   710    710   }
   711    711   
   712         -#ifdef SQLITE_ENABLE_NORMALIZE
   713         -
   714         -/*
   715         -** Attempt to estimate the final output buffer size needed for the fully
   716         -** normalized version of the specified SQL string.  This should take into
   717         -** account any potential expansion that could occur (e.g. via IN clauses
   718         -** being expanded, etc).  This size returned is the total number of bytes
   719         -** including the NUL terminator.
   720         -*/
   721         -static int estimateNormalizedSize(
   722         -  const char *zSql, /* The original SQL string */
   723         -  int nSql          /* Length of original SQL string */
   724         -){
   725         -  int nOut = nSql + 4;
   726         -  const char *z = zSql;
   727         -  while( nOut<nSql*5 ){
   728         -    while( z[0]!=0 && z[0]!='I' && z[0]!='i' ){ z++; }
   729         -    if( z[0]==0 ) break;
   730         -    z++;
   731         -    if( z[0]!='N' && z[0]!='n' ) break;
   732         -    z++;
   733         -    while( sqlite3Isspace(z[0]) ){ z++; }
   734         -    if( z[0]!='(' ) break;
   735         -    z++;
   736         -    nOut += 5; /* ?,?,? */
   737         -  }
   738         -  return nOut;
   739         -}
   740         -
   741         -/*
   742         -** Copy the current token into the output buffer while dealing with quoted
   743         -** identifiers.  By default, all letters will be converted into lowercase.
   744         -** If the bUpper flag is set, uppercase will be used.  The piOut argument
   745         -** will be used to update the target index into the output string.
   746         -*/
   747         -static void copyNormalizedToken(
   748         -  const char *zSql, /* The original SQL string */
   749         -  int iIn,          /* Current index into the original SQL string */
   750         -  int nToken,       /* Number of bytes in the current token */
   751         -  int tokenFlags,   /* Flags returned by the tokenizer */
   752         -  char *zOut,       /* The output string */
   753         -  int *piOut        /* Pointer to target index into the output string */
   754         -){
   755         -  int bQuoted = tokenFlags & SQLITE_TOKEN_QUOTED;
   756         -  int bKeyword = tokenFlags & SQLITE_TOKEN_KEYWORD;
   757         -  int j = *piOut, k = 0;
   758         -  for(; k<nToken; k++){
   759         -    if( bQuoted ){
   760         -      if( k==0 && iIn>0 ){
   761         -        zOut[j++] = '"';
   762         -        continue;
   763         -      }else if( k==nToken-1 ){
   764         -        zOut[j++] = '"';
   765         -        continue;
   766         -      }
   767         -    }
   768         -    if( bKeyword ){
   769         -      zOut[j++] = sqlite3Toupper(zSql[iIn+k]);
   770         -    }else{
   771         -      zOut[j++] = sqlite3Tolower(zSql[iIn+k]);
   772         -    }
   773         -  }
   774         -  *piOut = j;
   775         -}
   776         -
   777         -/*
   778         -** Compute a normalization of the SQL given by zSql[0..nSql-1].  Return
   779         -** the normalization in space obtained from sqlite3DbMalloc().  Or return
   780         -** NULL if anything goes wrong or if zSql is NULL.
   781         -*/
   782         -char *sqlite3Normalize(
   783         -  Vdbe *pVdbe,      /* VM being reprepared */
   784         -  const char *zSql, /* The original SQL string */
   785         -  int nSql          /* Size of the input string in bytes */
   786         -){
   787         -  sqlite3 *db;           /* Database handle. */
   788         -  char *z;               /* The output string */
   789         -  int nZ;                /* Size of the output string in bytes */
   790         -  int i;                 /* Next character to read from zSql[] */
   791         -  int j;                 /* Next character to fill in on z[] */
   792         -  int tokenType = 0;     /* Type of the next token */
   793         -  int prevTokenType = 0; /* Type of the previous token, except spaces */
   794         -  int n;                 /* Size of the next token */
   795         -  int nParen = 0;        /* Nesting level of parenthesis */
   796         -  int iStartIN = 0;      /* Start of RHS of IN operator in z[] */
   797         -  int nParenAtIN = 0;    /* Value of nParent at start of RHS of IN operator */
   798         -
   799         -  db = sqlite3VdbeDb(pVdbe);
   800         -  assert( db!=0 );
   801         -  if( zSql==0 ) return 0;
   802         -  nZ = estimateNormalizedSize(zSql, nSql);
   803         -  z = sqlite3DbMallocRawNN(db, nZ);
   804         -  if( z==0 ) goto normalizeError;
   805         -  for(i=j=0; i<nSql && zSql[i]; i+=n){
   806         -    int flags = 0;
   807         -    if( tokenType!=TK_SPACE ) prevTokenType = tokenType;
   808         -    n = sqlite3GetTokenNormalized((unsigned char*)zSql+i, &tokenType, &flags);
   809         -    switch( tokenType ){
   810         -      case TK_SPACE: {
   811         -        break;
   812         -      }
   813         -      case TK_ILLEGAL: {
   814         -        goto normalizeError;
   815         -      }
   816         -      case TK_STRING:
   817         -      case TK_INTEGER:
   818         -      case TK_FLOAT:
   819         -      case TK_VARIABLE:
   820         -      case TK_BLOB: {
   821         -        z[j++] = '?';
   822         -        break;
   823         -      }
   824         -      case TK_LP:
   825         -      case TK_RP: {
   826         -        if( tokenType==TK_LP ){
   827         -          nParen++;
   828         -          if( prevTokenType==TK_IN ){
   829         -            iStartIN = j;
   830         -            nParenAtIN = nParen;
   831         -          }
   832         -        }else{
   833         -          if( iStartIN>0 && nParen==nParenAtIN ){
   834         -            assert( iStartIN+6<nZ );
   835         -            memcpy(z+iStartIN+1, "?,?,?", 5);
   836         -            j = iStartIN+6;
   837         -            assert( nZ-1-j>=0 );
   838         -            assert( nZ-1-j<nZ );
   839         -            memset(z+j, 0, nZ-1-j);
   840         -            iStartIN = 0;
   841         -          }
   842         -          nParen--;
   843         -        }
   844         -        assert( nParen>=0 );
   845         -        /* Fall through */
   846         -      }
   847         -      case TK_MINUS:
   848         -      case TK_SEMI:
   849         -      case TK_PLUS:
   850         -      case TK_STAR:
   851         -      case TK_SLASH:
   852         -      case TK_REM:
   853         -      case TK_EQ:
   854         -      case TK_LE:
   855         -      case TK_NE:
   856         -      case TK_LSHIFT:
   857         -      case TK_LT:
   858         -      case TK_RSHIFT:
   859         -      case TK_GT:
   860         -      case TK_GE:
   861         -      case TK_BITOR:
   862         -      case TK_CONCAT:
   863         -      case TK_COMMA:
   864         -      case TK_BITAND:
   865         -      case TK_BITNOT:
   866         -      case TK_DOT:
   867         -      case TK_IN:
   868         -      case TK_IS:
   869         -      case TK_NOT:
   870         -      case TK_NULL:
   871         -      case TK_ID: {
   872         -        if( tokenType==TK_NULL ){
   873         -          if( prevTokenType==TK_IS || prevTokenType==TK_NOT ){
   874         -            /* NULL is a keyword in this case, not a literal value */
   875         -          }else{
   876         -            /* Here the NULL is a literal value */
   877         -            z[j++] = '?';
   878         -            break;
   879         -          }
   880         -        }
   881         -        if( j>0 && sqlite3IsIdChar(z[j-1]) && sqlite3IsIdChar(zSql[i]) ){
   882         -          z[j++] = ' ';
   883         -        }
   884         -        if( tokenType==TK_ID ){
   885         -          int i2 = i, n2 = n;
   886         -          if( nParen==nParenAtIN ) iStartIN = 0;
   887         -          if( flags&SQLITE_TOKEN_QUOTED ){ i2++; n2-=2; }
   888         -        }
   889         -        copyNormalizedToken(zSql, i, n, flags, z, &j);
   890         -        break;
   891         -      }
   892         -    }
   893         -  }
   894         -  assert( j<nZ && "one" );
   895         -  while( j>0 && z[j-1]==' ' ){ j--; }
   896         -  if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
   897         -  z[j] = 0;
   898         -  assert( j<nZ && "two" );
   899         -  return z;
   900         -
   901         -normalizeError:
   902         -  sqlite3DbFree(db, z);
   903         -  return 0;
   904         -}
   905         -#endif /* SQLITE_ENABLE_NORMALIZE */
   906    712   
   907    713   /*
   908    714   ** Rerun the compilation of a statement after a schema change.
   909    715   **
   910    716   ** If the statement is successfully recompiled, return SQLITE_OK. Otherwise,
   911    717   ** if the statement cannot be recompiled because another connection has
   912    718   ** locked the sqlite3_master table, return SQLITE_LOCKED. If any other error

Changes to src/resolve.c.

    76     76     db = pParse->db;
    77     77     pDup = sqlite3ExprDup(db, pOrig, 0);
    78     78     if( pDup!=0 ){
    79     79       if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery);
    80     80       if( pExpr->op==TK_COLLATE ){
    81     81         pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken);
    82     82       }
    83         -//    ExprSetProperty(pDup, EP_Alias);
    84     83   
    85     84       /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This 
    86     85       ** prevents ExprDelete() from deleting the Expr structure itself,
    87     86       ** allowing it to be repopulated by the memcpy() on the following line.
    88     87       ** The pExpr->u.zToken might point into memory that will be freed by the
    89     88       ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to
    90     89       ** make a copy of the token before doing the sqlite3DbFree().
................................................................................
   470    469     **
   471    470     ** Because no reference was made to outer contexts, the pNC->nRef
   472    471     ** fields are not changed in any context.
   473    472     */
   474    473     if( cnt==0 && zTab==0 ){
   475    474       assert( pExpr->op==TK_ID );
   476    475       if( ExprHasProperty(pExpr,EP_DblQuoted) ){
          476  +      /* If a double-quoted identifier does not match any known column name,
          477  +      ** then treat it as a string.
          478  +      **
          479  +      ** This hack was added in the early days of SQLite in a misguided attempt
          480  +      ** to be compatible with MySQL 3.x, which used double-quotes for strings.
          481  +      ** I now sorely regret putting in this hack. The effect of this hack is
          482  +      ** that misspelled identifier names are silently converted into strings
          483  +      ** rather than causing an error, to the frustration of countless
          484  +      ** programmers. To all those frustrated programmers, my apologies.
          485  +      **
          486  +      ** Someday, I hope to get rid of this hack. Unfortunately there is
          487  +      ** a huge amount of legacy SQL that uses it. So for now, we just
          488  +      ** issue a warning.
          489  +      */
          490  +      sqlite3_log(SQLITE_WARNING,
          491  +        "double-quoted string literal: \"%w\"", zCol);
          492  +#ifdef SQLITE_ENABLE_NORMALIZE
          493  +      sqlite3VdbeAddDblquoteStr(db, pParse->pVdbe, zCol);
          494  +#endif
   477    495         pExpr->op = TK_STRING;
   478    496         pExpr->y.pTab = 0;
   479    497         return WRC_Prune;
   480    498       }
   481    499       if( sqlite3ExprIdToTrueFalse(pExpr) ){
   482    500         return WRC_Prune;
   483    501       }

Changes to src/select.c.

  2073   2073   /*
  2074   2074   ** Given a SELECT statement, generate a Table structure that describes
  2075   2075   ** the result set of that SELECT.
  2076   2076   */
  2077   2077   Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){
  2078   2078     Table *pTab;
  2079   2079     sqlite3 *db = pParse->db;
  2080         -  int savedFlags;
         2080  +  u64 savedFlags;
  2081   2081   
  2082   2082     savedFlags = db->flags;
  2083         -  db->flags &= ~SQLITE_FullColNames;
         2083  +  db->flags &= ~(u64)SQLITE_FullColNames;
  2084   2084     db->flags |= SQLITE_ShortColNames;
  2085   2085     sqlite3SelectPrep(pParse, pSelect, 0);
  2086   2086     if( pParse->nErr ) return 0;
  2087   2087     while( pSelect->pPrior ) pSelect = pSelect->pPrior;
  2088   2088     db->flags = savedFlags;
  2089   2089     pTab = sqlite3DbMallocZero(db, sizeof(Table) );
  2090   2090     if( pTab==0 ){
................................................................................
  3457   3457           if( pSubst->isLeftJoin && pCopy->op!=TK_COLUMN ){
  3458   3458             memset(&ifNullRow, 0, sizeof(ifNullRow));
  3459   3459             ifNullRow.op = TK_IF_NULL_ROW;
  3460   3460             ifNullRow.pLeft = pCopy;
  3461   3461             ifNullRow.iTable = pSubst->iNewTable;
  3462   3462             pCopy = &ifNullRow;
  3463   3463           }
         3464  +        testcase( ExprHasProperty(pCopy, EP_Subquery) );
  3464   3465           pNew = sqlite3ExprDup(db, pCopy, 0);
  3465   3466           if( pNew && pSubst->isLeftJoin ){
  3466   3467             ExprSetProperty(pNew, EP_CanBeNull);
  3467   3468           }
  3468   3469           if( pNew && ExprHasProperty(pExpr,EP_FromJoin) ){
  3469   3470             pNew->iRightJoinTable = pExpr->iRightJoinTable;
  3470   3471             ExprSetProperty(pNew, EP_FromJoin);
................................................................................
  4021   4022         for(i=0; i<pOrderBy->nExpr; i++){
  4022   4023           pOrderBy->a[i].u.x.iOrderByCol = 0;
  4023   4024         }
  4024   4025         assert( pParent->pOrderBy==0 );
  4025   4026         pParent->pOrderBy = pOrderBy;
  4026   4027         pSub->pOrderBy = 0;
  4027   4028       }
  4028         -    pWhere = sqlite3ExprDup(db, pSub->pWhere, 0);
         4029  +    pWhere = pSub->pWhere;
         4030  +    pSub->pWhere = 0;
  4029   4031       if( isLeftJoin>0 ){
  4030   4032         setJoinExpr(pWhere, iNewParent);
  4031   4033       }
  4032   4034       pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere);
  4033   4035       if( db->mallocFailed==0 ){
  4034   4036         SubstContext x;
  4035   4037         x.pParse = pParse;

Changes to src/shell.c.in.

  1007   1007     u8 openMode;           /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
  1008   1008     u8 doXdgOpen;          /* Invoke start/open/xdg-open in output_reset() */
  1009   1009     u8 nEqpLevel;          /* Depth of the EQP output graph */
  1010   1010     u8 eTraceType;         /* SHELL_TRACE_* value for type of trace */
  1011   1011     unsigned mEqpLines;    /* Mask of veritical lines in the EQP output graph */
  1012   1012     int outCount;          /* Revert to stdout when reaching zero */
  1013   1013     int cnt;               /* Number of records displayed so far */
         1014  +  int lineno;            /* Line number of last line read from in */
         1015  +  FILE *in;              /* Read commands from this stream */
  1014   1016     FILE *out;             /* Write results here */
  1015   1017     FILE *traceOut;        /* Output for sqlite3_trace() */
  1016   1018     int nErr;              /* Number of errors seen */
  1017   1019     int mode;              /* An output mode setting */
  1018   1020     int modePrior;         /* Saved mode */
  1019   1021     int cMode;             /* temporary output mode for the current query */
  1020   1022     int normalMode;        /* Output mode before ".explain on" */
................................................................................
  1062   1064   */
  1063   1065   #define SHELL_OPEN_UNSPEC      0      /* No open-mode specified */
  1064   1066   #define SHELL_OPEN_NORMAL      1      /* Normal database file */
  1065   1067   #define SHELL_OPEN_APPENDVFS   2      /* Use appendvfs */
  1066   1068   #define SHELL_OPEN_ZIPFILE     3      /* Use the zipfile virtual table */
  1067   1069   #define SHELL_OPEN_READONLY    4      /* Open a normal database read-only */
  1068   1070   #define SHELL_OPEN_DESERIALIZE 5      /* Open using sqlite3_deserialize() */
         1071  +#define SHELL_OPEN_HEXDB       6      /* Use "dbtotxt" output as data source */
  1069   1072   
  1070   1073   /* Allowed values for ShellState.eTraceType
  1071   1074   */
  1072   1075   #define SHELL_TRACE_PLAIN      0      /* Show input SQL text */
  1073   1076   #define SHELL_TRACE_EXPANDED   1      /* Show expanded SQL text */
  1074   1077   #define SHELL_TRACE_NORMALIZED 2      /* Show normalized SQL text */
  1075   1078   
................................................................................
  3373   3376     "      http://sqlite.org/cli.html#sqlar_archive_support",
  3374   3377   #endif
  3375   3378   #ifndef SQLITE_OMIT_AUTHORIZATION
  3376   3379     ".auth ON|OFF             Show authorizer callbacks",
  3377   3380   #endif
  3378   3381     ".backup ?DB? FILE        Backup DB (default \"main\") to FILE",
  3379   3382     "       --append            Use the appendvfs",
         3383  +  "       --async             Write to FILE without a journal and without fsync()",
  3380   3384     ".bail on|off             Stop after hitting an error.  Default OFF",
  3381   3385     ".binary on|off           Turn binary output on or off.  Default OFF",
  3382   3386     ".cd DIRECTORY            Change the working directory to DIRECTORY",
  3383   3387     ".changes on|off          Show number of rows changed by SQL",
  3384   3388     ".check GLOB              Fail if output since .testcase does not match",
  3385   3389     ".clone NEWDB             Clone data into NEWDB from the existing database",
  3386   3390     ".databases               List names and files of attached databases",
................................................................................
  3439   3443     "       -e    Invoke system text editor",
  3440   3444     "       -x    Open in a spreadsheet",
  3441   3445     ".open ?OPTIONS? ?FILE?   Close existing database and reopen FILE",
  3442   3446     "     Options:",
  3443   3447     "        --append        Use appendvfs to append database to the end of FILE",
  3444   3448   #ifdef SQLITE_ENABLE_DESERIALIZE
  3445   3449     "        --deserialize   Load into memory useing sqlite3_deserialize()",
         3450  +  "        --hexdb         Load the output of \"dbtotxt\" as an in-memory database",
  3446   3451   #endif
  3447   3452     "        --new           Initialize FILE to an empty database",
  3448   3453     "        --readonly      Open FILE readonly",
  3449   3454     "        --zip           FILE is a ZIP archive",
  3450   3455     ".output ?FILE?           Send output to FILE or stdout if FILE is omitted",
  3451   3456     "     If FILE begins with '|' then open it as a pipe.",
  3452   3457     ".print STRING...         Print literal STRING",
................................................................................
  3588   3593       }
  3589   3594       sqlite3_free(zPat);
  3590   3595     }
  3591   3596     return n;
  3592   3597   }
  3593   3598   
  3594   3599   /* Forward reference */
  3595         -static int process_input(ShellState *p, FILE *in);
         3600  +static int process_input(ShellState *p);
  3596   3601   
  3597   3602   /*
  3598   3603   ** Read the content of file zName into memory obtained from sqlite3_malloc64()
  3599   3604   ** and return a pointer to the buffer. The caller is responsible for freeing
  3600   3605   ** the memory.
  3601   3606   **
  3602   3607   ** If parameter pnByte is not NULL, (*pnByte) is set to the number of bytes
................................................................................
  3718   3723         rc = SHELL_OPEN_ZIPFILE;
  3719   3724       }
  3720   3725     }
  3721   3726     fclose(f);
  3722   3727     return rc;  
  3723   3728   }
  3724   3729   
         3730  +#ifdef SQLITE_ENABLE_DESERIALIZE
         3731  +/*
         3732  +** Reconstruct an in-memory database using the output from the "dbtotxt"
         3733  +** program.  Read content from the file in p->zDbFilename.  If p->zDbFilename
         3734  +** is 0, then read from standard input.
         3735  +*/
         3736  +static unsigned char *readHexDb(ShellState *p, int *pnData){
         3737  +  unsigned char *a = 0;
         3738  +  int nLine;
         3739  +  int n = 0;
         3740  +  int pgsz = 0;
         3741  +  int iOffset = 0;
         3742  +  int j, k;
         3743  +  int rc;
         3744  +  FILE *in;
         3745  +  unsigned char x[16];
         3746  +  char zLine[1000];
         3747  +  if( p->zDbFilename ){
         3748  +    in = fopen(p->zDbFilename, "r");
         3749  +    if( in==0 ){
         3750  +      utf8_printf(stderr, "cannot open \"%s\" for reading\n", p->zDbFilename);
         3751  +      return 0;
         3752  +    }
         3753  +    nLine = 0;
         3754  +  }else{
         3755  +    in = p->in;
         3756  +    nLine = p->lineno;
         3757  +  }
         3758  +  *pnData = 0;
         3759  +  nLine++;
         3760  +  if( fgets(zLine, sizeof(zLine), in)==0 ) goto readHexDb_error;
         3761  +  rc = sscanf(zLine, "| size %d pagesize %d", &n, &pgsz);
         3762  +  if( rc!=2 ) goto readHexDb_error;
         3763  +  if( n<=0 ) goto readHexDb_error;
         3764  +  a = sqlite3_malloc( n );
         3765  +  if( a==0 ){
         3766  +    utf8_printf(stderr, "Out of memory!\n");
         3767  +    goto readHexDb_error;
         3768  +  }
         3769  +  memset(a, 0, n);
         3770  +  if( pgsz<512 || pgsz>65536 || (pgsz & (pgsz-1))!=0 ){
         3771  +    utf8_printf(stderr, "invalid pagesize\n");
         3772  +    goto readHexDb_error;
         3773  +  }
         3774  +  for(nLine++; fgets(zLine, sizeof(zLine), in)!=0; nLine++){
         3775  +    rc = sscanf(zLine, "| page %d offset %d", &j, &k);
         3776  +    if( rc==2 ){
         3777  +      iOffset = k;
         3778  +      continue;
         3779  +    }
         3780  +    if( strncmp(zLine, "| end ", 6)==0 ){
         3781  +      break;
         3782  +    }
         3783  +    rc = sscanf(zLine,"| %d: %hhx %hhx %hhx %hhx %hhx %hhx %hhx %hhx"
         3784  +                      "  %hhx %hhx %hhx %hhx %hhx %hhx %hhx %hhx",
         3785  +                &j, &x[0], &x[1], &x[2], &x[3], &x[4], &x[5], &x[6], &x[7],
         3786  +                &x[8], &x[9], &x[10], &x[11], &x[12], &x[13], &x[14], &x[15]);
         3787  +    if( rc==17 ){
         3788  +      k = iOffset+j;
         3789  +      if( k+16<=n ){
         3790  +        memcpy(a+k, x, 16);
         3791  +      }
         3792  +    }
         3793  +  }
         3794  +  *pnData = n;
         3795  +  if( in!=p->in ){
         3796  +    fclose(in);
         3797  +  }else{
         3798  +    p->lineno = nLine;
         3799  +  }
         3800  +  return a;
         3801  +
         3802  +readHexDb_error:
         3803  +  if( in!=stdin ){
         3804  +    fclose(in);
         3805  +  }else{
         3806  +    while( fgets(zLine, sizeof(zLine), p->in)!=0 ){
         3807  +      nLine++;
         3808  +      if(strncmp(zLine, "| end ", 6)==0 ) break;
         3809  +    }
         3810  +    p->lineno = nLine;
         3811  +  }
         3812  +  sqlite3_free(a);
         3813  +  utf8_printf(stderr,"Error on line %d of --hexdb input\n", nLine);
         3814  +  return 0;
         3815  +}
         3816  +#endif /* SQLITE_ENABLE_DESERIALIZE */
         3817  +
  3725   3818   /* Flags for open_db().
  3726   3819   **
  3727   3820   ** The default behavior of open_db() is to exit(1) if the database fails to
  3728   3821   ** open.  The OPEN_DB_KEEPALIVE flag changes that so that it prints an error
  3729   3822   ** but still returns without calling exit.
  3730   3823   **
  3731   3824   ** The OPEN_DB_ZIPFILE flag causes open_db() to prefer to open files as a
................................................................................
  3751   3844       }
  3752   3845       switch( p->openMode ){
  3753   3846         case SHELL_OPEN_APPENDVFS: {
  3754   3847           sqlite3_open_v2(p->zDbFilename, &p->db, 
  3755   3848              SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, "apndvfs");
  3756   3849           break;
  3757   3850         }
         3851  +      case SHELL_OPEN_HEXDB:
  3758   3852         case SHELL_OPEN_DESERIALIZE: {
  3759   3853           sqlite3_open(0, &p->db);
  3760   3854           break;
  3761   3855         }
  3762   3856         case SHELL_OPEN_ZIPFILE: {
  3763   3857           sqlite3_open(":memory:", &p->db);
  3764   3858           break;
................................................................................
  3805   3899       if( p->openMode==SHELL_OPEN_ZIPFILE ){
  3806   3900         char *zSql = sqlite3_mprintf(
  3807   3901            "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename);
  3808   3902         sqlite3_exec(p->db, zSql, 0, 0, 0);
  3809   3903         sqlite3_free(zSql);
  3810   3904       }
  3811   3905   #ifdef SQLITE_ENABLE_DESERIALIZE
  3812         -    else if( p->openMode==SHELL_OPEN_DESERIALIZE ){
         3906  +    else
         3907  +    if( p->openMode==SHELL_OPEN_DESERIALIZE || p->openMode==SHELL_OPEN_HEXDB ){
         3908  +      int rc;
  3813   3909         int nData = 0;
  3814         -      unsigned char *aData = (unsigned char*)readFile(p->zDbFilename, &nData);
  3815         -      int rc = sqlite3_deserialize(p->db, "main", aData, nData, nData,
         3910  +      unsigned char *aData;
         3911  +      if( p->openMode==SHELL_OPEN_DESERIALIZE ){
         3912  +        aData = (unsigned char*)readFile(p->zDbFilename, &nData);
         3913  +      }else{
         3914  +        aData = readHexDb(p, &nData);
         3915  +        if( aData==0 ){
         3916  +          utf8_printf(stderr, "Error in hexdb input\n");
         3917  +          return;
         3918  +        }
         3919  +      }
         3920  +      rc = sqlite3_deserialize(p->db, "main", aData, nData, nData,
  3816   3921                      SQLITE_DESERIALIZE_RESIZEABLE |
  3817   3922                      SQLITE_DESERIALIZE_FREEONCLOSE);
  3818   3923         if( rc ){
  3819   3924           utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc);
  3820   3925         }
  3821   3926       }
  3822   3927   #endif
................................................................................
  5836   5941      || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0)
  5837   5942     ){
  5838   5943       const char *zDestFile = 0;
  5839   5944       const char *zDb = 0;
  5840   5945       sqlite3 *pDest;
  5841   5946       sqlite3_backup *pBackup;
  5842   5947       int j;
         5948  +    int bAsync = 0;
  5843   5949       const char *zVfs = 0;
  5844   5950       for(j=1; j<nArg; j++){
  5845   5951         const char *z = azArg[j];
  5846   5952         if( z[0]=='-' ){
  5847   5953           if( z[1]=='-' ) z++;
  5848   5954           if( strcmp(z, "-append")==0 ){
  5849   5955             zVfs = "apndvfs";
         5956  +        }else
         5957  +        if( strcmp(z, "-async")==0 ){
         5958  +          bAsync = 1;
  5850   5959           }else
  5851   5960           {
  5852   5961             utf8_printf(stderr, "unknown option: %s\n", azArg[j]);
  5853   5962             return 1;
  5854   5963           }
  5855   5964         }else if( zDestFile==0 ){
  5856   5965           zDestFile = azArg[j];
  5857   5966         }else if( zDb==0 ){
  5858   5967           zDb = zDestFile;
  5859   5968           zDestFile = azArg[j];
  5860   5969         }else{
  5861         -        raw_printf(stderr, "Usage: .backup ?DB? ?--append? FILENAME\n");
         5970  +        raw_printf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n");
  5862   5971           return 1;
  5863   5972         }
  5864   5973       }
  5865   5974       if( zDestFile==0 ){
  5866   5975         raw_printf(stderr, "missing FILENAME argument on .backup\n");
  5867   5976         return 1;
  5868   5977       }
................................................................................
  5870   5979       rc = sqlite3_open_v2(zDestFile, &pDest, 
  5871   5980                     SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, zVfs);
  5872   5981       if( rc!=SQLITE_OK ){
  5873   5982         utf8_printf(stderr, "Error: cannot open \"%s\"\n", zDestFile);
  5874   5983         close_db(pDest);
  5875   5984         return 1;
  5876   5985       }
         5986  +    if( bAsync ){
         5987  +      sqlite3_exec(pDest, "PRAGMA synchronous=OFF; PRAGMA journal_mode=OFF;",
         5988  +                   0, 0, 0);
         5989  +    }
  5877   5990       open_db(p, 0);
  5878   5991       pBackup = sqlite3_backup_init(pDest, "main", p->db, zDb);
  5879   5992       if( pBackup==0 ){
  5880   5993         utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest));
  5881   5994         close_db(pDest);
  5882   5995         return 1;
  5883   5996       }
................................................................................
  6736   6849         }else if( optionMatch(z, "append") ){
  6737   6850           p->openMode = SHELL_OPEN_APPENDVFS;
  6738   6851         }else if( optionMatch(z, "readonly") ){
  6739   6852           p->openMode = SHELL_OPEN_READONLY;
  6740   6853   #ifdef SQLITE_ENABLE_DESERIALIZE
  6741   6854         }else if( optionMatch(z, "deserialize") ){
  6742   6855           p->openMode = SHELL_OPEN_DESERIALIZE;
  6743         -#endif
         6856  +      }else if( optionMatch(z, "hexdb") ){
         6857  +        p->openMode = SHELL_OPEN_HEXDB;
         6858  +#endif /* SQLITE_ENABLE_DESERIALIZE */
  6744   6859         }else if( z[0]=='-' ){
  6745   6860           utf8_printf(stderr, "unknown option: %s\n", z);
  6746   6861           rc = 1;
  6747   6862           goto meta_command_exit;
  6748   6863         }
  6749   6864       }
  6750   6865       /* If a filename is specified, try to open it first */
  6751   6866       zNewFilename = nArg>iName ? sqlite3_mprintf("%s", azArg[iName]) : 0;
  6752         -    if( zNewFilename ){
         6867  +    if( zNewFilename || p->openMode==SHELL_OPEN_HEXDB ){
  6753   6868         if( newFlag ) shellDeleteFile(zNewFilename);
  6754   6869         p->zDbFilename = zNewFilename;
  6755   6870         open_db(p, OPEN_DB_KEEPALIVE);
  6756   6871         if( p->db==0 ){
  6757   6872           utf8_printf(stderr, "Error: cannot open '%s'\n", zNewFilename);
  6758   6873           sqlite3_free(zNewFilename);
  6759   6874         }else{
................................................................................
  6861   6976     }else
  6862   6977   
  6863   6978     if( c=='q' && strncmp(azArg[0], "quit", n)==0 ){
  6864   6979       rc = 2;
  6865   6980     }else
  6866   6981   
  6867   6982     if( c=='r' && n>=3 && strncmp(azArg[0], "read", n)==0 ){
  6868         -    FILE *alt;
         6983  +    FILE *inSaved = p->in;
         6984  +    int savedLineno = p->lineno;
  6869   6985       if( nArg!=2 ){
  6870   6986         raw_printf(stderr, "Usage: .read FILE\n");
  6871   6987         rc = 1;
  6872   6988         goto meta_command_exit;
  6873   6989       }
  6874         -    alt = fopen(azArg[1], "rb");
  6875         -    if( alt==0 ){
         6990  +    p->in = fopen(azArg[1], "rb");
         6991  +    if( p->in==0 ){
  6876   6992         utf8_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]);
  6877   6993         rc = 1;
  6878   6994       }else{
  6879         -      rc = process_input(p, alt);
  6880         -      fclose(alt);
         6995  +      rc = process_input(p);
         6996  +      fclose(p->in);
  6881   6997       }
         6998  +    p->in = inSaved;
         6999  +    p->lineno = savedLineno;
  6882   7000     }else
  6883   7001   
  6884   7002     if( c=='r' && n>=3 && strncmp(azArg[0], "restore", n)==0 ){
  6885   7003       const char *zSrcFile;
  6886   7004       const char *zDb;
  6887   7005       sqlite3 *pSrc;
  6888   7006       sqlite3_backup *pBackup;
................................................................................
  8215   8333   ** is interactive - the user is typing it it.  Otherwise, input
  8216   8334   ** is coming from a file or device.  A prompt is issued and history
  8217   8335   ** is saved only if input is interactive.  An interrupt signal will
  8218   8336   ** cause this routine to exit immediately, unless input is interactive.
  8219   8337   **
  8220   8338   ** Return the number of errors.
  8221   8339   */
  8222         -static int process_input(ShellState *p, FILE *in){
         8340  +static int process_input(ShellState *p){
  8223   8341     char *zLine = 0;          /* A single input line */
  8224   8342     char *zSql = 0;           /* Accumulated SQL text */
  8225   8343     int nLine;                /* Length of current line */
  8226   8344     int nSql = 0;             /* Bytes of zSql[] used */
  8227   8345     int nAlloc = 0;           /* Allocated zSql[] space */
  8228   8346     int nSqlPrior = 0;        /* Bytes of zSql[] used by prior line */
  8229   8347     int rc;                   /* Error code */
  8230   8348     int errCnt = 0;           /* Number of errors seen */
  8231         -  int lineno = 0;           /* Current line number */
  8232   8349     int startline = 0;        /* Line number for start of current input */
  8233   8350   
  8234         -  while( errCnt==0 || !bail_on_error || (in==0 && stdin_is_interactive) ){
         8351  +  p->lineno = 0;
         8352  +  while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){
  8235   8353       fflush(p->out);
  8236         -    zLine = one_input_line(in, zLine, nSql>0);
         8354  +    zLine = one_input_line(p->in, zLine, nSql>0);
  8237   8355       if( zLine==0 ){
  8238   8356         /* End of input */
  8239         -      if( in==0 && stdin_is_interactive ) printf("\n");
         8357  +      if( p->in==0 && stdin_is_interactive ) printf("\n");
  8240   8358         break;
  8241   8359       }
  8242   8360       if( seenInterrupt ){
  8243         -      if( in!=0 ) break;
         8361  +      if( p->in!=0 ) break;
  8244   8362         seenInterrupt = 0;
  8245   8363       }
  8246         -    lineno++;
         8364  +    p->lineno++;
  8247   8365       if( nSql==0 && _all_whitespace(zLine) ){
  8248   8366         if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine);
  8249   8367         continue;
  8250   8368       }
  8251   8369       if( zLine && (zLine[0]=='.' || zLine[0]=='#') && nSql==0 ){
  8252   8370         if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine);
  8253   8371         if( zLine[0]=='.' ){
................................................................................
  8271   8389       }
  8272   8390       nSqlPrior = nSql;
  8273   8391       if( nSql==0 ){
  8274   8392         int i;
  8275   8393         for(i=0; zLine[i] && IsSpace(zLine[i]); i++){}
  8276   8394         assert( nAlloc>0 && zSql!=0 );
  8277   8395         memcpy(zSql, zLine+i, nLine+1-i);
  8278         -      startline = lineno;
         8396  +      startline = p->lineno;
  8279   8397         nSql = nLine-i;
  8280   8398       }else{
  8281   8399         zSql[nSql++] = '\n';
  8282   8400         memcpy(zSql+nSql, zLine, nLine+1);
  8283   8401         nSql += nLine;
  8284   8402       }
  8285   8403       if( nSql && line_contains_semicolon(&zSql[nSqlPrior], nSql-nSqlPrior)
  8286   8404                   && sqlite3_complete(zSql) ){
  8287         -      errCnt += runOneSqlLine(p, zSql, in, startline);
         8405  +      errCnt += runOneSqlLine(p, zSql, p->in, startline);
  8288   8406         nSql = 0;
  8289   8407         if( p->outCount ){
  8290   8408           output_reset(p);
  8291   8409           p->outCount = 0;
  8292   8410         }else{
  8293   8411           clearTempFile(p);
  8294   8412         }
  8295   8413       }else if( nSql && _all_whitespace(zSql) ){
  8296   8414         if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zSql);
  8297   8415         nSql = 0;
  8298   8416       }
  8299   8417     }
  8300   8418     if( nSql && !_all_whitespace(zSql) ){
  8301         -    errCnt += runOneSqlLine(p, zSql, in, startline);
         8419  +    errCnt += runOneSqlLine(p, zSql, p->in, startline);
  8302   8420     }
  8303   8421     free(zSql);
  8304   8422     free(zLine);
  8305   8423     return errCnt>0;
  8306   8424   }
  8307   8425   
  8308   8426   /*
................................................................................
  8383   8501   static void process_sqliterc(
  8384   8502     ShellState *p,                  /* Configuration data */
  8385   8503     const char *sqliterc_override   /* Name of config file. NULL to use default */
  8386   8504   ){
  8387   8505     char *home_dir = NULL;
  8388   8506     const char *sqliterc = sqliterc_override;
  8389   8507     char *zBuf = 0;
  8390         -  FILE *in = NULL;
         8508  +  FILE *inSaved = p->in;
         8509  +  int savedLineno = p->lineno;
  8391   8510   
  8392   8511     if (sqliterc == NULL) {
  8393   8512       home_dir = find_home_dir(0);
  8394   8513       if( home_dir==0 ){
  8395   8514         raw_printf(stderr, "-- warning: cannot find home directory;"
  8396   8515                         " cannot read ~/.sqliterc\n");
  8397   8516         return;
  8398   8517       }
  8399   8518       zBuf = sqlite3_mprintf("%s/.sqliterc",home_dir);
  8400   8519       sqliterc = zBuf;
  8401   8520     }
  8402         -  in = fopen(sqliterc,"rb");
  8403         -  if( in ){
         8521  +  p->in = fopen(sqliterc,"rb");
         8522  +  if( p->in ){
  8404   8523       if( stdin_is_interactive ){
  8405   8524         utf8_printf(stderr,"-- Loading resources from %s\n",sqliterc);
  8406   8525       }
  8407         -    process_input(p,in);
  8408         -    fclose(in);
         8526  +    process_input(p);
         8527  +    fclose(p->in);
  8409   8528     }
         8529  +  p->in = inSaved;
         8530  +  p->lineno = savedLineno;
  8410   8531     sqlite3_free(zBuf);
  8411   8532   }
  8412   8533   
  8413   8534   /*
  8414   8535   ** Show available command line options
  8415   8536   */
  8416   8537   static const char zOptions[] =
................................................................................
  9014   9135         }
  9015   9136         if( zHistory ){ shell_read_history(zHistory); }
  9016   9137   #if HAVE_READLINE || HAVE_EDITLINE
  9017   9138         rl_attempted_completion_function = readline_completion;
  9018   9139   #elif HAVE_LINENOISE
  9019   9140         linenoiseSetCompletionCallback(linenoise_completion);
  9020   9141   #endif
  9021         -      rc = process_input(&data, 0);
         9142  +      data.in = 0;
         9143  +      rc = process_input(&data);
  9022   9144         if( zHistory ){
  9023   9145           shell_stifle_history(2000);
  9024   9146           shell_write_history(zHistory);
  9025   9147           free(zHistory);
  9026   9148         }
  9027   9149       }else{
  9028         -      rc = process_input(&data, stdin);
         9150  +      data.in = stdin;
         9151  +      rc = process_input(&data);
  9029   9152       }
  9030   9153     }
  9031   9154     set_table_name(&data, 0);
  9032   9155     if( data.db ){
  9033   9156       session_close_all(&data);
  9034   9157       close_db(data.db);
  9035   9158     }

Changes to src/sqlite.h.in.

  2987   2987   ** ^The callback function registered by sqlite3_profile() is invoked
  2988   2988   ** as each SQL statement finishes.  ^The profile callback contains
  2989   2989   ** the original statement text and an estimate of wall-clock time
  2990   2990   ** of how long that statement took to run.  ^The profile callback
  2991   2991   ** time is in units of nanoseconds, however the current implementation
  2992   2992   ** is only capable of millisecond resolution so the six least significant
  2993   2993   ** digits in the time are meaningless.  Future versions of SQLite
  2994         -** might provide greater resolution on the profiler callback.  The
  2995         -** sqlite3_profile() function is considered experimental and is
  2996         -** subject to change in future versions of SQLite.
         2994  +** might provide greater resolution on the profiler callback.  Invoking
         2995  +** either [sqlite3_trace()] or [sqlite3_trace_v2()] will cancel the
         2996  +** profile callback.
  2997   2997   */
  2998   2998   SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*,
  2999   2999      void(*xTrace)(void*,const char*), void*);
  3000   3000   SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*,
  3001   3001      void(*xProfile)(void*,const char*,sqlite3_uint64), void*);
  3002   3002   
  3003   3003   /*
................................................................................
  3625   3625   ** and [sqlite3_prepare16_v3()] assume that the prepared statement will 
  3626   3626   ** be used just once or at most a few times and then destroyed using
  3627   3627   ** [sqlite3_finalize()] relatively soon. The current implementation acts
  3628   3628   ** on this hint by avoiding the use of [lookaside memory] so as not to
  3629   3629   ** deplete the limited store of lookaside memory. Future versions of
  3630   3630   ** SQLite may act on this hint differently.
  3631   3631   **
  3632         -** [[SQLITE_PREPARE_NORMALIZE]] ^(<dt>SQLITE_PREPARE_NORMALIZE</dt>
  3633         -** <dd>The SQLITE_PREPARE_NORMALIZE flag indicates that a normalized
  3634         -** representation of the SQL statement should be calculated and then
  3635         -** associated with the prepared statement, which can be obtained via
  3636         -** the [sqlite3_normalized_sql()] interface.)^  The semantics used to
  3637         -** normalize a SQL statement are unspecified and subject to change.
  3638         -** At a minimum, literal values will be replaced with suitable
  3639         -** placeholders.
         3632  +** [[SQLITE_PREPARE_NORMALIZE]] <dt>SQLITE_PREPARE_NORMALIZE</dt>
         3633  +** <dd>The SQLITE_PREPARE_NORMALIZE flag is a no-op. This flag used
         3634  +** to be required for any prepared statement that wanted to use the
         3635  +** [sqlite3_normalized_sql()] interface.  However, the
         3636  +** [sqlite3_normalized_sql()] interface is now available to all
         3637  +** prepared statements, regardless of whether or not they use this
         3638  +** flag.
  3640   3639   ** </dl>
  3641   3640   */
  3642   3641   #define SQLITE_PREPARE_PERSISTENT              0x01
  3643   3642   #define SQLITE_PREPARE_NORMALIZE               0x02
  3644   3643   
  3645   3644   /*
  3646   3645   ** CAPI3REF: Compiling An SQL Statement

Changes to src/sqliteInt.h.

  3982   3982   #define LOCATE_VIEW    0x01
  3983   3983   #define LOCATE_NOERR   0x02
  3984   3984   Table *sqlite3LocateTable(Parse*,u32 flags,const char*, const char*);
  3985   3985   Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *);
  3986   3986   Index *sqlite3FindIndex(sqlite3*,const char*, const char*);
  3987   3987   void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*);
  3988   3988   void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*);
  3989         -void sqlite3Vacuum(Parse*,Token*);
  3990         -int sqlite3RunVacuum(char**, sqlite3*, int);
         3989  +void sqlite3Vacuum(Parse*,Token*,Expr*);
         3990  +int sqlite3RunVacuum(char**, sqlite3*, int, sqlite3_value*);
  3991   3991   char *sqlite3NameFromToken(sqlite3*, Token*);
  3992   3992   int sqlite3ExprCompare(Parse*,Expr*, Expr*, int);
  3993   3993   int sqlite3ExprCompareSkip(Expr*, Expr*, int);
  3994   3994   int sqlite3ExprListCompare(ExprList*, ExprList*, int);
  3995   3995   int sqlite3ExprImpliesExpr(Parse*,Expr*, Expr*, int);
  3996   3996   int sqlite3ExprImpliesNonNullRow(Expr*,int);
  3997   3997   void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*);
................................................................................
  4252   4252   #endif
  4253   4253   void sqlite3RootPageMoved(sqlite3*, int, int, int);
  4254   4254   void sqlite3Reindex(Parse*, Token*, Token*);
  4255   4255   void sqlite3AlterFunctions(void);
  4256   4256   void sqlite3AlterRenameTable(Parse*, SrcList*, Token*);
  4257   4257   void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*);
  4258   4258   int sqlite3GetToken(const unsigned char *, int *);
  4259         -#ifdef SQLITE_ENABLE_NORMALIZE
  4260         -int sqlite3GetTokenNormalized(const unsigned char *, int *, int *);
  4261         -#endif
  4262   4259   void sqlite3NestedParse(Parse*, const char*, ...);
  4263   4260   void sqlite3ExpirePreparedStatements(sqlite3*, int);
  4264   4261   int sqlite3CodeSubselect(Parse*, Expr *, int, int);
  4265   4262   void sqlite3SelectPrep(Parse*, Select*, NameContext*);
  4266   4263   void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p);
  4267   4264   int sqlite3MatchSpanName(const char*, const char*, const char*, const char*);
  4268   4265   int sqlite3ResolveExprNames(NameContext*, Expr*);
................................................................................
  4413   4410   int sqlite3VtabBegin(sqlite3 *, VTable *);
  4414   4411   FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*);
  4415   4412   sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*);
  4416   4413   int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
  4417   4414   int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
  4418   4415   void sqlite3ParserReset(Parse*);
  4419   4416   #ifdef SQLITE_ENABLE_NORMALIZE
  4420         -char *sqlite3Normalize(Vdbe*, const char*, int);
         4417  +char *sqlite3Normalize(Vdbe*, const char*);
  4421   4418   #endif
  4422   4419   int sqlite3Reprepare(Vdbe*);
  4423   4420   void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*);
  4424   4421   CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *);
  4425   4422   int sqlite3TempInMemory(const sqlite3*);
  4426   4423   const char *sqlite3JournalModename(int);
  4427   4424   #ifndef SQLITE_OMIT_WAL

Changes to src/test1.c.

  7671   7671         zDb = Tcl_GetString(objv[2]);
  7672   7672       }
  7673   7673       rc = sqlite3_mmap_warm(db, zDb);
  7674   7674       Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
  7675   7675       return TCL_OK;
  7676   7676     }
  7677   7677   }
         7678  +
         7679  +/*
         7680  +** Usage:  decode_hexdb TEXT
         7681  +**
         7682  +** Example:   db deserialize [decode_hexdb $output_of_dbtotxt]
         7683  +**
         7684  +** This routine returns a byte-array for an SQLite database file that
         7685  +** is constructed from a text input which is the output of the "dbtotxt"
         7686  +** utility.
         7687  +*/
         7688  +static int SQLITE_TCLAPI test_decode_hexdb(
         7689  +  void * clientData,
         7690  +  Tcl_Interp *interp,
         7691  +  int objc,
         7692  +  Tcl_Obj *CONST objv[]
         7693  +){
         7694  +  const char *zIn = 0;
         7695  +  unsigned char *a = 0;
         7696  +  int n = 0;
         7697  +  int lineno = 0;
         7698  +  int i, iNext;
         7699  +  int iOffset = 0;
         7700  +  int j, k;
         7701  +  int rc;
         7702  +  unsigned char x[16];
         7703  +  if( objc!=2 ){
         7704  +    Tcl_WrongNumArgs(interp, 1, objv, "HEXDB");
         7705  +    return TCL_ERROR;
         7706  +  }
         7707  +  zIn = Tcl_GetString(objv[1]);
         7708  +  for(i=0; zIn[i]; i=iNext){
         7709  +    lineno++;
         7710  +    for(iNext=i; zIn[iNext] && zIn[iNext]!='\n'; iNext++){}
         7711  +    if( zIn[iNext]=='\n' ) iNext++;
         7712  +    while( zIn[i]==' ' || zIn[i]=='\t' ){ i++; }
         7713  +    if( a==0 ){
         7714  +      int pgsz;
         7715  +      rc = sscanf(zIn+i, "| size %d pagesize %d", &n, &pgsz);
         7716  +      if( rc!=2 ) continue;
         7717  +      if( n<512 ){
         7718  +        Tcl_AppendResult(interp, "bad 'size' field", (void*)0);
         7719  +        return TCL_ERROR;
         7720  +      }
         7721  +      a = malloc( n );
         7722  +      if( a==0 ){
         7723  +        Tcl_AppendResult(interp, "out of memory", (void*)0);
         7724  +        return TCL_ERROR;
         7725  +      }
         7726  +      memset(a, 0, n);
         7727  +      continue;
         7728  +    }
         7729  +    rc = sscanf(zIn+i, "| page %d offset %d", &j, &k);
         7730  +    if( rc==2 ){
         7731  +      iOffset = k;
         7732  +      continue;
         7733  +    }
         7734  +    rc = sscanf(zIn+i,"| %d: %hhx %hhx %hhx %hhx %hhx %hhx %hhx %hhx"
         7735  +                      "  %hhx %hhx %hhx %hhx %hhx %hhx %hhx %hhx",
         7736  +                &j, &x[0], &x[1], &x[2], &x[3], &x[4], &x[5], &x[6], &x[7],
         7737  +                &x[8], &x[9], &x[10], &x[11], &x[12], &x[13], &x[14], &x[15]);
         7738  +    if( rc==17 ){
         7739  +      k = iOffset+j;
         7740  +      if( k+16<=n ){
         7741  +        memcpy(a+k, x, 16);
         7742  +      }
         7743  +      continue;
         7744  +    }
         7745  +  }
         7746  +  Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(a, n));
         7747  +  free(a);
         7748  +  return TCL_OK;
         7749  +}
         7750  +
  7678   7751   
  7679   7752   /*
  7680   7753   ** Register commands with the TCL interpreter.
  7681   7754   */
  7682   7755   int Sqlitetest1_Init(Tcl_Interp *interp){
  7683   7756     extern int sqlite3_search_count;
  7684   7757     extern int sqlite3_found_count;
................................................................................
  7952   8025        { "sqlite3_snapshot_cmp_blob", test_snapshot_cmp_blob, 0 },
  7953   8026   #endif
  7954   8027        { "sqlite3_delete_database", test_delete_database, 0 },
  7955   8028        { "sqlite3_wal_info", test_wal_info, 0 },
  7956   8029        { "atomic_batch_write",      test_atomic_batch_write,     0   },
  7957   8030        { "sqlite3_mmap_warm",       test_mmap_warm,          0 },
  7958   8031        { "sqlite3_config_sorterref", test_config_sorterref,   0 },
         8032  +     { "decode_hexdb",             test_decode_hexdb,       0 },
  7959   8033     };
  7960   8034     static int bitmask_size = sizeof(Bitmask)*8;
  7961   8035     static int longdouble_size = sizeof(LONGDOUBLE_TYPE);
  7962   8036     int i;
  7963   8037     extern int sqlite3_sync_count, sqlite3_fullsync_count;
  7964   8038     extern int sqlite3_opentemp_count;
  7965   8039     extern int sqlite3_like_count;

Changes to src/tokenize.c.

   541    541       }
   542    542     }
   543    543     while( IdChar(z[i]) ){ i++; }
   544    544     *tokenType = TK_ID;
   545    545     return i;
   546    546   }
   547    547   
   548         -#ifdef SQLITE_ENABLE_NORMALIZE
   549         -/*
   550         -** Return the length (in bytes) of the token that begins at z[0].
   551         -** Store the token type in *tokenType before returning.  If flags has
   552         -** SQLITE_TOKEN_NORMALIZE flag enabled, use the identifier token type
   553         -** for keywords.  Add SQLITE_TOKEN_QUOTED to flags if the token was
   554         -** actually a quoted identifier.  Add SQLITE_TOKEN_KEYWORD to flags
   555         -** if the token was recognized as a keyword; this is useful when the
   556         -** SQLITE_TOKEN_NORMALIZE flag is used, because it enables the caller
   557         -** to differentiate between a keyword being treated as an identifier
   558         -** (for normalization purposes) and an actual identifier.
   559         -*/
   560         -int sqlite3GetTokenNormalized(
   561         -  const unsigned char *z,
   562         -  int *tokenType,
   563         -  int *flags
   564         -){
   565         -  int n;
   566         -  unsigned char iClass = aiClass[*z];
   567         -  if( iClass==CC_KYWD ){
   568         -    int i;
   569         -    for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
   570         -    if( IdChar(z[i]) ){
   571         -      /* This token started out using characters that can appear in keywords,
   572         -      ** but z[i] is a character not allowed within keywords, so this must
   573         -      ** be an identifier instead */
   574         -      i++;
   575         -      while( IdChar(z[i]) ){ i++; }
   576         -      *tokenType = TK_ID;
   577         -      return i;
   578         -    }
   579         -    *tokenType = TK_ID;
   580         -    n = keywordCode((char*)z, i, tokenType);
   581         -    /* If the token is no longer considered to be an identifier, then it is a
   582         -    ** keyword of some kind.  Make the token back into an identifier and then
   583         -    ** set the SQLITE_TOKEN_KEYWORD flag.  Several non-identifier tokens are
   584         -    ** used verbatim, including IN, IS, NOT, and NULL. */
   585         -    switch( *tokenType ){
   586         -      case TK_ID: {
   587         -        /* do nothing, handled by caller */
   588         -        break;
   589         -      }
   590         -      case TK_IN:
   591         -      case TK_IS:
   592         -      case TK_NOT:
   593         -      case TK_NULL: {
   594         -        *flags |= SQLITE_TOKEN_KEYWORD;
   595         -        break;
   596         -      }
   597         -      default: {
   598         -        *tokenType = TK_ID;
   599         -        *flags |= SQLITE_TOKEN_KEYWORD;
   600         -        break;
   601         -      }
   602         -    }
   603         -  }else{
   604         -    n = sqlite3GetToken(z, tokenType);
   605         -    /* If the token is considered to be an identifier and the character class
   606         -    ** of the first character is a quote, set the SQLITE_TOKEN_QUOTED flag. */
   607         -    if( *tokenType==TK_ID && (iClass==CC_QUOTE || iClass==CC_QUOTE2) ){
   608         -      *flags |= SQLITE_TOKEN_QUOTED;
   609         -    }
   610         -  }
   611         -  return n;
   612         -}
   613         -#endif /* SQLITE_ENABLE_NORMALIZE */
   614         -
   615    548   /*
   616    549   ** Run the parser on the given SQL string.  The parser structure is
   617    550   ** passed in.  An SQLITE_ status code is returned.  If an error occurs
   618    551   ** then an and attempt is made to write an error message into 
   619    552   ** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that
   620    553   ** error message.
   621    554   */
................................................................................
   777    710       Table *p = pParse->pZombieTab;
   778    711       pParse->pZombieTab = p->pNextZombie;
   779    712       sqlite3DeleteTable(db, p);
   780    713     }
   781    714     assert( nErr==0 || pParse->rc!=SQLITE_OK );
   782    715     return nErr;
   783    716   }
          717  +
          718  +
          719  +#ifdef SQLITE_ENABLE_NORMALIZE
          720  +/*
          721  +** Insert a single space character into pStr if the current string
          722  +** ends with an identifier
          723  +*/
          724  +static void addSpaceSeparator(sqlite3_str *pStr){
          725  +  if( pStr->nChar && sqlite3IsIdChar(pStr->zText[pStr->nChar-1]) ){
          726  +    sqlite3_str_append(pStr, " ", 1);
          727  +  }
          728  +}
          729  +
          730  +/*
          731  +** Compute a normalization of the SQL given by zSql[0..nSql-1].  Return
          732  +** the normalization in space obtained from sqlite3DbMalloc().  Or return
          733  +** NULL if anything goes wrong or if zSql is NULL.
          734  +*/
          735  +char *sqlite3Normalize(
          736  +  Vdbe *pVdbe,       /* VM being reprepared */
          737  +  const char *zSql   /* The original SQL string */
          738  +){
          739  +  sqlite3 *db;       /* The database connection */
          740  +  int i;             /* Next unread byte of zSql[] */
          741  +  int n;             /* length of current token */
          742  +  int tokenType;     /* type of current token */
          743  +  int prevType;      /* Previous non-whitespace token */
          744  +  int nParen;        /* Number of nested levels of parentheses */
          745  +  int iStartIN;      /* Start of RHS of IN operator in z[] */
          746  +  int nParenAtIN;    /* Value of nParent at start of RHS of IN operator */
          747  +  int j;             /* Bytes of normalized SQL generated so far */
          748  +  sqlite3_str *pStr; /* The normalized SQL string under construction */
          749  +
          750  +  db = sqlite3VdbeDb(pVdbe);
          751  +  tokenType = -1;
          752  +  nParen = iStartIN = nParenAtIN = 0;
          753  +  pStr = sqlite3_str_new(db);
          754  +  assert( pStr!=0 );  /* sqlite3_str_new() never returns NULL */
          755  +  for(i=0; zSql[i] && pStr->accError==0; i+=n){
          756  +    if( tokenType!=TK_SPACE ){
          757  +      prevType = tokenType;
          758  +    }
          759  +    n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType);
          760  +    if( NEVER(n<=0) ) break;
          761  +    switch( tokenType ){
          762  +      case TK_SPACE: {
          763  +        break;
          764  +      }
          765  +      case TK_NULL: {
          766  +        if( prevType==TK_IS || prevType==TK_NOT ){
          767  +          sqlite3_str_append(pStr, " NULL", 5);
          768  +          break;
          769  +        }
          770  +        /* Fall through */
          771  +      }
          772  +      case TK_STRING:
          773  +      case TK_INTEGER:
          774  +      case TK_FLOAT:
          775  +      case TK_VARIABLE:
          776  +      case TK_BLOB: {
          777  +        sqlite3_str_append(pStr, "?", 1);
          778  +        break;
          779  +      }
          780  +      case TK_LP: {
          781  +        nParen++;
          782  +        if( prevType==TK_IN ){
          783  +          iStartIN = pStr->nChar;
          784  +          nParenAtIN = nParen;
          785  +        }
          786  +        sqlite3_str_append(pStr, "(", 1);
          787  +        break;
          788  +      }
          789  +      case TK_RP: {
          790  +        if( iStartIN>0 && nParen==nParenAtIN ){
          791  +          assert( pStr->nChar>=iStartIN );
          792  +          pStr->nChar = iStartIN+1;
          793  +          sqlite3_str_append(pStr, "?,?,?", 5);
          794  +          iStartIN = 0;
          795  +        }
          796  +        nParen--;
          797  +        sqlite3_str_append(pStr, ")", 1);
          798  +        break;
          799  +      }
          800  +      case TK_ID: {
          801  +        iStartIN = 0;
          802  +        j = pStr->nChar;
          803  +        if( sqlite3Isquote(zSql[i]) ){
          804  +          char *zId = sqlite3DbStrNDup(db, zSql+i, n);
          805  +          int nId;
          806  +          int eType = 0;
          807  +          if( zId==0 ) break;
          808  +          sqlite3Dequote(zId);
          809  +          if( zSql[i]=='"' && sqlite3VdbeUsesDoubleQuotedString(pVdbe, zId) ){
          810  +            sqlite3_str_append(pStr, "?", 1);
          811  +            sqlite3DbFree(db, zId);
          812  +            break;
          813  +          }
          814  +          nId = sqlite3Strlen30(zId);
          815  +          if( sqlite3GetToken((u8*)zId, &eType)==nId && eType==TK_ID ){
          816  +            addSpaceSeparator(pStr);
          817  +            sqlite3_str_append(pStr, zId, nId);
          818  +          }else{
          819  +            sqlite3_str_appendf(pStr, "\"%w\"", zId);
          820  +          }
          821  +          sqlite3DbFree(db, zId);
          822  +        }else{
          823  +          addSpaceSeparator(pStr);
          824  +          sqlite3_str_append(pStr, zSql+i, n);
          825  +        }
          826  +        while( j<pStr->nChar ){
          827  +          pStr->zText[j] = sqlite3Tolower(pStr->zText[j]);
          828  +          j++;
          829  +        }
          830  +        break;
          831  +      }
          832  +      case TK_SELECT: {
          833  +        iStartIN = 0;
          834  +        /* fall through */
          835  +      }
          836  +      default: {
          837  +        if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr);
          838  +        j = pStr->nChar;
          839  +        sqlite3_str_append(pStr, zSql+i, n);
          840  +        while( j<pStr->nChar ){
          841  +          pStr->zText[j] = sqlite3Toupper(pStr->zText[j]);
          842  +          j++;
          843  +        }
          844  +        break;
          845  +      }
          846  +    }
          847  +  }
          848  +  if( tokenType!=TK_SEMI ) sqlite3_str_append(pStr, ";", 1);
          849  +  return sqlite3_str_finish(pStr);
          850  +}
          851  +#endif /* SQLITE_ENABLE_NORMALIZE */

Changes to src/vacuum.c.

    98     98   ** the copy of step (3) were replaced by deleting the original database
    99     99   ** and renaming the transient database as the original.  But that will
   100    100   ** not work if other processes are attached to the original database.
   101    101   ** And a power loss in between deleting the original and renaming the
   102    102   ** transient would cause the database file to appear to be deleted
   103    103   ** following reboot.
   104    104   */
   105         -void sqlite3Vacuum(Parse *pParse, Token *pNm){
          105  +void sqlite3Vacuum(Parse *pParse, Token *pNm, Expr *pInto){
   106    106     Vdbe *v = sqlite3GetVdbe(pParse);
   107    107     int iDb = 0;
   108         -  if( v==0 ) return;
          108  +  if( v==0 ) goto build_vacuum_end;
   109    109     if( pNm ){
   110    110   #ifndef SQLITE_BUG_COMPATIBLE_20160819
   111    111       /* Default behavior:  Report an error if the argument to VACUUM is
   112    112       ** not recognized */
   113    113       iDb = sqlite3TwoPartName(pParse, pNm, pNm, &pNm);
   114         -    if( iDb<0 ) return;
          114  +    if( iDb<0 ) goto build_vacuum_end;
   115    115   #else
   116    116       /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments
   117    117       ** to VACUUM are silently ignored.  This is a back-out of a bug fix that
   118    118       ** occurred on 2016-08-19 (https://www.sqlite.org/src/info/083f9e6270).
   119    119       ** The buggy behavior is required for binary compatibility with some
   120    120       ** legacy applications. */
   121    121       iDb = sqlite3FindDb(pParse->db, pNm);
   122    122       if( iDb<0 ) iDb = 0;
   123    123   #endif
   124    124     }
   125    125     if( iDb!=1 ){
   126         -    sqlite3VdbeAddOp1(v, OP_Vacuum, iDb);
          126  +    int iIntoReg = 0;
          127  +    if( pInto ){
          128  +      iIntoReg = ++pParse->nMem;
          129  +      sqlite3ExprCode(pParse, pInto, iIntoReg);
          130  +    }
          131  +    sqlite3VdbeAddOp2(v, OP_Vacuum, iDb, iIntoReg);
   127    132       sqlite3VdbeUsesBtree(v, iDb);
   128    133     }
          134  +build_vacuum_end:
          135  +  sqlite3ExprDelete(pParse->db, pInto);
   129    136     return;
   130    137   }
   131    138   
   132    139   /*
   133    140   ** This routine implements the OP_Vacuum opcode of the VDBE.
   134    141   */
   135         -int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){
          142  +int sqlite3RunVacuum(
          143  +  char **pzErrMsg,        /* Write error message here */
          144  +  sqlite3 *db,            /* Database connection */
          145  +  int iDb,                /* Which attached DB to vacuum */
          146  +  sqlite3_value *pOut     /* Write results here, if not NULL */
          147  +){
   136    148     int rc = SQLITE_OK;     /* Return code from service routines */
   137    149     Btree *pMain;           /* The database being vacuumed */
   138    150     Btree *pTemp;           /* The temporary database we vacuum into */
   139         -  u16 saved_mDbFlags;     /* Saved value of db->mDbFlags */
   140         -  u32 saved_flags;        /* Saved value of db->flags */
          151  +  u32 saved_mDbFlags;     /* Saved value of db->mDbFlags */
          152  +  u64 saved_flags;        /* Saved value of db->flags */
   141    153     int saved_nChange;      /* Saved value of db->nChange */
   142    154     int saved_nTotalChange; /* Saved value of db->nTotalChange */
   143    155     u8 saved_mTrace;        /* Saved trace settings */
   144    156     Db *pDb = 0;            /* Database to detach at end of vacuum */
   145    157     int isMemDb;            /* True if vacuuming a :memory: database */
   146    158     int nRes;               /* Bytes of reserved space at the end of each page */
   147    159     int nDb;                /* Number of attached databases */
   148    160     const char *zDbMain;    /* Schema name of database to vacuum */
          161  +  const char *zOut;       /* Name of output file */
   149    162   
   150    163     if( !db->autoCommit ){
   151    164       sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction");
   152    165       return SQLITE_ERROR;
   153    166     }
   154    167     if( db->nVdbeActive>1 ){
   155    168       sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress");
   156    169       return SQLITE_ERROR;
   157    170     }
          171  +  if( pOut ){
          172  +    if( sqlite3_value_type(pOut)!=SQLITE_TEXT ){
          173  +      sqlite3SetString(pzErrMsg, db, "non-text filename");
          174  +      return SQLITE_ERROR;
          175  +    }
          176  +    zOut = (const char*)sqlite3_value_text(pOut);
          177  +  }else{
          178  +    zOut = "";
          179  +  }
   158    180   
   159    181     /* Save the current value of the database flags so that it can be 
   160    182     ** restored before returning. Then set the writable-schema flag, and
   161    183     ** disable CHECK and foreign key constraints.  */
   162    184     saved_flags = db->flags;
   163    185     saved_mDbFlags = db->mDbFlags;
   164    186     saved_nChange = db->nChange;
   165    187     saved_nTotalChange = db->nTotalChange;
   166    188     saved_mTrace = db->mTrace;
   167    189     db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks;
   168    190     db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum;
   169         -  db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder
          191  +  db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder
   170    192                      | SQLITE_Defensive | SQLITE_CountRows);
   171    193     db->mTrace = 0;
   172    194   
   173    195     zDbMain = db->aDb[iDb].zDbSName;
   174    196     pMain = db->aDb[iDb].pBt;
   175    197     isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain));
   176    198   
................................................................................
   185    207     ** that actually made the VACUUM run slower.  Very little journalling
   186    208     ** actually occurs when doing a vacuum since the vacuum_db is initially
   187    209     ** empty.  Only the journal header is written.  Apparently it takes more
   188    210     ** time to parse and run the PRAGMA to turn journalling off than it does
   189    211     ** to write the journal header file.
   190    212     */
   191    213     nDb = db->nDb;
   192         -  rc = execSql(db, pzErrMsg, "ATTACH''AS vacuum_db");
          214  +  rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS vacuum_db", zOut);
   193    215     if( rc!=SQLITE_OK ) goto end_of_vacuum;
   194    216     assert( (db->nDb-1)==nDb );
   195    217     pDb = &db->aDb[nDb];
   196    218     assert( strcmp(pDb->zDbSName,"vacuum_db")==0 );
   197    219     pTemp = pDb->pBt;
   198         -
   199         -  /* The call to execSql() to attach the temp database has left the file
   200         -  ** locked (as there was more than one active statement when the transaction
   201         -  ** to read the schema was concluded. Unlock it here so that this doesn't
   202         -  ** cause problems for the call to BtreeSetPageSize() below.  */
   203         -  sqlite3BtreeCommit(pTemp);
   204         -
          220  +  if( pOut ){
          221  +    sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp));
          222  +    i64 sz = 0;
          223  +    if( id->pMethods!=0 && (sqlite3OsFileSize(id, &sz)!=SQLITE_OK || sz>0) ){
          224  +      rc = SQLITE_ERROR;
          225  +      sqlite3SetString(pzErrMsg, db, "output file already exists");
          226  +      goto end_of_vacuum;
          227  +    }
          228  +  }
   205    229     nRes = sqlite3BtreeGetOptimalReserve(pMain);
   206    230   
   207    231     /* A VACUUM cannot change the pagesize of an encrypted database. */
   208    232   #ifdef SQLITE_HAS_CODEC
   209    233     if( db->nextPagesize ){
   210    234       extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*);
   211    235       int nKey;
................................................................................
   221    245   
   222    246     /* Begin a transaction and take an exclusive lock on the main database
   223    247     ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below,
   224    248     ** to ensure that we do not try to change the page-size on a WAL database.
   225    249     */
   226    250     rc = execSql(db, pzErrMsg, "BEGIN");
   227    251     if( rc!=SQLITE_OK ) goto end_of_vacuum;
   228         -  rc = sqlite3BtreeBeginTrans(pMain, 2, 0);
          252  +  rc = sqlite3BtreeBeginTrans(pMain, pOut==0 ? 2 : 0, 0);
   229    253     if( rc!=SQLITE_OK ) goto end_of_vacuum;
   230    254   
   231    255     /* Do not attempt to change the page size for a WAL database */
   232    256     if( sqlite3PagerGetJournalMode(sqlite3BtreePager(pMain))
   233    257                                                  ==PAGER_JOURNALMODE_WAL ){
   234    258       db->nextPagesize = 0;
   235    259     }
................................................................................
   316    340          BTREE_DEFAULT_CACHE_SIZE, 0,  /* Preserve the default page cache size */
   317    341          BTREE_TEXT_ENCODING,      0,  /* Preserve the text encoding */
   318    342          BTREE_USER_VERSION,       0,  /* Preserve the user version */
   319    343          BTREE_APPLICATION_ID,     0,  /* Preserve the application id */
   320    344       };
   321    345   
   322    346       assert( 1==sqlite3BtreeIsInTrans(pTemp) );
   323         -    assert( 1==sqlite3BtreeIsInTrans(pMain) );
          347  +    assert( pOut!=0 || 1==sqlite3BtreeIsInTrans(pMain) );
   324    348   
   325    349       /* Copy Btree meta values */
   326    350       for(i=0; i<ArraySize(aCopy); i+=2){
   327    351         /* GetMeta() and UpdateMeta() cannot fail in this context because
   328    352         ** we already have page 1 loaded into cache and marked dirty. */
   329    353         sqlite3BtreeGetMeta(pMain, aCopy[i], &meta);
   330    354         rc = sqlite3BtreeUpdateMeta(pTemp, aCopy[i], meta+aCopy[i+1]);
   331    355         if( NEVER(rc!=SQLITE_OK) ) goto end_of_vacuum;
   332    356       }
   333    357   
   334         -    rc = sqlite3BtreeCopyFile(pMain, pTemp);
          358  +    if( pOut==0 ){
          359  +      rc = sqlite3BtreeCopyFile(pMain, pTemp);
          360  +    }
   335    361       if( rc!=SQLITE_OK ) goto end_of_vacuum;
   336    362       rc = sqlite3BtreeCommit(pTemp);
   337    363       if( rc!=SQLITE_OK ) goto end_of_vacuum;
   338    364   #ifndef SQLITE_OMIT_AUTOVACUUM
   339         -    sqlite3BtreeSetAutoVacuum(pMain, sqlite3BtreeGetAutoVacuum(pTemp));
          365  +    if( pOut==0 ){
          366  +      sqlite3BtreeSetAutoVacuum(pMain, sqlite3BtreeGetAutoVacuum(pTemp));
          367  +    }
   340    368   #endif
   341    369     }
   342    370   
   343    371     assert( rc==SQLITE_OK );
   344         -  rc = sqlite3BtreeSetPageSize(pMain, sqlite3BtreeGetPageSize(pTemp), nRes,1);
          372  +  if( pOut==0 ){
          373  +    rc = sqlite3BtreeSetPageSize(pMain, sqlite3BtreeGetPageSize(pTemp), nRes,1);
          374  +  }
   345    375   
   346    376   end_of_vacuum:
   347    377     /* Restore the original value of db->flags */
   348    378     db->init.iDb = 0;
   349    379     db->mDbFlags = saved_mDbFlags;
   350    380     db->flags = saved_flags;
   351    381     db->nChange = saved_nChange;

Changes to src/vdbe.c.

  6711   6711     sqlite3VdbeChangeEncoding(pOut, encoding);
  6712   6712     if( rc ) goto abort_due_to_error;
  6713   6713     break;
  6714   6714   };
  6715   6715   #endif /* SQLITE_OMIT_PRAGMA */
  6716   6716   
  6717   6717   #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
  6718         -/* Opcode: Vacuum P1 * * * *
         6718  +/* Opcode: Vacuum P1 P2 * * *
  6719   6719   **
  6720   6720   ** Vacuum the entire database P1.  P1 is 0 for "main", and 2 or more
  6721   6721   ** for an attached database.  The "temp" database may not be vacuumed.
         6722  +**
         6723  +** If P2 is not zero, then it is a register holding a string which is
         6724  +** the file into which the result of vacuum should be written.  When
         6725  +** P2 is zero, the vacuum overwrites the original database.
  6722   6726   */
  6723   6727   case OP_Vacuum: {
  6724   6728     assert( p->readOnly==0 );
  6725         -  rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1);
         6729  +  rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1,
         6730  +                        pOp->p2 ? &aMem[pOp->p2] : 0);
  6726   6731     if( rc ) goto abort_due_to_error;
  6727   6732     break;
  6728   6733   }
  6729   6734   #endif
  6730   6735   
  6731   6736   #if !defined(SQLITE_OMIT_AUTOVACUUM)
  6732   6737   /* Opcode: IncrVacuum P1 P2 * * *
................................................................................
  7121   7126     assert( pName->flags & MEM_Str );
  7122   7127     testcase( pName->enc==SQLITE_UTF8 );
  7123   7128     testcase( pName->enc==SQLITE_UTF16BE );
  7124   7129     testcase( pName->enc==SQLITE_UTF16LE );
  7125   7130     rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8);
  7126   7131     if( rc ) goto abort_due_to_error;
  7127   7132     rc = pVtab->pModule->xRename(pVtab, pName->z);
  7128         -  if( isLegacy==0 ) db->flags &= ~SQLITE_LegacyAlter;
         7133  +  if( isLegacy==0 ) db->flags &= ~(u64)SQLITE_LegacyAlter;
  7129   7134     sqlite3VtabImportErrmsg(p, pVtab);
  7130   7135     p->expired = 0;
  7131   7136     if( rc ) goto abort_due_to_error;
  7132   7137     break;
  7133   7138   }
  7134   7139   #endif
  7135   7140   

Changes to src/vdbe.h.

   247    247   int sqlite3VdbeReset(Vdbe*);
   248    248   void sqlite3VdbeSetNumCols(Vdbe*,int);
   249    249   int sqlite3VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*));
   250    250   void sqlite3VdbeCountChanges(Vdbe*);
   251    251   sqlite3 *sqlite3VdbeDb(Vdbe*);
   252    252   u8 sqlite3VdbePrepareFlags(Vdbe*);
   253    253   void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, u8);
          254  +#ifdef SQLITE_ENABLE_NORMALIZE
          255  +void sqlite3VdbeAddDblquoteStr(sqlite3*,Vdbe*,const char*);
          256  +int sqlite3VdbeUsesDoubleQuotedString(Vdbe*,const char*);
          257  +#endif
   254    258   void sqlite3VdbeSwap(Vdbe*,Vdbe*);
   255    259   VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*);
   256    260   sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8);
   257    261   void sqlite3VdbeSetVarmask(Vdbe*, int);
   258    262   #ifndef SQLITE_OMIT_TRACE
   259    263     char *sqlite3VdbeExpandSql(Vdbe*, const char*);
   260    264   #endif

Changes to src/vdbeInt.h.

   331    331   };
   332    332   
   333    333   /* A bitfield type for use inside of structures.  Always follow with :N where
   334    334   ** N is the number of bits.
   335    335   */
   336    336   typedef unsigned bft;  /* Bit Field Type */
   337    337   
          338  +/* The ScanStatus object holds a single value for the
          339  +** sqlite3_stmt_scanstatus() interface.
          340  +*/
   338    341   typedef struct ScanStatus ScanStatus;
   339    342   struct ScanStatus {
   340    343     int addrExplain;                /* OP_Explain for loop */
   341    344     int addrLoop;                   /* Address of "loops" counter */
   342    345     int addrVisit;                  /* Address of "rows visited" counter */
   343    346     int iSelectID;                  /* The "Select-ID" for this loop */
   344    347     LogEst nEst;                    /* Estimated output rows per loop */
   345    348     char *zName;                    /* Name of table or index */
   346    349   };
          350  +
          351  +/* The DblquoteStr object holds the text of a double-quoted
          352  +** string for a prepared statement.  A linked list of these objects
          353  +** is constructed during statement parsing and is held on Vdbe.pDblStr.
          354  +** When computing a normalized SQL statement for an SQL statement, that
          355  +** list is consulted for each double-quoted identifier to see if the
          356  +** identifier should really be a string literal.
          357  +*/
          358  +typedef struct DblquoteStr DblquoteStr;
          359  +struct DblquoteStr {
          360  +  DblquoteStr *pNextStr;   /* Next string literal in the list */
          361  +  char z[8];               /* Dequoted value for the string */
          362  +};
   347    363   
   348    364   /*
   349    365   ** An instance of the virtual machine.  This structure contains the complete
   350    366   ** state of the virtual machine.
   351    367   **
   352    368   ** The "sqlite3_stmt" structure pointer that is returned by sqlite3_prepare()
   353    369   ** is really a pointer to an instance of this structure.
................................................................................
   360    376     u32 magic;              /* Magic number for sanity checking */
   361    377     int nMem;               /* Number of memory locations currently allocated */
   362    378     int nCursor;            /* Number of slots in apCsr[] */
   363    379     u32 cacheCtr;           /* VdbeCursor row cache generation counter */
   364    380     int pc;                 /* The program counter */
   365    381     int rc;                 /* Value to return */
   366    382     int nChange;            /* Number of db changes made since last reset */
   367         -  int iStatement;         /* Statement number (or 0 if has not opened stmt) */
          383  +  int iStatement;         /* Statement number (or 0 if has no opened stmt) */
   368    384     i64 iCurrentTime;       /* Value of julianday('now') for this statement */
   369    385     i64 nFkConstraint;      /* Number of imm. FK constraints this VM */
   370    386     i64 nStmtDefCons;       /* Number of def. constraints when stmt started */
   371    387     i64 nStmtDefImmCons;    /* Number of def. imm constraints when stmt started */
   372    388   
   373    389     /* When allocating a new Vdbe object, all of the fields below should be
   374    390     ** initialized to zero or NULL */
................................................................................
   404    420     bft bIsReader:1;        /* True for statements that read */
   405    421     yDbMask btreeMask;      /* Bitmask of db->aDb[] entries referenced */
   406    422     yDbMask lockMask;       /* Subset of btreeMask that requires a lock */
   407    423     u32 aCounter[7];        /* Counters used by sqlite3_stmt_status() */
   408    424     char *zSql;             /* Text of the SQL statement that generated this */
   409    425   #ifdef SQLITE_ENABLE_NORMALIZE
   410    426     char *zNormSql;         /* Normalization of the associated SQL statement */
          427  +  DblquoteStr *pDblStr;   /* List of double-quoted string literals */
   411    428   #endif
   412    429     void *pFree;            /* Free this when deleting the vdbe */
   413    430     VdbeFrame *pFrame;      /* Parent frame */
   414    431     VdbeFrame *pDelFrame;   /* List of frame objects to free on VM reset */
   415    432     int nFrame;             /* Number of frames in pFrame list */
   416    433     u32 expmask;            /* Binding to these vars invalidates VM */
   417    434     SubProgram *pProgram;   /* Linked list of all sub-programs used by VM */

Changes to src/vdbeapi.c.

   581    581     /* Check that malloc() has not failed. If it has, return early. */
   582    582     db = p->db;
   583    583     if( db->mallocFailed ){
   584    584       p->rc = SQLITE_NOMEM;
   585    585       return SQLITE_NOMEM_BKPT;
   586    586     }
   587    587   
   588         -  if( p->pc<=0 && p->expired ){
          588  +  if( p->pc<0 && p->expired ){
   589    589       p->rc = SQLITE_SCHEMA;
   590    590       rc = SQLITE_ERROR;
   591    591       goto end_of_step;
   592    592     }
   593    593     if( p->pc<0 ){
   594    594       /* If there are no other statements currently running, then
   595    595       ** reset the interrupt flag.  This prevents a call to sqlite3_interrupt
................................................................................
   658    658     ** contains the value that would be returned if sqlite3_finalize() 
   659    659     ** were called on statement p.
   660    660     */
   661    661     assert( rc==SQLITE_ROW  || rc==SQLITE_DONE   || rc==SQLITE_ERROR 
   662    662          || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE
   663    663     );
   664    664     assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp );
   665         -  if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 
   666         -   && rc!=SQLITE_ROW 
   667         -   && rc!=SQLITE_DONE 
          665  +  if( rc!=SQLITE_ROW 
          666  +   && rc!=SQLITE_DONE
          667  +   && (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0
   668    668     ){
   669    669       /* If this statement was prepared using saved SQL and an 
   670    670       ** error has occurred, then return the error code in p->rc to the
   671    671       ** caller. Set the error code in the database handle to the same value.
   672    672       */ 
   673    673       rc = sqlite3VdbeTransferError(p);
   674    674     }
................................................................................
  1282   1282       sqlite3_mutex_leave(p->db->mutex);
  1283   1283       return SQLITE_RANGE;
  1284   1284     }
  1285   1285     i--;
  1286   1286     pVar = &p->aVar[i];
  1287   1287     sqlite3VdbeMemRelease(pVar);
  1288   1288     pVar->flags = MEM_Null;
  1289         -  sqlite3Error(p->db, SQLITE_OK);
         1289  +  p->db->errCode = SQLITE_OK;
  1290   1290   
  1291   1291     /* If the bit corresponding to this variable in Vdbe.expmask is set, then 
  1292   1292     ** binding a new value to this variable invalidates the current query plan.
  1293   1293     **
  1294   1294     ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host
  1295   1295     ** parameter in the WHERE clause might influence the choice of query plan
  1296   1296     ** for a statement, then the statement will be automatically recompiled,
................................................................................
  1709   1709   #ifdef SQLITE_ENABLE_NORMALIZE
  1710   1710   /*
  1711   1711   ** Return the normalized SQL associated with a prepared statement.
  1712   1712   */
  1713   1713   const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){
  1714   1714     Vdbe *p = (Vdbe *)pStmt;
  1715   1715     if( p==0 ) return 0;
  1716         -  if( p->zNormSql==0 && p->zSql!=0 ){
         1716  +  if( p->zNormSql==0 && ALWAYS(p->zSql!=0) ){
  1717   1717       sqlite3_mutex_enter(p->db->mutex);
  1718         -    p->zNormSql = sqlite3Normalize(p, p->zSql, sqlite3Strlen30(p->zSql));
         1718  +    p->zNormSql = sqlite3Normalize(p, p->zSql);
  1719   1719       sqlite3_mutex_leave(p->db->mutex);
  1720   1720     }
  1721   1721     return p->zNormSql;
  1722   1722   }
  1723   1723   #endif /* SQLITE_ENABLE_NORMALIZE */
  1724   1724   
  1725   1725   #ifdef SQLITE_ENABLE_PREUPDATE_HOOK

Changes to src/vdbeaux.c.

    60     60     if( p==0 ) return;
    61     61     p->prepFlags = prepFlags;
    62     62     if( (prepFlags & SQLITE_PREPARE_SAVESQL)==0 ){
    63     63       p->expmask = 0;
    64     64     }
    65     65     assert( p->zSql==0 );
    66     66     p->zSql = sqlite3DbStrNDup(p->db, z, n);
           67  +}
           68  +
           69  +#ifdef SQLITE_ENABLE_NORMALIZE
           70  +/*
           71  +** Add a new element to the Vdbe->pDblStr list.
           72  +*/
           73  +void sqlite3VdbeAddDblquoteStr(sqlite3 *db, Vdbe *p, const char *z){
           74  +  if( p ){
           75  +    int n = sqlite3Strlen30(z);
           76  +    DblquoteStr *pStr = sqlite3DbMallocRawNN(db,
           77  +                            sizeof(*pStr)+n+1-sizeof(pStr->z));
           78  +    if( pStr ){
           79  +      pStr->pNextStr = p->pDblStr;
           80  +      p->pDblStr = pStr;
           81  +      memcpy(pStr->z, z, n+1);
           82  +    }
           83  +  }
           84  +}
           85  +#endif
           86  +
    67     87   #ifdef SQLITE_ENABLE_NORMALIZE
    68         -  assert( p->zNormSql==0 );
    69         -  if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){
    70         -    p->zNormSql = sqlite3Normalize(p, p->zSql, n);
    71         -    assert( p->zNormSql!=0 || p->db->mallocFailed );
           88  +/*
           89  +** zId of length nId is a double-quoted identifier.  Check to see if
           90  +** that identifier is really used as a string literal.
           91  +*/
           92  +int sqlite3VdbeUsesDoubleQuotedString(
           93  +  Vdbe *pVdbe,            /* The prepared statement */
           94  +  const char *zId         /* The double-quoted identifier, already dequoted */
           95  +){
           96  +  DblquoteStr *pStr;
           97  +  assert( zId!=0 );
           98  +  if( pVdbe->pDblStr==0 ) return 0;
           99  +  for(pStr=pVdbe->pDblStr; pStr; pStr=pStr->pNextStr){
          100  +    if( strcmp(zId, pStr->z)==0 ) return 1;
    72    101     }
          102  +  return 0;
          103  +}
    73    104   #endif
    74         -}
    75    105   
    76    106   /*
    77    107   ** Swap all content between two VDBE structures.
    78    108   */
    79    109   void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){
    80    110     Vdbe tmp, *pTmp;
    81    111     char *zTmp;
................................................................................
    88    118     pB->pNext = pTmp;
    89    119     pTmp = pA->pPrev;
    90    120     pA->pPrev = pB->pPrev;
    91    121     pB->pPrev = pTmp;
    92    122     zTmp = pA->zSql;
    93    123     pA->zSql = pB->zSql;
    94    124     pB->zSql = zTmp;
    95         -#ifdef SQLITE_ENABLE_NORMALIZE
          125  +#if 0
    96    126     zTmp = pA->zNormSql;
    97    127     pA->zNormSql = pB->zNormSql;
    98    128     pB->zNormSql = zTmp;
    99    129   #endif
   100    130     pB->expmask = pA->expmask;
   101    131     pB->prepFlags = pA->prepFlags;
   102    132     memcpy(pB->aCounter, pA->aCounter, sizeof(pB->aCounter));
................................................................................
  2869   2899           }else if( rc!=SQLITE_OK ){
  2870   2900             p->rc = rc;
  2871   2901             sqlite3RollbackAll(db, SQLITE_OK);
  2872   2902             p->nChange = 0;
  2873   2903           }else{
  2874   2904             db->nDeferredCons = 0;
  2875   2905             db->nDeferredImmCons = 0;
  2876         -          db->flags &= ~SQLITE_DeferFKs;
         2906  +          db->flags &= ~(u64)SQLITE_DeferFKs;
  2877   2907             sqlite3CommitInternalChanges(db);
  2878   2908           }
  2879   2909         }else{
  2880   2910           sqlite3RollbackAll(db, SQLITE_OK);
  2881   2911           p->nChange = 0;
  2882   2912         }
  2883   2913         db->nStatement = 0;
................................................................................
  3186   3216       sqlite3DbFree(db, p->pFree);
  3187   3217     }
  3188   3218     vdbeFreeOpArray(db, p->aOp, p->nOp);
  3189   3219     sqlite3DbFree(db, p->aColName);
  3190   3220     sqlite3DbFree(db, p->zSql);
  3191   3221   #ifdef SQLITE_ENABLE_NORMALIZE
  3192   3222     sqlite3DbFree(db, p->zNormSql);
         3223  +  {
         3224  +    DblquoteStr *pThis, *pNext;
         3225  +    for(pThis=p->pDblStr; pThis; pThis=pNext){
         3226  +      pNext = pThis->pNextStr;
         3227  +      sqlite3DbFree(db, pThis);
         3228  +    }
         3229  +  }
  3193   3230   #endif
  3194   3231   #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
  3195   3232     {
  3196   3233       int i;
  3197   3234       for(i=0; i<p->nScan; i++){
  3198   3235         sqlite3DbFree(db, p->aScan[i].zName);
  3199   3236       }

Changes to src/whereexpr.c.

   773    773       /* Search for a table and column that appears on one side or the
   774    774       ** other of the == operator in every subterm.  That table and column
   775    775       ** will be recorded in iCursor and iColumn.  There might not be any
   776    776       ** such table and column.  Set okToChngToIN if an appropriate table
   777    777       ** and column is found but leave okToChngToIN false if not found.
   778    778       */
   779    779       for(j=0; j<2 && !okToChngToIN; j++){
          780  +      Expr *pLeft = 0;
   780    781         pOrTerm = pOrWc->a;
   781    782         for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){
   782    783           assert( pOrTerm->eOperator & WO_EQ );
   783    784           pOrTerm->wtFlags &= ~TERM_OR_OK;
   784    785           if( pOrTerm->leftCursor==iCursor ){
   785    786             /* This is the 2-bit case and we are on the second iteration and
   786    787             ** current term is from the first iteration.  So skip this term. */
................................................................................
   796    797             testcase( pOrTerm->wtFlags & TERM_COPIED );
   797    798             testcase( pOrTerm->wtFlags & TERM_VIRTUAL );
   798    799             assert( pOrTerm->wtFlags & (TERM_COPIED|TERM_VIRTUAL) );
   799    800             continue;
   800    801           }
   801    802           iColumn = pOrTerm->u.leftColumn;
   802    803           iCursor = pOrTerm->leftCursor;
          804  +        pLeft = pOrTerm->pExpr->pLeft;
   803    805           break;
   804    806         }
   805    807         if( i<0 ){
   806    808           /* No candidate table+column was found.  This can only occur
   807    809           ** on the second iteration */
   808    810           assert( j==1 );
   809    811           assert( IsPowerOfTwo(chngToIN) );
................................................................................
   815    817         /* We have found a candidate table and column.  Check to see if that
   816    818         ** table and column is common to every term in the OR clause */
   817    819         okToChngToIN = 1;
   818    820         for(; i>=0 && okToChngToIN; i--, pOrTerm++){
   819    821           assert( pOrTerm->eOperator & WO_EQ );
   820    822           if( pOrTerm->leftCursor!=iCursor ){
   821    823             pOrTerm->wtFlags &= ~TERM_OR_OK;
   822         -        }else if( pOrTerm->u.leftColumn!=iColumn ){
          824  +        }else if( pOrTerm->u.leftColumn!=iColumn || (iColumn==XN_EXPR 
          825  +               && sqlite3ExprCompare(pParse, pOrTerm->pExpr->pLeft, pLeft, -1)
          826  +        )){
   823    827             okToChngToIN = 0;
   824    828           }else{
   825    829             int affLeft, affRight;
   826    830             /* If the right-hand side is also a column, then the affinities
   827    831             ** of both right and left sides must be such that no type
   828    832             ** conversions are required on the right.  (Ticket #2249)
   829    833             */

Changes to src/window.c.

  2129   2129   Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p){
  2130   2130     Window *pNew = 0;
  2131   2131     if( ALWAYS(p) ){
  2132   2132       pNew = sqlite3DbMallocZero(db, sizeof(Window));
  2133   2133       if( pNew ){
  2134   2134         pNew->zName = sqlite3DbStrDup(db, p->zName);
  2135   2135         pNew->pFilter = sqlite3ExprDup(db, p->pFilter, 0);
         2136  +      pNew->pFunc = p->pFunc;
  2136   2137         pNew->pPartition = sqlite3ExprListDup(db, p->pPartition, 0);
  2137   2138         pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, 0);
  2138   2139         pNew->eType = p->eType;
  2139   2140         pNew->eEnd = p->eEnd;
  2140   2141         pNew->eStart = p->eStart;
  2141   2142         pNew->pStart = sqlite3ExprDup(db, p->pStart, 0);
  2142   2143         pNew->pEnd = sqlite3ExprDup(db, p->pEnd, 0);

Added test/dbfuzz001.test.

            1  +# 2012-12-13
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# Test cases for corrupt database files.
           13  +
           14  +set testdir [file dirname $argv0]
           15  +source $testdir/tester.tcl
           16  +
           17  +ifcapable !deserialize {
           18  +  finish_test
           19  +  return
           20  +}
           21  +database_may_be_corrupt
           22  +
           23  +# In the following database file, there is 384 bytes of free space
           24  +# on page 8 that does not appear on the freeblock list.
           25  +#
           26  +do_test dbfuzz001-100 {
           27  +  sqlite3 db {}
           28  +  db deserialize [decode_hexdb {
           29  +    | size 5632 pagesize 512 filename c4.db
           30  +    | page 1 offset 0
           31  +    |      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
           32  +    |     16: 02 00 01 01 00 40 20 20 00 00 00 02 00 00 00 0b   .....@  ........
           33  +    |     32: 00 00 00 06 00 00 00 01 00 00 00 28 00 00 00 04   ...........(....
           34  +    |     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
           35  +    |     80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02   ................
           36  +    |     96: 00 2e 30 38 0d 00 00 00 06 01 06 00 01 da 01 b0   ..08............
           37  +    |    112: 01 56 01 86 01 2a 01 06 00 00 00 00 00 00 00 00   .V...*..........
           38  +    |    256: 00 00 00 00 00 00 22 07 06 17 11 11 01 31 74 61   ......"......1ta
           39  +    |    272: 62 6c 65 74 34 74 34 07 43 52 45 41 54 45 20 54   blet4t4.CREATE T
           40  +    |    288: 41 42 4c 45 20 74 34 28 78 29 2a 06 06 17 13 11   ABLE t4(x)*.....
           41  +    |    304: 01 3f 69 6e 64 65 78 00 00 00 00 00 00 00 00 00   .?index.........
           42  +    |    336: 20 74 33 28 78 29 2e 04 06 17 15 11 01 45 69 6e    t3(x).......Ein
           43  +    |    352: 64 65 78 74 32 63 64 74 32 05 43 52 45 41 54 45   dext2cdt2.CREATE
           44  +    |    368: 20 49 4e 44 45 58 20 74 32 63 64 20 4f 4e 20 74    INDEX t2cd ON t
           45  +    |    384: 32 28 63 2c 64 29 28 05 06 17 11 11 01 3d 74 61   2(c,d)(......=ta
           46  +    |    400: 62 6c 65 74 33 74 33 04 43 52 45 41 54 45 20 54   blet3t3.CREATE T
           47  +    |    416: 41 42 4c 45 20 74 33 28 63 2c 78 2c 65 2c 66 29   ABLE t3(c,x,e,f)
           48  +    |    432: 28 02 06 17 11 11 01 3d 74 61 62 6c 65 74 32 74   (......=tablet2t
           49  +    |    448: 32 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 74   2.CREATE TABLE t
           50  +    |    464: 32 28 63 2c 64 2c 65 2c 66 29 24 01 06 17 11 11   2(c,d,e,f)$.....
           51  +    |    480: 01 35 74 61 62 6c 65 74 31 74 31 02 43 52 45 41   .5tablet1t1.CREA
           52  +    |    496: 54 45 20 54 41 42 4c 45 20 74 31 28 61 2c 62 29   TE TABLE t1(a,b)
           53  +    | page 2 offset 512
           54  +    |      0: 0d 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00   ................
           55  +    | page 3 offset 1024
           56  +    |      0: 0d 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00   ................
           57  +    | page 4 offset 1536
           58  +    |      0: 05 00 00 00 03 01 f1 00 00 00 00 0b 01 fb 01 f6   ................
           59  +    |     16: 01 f1 00 16 00 00 09 06 05 01 01 01 01 04 04 03   ................
           60  +    |     32: 03 07 05 05 01 01 09 09 02 02 19 04 05 17 17 17   ................
           61  +    |     48: 17 73 65 76 65 6e 65 69 67 68 74 65 69 67 68 74   .seveneighteight
           62  +    |     64: 73 65 76 65 6e 25 03 05 07 07 07 07 40 14 00 00   seven%......@...
           63  +    |     80: 00 00 00 00 40 18 00 00 00 00 00 00 40 18 00 00   ....@.......@...
           64  +    |     96: 00 00 00 00 40 14 00 00 00 00 00 00 09 02 05 01   ....@...........
           65  +    |    112: 01 01 01 03 04 04 03 07 01 05 09 01 01 09 02 02   ................
           66  +    |    352: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1a   ................
           67  +    |    496: 00 00 00 00 0a 3e 00 00 00 09 21 00 00 00 08 06   .....>....!.....
           68  +    | page 5 offset 2048
           69  +    |      0: 0a 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00   ................
           70  +    | page 7 offset 3072
           71  +    |      0: 0d 00 00 00 08 01 c2 00 01 fb 01 f6 01 f1 01 ec   ................
           72  +    |     16: 01 e0 01 d4 01 cb 01 c2 00 00 00 00 00 00 00 00   ................
           73  +    |     96: 00 00 00 00 13 00 00 00 00 00 00 00 00 00 00 00   ................
           74  +    |    224: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02   ................
           75  +    |    288: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03   ................
           76  +    |    448: 00 00 07 08 02 17 65 69 67 68 74 07 07 02 17 65   ......eight....e
           77  +    |    464: 69 67 68 74 0a 06 02 07 40 18 00 00 00 00 00 00   ight....@.......
           78  +    |    480: 0a 05 02 07 40 18 00 00 00 00 00 00 03 04 02 01   ....@...........
           79  +    |    496: 04 03 03 02 01 04 03 02 02 01 02 03 01 02 01 02   ................
           80  +    | page 8 offset 3584
           81  +    |      0: 0d 00 21 00 01 00 16 00 00 16 00 16 00 16 00 16   ..!.............
           82  +    |     16: 00 16 00 16 00 00 09 06 05 01 01 01 01 04 04 03   ................
           83  +    |     32: 03 00 00 00 5f 01 09 09 02 02 00 00 00 56 17 17   ...._........V..
           84  +    |     48: 17 73 65 76 65 6e 65 69 67 68 74 65 69 67 68 74   .seveneighteight
           85  +    |     64: 73 65 76 65 6e 00 00 00 3b 07 07 07 40 14 00 00   seven...;...@...
           86  +    |     80: 00 00 00 00 40 18 00 00 00 00 00 00 40 18 00 00   ....@.......@...
           87  +    |     96: 00 00 00 00 40 14 00 00 00 00 00 00 00 00 00 14   ....@...........
           88  +    |    112: 01 01 01 03 04 04 03 00 00 00 09 01 01 09 02 02   ................
           89  +    |    352: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1a   ................
           90  +    | page 9 offset 4096
           91  +    |      0: 0d 00 00 00 1b 00 47 00 01 d9 01 be 01 af 01 a0   ......G.........
           92  +    |     16: 01 91 01 82 01 73 01 64 01 55 01 46 01 37 01 28   .....s.d.U.F.7.(
           93  +    |     32: 01 19 01 0a 00 fb 00 ec 00 dd 00 ce 00 bf 00 b0   ................
           94  +    |     48: 00 a1 00 92 00 83 00 74 00 65 00 56 00 47 00 00   .......t.e.V.G..
           95  +    |     64: 00 00 00 00 00 00 00 0d 21 00 00 48 01 54 00 01   ........!..H.T..
           96  +    |     80: f7 01 ec 01 c5 01 0d 20 00 00 48 01 54 00 01 f7   ....... ..H.T...
           97  +    |     96: 01 ec 01 c5 01 0d 1f 00 00 48 01 54 00 01 f7 01   .........H.T....
           98  +    |    112: ec 01 c5 01 0d 1e 00 00 48 01 54 00 01 f7 01 ec   ........H.T.....
           99  +    |    128: 01 c5 01 0d 1d 00 00 48 01 54 00 01 f7 01 ec 01   .......H.T......
          100  +    |    144: c5 01 0d 1c 00 00 48 01 54 00 01 f7 01 ec 01 c5   ......H.T.......
          101  +    |    160: 01 0d 1b 00 00 48 01 54 00 01 f7 01 ec 01 c5 01   .....H.T........
          102  +    |    176: 0d 1a 00 00 48 01 54 00 01 f7 01 ec 01 c5 01 0d   ....H.T.........
          103  +    |    192: 19 00 00 48 01 54 00 01 f7 01 ec 01 c5 01 0d 18   ...H.T..........
          104  +    |    208: 00 00 48 01 54 00 01 f7 01 ec 01 c5 01 0d 17 00   ..H.T...........
          105  +    |    224: 00 48 01 54 00 01 f7 01 ec 01 c5 01 0d 16 00 00   .H.T............
          106  +    |    240: 48 01 54 00 01 f7 01 ec 01 c5 01 0d 15 00 00 48   H.T............H
          107  +    |    256: 01 54 00 01 f7 01 ec 01 c5 01 0d 14 00 00 48 01   .T............H.
          108  +    |    272: 54 00 01 f7 01 ec 01 c5 01 0d 13 00 00 48 01 54   T............H.T
          109  +    |    288: 00 01 f7 01 ec 01 c5 01 0d 12 00 00 48 01 54 00   ............H.T.
          110  +    |    304: 01 f7 01 ec 01 c5 01 0d 11 00 00 48 01 54 00 01   ...........H.T..
          111  +    |    320: f7 01 ec 01 c5 01 0d 10 00 00 48 01 54 00 01 f7   ..........H.T...
          112  +    |    336: 01 ec 01 c5 01 0d 0f 00 00 48 01 54 00 01 f7 01   .........H.T....
          113  +    |    352: ec 01 c5 01 0d 0e 00 00 48 01 54 00 01 f7 01 ec   ........H.T.....
          114  +    |    368: 01 c5 01 0d 0d 00 00 48 01 54 00 01 f7 01 ec 01   .......H.T......
          115  +    |    384: c5 01 0d 0c 00 00 48 01 54 00 01 f7 01 ec 01 c5   ......H.T.......
          116  +    |    400: 01 0d 0b 00 00 48 01 54 00 01 f7 01 ec 01 c5 01   .....H.T........
          117  +    |    416: 0d 0a 00 00 48 01 54 00 01 f7 01 ec 01 c5 01 0d   ....H.T.........
          118  +    |    432: 09 00 00 48 01 54 00 01 f7 01 ec 01 c5 01 19 08   ...H.T..........
          119  +    |    448: 05 17 17 17 17 65 69 67 68 74 65 69 67 68 74 73   .....eighteights
          120  +    |    464: 65 76 65 6e 73 65 76 65 6e 25 07 05 07 07 07 07   evenseven%......
          121  +    |    480: 40 18 00 00 00 00 00 00 40 18 00 00 00 00 00 00   @.......@.......
          122  +    |    496: 40 14 00 00 00 00 00 00 40 14 00 00 00 00 00 00   @.......@.......
          123  +    | page 10 offset 4608
          124  +    |      0: 0d 00 00 00 1d 00 4d 00 01 f1 01 e2 01 d3 01 c4   ......M.........
          125  +    |     16: 01 b5 01 a6 01 97 01 88 01 79 01 6a 01 5b 01 4c   .........y.j.[.L
          126  +    |     32: 01 3d 01 2e 01 1f 01 10 01 01 00 f2 00 e3 00 d4   .=..............
          127  +    |     48: 00 c5 00 b6 00 a7 00 98 00 89 00 7a 00 6b 00 5c   ...........z.k.\
          128  +    |     64: 00 4d 00 00 00 00 00 00 00 00 00 00 00 0d 3e 00   .M............>.
          129  +    |     80: 00 48 01 54 00 01 f7 01 ec 01 c5 01 0d 3d 00 00   .H.T.........=..
          130  +    |     96: 48 01 54 00 01 f7 01 ec 01 c5 01 0d 3c 00 00 48   H.T.........<..H
          131  +    |    112: 01 54 00 01 f7 01 ec 01 c5 01 0d 3b 00 00 48 01   .T.........;..H.
          132  +    |    128: 54 00 01 f7 01 ec 01 c5 01 0d 3a 00 00 48 01 54   T.........:..H.T
          133  +    |    144: 00 01 f7 01 ec 01 c5 01 0d 39 00 00 48 01 54 00   .........9..H.T.
          134  +    |    160: 01 f7 01 ec 01 c5 01 0d 38 00 00 48 01 54 00 01   ........8..H.T..
          135  +    |    176: f7 01 ec 01 c5 01 0d 37 00 00 48 01 54 00 01 f7   .......7..H.T...
          136  +    |    192: 01 ec 01 c5 01 0d 36 00 00 48 01 54 00 01 f7 01   ......6..H.T....
          137  +    |    208: ec 01 c5 01 0d 35 00 00 48 01 54 00 01 f7 01 ec   .....5..H.T.....
          138  +    |    224: 01 c5 01 0d 34 00 00 48 01 54 00 01 f7 01 ec 01   ....4..H.T......
          139  +    |    240: c5 01 0d 33 00 00 48 01 54 00 01 f7 01 ec 01 c5   ...3..H.T.......
          140  +    |    256: 01 0d 32 00 00 48 01 54 00 01 f7 01 ec 01 c5 01   ..2..H.T........
          141  +    |    272: 0d 31 00 00 48 01 54 00 01 f7 01 ec 01 c5 01 0d   .1..H.T.........
          142  +    |    288: 30 00 00 48 01 54 00 01 f7 01 ec 01 c5 01 0d 2f   0..H.T........./
          143  +    |    304: 00 00 48 01 54 00 01 f7 01 ec 01 c5 01 0d 2e 00   ..H.T...........
          144  +    |    320: 00 48 01 54 00 01 f7 01 ec 01 c5 01 0d 2d 00 00   .H.T.........-..
          145  +    |    336: 48 01 54 00 01 f7 01 ec 01 c5 01 0d 2c 00 00 48   H.T.........,..H
          146  +    |    352: 01 54 00 01 f7 01 ec 01 c5 01 0d 2b 00 00 48 01   .T.........+..H.
          147  +    |    368: 54 00 01 f7 01 ec 01 c5 01 0d 2a 00 00 48 01 54   T.........*..H.T
          148  +    |    384: 00 01 f7 01 ec 01 c5 01 0d 29 00 00 48 01 54 00   .........)..H.T.
          149  +    |    400: 01 f7 01 ec 01 c5 01 0d 28 00 00 48 01 54 00 01   ........(..H.T..
          150  +    |    416: f7 01 ec 01 c5 01 0d 27 00 00 48 01 54 00 01 f7   .......'..H.T...
          151  +    |    432: 01 ec 01 c5 01 0d 26 00 00 48 01 54 00 01 f7 01   ......&..H.T....
          152  +    |    448: ec 01 c5 01 0d 25 00 00 48 01 54 00 01 f7 01 ec   .....%..H.T.....
          153  +    |    464: 01 c5 01 0d 24 00 00 48 01 54 00 01 f7 01 ec 01   ....$..H.T......
          154  +    |    480: c5 01 0d 23 00 00 48 01 54 00 01 f7 01 ec 01 c5   ...#..H.T.......
          155  +    |    496: 01 0d 22 00 00 48 01 54 00 01 f7 01 ec 01 c5 01   .."..H.T........
          156  +    | page 11 offset 5120
          157  +    |      0: 0d 00 00 00 0a 01 6a 00 01 f1 01 e2 01 d3 01 c4   ......j.........
          158  +    |     16: 01 b5 01 a6 01 97 01 88 01 79 01 6a 00 00 00 00   .........y.j....
          159  +    |    352: 00 00 00 00 00 00 00 00 00 00 0d 48 00 00 48 01   ...........H..H.
          160  +    |    368: 54 00 01 f7 01 ec 01 c5 01 0d 47 00 00 48 01 54   T.........G..H.T
          161  +    |    384: 00 01 f7 01 ec 01 c5 01 0d 46 00 00 48 01 54 00   .........F..H.T.
          162  +    |    400: 01 f7 01 ec 01 c5 01 0d 45 00 00 48 01 54 00 01   ........E..H.T..
          163  +    |    416: f7 01 ec 01 c5 01 0d 44 00 00 48 01 54 00 01 f7   .......D..H.T...
          164  +    |    432: 01 ec 01 c5 01 0d 43 00 00 48 01 54 00 01 f7 01   ......C..H.T....
          165  +    |    448: ec 01 c5 01 0d 42 00 00 48 01 54 00 01 f7 01 ec   .....B..H.T.....
          166  +    |    464: 01 c5 01 0d 41 00 00 48 01 54 00 01 f7 01 ec 01   ....A..H.T......
          167  +    |    480: c5 01 0d 40 00 00 48 01 54 00 01 f7 01 ec 01 c5   ...@..H.T.......
          168  +    |    496: 01 0d 3f 00 00 48 01 54 00 01 f7 01 ec 01 c5 01   ..?..H.T........
          169  +    | end c4.db
          170  +  }]
          171  +  db eval {PRAGMA integrity_check}
          172  +} {/Fragmentation of 384 bytes reported as 0 on page 8/}
          173  +
          174  +# The DELETE query below deletes the very last cell from page 8.
          175  +# Prior to a certain fix to sqlite3BtreeDelete() and because of the
          176  +# corruption to the freeblock list on page 8, this would fail to
          177  +# cause a rebalance operation, which would leave the btree in a weird
          178  +# state that would lead to segfaults and or assertion faults.
          179  +#
          180  +do_execsql_test dbfuzz001-110 {
          181  +  DELETE FROM t3 WHERE x IS NOT NULL AND +rowid=6;
          182  +} {}
          183  +
          184  +# This is a dbfuzz2-generate test case that can cause a page with
          185  +# pPage->nCell==0 to enter the balancer.
          186  +#
          187  +do_test dbfuzz001-200 {
          188  +  db deserialize [decode_hexdb {
          189  +    | size 3076 pagesize 512 filename c03.db
          190  +    | page 1 offset 0
          191  +    |      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
          192  +    |     16: 02 00 01 01 00 40 20 20 00 00 00 0c 00 00 00 07   .....@  ........
          193  +    |     32: 00 00 00 00 00 00 00 00 00 00 00 08 00 00 00 04   ................
          194  +    |     48: 00 00 00 00 00 00 00 03 e8 00 00 01 00 00 00 00   ................
          195  +    |     80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0c   ................
          196  +    |     96: 00 2e 2c 50 0d 00 00 00 06 01 06 00 01 da 01 b0   ..,P............
          197  +    |    112: 01 56 01 86 01 2a 01 06 00 00 00 00 00 00 00 00   .V...*..........
          198  +    |    128: 00 00 00 00 00 00 00 00 ef 00 00 00 00 00 00 00   ................
          199  +    |    192: 00 7f 00 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
          200  +    |    224: 00 00 00 00 00 00 00 00 00 00 00 00 00 ff e9 00   ................
          201  +    |    256: 00 00 00 00 00 00 22 07 06 17 11 11 01 31 74 61   ......"......1ta
          202  +    |    272: 62 6c 65 74 34 74 34 07 43 52 45 41 54 45 20 54   blet4t4.CREATE T
          203  +    |    288: 41 42 4c 45 20 74 34 28 78 29 2a 06 06 17 13 11   ABLE t4(x)*.....
          204  +    |    304: 01 3f 69 6e 64 65 78 74 33 78 74 33 06 43 52 45   .?indext3xt3.CRE
          205  +    |    320: 41 54 45 20 49 4e 44 45 58 20 74 33 64 20 4f 4e   ATE INDEX t3d ON
          206  +    |    336: 20 74 33 28 78 29 2e 04 06 17 15 11 01 45 69 6e    t3(x).......Ein
          207  +    |    352: 64 65 78 74 32 63 64 74 32 05 43 52 45 41 54 45   dext2cdt2.CREATE
          208  +    |    368: 20 49 4e 44 45 58 20 74 32 63 64 20 4f 4e 20 74    INDEX t2cd ON t
          209  +    |    384: 32 28 63 2c 64 29 28 05 06 17 11 11 01 3d 74 61   2(c,d)(......=ta
          210  +    |    400: 62 6c 65 74 33 74 33 04 43 52 45 41 54 45 20 54   blet3t3.CREATE T
          211  +    |    416: 41 42 4c 45 20 74 33 28 63 2c 78 2c 65 2c 66 29   ABLE t3(c,x,e,f)
          212  +    |    432: 28 02 06 17 11 11 01 3d 74 61 62 6c 65 74 32 74   (......=tablet2t
          213  +    |    448: 32 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 74   2.CREATE TABLE t
          214  +    |    464: 32 28 63 2c 64 2c 65 2c 66 29 24 01 06 17 11 11   2(c,d,e,f)$.....
          215  +    |    480: 01 35 74 61 62 6c 65 74 31 74 31 02 43 52 45 41   .5tablet1t1.CREA
          216  +    |    496: 54 45 20 54 41 42 4c 45 20 74 31 28 61 2c 62 29   TE TABLE t1(a,b)
          217  +    | page 2 offset 512
          218  +    |      0: 0d 00 00 00 04 01 cf 00 01 fa 01 f3 01 de 01 cf   ................
          219  +    |    176: 10 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
          220  +    |    256: 00 00 14 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
          221  +    |    368: 00 00 00 00 00 00 00 00 00 00 00 00 1e 00 00 00   ................
          222  +    |    416: 00 00 00 1b 00 00 00 00 04 00 00 00 00 00 00 00   ................
          223  +    |    448: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0d   ................
          224  +    |    464: 04 03 17 17 73 65 76 65 6e 65 69 67 68 74 13 03   ....seveneight..
          225  +    |    480: 03 07 07 40 14 00 00 00 00 00 00 40 18 00 00 00   ...@.......@....
          226  +    |    496: 00 00 00 05 02 03 01 01 03 04 04 01 03 09 01 02   ................
          227  +    | page 3 offset 1024
          228  +    |      0: 0d 00 00 00 08 01 54 00 01 f7 01 ec 01 c5 01 aa   ......T.........
          229  +    |     16: 01 a1 01 96 01 6f 01 54 00 00 00 00 00 00 00 00   .....o.T........
          230  +    |     32: 00 00 00 00 00 00 00 03 e8 00 00 00 00 00 00 00   ................
          231  +    |    336: 00 00 00 00 19 08 05 16 17 17 17 65 69 67 68 74   ...........eight
          232  +    |    352: 65 69 67 68 74 73 65 76 65 6e 73 65 76 ff ff ff   eightsevensev...
          233  +    |    368: 0e 05 07 07 07 07 40 18 00 00 00 00 00 00 40 18   ......@.......@.
          234  +    |    384: 00 00 00 00 00 00 40 14 00 00 00 00 00 00 40 14   ......@.......@.
          235  +    |    400: 00 00 00 00 00 00 09 06 05 01 01 01 01 04 04 03   ................
          236  +    |    416: 03 07 05 05 01 01 09 09 02 02 19 04 05 17 17 17   ................
          237  +    |    432: 17 73 65 6f 65 6e 65 69 67 68 74 65 69 67 68 74   .seoeneighteight
          238  +    |    448: 73 65 76 65 6e 25 03 05 07 07 07 07 40 14 00 00   seven%......@...
          239  +    |    464: 00 00 00 00 40 18 00 00 00 00 00 00 40 18 00 00   ....@.......@...
          240  +    |    480: 00 00 00 00 40 14 00 00 00 00 00 00 09 02 05 01   ....@...........
          241  +    |    496: 01 01 01 03 04 04 03 07 01 05 09 01 01 09 02 02   ................
          242  +    | page 4 offset 1536
          243  +    |      0: 0d 00 00 00 00 00 10 00 00 00 00 00 00 00 00 00   ................
          244  +    |    160: 00 00 00 ea 00 00 00 00 00 00 00 00 00 00 00 00   ................
          245  +    |    336: 00 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00   ............ ...
          246  +    | page 5 offset 2048
          247  +    |      0: 0a 00 00 00 08 01 96 00 01 fa 01 c4 01 f2 01 bc   ................
          248  +    |     16: 01 dc 01 a6 01 96 01 cc 00 00 00 00 00 00 00 00   ................
          249  +    |     48: 00 00 00 00 00 00 00 00 00 00 10 00 00 00 00 00   ................
          250  +    |    288: 00 00 00 00 00 00 00 00 00 64 00 00 00 2b 00 00   .........d...+..
          251  +    |    400: 00 00 00 00 00 00 0f 04 17 17 01 65 69 67 68 74   ...........eight
          252  +    |    416: 65 69 6f 68 74 08 15 04 07 07 01 40 18 00 00 00   eioht......@....
          253  +    |    432: 00 00 00 40 18 00 00 00 00 00 00 07 07 04 01 01   ...@............
          254  +    |    448: 01 04 04 06 07 04 01 01 01 02 02 05 0f 04 17 17   ................
          255  +    |    464: 01 73 65 76 65 6e 65 69 67 68 74 04 15 04 07 07   .seveneight.....
          256  +    |    480: 01 40 14 00 00 00 00 00 00 40 18 00 00 00 00 00   .@.......@......
          257  +    |    496: 00 03 07 04 01 01 01 03 04 02 05 04 09 01 09 02   ................
          258  +    | page 6 offset 2560
          259  +    |      0: 0a 00 00 00 00 02 00 00 00 00 00 00 00 0d 00 00   ................
          260  +    |     16: 00 08 01 c2 00 01 fb 01 f6 01 f1 01 ec 01 e0 01   ................
          261  +    |     32: d4 01 cb 01 c2 00 00 00 00 00 00 00 00 00 00 00   ................
          262  +    |    160: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00   ................
          263  +    |    448: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07   ................
          264  +    |    464: 08 02 17 65 69 67 68 74 07 07 02 17 65 69 67 68   ...eight....eigh
          265  +    |    480: 74 0a 06 02 07 40 18 00 00 00 00 00 00 0a 05 02   t....@..........
          266  +    |    496: 07 40 18 00 04 02 01 04 03 03 02 01 04 03 02 02   .@..............
          267  +    | end x/c03.db
          268  +  }]
          269  +  catchsql {INSERT INTO t3 SELECT * FROM t2;}
          270  +} {1 {database disk image is malformed}}
          271  +
          272  +
          273  +do_test dbfuzz001-110 {
          274  +  sqlite3 db {}
          275  +  db deserialize [decode_hexdb {
          276  +| size 3584 pagesize 512 filename x/c02.db
          277  +| page 1 offset 0
          278  +|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
          279  +|     16: 02 00 01 01 00 40 20 20 00 00 00 0c 00 00 00 07   .....@  ........
          280  +|     32: 00 00 00 00 00 00 00 00 00 00 00 08 00 00 00 04   ................
          281  +|     48: 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 00   ................
          282  +|     80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0c   ................
          283  +|     96: 00 2e 2c 50 0d 00 00 00 06 01 06 00 01 da 01 b0   ..,P............
          284  +|    112: 01 56 01 86 01 2a 01 06 00 00 00 00 00 00 00 00   .V...*..........
          285  +|    256: 00 00 00 00 00 00 22 07 06 17 11 11 01 31 74 61   ......"......1ta
          286  +|    272: 62 6c 65 74 34 74 34 07 43 52 45 41 54 45 20 54   blet4t4.CREATE T
          287  +|    288: 41 42 4c 45 20 74 34 28 78 29 2a 06 06 17 13 11   ABLE t4(x)*.....
          288  +|    304: 01 3f 69 6e 64 65 78 74 33 78 74 33 05 43 52 45   .?indext3xt3.CRE
          289  +|    320: 41 54 45 20 49 4e 44 45 58 20 74 33 78 20 4f 4e   ATE INDEX t3x ON
          290  +|    336: 20 74 33 28 78 29 2e 04 06 17 15 11 01 45 69 6e    t3(x).......Ein
          291  +|    352: 64 65 78 74 32 63 64 74 32 05 43 52 45 41 54 45   dext2cdt2.CREATE
          292  +|    368: 20 49 4e 44 45 58 20 74 32 63 64 20 4f 4e 20 74    INDEX t2cd ON t
          293  +|    384: 32 28 63 2c 64 29 28 05 06 17 11 11 01 3d 74 61   2(c,d)(......=ta
          294  +|    400: 62 6c 65 74 33 74 33 07 43 52 45 41 54 45 20 54   blet3t3.CREATE T
          295  +|    416: 41 42 4c 45 20 74 33 28 63 2c 78 2c 65 2c 66 29   ABLE t3(c,x,e,f)
          296  +|    432: 28 02 06 17 11 11 01 3d 74 61 74 65 6c 03 62 74   (......=tatel.bt
          297  +|    448: 32 32 43 52 45 41 54 45 20 54 41 42 4c 45 20 74   22CREATE TABLE t
          298  +|    464: 32 28 63 2c 64 2c 65 2c 66 29 24 01 06 17 11 11   2(c,d,e,f)$.....
          299  +|    480: 01 35 74 61 62 6c 65 74 31 74 31 02 43 52 45 41   .5tablet1t1.CREA
          300  +|    496: 54 45 20 54 41 42 4c 45 20 74 31 28 61 2c 62 29   TE TABLE t1(a,b)
          301  +| page 2 offset 512
          302  +|      0: 0d 00 00 00 04 01 cf 00 01 fa 01 f3 01 de 01 cf   ................
          303  +|    160: 00 00 20 00 00 00 00 00 00 00 00 00 00 00 00 00   .. .............
          304  +|    448: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0d   ................
          305  +|    464: 04 03 17 17 73 65 76 65 6e 65 69 67 68 74 13 03   ....seveneight..
          306  +|    480: 03 07 07 40 14 00 00 00 00 00 00 40 18 00 00 00   ...@.......@....
          307  +|    496: 00 00 00 05 02 03 01 01 03 04 04 01 03 09 01 02   ................
          308  +| page 3 offset 1024
          309  +|      0: 0d 00 00 00 08 01 54 00 01 f7 01 ec 01 c5 01 aa   ......T.........
          310  +|     16: 01 a1 01 96 01 6f 01 54 00 00 00 00 00 00 00 00   .....o.T........
          311  +|    112: 00 00 dd 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
          312  +|    336: 00 00 00 00 19 08 05 17 17 17 17 65 69 67 68 74   ...........eight
          313  +|    352: 65 69 67 68 74 73 65 76 65 6e 73 65 76 65 6e 25   eightsevenseven%
          314  +|    368: 07 05 07 07 07 07 40 18 00 00 00 00 00 00 40 18   ......@.......@.
          315  +|    384: 00 00 00 00 00 00 40 14 00 00 00 00 00 00 40 14   ......@.......@.
          316  +|    400: 00 00 00 00 00 00 09 06 05 01 01 01 01 04 04 03   ................
          317  +|    416: 03 07 05 05 01 01 09 09 02 02 19 04 05 17 17 17   ................
          318  +|    432: 17 73 65 76 65 6e 65 69 67 68 74 65 69 67 68 74   .seveneighteight
          319  +|    448: 73 65 76 65 6e 25 03 05 07 07 07 07 40 14 00 00   seven%......@...
          320  +|    464: 00 00 00 00 40 18 00 00 00 00 00 00 40 18 00 00   ....@.......@...
          321  +|    480: 00 00 00 00 40 14 00 00 00 00 00 00 09 02 05 01   ....@...........
          322  +|    496: 01 01 01 03 04 04 03 07 01 05 09 01 01 09 02 02   ................
          323  +| page 4 offset 1536
          324  +|      0: 0d 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00   ................
          325  +|    192: 00 00 00 00 00 00 7f 00 00 00 00 00 00 00 00 00   ................
          326  +|    208: 00 e5 00 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
          327  +| page 5 offset 2048
          328  +|      0: 0a 00 00 00 08 01 96 00 01 fa 01 c4 01 f2 01 bc   ................
          329  +|     16: 01 dc 01 a6 01 96 01 cc 00 00 00 00 00 00 00 00   ................
          330  +|    240: 00 00 00 00 00 00 00 00 00 00 00 00 00 0e 00 00   ................
          331  +|    400: 00 00 00 00 00 00 0f 04 17 07 01 65 69 67 68 74   ...........eight
          332  +|    416: 65 69 67 68 74 08 15 04 07 07 01 40 18 00 00 00   eight......@....
          333  +|    432: 00 00 00 40 18 00 00 00 00 00 00 07 07 04 01 01   ...@............
          334  +|    448: 01 04 04 06 07 04 01 01 01 02 02 05 0f 04 17 17   ................
          335  +|    464: 01 73 65 76 65 6e 65 69 67 68 74 04 15 04 07 07   .seveneight.....
          336  +|    480: 01 40 14 00 00 00 00 00 00 40 18 00 00 00 00 00   .@.......@......
          337  +|    496: 00 03 07 04 01 01 01 03 04 02 05 04 09 01 09 02   ................
          338  +| page 6 offset 2560
          339  +|      0: 0a 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00   ................
          340  +|    464: 00 00 00 00 00 00 00 00 00 00 7f 00 00 00 00 00   ................
          341  +| page 7 offset 3072
          342  +|      0: 0d 00 00 00 08 01 c2 00 01 fb 01 f6 01 f1 01 ec   ................
          343  +|     16: 01 e0 01 d4 01 cb 01 c2 00 00 00 00 00 00 00 00   ................
          344  +|    448: 00 00 07 08 02 17 65 69 67 68 74 07 07 02 17 65   ......eight....e
          345  +|    464: 69 67 68 74 0a 06 02 07 40 18 00 00 00 00 00 00   ight....@.......
          346  +|    480: 0a 05 02 07 40 18 00 00 00 00 00 00 03 04 02 01   ....@...........
          347  +|    496: 04 03 03 02 01 04 03 02 02 01 02 03 01 02 01 02   ................
          348  +| end x/c02.db
          349  +  }]
          350  +  execsql {
          351  +    DELETE FROM t3 WHERE x IN (SELECT x FROM t4);
          352  +  }
          353  +} {}
          354  +
          355  +finish_test

Changes to test/dbfuzz2.c.

    74     74     int rc;
    75     75     int i;
    76     76   
    77     77     if( eVerbosity>=1 ){
    78     78       printf("************** nByte=%d ***************\n", (int)nByte);
    79     79       fflush(stdout);
    80     80     }
           81  +  if( sqlite3_initialize() ) return 0;
    81     82     rc = sqlite3_open(0, &db);
    82     83     if( rc ) return 1;
    83     84     a = sqlite3_malloc64(nByte+1);
    84     85     if( a==0 ) return 1;
    85     86     memcpy(a, aData, nByte);
    86     87     sqlite3_deserialize(db, "main", a, nByte, nByte,
    87     88           SQLITE_DESERIALIZE_RESIZEABLE |

Added test/fuzz4.test.

            1  +# 2018-12-12
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# 
           12  +# Test cases found by Matthew Denton's fuzzer at Chrome.
           13  +#
           14  +
           15  +
           16  +set testdir [file dirname $argv0]
           17  +source $testdir/tester.tcl
           18  +
           19  +do_execsql_test fuzz4-100 {
           20  +  CREATE TABLE Table0 (Col0  NOT NULL DEFAULT (CURRENT_TIME IS 1 > 1));
           21  +  INSERT OR REPLACE INTO Table0 DEFAULT VALUES ;
           22  +  SELECT * FROM Table0;
           23  +} {0}
           24  +
           25  +do_execsql_test fuzz4-110 {
           26  +  CREATE TABLE Table1(
           27  +    Col0 TEXT DEFAULT (CASE WHEN 1 IS 3530822107858468864 
           28  +                            THEN 1 ELSE quote(1) IS 3530822107858468864 END)
           29  +  );
           30  +  INSERT INTO Table1 DEFAULT VALUES;
           31  +  SELECT * FROM Table1;
           32  +} {0}
           33  +
           34  +do_execsql_test fuzz4-200 {
           35  +  CREATE TABLE Table2a(
           36  +     Col0  NOT NULL   DEFAULT (CURRENT_TIME IS 1  IS NOT 1  > 1)
           37  +  );
           38  +  INSERT OR REPLACE INTO Table2a DEFAULT VALUES;
           39  +  SELECT * FROM Table2a;
           40  +} {0}
           41  +
           42  +do_execsql_test fuzz4-210 {
           43  +  CREATE TABLE Table2b (Col0  NOT NULL  DEFAULT (CURRENT_TIME  IS NOT FALSE)) ;
           44  +  INSERT OR REPLACE INTO Table2b DEFAULT VALUES ;
           45  +  SELECT * FROM Table2b;
           46  +} {1}
           47  +
           48  +do_execsql_test fuzz4-300 {
           49  +  CREATE TABLE Table3 (Col0 DEFAULT (CURRENT_TIMESTAMP BETWEEN 1 AND 1));
           50  +  INSERT INTO Table3 DEFAULT VALUES;
           51  +  SELECT * FROM Table3;
           52  +} {0}
           53  +
           54  +do_execsql_test fuzz4-400 {
           55  +  CREATE TABLE Table4 (Col0 DEFAULT (1 BETWEEN CURRENT_TIMESTAMP AND 1));
           56  +  INSERT INTO Table4 DEFAULT VALUES;
           57  +  SELECT * FROM Table4;
           58  +} {0}
           59  +
           60  +do_execsql_test fuzz4-500 {
           61  +  CREATE TABLE Table5 (Col0 DEFAULT (1 BETWEEN 1 AND CURRENT_TIMESTAMP));
           62  +  INSERT INTO Table5 DEFAULT VALUES;
           63  +  SELECT * FROM Table5;
           64  +} {1}
           65  +
           66  +do_execsql_test fuzz4-600 {
           67  +  CREATE TEMPORARY TABLE Table6(
           68  +    Col0 DEFAULT (CASE x'5d' WHEN 1 THEN
           69  +        CASE CURRENT_TIMESTAMP WHEN 1 THEN 1 ELSE 1 END
           70  +        ELSE CASE WHEN 1 THEN FALSE END  END )
           71  +  );
           72  +  INSERT INTO temp.Table6 DEFAULT VALUES ;
           73  +  SELECT * FROM Table6;
           74  +} {0}
           75  +do_execsql_test fuzz4-610 {
           76  +  WITH TableX AS (SELECT DISTINCT * ORDER BY 1  , 1 COLLATE RTRIM)
           77  +      DELETE FROM Table6  WHERE Col0 || +8388608  ;
           78  +  SELECT * FROM Table6;
           79  +} {}
           80  +
           81  +
           82  +finish_test

Changes to test/fuzzcheck.c.

   443    443     if( iOfst<0 || iOfst>=pVFile->sz ){
   444    444       memset(pData, 0, iAmt);
   445    445       return SQLITE_IOERR_SHORT_READ;
   446    446     }
   447    447     if( iOfst+iAmt>pVFile->sz ){
   448    448       memset(pData, 0, iAmt);
   449    449       iAmt = (int)(pVFile->sz - iOfst);
   450         -    memcpy(pData, pVFile->a, iAmt);
          450  +    memcpy(pData, pVFile->a + iOfst, iAmt);
   451    451       return SQLITE_IOERR_SHORT_READ;
   452    452     }
   453    453     memcpy(pData, pVFile->a + iOfst, iAmt);
   454    454     return SQLITE_OK;
   455    455   }
   456    456   static int inmemWrite(
   457    457     sqlite3_file *pFile,   /* Write to this file */

Changes to test/fuzzdata7.db.

cannot compute difference between binary files

Changes to test/index6.test.

   385    385     CREATE TABLE t11(a,b,c);
   386    386     CREATE INDEX t11x ON t11(a) WHERE b<>99;
   387    387     EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99;
   388    388   } {/USING INDEX t11x/}
   389    389   do_execsql_test index6-11.2 {
   390    390     EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99 AND c<>98;
   391    391   } {/USING INDEX t11x/}
   392         -  
   393    392   
          393  +# 2018-12-08
          394  +# Ticket https://www.sqlite.org/src/info/1d958d90596593a7
          395  +# NOT IN operator fails when using a partial index.
          396  +#
          397  +do_execsql_test index6-12.1 {
          398  +  DROP TABLE IF EXISTS t1;
          399  +  DROP TABLE IF EXISTS t2;
          400  +  CREATE TABLE t1(a,b);
          401  +  INSERT INTO t1 VALUES(1,1);
          402  +  INSERT INTO t1 VALUES(2,2);
          403  +  CREATE TABLE t2(x);
          404  +  INSERT INTO t2 VALUES(1);
          405  +  INSERT INTO t2 VALUES(2);
          406  +  SELECT 'one', * FROM t2 WHERE x NOT IN (SELECT a FROM t1);
          407  +  CREATE INDEX t1a ON t1(a) WHERE b=1;
          408  +  SELECT 'two', * FROM t2 WHERE x NOT IN (SELECT a FROM t1);
          409  +} {}
          410  +do_execsql_test index6-12.2 {
          411  +  SELECT x FROM t2 WHERE x IN (SELECT a FROM t1) ORDER BY +x;
          412  +} {1 2}
   394    413   finish_test

Changes to test/indexexpr2.test.

   226    226       SELECT sqlite_master.name 
   227    227         FROM sqlite_master, explain('UPDATE t2 SET c=c+1, f=NULL')
   228    228        WHERE explain.opcode LIKE 'Open%'
   229    229          AND sqlite_master.rootpage=explain.p2
   230    230        ORDER BY 1;
   231    231     } {t2 t2abc t2cd t2def}
   232    232   }
          233  +
          234  +#-------------------------------------------------------------------------
          235  +# Test that ticket [d96eba87] has been fixed.
          236  +#
          237  +do_execsql_test 5.0 {
          238  +  CREATE TABLE t5(a INTEGER, b INTEGER);
          239  +  INSERT INTO t5 VALUES(2, 4), (3, 9);
          240  +}
          241  +do_execsql_test 5.1 {
          242  +  SELECT * FROM t5 WHERE abs(a)=2 or abs(b)=9;
          243  +} {2 4 3 9}
          244  +do_execsql_test 5.2 {
          245  +  CREATE INDEX t5a ON t5( abs(a) );
          246  +  CREATE INDEX t5b ON t5( abs(b) );
          247  +}
          248  +do_execsql_test 5.4 {
          249  +  SELECT * FROM t5 WHERE abs(a)=2 or abs(b)=9;
          250  +} {2 4 3 9}
          251  +
   233    252   
   234    253   
   235    254   finish_test

Changes to test/normalize.test.

   203    203     {SELECT a FROM t1 WHERE x IN (1,2,3) AND hex8('abc');}
   204    204     0x2
   205    205     {0 {SELECT a FROM t1 WHERE x IN(?,?,?)AND hex8(?);}}
   206    206   
   207    207     430
   208    208     {SELECT "a" FROM t1 WHERE "x" IN ("1","2",'3');}
   209    209     0x2
   210         -  {0 {SELECT"a"FROM t1 WHERE"x"IN("1","2",?);}}
          210  +  {0 {SELECT a FROM t1 WHERE x IN(?,?,?);}}
   211    211   
   212    212     440
   213    213     {SELECT 'a' FROM t1 WHERE 'x';}
   214    214     0x2
   215    215     {0 {SELECT?FROM t1 WHERE?;}}
   216    216   
   217    217     450
   218    218     {SELECT [a] FROM t1 WHERE [x];}
   219    219     0x2
   220         -  {0 {SELECT"a"FROM t1 WHERE"x";}}
          220  +  {0 {SELECT a FROM t1 WHERE x;}}
   221    221   
   222    222     460
   223    223     {SELECT * FROM t1 WHERE x IN (x);}
   224    224     0x2
   225    225     {0 {SELECT*FROM t1 WHERE x IN(x);}}
   226    226   
   227    227     470
................................................................................
   228    228     {SELECT * FROM t1 WHERE x IN (x,a);}
   229    229     0x2
   230    230     {0 {SELECT*FROM t1 WHERE x IN(x,a);}}
   231    231   
   232    232     480
   233    233     {SELECT * FROM t1 WHERE x IN ([x],"a");}
   234    234     0x2
   235         -  {0 {SELECT*FROM t1 WHERE x IN("x","a");}}
          235  +  {0 {SELECT*FROM t1 WHERE x IN(x,a);}}
   236    236   
   237    237     500
   238    238     {SELECT * FROM t1 WHERE x IN ([x],"a",'b',sqlite_version());}
   239    239     0x2
   240         -  {0 {SELECT*FROM t1 WHERE x IN("x","a",?,sqlite_version());}}
          240  +  {0 {SELECT*FROM t1 WHERE x IN(x,a,?,sqlite_version());}}
   241    241   
   242    242     520
   243    243     {SELECT * FROM t1 WHERE x IN (SELECT x FROM t1);}
   244    244     0x2
   245    245     {0 {SELECT*FROM t1 WHERE x IN(SELECT x FROM t1);}}
   246    246   
   247    247     540
   248    248     {SELECT * FROM t1 WHERE x IN ((SELECT x FROM t1));}
   249    249     0x2
   250         -  {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}
          250  +  {0 {SELECT*FROM t1 WHERE x IN((SELECT x FROM t1));}}
   251    251   
   252    252     550
   253    253     {SELECT a, a+1, a||'b', a+"b" FROM t1;}
   254    254     0x2
   255         -  {0 {SELECT a,a+?,a||?,a+"b"FROM t1;}}
          255  +  {0 {SELECT a,a+?,a||?,a+b FROM t1;}}
   256    256   
   257    257     570
   258    258     {SELECT * FROM t1 WHERE x IN (1);}
   259    259     0x2
   260    260     {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}
   261    261   
   262    262     580
................................................................................
   312    312     {SELECT "col f", [col f] FROM t1;}
   313    313     0x2
   314    314     {0 {SELECT"col f","col f"FROM t1;}}
   315    315   
   316    316     680
   317    317     {SELECT a, "col f" FROM t1 LEFT OUTER JOIN t2 ON [t1].[col f] == [t2].[col y];}
   318    318     0x2
   319         -  {0 {SELECT a,"col f"FROM t1 LEFT OUTER JOIN t2 ON"t1"."col f"=="t2"."col y";}}
          319  +  {0 {SELECT a,"col f"FROM t1 LEFT OUTER JOIN t2 ON t1."col f"==t2."col y";}}
   320    320   
   321    321     690
   322    322     {SELECT * FROM ( WITH x AS ( SELECT * FROM t1 WHERE x IN ( 1)) SELECT 10);}
   323    323     0x2
   324    324     {0 {SELECT*FROM(WITH x AS(SELECT*FROM t1 WHERE x IN(?,?,?))SELECT?);}}
   325    325   
   326    326     700
................................................................................
   342    342     {SELECT x FROM t1 WHERE x = NULL;}
   343    343     0x2
   344    344     {0 {SELECT x FROM t1 WHERE x=?;}}
   345    345   
   346    346     760
   347    347     {SELECT x FROM t1 WHERE x IN ([x] IS NOT NULL, NULL, 1, 'a', "b", x'00');}
   348    348     0x2
   349         -  {0 {SELECT x FROM t1 WHERE x IN("x"IS NOT NULL,?,?,?,"b",?);}}
          349  +  {0 {SELECT x FROM t1 WHERE x IN(x IS NOT NULL,?,?,?,b,?);}}
   350    350   } {
   351    351     do_test $tnum {
   352    352       set code [catch {
   353    353         set STMT [sqlite3_prepare_v3 $DB $sql -1 $flags TAIL]
   354    354         sqlite3_normalized_sql $STMT
   355    355       } res]
   356    356       if {[info exists STMT]} {

Changes to test/shell1.test.

   252    252   } {0 {}}
   253    253   do_test shell1-3.1.3 {
   254    254     catchcmd "test.db" ".backup FOO BAR"
   255    255   } {1 {Error: unknown database FOO}}
   256    256   do_test shell1-3.1.4 {
   257    257     # too many arguments
   258    258     catchcmd "test.db" ".backup FOO BAR BAD"
   259         -} {1 {Usage: .backup ?DB? ?--append? FILENAME}}
          259  +} {1 {Usage: .backup ?DB? ?OPTIONS? FILENAME}}
   260    260   
   261    261   # .bail ON|OFF           Stop after hitting an error.  Default OFF
   262    262   do_test shell1-3.2.1 {
   263    263     catchcmd "test.db" ".bail"
   264    264   } {1 {Usage: .bail on|off}}
   265    265   do_test shell1-3.2.2 {
   266    266     catchcmd "test.db" ".bail ON"

Added test/vacuum-into.test.

            1  +# 2018-12-07
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this file is testing the VACUUM INTO statement.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +
           18  +# If the VACUUM statement is disabled in the current build, skip all
           19  +# the tests in this file.
           20  +#
           21  +ifcapable {!vacuum} {
           22  +  omit_test vacuum.test {Compiled with SQLITE_OMIT_VACUUM}
           23  +  finish_test
           24  +  return
           25  +}
           26  +
           27  +forcedelete out.db
           28  +do_execsql_test vacuum-into-100 {
           29  +  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
           30  +  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
           31  +  INSERT INTO t1(a,b) SELECT x, randomblob(600) FROM c;
           32  +  CREATE INDEX t1b ON t1(b);
           33  +  DELETE FROM t1 WHERE a%2;
           34  +  SELECT count(*), sum(a), sum(length(b)) FROM t1;
           35  +} {50 2550 30000}
           36  +do_execsql_test vacuum-into-110 {
           37  +  VACUUM main INTO 'out.db';
           38  +} {}
           39  +sqlite3 db2 out.db
           40  +do_test vacuum-into-120 {
           41  +  db2 eval {SELECT count(*), sum(a), sum(length(b)) FROM t1}
           42  +} {50 2550 30000}
           43  +do_catchsql_test vacuum-into-130 {
           44  +  VACUUM INTO 'out.db';
           45  +} {1 {output file already exists}}
           46  +forcedelete out2.db
           47  +do_catchsql_test vacuum-into-140 {
           48  +  VACUUM INTO 'out2.db';
           49  +} {0 {}}
           50  +do_catchsql_test vacuum-into-150 {
           51  +  VACUUM INTO 'out2.db';
           52  +} {1 {output file already exists}}
           53  +
           54  +do_catchsql_test vacuum-into-200 {
           55  +  VACUUM main INTO ':memory:';
           56  +} {0 {}}
           57  +
           58  +# The INTO argument can be an arbitrary expression.
           59  +#
           60  +do_execsql_test vacuum-into-300 {
           61  +  CREATE TABLE t2(name TEXT);
           62  +  INSERT INTO t2 VALUES(':memory:');
           63  +  VACUUM main INTO (SELECT name FROM t2);
           64  +} {}
           65  +do_catchsql_test vacuum-into-310 {
           66  +  VACUUM INTO null;
           67  +} {1 {non-text filename}}
           68  +
           69  +finish_test

Changes to test/window1.test.

   589    589   
   590    590   do_execsql_test 13.5 {
   591    591     SELECT a, rank() OVER(ORDER BY b) FROM t1
   592    592       INTERSECT 
   593    593     SELECT a, rank() OVER(ORDER BY b DESC) FROM t1;
   594    594   } {
   595    595   }
          596  +
          597  +# 2018-12-06
          598  +# https://www.sqlite.org/src/info/f09fcd17810f65f7
          599  +# Assertion fault when window functions are used.
          600  +#
          601  +# Root cause is the query flattener invoking sqlite3ExprDup() on
          602  +# expressions that contain subqueries with window functions.  The
          603  +# sqlite3ExprDup() routine is not making correctly initializing
          604  +# Select.pWin field of the subqueries.
          605  +#
          606  +sqlite3 db :memory:
          607  +do_execsql_test 14.0 {
          608  +  SELECT * FROM(
          609  +    SELECT * FROM (SELECT 1 AS c) WHERE c IN (
          610  +        SELECT (row_number() OVER()) FROM (VALUES (0))
          611  +    )
          612  +  );
          613  +} {1}
          614  +do_execsql_test 14.1 {
          615  +  CREATE TABLE t1(x); INSERT INTO t1(x) VALUES(12345);
          616  +  CREATE TABLE t2(c); INSERT INTO t2(c) VALUES(1);
          617  +  SELECT y, y+1, y+2 FROM (
          618  +    SELECT c IN (
          619  +      SELECT (row_number() OVER()) FROM t1
          620  +    ) AS y FROM t2
          621  +  );
          622  +} {1 2 3}
   596    623   
   597    624   finish_test

Added tool/dbtotxt.c.

            1  +/*
            2  +** Copyright 2008 D. Richard Hipp and Hipp, Wyrick & Company, Inc.
            3  +** All Rights Reserved
            4  +**
            5  +******************************************************************************
            6  +**
            7  +** This file implements a stand-alone utility program that converts
            8  +** a binary file (usually an SQLite database) into a text format that
            9  +** is compact and friendly to human-readers.
           10  +**
           11  +** Usage:
           12  +**
           13  +**         dbtotxt [--pagesize N] FILENAME
           14  +**
           15  +** The translation of the database appears on standard output.  If the
           16  +** --pagesize command-line option is omitted, then the page size is taken
           17  +** from the database header.
           18  +**
           19  +** Compactness is achieved by suppressing lines of all zero bytes.  This
           20  +** works well at compressing test databases that are mostly empty.  But
           21  +** the output will probably be lengthy for a real database containing lots
           22  +** of real content.  For maximum compactness, it is suggested that test
           23  +** databases be constructed with "zeroblob()" rather than "randomblob()"
           24  +** used for filler content and with "PRAGMA secure_delete=ON" selected to
           25  +** zero-out deleted content.
           26  +*/
           27  +#include <stdio.h>
           28  +#include <string.h>
           29  +#include <stdlib.h>
           30  + 
           31  +/* Return true if the line is all zeros */
           32  +static int allZero(unsigned char *aLine){
           33  +  int i;
           34  +  for(i=0; i<16 && aLine[i]==0; i++){}
           35  +  return i==16;
           36  +}
           37  +
           38  +int main(int argc, char **argv){
           39  +  int pgsz = 0;               /* page size */
           40  +  long szFile;                /* Size of the input file in bytes */
           41  +  FILE *in;                   /* Input file */
           42  +  int i, j;                   /* Loop counters */
           43  +  int nErr = 0;               /* Number of errors */
           44  +  const char *zInputFile = 0; /* Name of the input file */
           45  +  const char *zBaseName = 0;  /* Base name of the file */
           46  +  int lastPage = 0;           /* Last page number shown */
           47  +  int iPage;                  /* Current page number */
           48  +  unsigned char aLine[16];    /* A single line of the file */
           49  +  unsigned char aHdr[100];    /* File header */
           50  +  for(i=1; i<argc; i++){
           51  +    if( argv[i][0]=='-' ){
           52  +      const char *z = argv[i];
           53  +      z++;
           54  +      if( z[0]=='-' ) z++;
           55  +      if( strcmp(z,"pagesize")==0 ){
           56  +        i++;
           57  +        pgsz = atoi(argv[i]);
           58  +        if( pgsz<512 || pgsz>65536 || (pgsz&(pgsz-1))!=0 ){
           59  +          fprintf(stderr, "Page size must be a power of two between"
           60  +                          " 512 and 65536.\n");
           61  +          nErr++;
           62  +        }
           63  +        continue;
           64  +      }
           65  +      fprintf(stderr, "Unknown option: %s\n", argv[i]);
           66  +      nErr++;
           67  +    }else if( zInputFile ){
           68  +      fprintf(stderr, "Already using a different input file: [%s]\n", argv[i]);
           69  +      nErr++;
           70  +    }else{
           71  +      zInputFile = argv[i];
           72  +    }
           73  +  }
           74  +  if( zInputFile==0 ){
           75  +    fprintf(stderr, "No input file specified.\n");
           76  +    nErr++;
           77  +  }
           78  +  if( nErr ){
           79  +    fprintf(stderr, "Usage: %s [--pagesize N] FILENAME\n", argv[0]);
           80  +    exit(1);
           81  +  }
           82  +  in = fopen(zInputFile, "rb");
           83  +  if( in==0 ){
           84  +    fprintf(stderr, "Cannot open input file [%s]\n", zInputFile);
           85  +    exit(1);
           86  +  }
           87  +  fseek(in, 0, SEEK_END);
           88  +  szFile = ftell(in);
           89  +  rewind(in);
           90  +  if( szFile<512 ){
           91  +    fprintf(stderr, "File too short. Minimum size is 512 bytes.\n");
           92  +    exit(1);
           93  +  }
           94  +  if( fread(aHdr, 100, 1, in)!=1 ){
           95  +    fprintf(stderr, "Cannot read file header\n");
           96  +    exit(1);
           97  +  }
           98  +  rewind(in);
           99  +  if( pgsz==0 ){
          100  +    pgsz = (aHdr[16]<<8) | aHdr[17];
          101  +    if( pgsz==1 ) pgsz = 65536;
          102  +    if( pgsz<512 || (pgsz&(pgsz-1))!=0 ){
          103  +      fprintf(stderr, "Invalid page size in header: %d\n", pgsz);
          104  +      exit(1);
          105  +    }
          106  +  }
          107  +  zBaseName = zInputFile;
          108  +  for(i=0; zInputFile[i]; i++){
          109  +    if( zInputFile[i]=='/' && zInputFile[i+1]!=0 ) zBaseName = zInputFile+1;
          110  +  }
          111  +  printf("| size %d pagesize %d filename %s\n",(int)szFile,pgsz,zBaseName);
          112  +  for(i=0; i<szFile; i+=16){
          113  +    int got = (int)fread(aLine, 1, 16, in);
          114  +    if( got!=16 ){
          115  +      static int once = 1;
          116  +      if( once ){
          117  +        fprintf(stderr, "Could not read input file starting at byte %d\n",
          118  +                         i+got);
          119  +      }
          120  +      memset(aLine+got, 0, 16-got);
          121  +    }
          122  +    if( allZero(aLine) ) continue;
          123  +    iPage = i/pgsz + 1;
          124  +    if( lastPage!=iPage ){
          125  +      printf("| page %d offset %d\n", iPage, (iPage-1)*pgsz);
          126  +      lastPage = iPage;
          127  +    }
          128  +    printf("|  %5d:", i-(iPage-1)*pgsz);
          129  +    for(j=0; j<16; j++) printf(" %02x", aLine[j]);
          130  +    printf("   ");
          131  +    for(j=0; j<16; j++){
          132  +      char c = aLine[j];
          133  +      fputc(c>=0x20 && c<=0x7e ? c : '.', stdout);
          134  +    }
          135  +    fputc('\n', stdout);
          136  +  }
          137  +  fclose(in);
          138  +  printf("| end %s\n", zBaseName);
          139  +  return 0;
          140  +}

Added tool/dbtotxt.md.

            1  +<h1 align="center">The dbtotxt Tool</h1>
            2  +
            3  +The dbtotxt utility program reads an SQLite database file and writes its
            4  +raw binary content to screen as a hex dump for testing and debugging
            5  +purposes.
            6  +
            7  +The hex-dump output is formatted in such a way as to be easily readable
            8  +both by humans and by software.  The dbtotxt utility has long been a part
            9  +of the TH3 test suite.  The output of dbtotxt can be embedded in TH3 test
           10  +scripts and used to generate very specific database files, perhaps with
           11  +deliberately introduced corruption.  The cov1/corrupt*.test modules in
           12  +TH3 make extensive use of dbtotxt.
           13  +
           14  +More recently (2018-12-13) the dbtotxt utility has been added to the SQLite 
           15  +core and the command-line shell (CLI) has been augmented to be able to read 
           16  +dbtotxt output.  The CLI dot-command is:
           17  +
           18  +>     .open --hexdb  ?OPTIONAL-FILENAME?
           19  +
           20  +If the OPTIONAL-FILENAME is included, then content is read from that file.
           21  +If OPTIONAL-FILENAME is omitted, then the text is taken from the input stream,
           22  +terminated by the "| end" line of the dbtotxt text.  This allows small test
           23  +databases to be embedded directly in scripts.  Consider this example:
           24  +
           25  +>
           26  +    .open --hexdb
           27  +    | size 8192 pagesize 4096 filename x9.db
           28  +    | page 1 offset 0
           29  +    |      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
           30  +    |     16: 10 00 01 01 00 40 20 20 00 00 00 04 00 00 00 02   .....@  ........
           31  +    |     32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04   ................
           32  +    |     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
           33  +    |     80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04   ................
           34  +    |     96: 00 2e 30 38 0d 00 00 00 01 0f c0 00 0f c0 00 00   ..08............
           35  +    |   4032: 3e 01 06 17 11 11 01 69 74 61 62 6c 65 74 31 74   >......itablet1t
           36  +    |   4048: 31 02 43 52 45 41 54 45 20 54 41 42 4c 45 20 74   1.CREATE TABLE t
           37  +    |   4064: 31 28 78 2c 79 20 44 45 46 41 55 4c 54 20 78 27   1(x,y DEFAULT x'
           38  +    |   4080: 66 66 27 2c 7a 20 44 45 46 41 55 4c 54 20 30 29   ff',z DEFAULT 0)
           39  +    | page 2 offset 4096
           40  +    |      0: 0d 08 14 00 04 00 10 00 0e 05 0a 0f 04 15 00 10   ................
           41  +    |     16: 88 02 03 05 90 04 0e 08 00 00 00 00 00 00 00 00   ................
           42  +    |   1040: 00 00 00 00 ff 87 7c 02 05 8f 78 0e 08 00 00 00   ......|...x.....
           43  +    |   2064: 00 00 00 ff 0c 0a 01 fb 00 00 00 00 00 00 00 00   ................
           44  +    |   2560: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 83   ................
           45  +    |   2576: 78 01 05 87 70 0e 08 00 00 00 00 00 00 00 00 00   x...p...........
           46  +    |   3072: 00 00 00 00 00 00 00 00 00 ff 00 00 01 fb 00 00   ................
           47  +    |   3584: 00 00 00 00 00 83 78 00 05 87 70 0e 08 00 00 00   ......x...p.....
           48  +    |   4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff   ................
           49  +    | end x9.db
           50  +    SELECT rowid FROM t1;
           51  +    PRAGMA integrity_check;
           52  +
           53  +You can run this script to see that the database file is correctly decoded 
           54  +and loaded.  Furthermore, you can make subtle corruptions to the input
           55  +database simply by editing the hexadecimal description, then rerun the
           56  +script to verify that SQLite correctly handles the corruption.