/ Check-in [f0734017]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge recent fixes and enhancements from trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | apple-osx
Files: files | file ages | folders
SHA1: f073401792c0a093d4324fc2636170f108656976
User & Date: drh 2016-02-09 15:50:33
Context
2016-02-13
17:35
Merge changes from the 3.11.0 release candidate. check-in: d198a6db user: drh tags: apple-osx
2016-02-09
15:50
Merge recent fixes and enhancements from trunk. check-in: f0734017 user: drh tags: apple-osx
02:12
Make sure every co-routines has its own set of temporary registers and does not share temporaries, since a co-routine might expect the content of a temporary register to be preserved across an OP_Yield. Proposed fix for ticket [d06a25c84454a]. check-in: ca72be86 user: drh tags: trunk
2016-02-05
14:29
Merge OOM handling optimizations and PRAGMA synchronous=EXTRA as well as other enhancements from trunk. check-in: 201fcbee user: drh tags: apple-osx
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   590    590   sqlite3$(TEXE):	$(TOP)/src/shell.c libsqlite3.la sqlite3.h
   591    591   	$(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \
   592    592   		$(TOP)/src/shell.c libsqlite3.la \
   593    593   		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
   594    594   
   595    595   sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
   596    596   	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS)
          597  +
          598  +srcck1$(BEXE):	$(TOP)/tool/srcck1.c
          599  +	$(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c
          600  +
          601  +sourcetest:	srcck1$(BEXE) sqlite3.c
          602  +	./srcck1 sqlite3.c
   597    603   
   598    604   fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
   599    605   	$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
   600    606   	  $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)
   601    607   
   602    608   fuzzcheck$(TEXE):	$(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h
   603    609   	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS)
................................................................................
  1091   1097   #
  1092   1098   quicktest:	./testfixture$(TEXE)
  1093   1099   	./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS)
  1094   1100   
  1095   1101   # This is the common case.  Run many tests that do not take too long,
  1096   1102   # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
  1097   1103   #
  1098         -test:	$(TESTPROGS) fastfuzztest
         1104  +test:	$(TESTPROGS) sourcetest fastfuzztest
  1099   1105   	./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
  1100   1106   
  1101   1107   # Run a test using valgrind.  This can take a really long time
  1102   1108   # because valgrind is so much slower than a native machine.
  1103   1109   #
  1104   1110   valgrindtest:	$(TESTPROGS) valgrindfuzz
  1105   1111   	OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind $(TESTOPTS)
................................................................................
  1227   1233   	rm -f sqlite3.c
  1228   1234   	rm -f sqlite3rc.h
  1229   1235   	rm -f shell.c sqlite3ext.h
  1230   1236   	rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c
  1231   1237   	rm -f sqlite-*-output.vsix
  1232   1238   	rm -f mptester mptester.exe
  1233   1239   	rm -f rbu rbu.exe
         1240  +	rm -f srcck1 srcck1.exe
  1234   1241   	rm -f fuzzershell fuzzershell.exe
  1235   1242   	rm -f fuzzcheck fuzzcheck.exe
  1236   1243   	rm -f sqldiff sqldiff.exe
  1237   1244   	rm -f fts5.* fts5parse.*
  1238   1245   
  1239   1246   distclean:	clean
  1240   1247   	rm -f config.h config.log config.status libtool Makefile sqlite3.pc

Changes to Makefile.msc.

   262    262   EXT_FEATURE_FLAGS =
   263    263   !ENDIF
   264    264   !ENDIF
   265    265   
   266    266   ###############################################################################
   267    267   ############################### END OF OPTIONS ################################
   268    268   ###############################################################################
          269  +
          270  +# When compiling for the Windows 10 platform, the PLATFORM macro must be set
          271  +# to an appropriate value (e.g. x86, x64, arm, arm64, etc).
          272  +#
          273  +!IF $(FOR_WIN10)!=0
          274  +!IFNDEF PLATFORM
          275  +!ERROR Using the FOR_WIN10 option requires a value for PLATFORM.
          276  +!ENDIF
          277  +!ENDIF
   269    278   
   270    279   # This assumes that MSVC is always installed in 32-bit Program Files directory
   271    280   # and sets the variable for use in locating other 32-bit installs accordingly.
   272    281   #
   273    282   PROGRAMFILES_X86 = $(VCINSTALLDIR)\..\..
   274    283   PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\)
   275    284   
................................................................................
   293    302   # compiler binary for the target platform.  If it is not defined, simply define
   294    303   # it to the legacy default value 'rc.exe'.
   295    304   #
   296    305   !IFNDEF RC
   297    306   RC = rc.exe
   298    307   !ENDIF
   299    308   
   300         -# Check for the MSVC runtime library path macro.  Othertise, this value will
          309  +# Check for the MSVC runtime library path macro.  Otherwise, this value will
   301    310   # default to the 'lib' directory underneath the MSVC installation directory.
   302    311   #
   303    312   !IFNDEF CRTLIBPATH
   304    313   CRTLIBPATH = $(VCINSTALLDIR)\lib
   305    314   !ENDIF
   306    315   
   307    316   CRTLIBPATH = $(CRTLIBPATH:\\=\)
................................................................................
   330    339   !ELSEIF $(XCOMPILE)!=0
   331    340   NCC = "$(VCINSTALLDIR)\bin\$(CC)"
   332    341   NCC = $(NCC:\\=\)
   333    342   !ELSE
   334    343   NCC = $(CC)
   335    344   !ENDIF
   336    345   
   337         -# Check for the MSVC native runtime library path macro.  Othertise,
          346  +# Check for the MSVC native runtime library path macro.  Otherwise,
   338    347   # this value will default to the 'lib' directory underneath the MSVC
   339    348   # installation directory.
   340    349   #
   341    350   !IFNDEF NCRTLIBPATH
   342    351   NCRTLIBPATH = $(VCINSTALLDIR)\lib
   343    352   !ENDIF
   344    353   
   345    354   NCRTLIBPATH = $(NCRTLIBPATH:\\=\)
   346    355   
   347         -# Check for the Platform SDK library path macro.  Othertise, this
          356  +# Check for the Platform SDK library path macro.  Otherwise, this
   348    357   # value will default to the 'lib' directory underneath the Windows
   349    358   # SDK installation directory (the environment variable used appears
   350    359   # to be available when using Visual C++ 2008 or later via the
   351    360   # command line).
   352    361   #
   353    362   !IFNDEF NSDKLIBPATH
   354    363   NSDKLIBPATH = $(WINDOWSSDKDIR)\lib
   355    364   !ENDIF
   356    365   
   357    366   NSDKLIBPATH = $(NSDKLIBPATH:\\=\)
          367  +
          368  +# Check for the UCRT library path macro.  Otherwise, this value will
          369  +# default to the version-specific, platform-specific 'lib' directory
          370  +# underneath the Windows SDK installation directory.
          371  +#
          372  +!IFNDEF UCRTLIBPATH
          373  +UCRTLIBPATH = $(WINDOWSSDKDIR)\lib\$(WINDOWSSDKLIBVERSION)\ucrt\$(PLATFORM)
          374  +!ENDIF
          375  +
          376  +UCRTLIBPATH = $(UCRTLIBPATH:\\=\)
   358    377   
   359    378   # C compiler and options for use in building executables that
   360    379   # will run on the platform that is doing the build.
   361    380   #
   362    381   !IF $(USE_FULLWARN)!=0
   363    382   BCC = $(NCC) -nologo -W4 $(CCOPTS) $(BCCOPTS)
   364    383   !ELSE
................................................................................
   537    556   !ENDIF
   538    557   
   539    558   # Also, we need to dynamically link to the correct MSVC runtime
   540    559   # when compiling for WinRT (e.g. debug or release) OR if the
   541    560   # USE_CRT_DLL option is set to force dynamically linking to the
   542    561   # MSVC runtime library.
   543    562   #
   544         -!IF $(FOR_WINRT)!=0 || $(FOR_WIN10)!=0 || $(USE_CRT_DLL)!=0
          563  +!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0
   545    564   !IF $(DEBUG)>1
   546    565   TCC = $(TCC) -MDd
   547    566   BCC = $(BCC) -MDd
   548    567   !ELSE
   549    568   TCC = $(TCC) -MD
   550    569   BCC = $(BCC) -MD
   551    570   !ENDIF
................................................................................
   884    903   LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)"
   885    904   !ENDIF
   886    905   LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE
   887    906   LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib
   888    907   LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib
   889    908   !ENDIF
   890    909   
   891         -# When compiling for UAP, some extra linker options are also required.
          910  +# When compiling for UAP or the Windows 10 platform, some extra linker
          911  +# options are also required.
   892    912   #
   893    913   !IF $(FOR_UAP)!=0 || $(FOR_WIN10)!=0
   894    914   LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE /NODEFAULTLIB:kernel32.lib
   895    915   LTLINKOPTS = $(LTLINKOPTS) mincore.lib
   896    916   !IFDEF PSDKLIBPATH
   897    917   LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(PSDKLIBPATH)"
   898    918   !ENDIF
   899    919   !ENDIF
          920  +
          921  +!IF $(FOR_WIN10)!=0
          922  +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(UCRTLIBPATH)"
          923  +!IF $(DEBUG)>1
          924  +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrtd.lib /DEFAULTLIB:ucrtd.lib
          925  +!ELSE
          926  +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib
          927  +!ENDIF
          928  +!ENDIF
   900    929   
   901    930   # If either debugging or symbols are enabled, enable PDBs.
   902    931   #
   903    932   !IF $(DEBUG)>1 || $(SYMBOLS)!=0
   904    933   LDFLAGS = /DEBUG $(LDOPTS)
   905    934   !ELSE
   906    935   LDFLAGS = $(LDOPTS)
................................................................................
  1388   1417   $(SQLITE3EXE):	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
  1389   1418   	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \
  1390   1419   		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
  1391   1420   
  1392   1421   # <<mark>>
  1393   1422   sqldiff.exe:	$(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
  1394   1423   	$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         1424  +
         1425  +srcck1.exe:	$(TOP)\tool\srcck1.c
         1426  +	$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c
         1427  +
         1428  +sourcetest:	srcck1.exe sqlite3.c
         1429  +	srcck1.exe sqlite3.c
  1395   1430   
  1396   1431   fuzzershell.exe:	$(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H)
  1397   1432   	$(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1398   1433   
  1399   1434   fuzzcheck.exe:	$(TOP)\test\fuzzcheck.c $(SQLITE3C) $(SQLITE3H)
  1400   1435   	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1401   1436   
................................................................................
  1912   1947   	.\fuzzcheck.exe $(FUZZDATA)
  1913   1948   
  1914   1949   fastfuzztest:	fuzzcheck.exe
  1915   1950   	.\fuzzcheck.exe --limit-mem 100M $(FUZZDATA)
  1916   1951   
  1917   1952   # Minimal testing that runs in less than 3 minutes (on a fast machine)
  1918   1953   #
  1919         -quicktest:	testfixture.exe
         1954  +quicktest:	testfixture.exe sourcetest
  1920   1955   	@set PATH=$(LIBTCLPATH);$(PATH)
  1921   1956   	.\testfixture.exe $(TOP)\test\extraquick.test $(TESTOPTS)
  1922   1957   
  1923   1958   # This is the common case.  Run many tests that do not take too long,
  1924   1959   # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
  1925   1960   #
  1926         -test:	$(TESTPROGS) fastfuzztest
         1961  +test:	$(TESTPROGS) sourcetest fastfuzztest
  1927   1962   	@set PATH=$(LIBTCLPATH);$(PATH)
  1928   1963   	.\testfixture.exe $(TOP)\test\veryquick.test $(TESTOPTS)
  1929   1964   
  1930   1965   smoketest:	$(TESTPROGS)
  1931   1966   	@set PATH=$(LIBTCLPATH);$(PATH)
  1932   1967   	.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)
  1933   1968   
................................................................................
  2006   2041   	-rmdir /Q/S tsrc 2>NUL
  2007   2042   	del /Q .target_source 2>NUL
  2008   2043   	del /Q tclsqlite3.exe 2>NUL
  2009   2044   	del /Q testloadext.dll 2>NUL
  2010   2045   	del /Q testfixture.exe test.db 2>NUL
  2011   2046   	del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL
  2012   2047   	del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL
  2013         -	del /Q mptester.exe wordcount.exe rbu.exe 2>NUL
         2048  +	del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL
  2014   2049   	del /Q $(SQLITE3EXE) $(SQLITE3DLL) sqlite3.def 2>NUL
  2015   2050   	del /Q sqlite3.c sqlite3-*.c 2>NUL
  2016   2051   	del /Q sqlite3rc.h 2>NUL
  2017   2052   	del /Q shell.c sqlite3ext.h 2>NUL
  2018   2053   	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
  2019   2054   	del /Q sqlite-*-output.vsix 2>NUL
  2020   2055   	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL
  2021   2056   	del /Q fts5.* fts5parse.* 2>NUL
  2022   2057   # <</mark>>

Changes to autoconf/Makefile.msc.

   247    247   EXT_FEATURE_FLAGS =
   248    248   !ENDIF
   249    249   !ENDIF
   250    250   
   251    251   ###############################################################################
   252    252   ############################### END OF OPTIONS ################################
   253    253   ###############################################################################
          254  +
          255  +# When compiling for the Windows 10 platform, the PLATFORM macro must be set
          256  +# to an appropriate value (e.g. x86, x64, arm, arm64, etc).
          257  +#
          258  +!IF $(FOR_WIN10)!=0
          259  +!IFNDEF PLATFORM
          260  +!ERROR Using the FOR_WIN10 option requires a value for PLATFORM.
          261  +!ENDIF
          262  +!ENDIF
   254    263   
   255    264   # This assumes that MSVC is always installed in 32-bit Program Files directory
   256    265   # and sets the variable for use in locating other 32-bit installs accordingly.
   257    266   #
   258    267   PROGRAMFILES_X86 = $(VCINSTALLDIR)\..\..
   259    268   PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\)
   260    269   
................................................................................
   278    287   # compiler binary for the target platform.  If it is not defined, simply define
   279    288   # it to the legacy default value 'rc.exe'.
   280    289   #
   281    290   !IFNDEF RC
   282    291   RC = rc.exe
   283    292   !ENDIF
   284    293   
   285         -# Check for the MSVC runtime library path macro.  Othertise, this value will
          294  +# Check for the MSVC runtime library path macro.  Otherwise, this value will
   286    295   # default to the 'lib' directory underneath the MSVC installation directory.
   287    296   #
   288    297   !IFNDEF CRTLIBPATH
   289    298   CRTLIBPATH = $(VCINSTALLDIR)\lib
   290    299   !ENDIF
   291    300   
   292    301   CRTLIBPATH = $(CRTLIBPATH:\\=\)
................................................................................
   315    324   !ELSEIF $(XCOMPILE)!=0
   316    325   NCC = "$(VCINSTALLDIR)\bin\$(CC)"
   317    326   NCC = $(NCC:\\=\)
   318    327   !ELSE
   319    328   NCC = $(CC)
   320    329   !ENDIF
   321    330   
   322         -# Check for the MSVC native runtime library path macro.  Othertise,
          331  +# Check for the MSVC native runtime library path macro.  Otherwise,
   323    332   # this value will default to the 'lib' directory underneath the MSVC
   324    333   # installation directory.
   325    334   #
   326    335   !IFNDEF NCRTLIBPATH
   327    336   NCRTLIBPATH = $(VCINSTALLDIR)\lib
   328    337   !ENDIF
   329    338   
   330    339   NCRTLIBPATH = $(NCRTLIBPATH:\\=\)
   331    340   
   332         -# Check for the Platform SDK library path macro.  Othertise, this
          341  +# Check for the Platform SDK library path macro.  Otherwise, this
   333    342   # value will default to the 'lib' directory underneath the Windows
   334    343   # SDK installation directory (the environment variable used appears
   335    344   # to be available when using Visual C++ 2008 or later via the
   336    345   # command line).
   337    346   #
   338    347   !IFNDEF NSDKLIBPATH
   339    348   NSDKLIBPATH = $(WINDOWSSDKDIR)\lib
   340    349   !ENDIF
   341    350   
   342    351   NSDKLIBPATH = $(NSDKLIBPATH:\\=\)
          352  +
          353  +# Check for the UCRT library path macro.  Otherwise, this value will
          354  +# default to the version-specific, platform-specific 'lib' directory
          355  +# underneath the Windows SDK installation directory.
          356  +#
          357  +!IFNDEF UCRTLIBPATH
          358  +UCRTLIBPATH = $(WINDOWSSDKDIR)\lib\$(WINDOWSSDKLIBVERSION)\ucrt\$(PLATFORM)
          359  +!ENDIF
          360  +
          361  +UCRTLIBPATH = $(UCRTLIBPATH:\\=\)
   343    362   
   344    363   # C compiler and options for use in building executables that
   345    364   # will run on the platform that is doing the build.
   346    365   #
   347    366   !IF $(USE_FULLWARN)!=0
   348    367   BCC = $(NCC) -nologo -W4 $(CCOPTS) $(BCCOPTS)
   349    368   !ELSE
................................................................................
   522    541   !ENDIF
   523    542   
   524    543   # Also, we need to dynamically link to the correct MSVC runtime
   525    544   # when compiling for WinRT (e.g. debug or release) OR if the
   526    545   # USE_CRT_DLL option is set to force dynamically linking to the
   527    546   # MSVC runtime library.
   528    547   #
   529         -!IF $(FOR_WINRT)!=0 || $(FOR_WIN10)!=0 || $(USE_CRT_DLL)!=0
          548  +!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0
   530    549   !IF $(DEBUG)>1
   531    550   TCC = $(TCC) -MDd
   532    551   BCC = $(BCC) -MDd
   533    552   !ELSE
   534    553   TCC = $(TCC) -MD
   535    554   BCC = $(BCC) -MD
   536    555   !ENDIF
................................................................................
   779    798   LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)"
   780    799   !ENDIF
   781    800   LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE
   782    801   LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib
   783    802   LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib
   784    803   !ENDIF
   785    804   
   786         -# When compiling for UAP, some extra linker options are also required.
          805  +# When compiling for UAP or the Windows 10 platform, some extra linker
          806  +# options are also required.
   787    807   #
   788    808   !IF $(FOR_UAP)!=0 || $(FOR_WIN10)!=0
   789    809   LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE /NODEFAULTLIB:kernel32.lib
   790    810   LTLINKOPTS = $(LTLINKOPTS) mincore.lib
   791    811   !IFDEF PSDKLIBPATH
   792    812   LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(PSDKLIBPATH)"
   793    813   !ENDIF
   794    814   !ENDIF
          815  +
          816  +!IF $(FOR_WIN10)!=0
          817  +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(UCRTLIBPATH)"
          818  +!IF $(DEBUG)>1
          819  +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrtd.lib /DEFAULTLIB:ucrtd.lib
          820  +!ELSE
          821  +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib
          822  +!ENDIF
          823  +!ENDIF
   795    824   
   796    825   # If either debugging or symbols are enabled, enable PDBs.
   797    826   #
   798    827   !IF $(DEBUG)>1 || $(SYMBOLS)!=0
   799    828   LDFLAGS = /DEBUG $(LDOPTS)
   800    829   !ELSE
   801    830   LDFLAGS = $(LDOPTS)

Changes to ext/fts3/fts3_tokenizer.c.

    63     63   
    64     64     pHash = (Fts3Hash *)sqlite3_user_data(context);
    65     65   
    66     66     zName = sqlite3_value_text(argv[0]);
    67     67     nName = sqlite3_value_bytes(argv[0])+1;
    68     68   
    69     69     if( argc==2 ){
           70  +#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
    70     71       void *pOld;
    71     72       int n = sqlite3_value_bytes(argv[1]);
    72     73       if( zName==0 || n!=sizeof(pPtr) ){
    73     74         sqlite3_result_error(context, "argument type mismatch", -1);
    74     75         return;
    75     76       }
    76     77       pPtr = *(void **)sqlite3_value_blob(argv[1]);
    77     78       pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
    78     79       if( pOld==pPtr ){
    79     80         sqlite3_result_error(context, "out of memory", -1);
    80     81         return;
    81     82       }
    82         -  }else{
           83  +#else
           84  +    sqlite3_result_error(context, "fts3tokenize: " 
           85  +        "disabled - rebuild with -DSQLITE_ENABLE_FTS3_TOKENIZER", -1
           86  +    );
           87  +    return;
           88  +#endif /* SQLITE_ENABLE_FTS3_TOKENIZER */
           89  +  }else
           90  +  {
    83     91       if( zName ){
    84     92         pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
    85     93       }
    86     94       if( !pPtr ){
    87     95         char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
    88     96         sqlite3_result_error(context, zErr, -1);
    89     97         sqlite3_free(zErr);
................................................................................
   324    332       sqlite3_result_error(context, zErr, -1);
   325    333     }else{
   326    334       sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
   327    335     }
   328    336     Tcl_DecrRefCount(pRet);
   329    337   }
   330    338   
          339  +#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
   331    340   static
   332    341   int registerTokenizer(
   333    342     sqlite3 *db, 
   334    343     char *zName, 
   335    344     const sqlite3_tokenizer_module *p
   336    345   ){
   337    346     int rc;
................................................................................
   345    354   
   346    355     sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
   347    356     sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
   348    357     sqlite3_step(pStmt);
   349    358   
   350    359     return sqlite3_finalize(pStmt);
   351    360   }
          361  +#endif /* SQLITE_ENABLE_FTS3_TOKENIZER */
          362  +
   352    363   
   353    364   static
   354    365   int queryTokenizer(
   355    366     sqlite3 *db, 
   356    367     char *zName,  
   357    368     const sqlite3_tokenizer_module **pp
   358    369   ){
................................................................................
   416    427     assert( p1==p2 );
   417    428     rc = queryTokenizer(db, "nosuchtokenizer", &p2);
   418    429     assert( rc==SQLITE_ERROR );
   419    430     assert( p2==0 );
   420    431     assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
   421    432   
   422    433     /* Test the storage function */
          434  +#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
   423    435     rc = registerTokenizer(db, "nosuchtokenizer", p1);
   424    436     assert( rc==SQLITE_OK );
   425    437     rc = queryTokenizer(db, "nosuchtokenizer", &p2);
   426    438     assert( rc==SQLITE_OK );
   427    439     assert( p2==p1 );
          440  +#endif
   428    441   
   429    442     sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
   430    443   }
   431    444   
   432    445   #endif
   433    446   
   434    447   /*

Changes to ext/fts5/fts5Int.h.

   278    278   int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);
   279    279   
   280    280   typedef struct Fts5PoslistWriter Fts5PoslistWriter;
   281    281   struct Fts5PoslistWriter {
   282    282     i64 iPrev;
   283    283   };
   284    284   int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
          285  +void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
   285    286   
   286    287   int sqlite3Fts5PoslistNext64(
   287    288     const u8 *a, int n,             /* Buffer containing poslist */
   288    289     int *pi,                        /* IN/OUT: Offset within a[] */
   289    290     i64 *piOff                      /* IN/OUT: Current offset */
   290    291   );
   291    292   

Changes to ext/fts5/fts5_buffer.c.

    12     12   */
    13     13   
    14     14   
    15     15   
    16     16   #include "fts5Int.h"
    17     17   
    18     18   int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){
    19         -  u32 nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
    20         -  u8 *pNew;
    21         -  while( nNew<nByte ){
    22         -    nNew = nNew * 2;
    23         -  }
    24         -  pNew = sqlite3_realloc(pBuf->p, nNew);
    25         -  if( pNew==0 ){
    26         -    *pRc = SQLITE_NOMEM;
    27         -    return 1;
    28         -  }else{
    29         -    pBuf->nSpace = nNew;
    30         -    pBuf->p = pNew;
           19  +  if( (u32)pBuf->nSpace<nByte ){
           20  +    u32 nNew = pBuf->nSpace ? pBuf->nSpace : 64;
           21  +    u8 *pNew;
           22  +    while( nNew<nByte ){
           23  +      nNew = nNew * 2;
           24  +    }
           25  +    pNew = sqlite3_realloc(pBuf->p, nNew);
           26  +    if( pNew==0 ){
           27  +      *pRc = SQLITE_NOMEM;
           28  +      return 1;
           29  +    }else{
           30  +      pBuf->nSpace = nNew;
           31  +      pBuf->p = pNew;
           32  +    }
    31     33     }
    32     34     return 0;
    33     35   }
    34     36   
    35     37   
    36     38   /*
    37     39   ** Encode value iVal as an SQLite varint and append it to the buffer object
................................................................................
   203    205   ){
   204    206     memset(pIter, 0, sizeof(*pIter));
   205    207     pIter->a = a;
   206    208     pIter->n = n;
   207    209     sqlite3Fts5PoslistReaderNext(pIter);
   208    210     return pIter->bEof;
   209    211   }
          212  +
          213  +/*
          214  +** Append position iPos to the position list being accumulated in buffer
          215  +** pBuf, which must be already be large enough to hold the new data.
          216  +** The previous position written to this list is *piPrev. *piPrev is set
          217  +** to iPos before returning.
          218  +*/
          219  +void sqlite3Fts5PoslistSafeAppend(
          220  +  Fts5Buffer *pBuf, 
          221  +  i64 *piPrev, 
          222  +  i64 iPos
          223  +){
          224  +  static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
          225  +  if( (iPos & colmask) != (*piPrev & colmask) ){
          226  +    pBuf->p[pBuf->n++] = 1;
          227  +    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
          228  +    *piPrev = (iPos & colmask);
          229  +  }
          230  +  pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2);
          231  +  *piPrev = iPos;
          232  +}
   210    233   
   211    234   int sqlite3Fts5PoslistWriterAppend(
   212    235     Fts5Buffer *pBuf, 
   213    236     Fts5PoslistWriter *pWriter,
   214    237     i64 iPos
   215    238   ){
   216         -  static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
   217         -  int rc = SQLITE_OK;
   218         -  if( 0==fts5BufferGrow(&rc, pBuf, 5+5+5) ){
   219         -    if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
   220         -      pBuf->p[pBuf->n++] = 1;
   221         -      pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
   222         -      pWriter->iPrev = (iPos & colmask);
   223         -    }
   224         -    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2);
   225         -    pWriter->iPrev = iPos;
   226         -  }
   227         -  return rc;
          239  +  int rc;
          240  +  if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc;
          241  +  sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos);
          242  +  return SQLITE_OK;
   228    243   }
   229    244   
   230    245   void *sqlite3Fts5MallocZero(int *pRc, int nByte){
   231    246     void *pRet = 0;
   232    247     if( *pRc==SQLITE_OK ){
   233    248       pRet = sqlite3_malloc(nByte);
   234    249       if( pRet==0 && nByte>0 ){

Changes to ext/fts5/fts5_index.c.

   514    514   
   515    515     /* Invoked to set output variables. */
   516    516     void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*);
   517    517   
   518    518     int nSeg;                       /* Size of aSeg[] array */
   519    519     int bRev;                       /* True to iterate in reverse order */
   520    520     u8 bSkipEmpty;                  /* True to skip deleted entries */
   521         -  u8 bFiltered;                   /* True if column-filter already applied */
   522    521   
   523    522     i64 iSwitchRowid;               /* Firstest rowid of other than aFirst[1] */
   524    523     Fts5CResult *aFirst;            /* Current merge state (see above) */
   525    524     Fts5SegIter aSeg[1];            /* Array of segment iterators */
   526    525   };
   527    526   
   528    527   
................................................................................
  2026   2025       int iPoslist;
  2027   2026       if( pIter->iTermLeafPgno==pIter->iLeafPgno ){
  2028   2027         iPoslist = pIter->iTermLeafOffset;
  2029   2028       }else{
  2030   2029         iPoslist = 4;
  2031   2030       }
  2032   2031       fts5IndexSkipVarint(pLeaf->p, iPoslist);
  2033         -    assert( p->pConfig->eDetail==FTS5_DETAIL_NONE || iPoslist==(
  2034         -        pIter->iLeafOffset - sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel)
  2035         -    ));
  2036   2032       pIter->iLeafOffset = iPoslist;
  2037   2033   
  2038   2034       /* If this condition is true then the largest rowid for the current
  2039   2035       ** term may not be stored on the current page. So search forward to
  2040   2036       ** see where said rowid really is.  */
  2041   2037       if( pIter->iEndofDoclist>=pLeaf->szLeaf ){
  2042   2038         int pgno;
................................................................................
  3091   3087   ** column filters are specified.
  3092   3088   */
  3093   3089   static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){
  3094   3090     pIter->base.iRowid = pSeg->iRowid;
  3095   3091     pIter->base.nData = pSeg->nPos;
  3096   3092   
  3097   3093     assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE );
  3098         -  assert( pIter->pColset==0 || pIter->bFiltered );
         3094  +  assert( pIter->pColset==0 );
  3099   3095   
  3100   3096     if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
  3101   3097       /* All data is stored on the current page. Populate the output 
  3102   3098       ** variables to point into the body of the page object. */
  3103   3099       pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  3104   3100     }else{
  3105   3101       /* The data is distributed over two or more pages. Copy it into the
................................................................................
  3208   3204   static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){
  3209   3205     if( *pRc==SQLITE_OK ){
  3210   3206       Fts5Config *pConfig = pIter->pIndex->pConfig;
  3211   3207       if( pConfig->eDetail==FTS5_DETAIL_NONE ){
  3212   3208         pIter->xSetOutputs = fts5IterSetOutputs_None;
  3213   3209       }
  3214   3210   
  3215         -    else if( pIter->pColset==0 || pIter->bFiltered ){
         3211  +    else if( pIter->pColset==0 ){
  3216   3212         pIter->xSetOutputs = fts5IterSetOutputs_Nocolset;
  3217   3213       }
  3218   3214   
  3219   3215       else if( pConfig->eDetail==FTS5_DETAIL_FULL ){
  3220   3216         pIter->xSetOutputs = fts5IterSetOutputs_Full;
  3221   3217       }
  3222   3218   
................................................................................
  3355   3351     Fts5Iter **ppOut                /* New object */
  3356   3352   ){
  3357   3353     Fts5Iter *pNew;
  3358   3354     pNew = fts5MultiIterAlloc(p, 2);
  3359   3355     if( pNew ){
  3360   3356       Fts5SegIter *pIter = &pNew->aSeg[1];
  3361   3357   
  3362         -    pNew->bFiltered = 1;
  3363   3358       pIter->flags = FTS5_SEGITER_ONETERM;
  3364   3359       if( pData->szLeaf>0 ){
  3365   3360         pIter->pLeaf = pData;
  3366   3361         pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid);
  3367   3362         pIter->iEndofDoclist = pData->nn;
  3368   3363         pNew->aFirst[1].iFirst = 1;
  3369   3364         if( bDesc ){
................................................................................
  4701   4696   */
  4702   4697   static void fts5MergePrefixLists(
  4703   4698     Fts5Index *p,                   /* FTS5 backend object */
  4704   4699     Fts5Buffer *p1,                 /* First list to merge */
  4705   4700     Fts5Buffer *p2                  /* Second list to merge */
  4706   4701   ){
  4707   4702     if( p2->n ){
  4708         -    if( p1->n==0 ){
  4709         -      fts5BufferSwap(p1, p2);
  4710         -    }else{
  4711         -      i64 iLastRowid = 0;
  4712         -      Fts5DoclistIter i1;
  4713         -      Fts5DoclistIter i2;
  4714         -      Fts5Buffer out = {0, 0, 0};
  4715         -      Fts5Buffer tmp = {0, 0, 0};
  4716         -
  4717         -      if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return;
  4718         -      fts5DoclistIterInit(p1, &i1);
  4719         -      fts5DoclistIterInit(p2, &i2);
  4720         -
  4721         -      while( 1 ){
  4722         -
  4723         -        if( i1.iRowid<i2.iRowid ){
  4724         -          /* Copy entry from i1 */
  4725         -          fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
  4726         -          fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist+i1.nSize);
  4727         -          fts5DoclistIterNext(&i1);
  4728         -          if( i1.aPoslist==0 ) break;
  4729         -        }
  4730         -        else if( i2.iRowid!=i1.iRowid ){
  4731         -          /* Copy entry from i2 */
  4732         -          fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
  4733         -          fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist+i2.nSize);
  4734         -          fts5DoclistIterNext(&i2);
  4735         -          if( i2.aPoslist==0 ) break;
  4736         -        }
  4737         -        else{
  4738         -          i64 iPos1 = 0;
  4739         -          i64 iPos2 = 0;
  4740         -          int iOff1 = 0;
  4741         -          int iOff2 = 0;
  4742         -          u8 *a1 = &i1.aPoslist[i1.nSize];
  4743         -          u8 *a2 = &i2.aPoslist[i2.nSize];
  4744         -
  4745         -          Fts5PoslistWriter writer;
  4746         -          memset(&writer, 0, sizeof(writer));
  4747         -
  4748         -          /* Merge the two position lists. */ 
  4749         -          fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
  4750         -          fts5BufferZero(&tmp);
  4751         -
         4703  +    i64 iLastRowid = 0;
         4704  +    Fts5DoclistIter i1;
         4705  +    Fts5DoclistIter i2;
         4706  +    Fts5Buffer out = {0, 0, 0};
         4707  +    Fts5Buffer tmp = {0, 0, 0};
         4708  +
         4709  +    if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return;
         4710  +    fts5DoclistIterInit(p1, &i1);
         4711  +    fts5DoclistIterInit(p2, &i2);
         4712  +
         4713  +    while( 1 ){
         4714  +      if( i1.iRowid<i2.iRowid ){
         4715  +        /* Copy entry from i1 */
         4716  +        fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
         4717  +        fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist+i1.nSize);
         4718  +        fts5DoclistIterNext(&i1);
         4719  +        if( i1.aPoslist==0 ) break;
         4720  +      }
         4721  +      else if( i2.iRowid!=i1.iRowid ){
         4722  +        /* Copy entry from i2 */
         4723  +        fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
         4724  +        fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist+i2.nSize);
         4725  +        fts5DoclistIterNext(&i2);
         4726  +        if( i2.aPoslist==0 ) break;
         4727  +      }
         4728  +      else{
         4729  +        /* Merge the two position lists. */ 
         4730  +        i64 iPos1 = 0;
         4731  +        i64 iPos2 = 0;
         4732  +        int iOff1 = 0;
         4733  +        int iOff2 = 0;
         4734  +        u8 *a1 = &i1.aPoslist[i1.nSize];
         4735  +        u8 *a2 = &i2.aPoslist[i2.nSize];
         4736  +
         4737  +        i64 iPrev = 0;
         4738  +        Fts5PoslistWriter writer;
         4739  +        memset(&writer, 0, sizeof(writer));
         4740  +
         4741  +        fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
         4742  +        fts5BufferZero(&tmp);
         4743  +        sqlite3Fts5BufferSize(&p->rc, &tmp, i1.nPoslist + i2.nPoslist);
         4744  +        if( p->rc ) break;
         4745  +
         4746  +        sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
         4747  +        sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
         4748  +        assert( iPos1>=0 && iPos2>=0 );
         4749  +
         4750  +        if( iPos1<iPos2 ){
         4751  +          sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
  4752   4752             sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
         4753  +        }else{
         4754  +          sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
  4753   4755             sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
         4756  +        }
  4754   4757   
  4755         -          while( iPos1>=0 || iPos2>=0 ){
  4756         -            i64 iNew;
  4757         -            if( iPos2<0 || (iPos1>=0 && iPos1<iPos2) ){
  4758         -              iNew = iPos1;
  4759         -              sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
  4760         -            }else{
  4761         -              iNew = iPos2;
  4762         -              sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
  4763         -              if( iPos1==iPos2 ){
  4764         -                sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1,&iPos1);
         4758  +        if( iPos1>=0 && iPos2>=0 ){
         4759  +          while( 1 ){
         4760  +            if( iPos1<iPos2 ){
         4761  +              if( iPos1!=iPrev ){
         4762  +                sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
  4765   4763                 }
  4766         -            }
  4767         -            if( iNew!=writer.iPrev || tmp.n==0 ){
  4768         -              p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
  4769         -              if( p->rc ) goto error_out;
         4764  +              sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
         4765  +              if( iPos1<0 ) break;
         4766  +            }else{
         4767  +              assert( iPos2!=iPrev );
         4768  +              sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
         4769  +              sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
         4770  +              if( iPos2<0 ) break;
  4770   4771               }
  4771   4772             }
         4773  +        }
  4772   4774   
  4773         -          /* WRITEPOSLISTSIZE */
  4774         -          fts5BufferSafeAppendVarint(&out, tmp.n * 2);
  4775         -          fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n);
  4776         -          fts5DoclistIterNext(&i1);
  4777         -          fts5DoclistIterNext(&i2);
  4778         -          if( i1.aPoslist==0 || i2.aPoslist==0 ) break;
         4775  +        if( iPos1>=0 ){
         4776  +          if( iPos1!=iPrev ){
         4777  +            sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
         4778  +          }
         4779  +          fts5BufferSafeAppendBlob(&tmp, &a1[iOff1], i1.nPoslist-iOff1);
         4780  +        }else{
         4781  +          assert( iPos2>=0 && iPos2!=iPrev );
         4782  +          sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
         4783  +          fts5BufferSafeAppendBlob(&tmp, &a2[iOff2], i2.nPoslist-iOff2);
  4779   4784           }
         4785  +
         4786  +        /* WRITEPOSLISTSIZE */
         4787  +        fts5BufferSafeAppendVarint(&out, tmp.n * 2);
         4788  +        fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n);
         4789  +        fts5DoclistIterNext(&i1);
         4790  +        fts5DoclistIterNext(&i2);
         4791  +        if( i1.aPoslist==0 || i2.aPoslist==0 ) break;
  4780   4792         }
         4793  +    }
  4781   4794   
  4782         -      if( i1.aPoslist ){
  4783         -        fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
  4784         -        fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.aEof - i1.aPoslist);
  4785         -      }
  4786         -      else if( i2.aPoslist ){
  4787         -        fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
  4788         -        fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist);
  4789         -      }
         4795  +    if( i1.aPoslist ){
         4796  +      fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
         4797  +      fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.aEof - i1.aPoslist);
         4798  +    }
         4799  +    else if( i2.aPoslist ){
         4800  +      fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
         4801  +      fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist);
         4802  +    }
  4790   4803   
  4791         -     error_out:
  4792         -      fts5BufferSet(&p->rc, p1, out.n, out.p);
  4793         -      fts5BufferFree(&tmp);
  4794         -      fts5BufferFree(&out);
  4795         -    }
         4804  +    fts5BufferSet(&p->rc, p1, out.n, out.p);
         4805  +    fts5BufferFree(&tmp);
         4806  +    fts5BufferFree(&out);
  4796   4807     }
  4797   4808   }
  4798   4809   
  4799   4810   static void fts5SetupPrefixIter(
  4800   4811     Fts5Index *p,                   /* Index to read from */
  4801   4812     int bDesc,                      /* True for "ORDER BY rowid DESC" */
  4802   4813     const u8 *pToken,               /* Buffer containing prefix to match */
................................................................................
  4932   4943   ** to the database. Additionally, assume that the contents of the %_data
  4933   4944   ** table may have changed on disk. So any in-memory caches of %_data 
  4934   4945   ** records must be invalidated.
  4935   4946   */
  4936   4947   int sqlite3Fts5IndexRollback(Fts5Index *p){
  4937   4948     fts5CloseReader(p);
  4938   4949     fts5IndexDiscardData(p);
  4939         -  assert( p->rc==SQLITE_OK );
         4950  +  /* assert( p->rc==SQLITE_OK ); */
  4940   4951     return SQLITE_OK;
  4941   4952   }
  4942   4953   
  4943   4954   /*
  4944   4955   ** The %_data table is completely empty when this function is called. This
  4945   4956   ** function populates it with the initial structure objects for each index,
  4946   4957   ** and the initial version of the "averages" record (a zero-byte blob).
................................................................................
  5152   5163           fts5StructureRelease(pStruct);
  5153   5164         }
  5154   5165       }else{
  5155   5166         /* Scan multiple terms in the main index */
  5156   5167         int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0;
  5157   5168         buf.p[0] = FTS5_MAIN_PREFIX;
  5158   5169         fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet);
         5170  +      assert( pRet->pColset==0 );
  5159   5171         fts5IterSetOutputCb(&p->rc, pRet);
  5160   5172         if( p->rc==SQLITE_OK ){
  5161   5173           Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst];
  5162         -        if( p->rc==SQLITE_OK && pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg);
         5174  +        if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg);
  5163   5175         }
  5164   5176       }
  5165   5177   
  5166   5178       if( p->rc ){
  5167   5179         sqlite3Fts5IterClose(&pRet->base);
  5168   5180         pRet = 0;
  5169   5181         fts5CloseReader(p);

Changes to ext/fts5/fts5_main.c.

   799    799   **
   800    800   ** Return SQLITE_OK if nothing goes wrong.  SQLITE_OK is returned
   801    801   ** even if we reach end-of-file.  The fts5EofMethod() will be called
   802    802   ** subsequently to determine whether or not an EOF was hit.
   803    803   */
   804    804   static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
   805    805     Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
   806         -  int rc = SQLITE_OK;
          806  +  int rc;
   807    807   
   808    808     assert( (pCsr->ePlan<3)==
   809    809             (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) 
   810    810     );
   811    811     assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) );
   812    812   
   813    813     if( pCsr->ePlan<3 ){
................................................................................
   816    816       rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid);
   817    817       CsrFlagSet(pCsr, sqlite3Fts5ExprEof(pCsr->pExpr));
   818    818       fts5CsrNewrow(pCsr);
   819    819     }else{
   820    820       switch( pCsr->ePlan ){
   821    821         case FTS5_PLAN_SPECIAL: {
   822    822           CsrFlagSet(pCsr, FTS5CSR_EOF);
          823  +        rc = SQLITE_OK;
   823    824           break;
   824    825         }
   825    826     
   826    827         case FTS5_PLAN_SORTED_MATCH: {
   827    828           rc = fts5SorterNext(pCsr);
   828    829           break;
   829    830         }

Changes to ext/fts5/fts5_varint.c.

   329    329       return 2;
   330    330     }
   331    331     return fts5PutVarint64(p,v);
   332    332   }
   333    333   
   334    334   
   335    335   int sqlite3Fts5GetVarintLen(u32 iVal){
          336  +#if 0
   336    337     if( iVal<(1 << 7 ) ) return 1;
          338  +#endif
          339  +  assert( iVal>=(1 << 7) );
   337    340     if( iVal<(1 << 14) ) return 2;
   338    341     if( iVal<(1 << 21) ) return 3;
   339    342     if( iVal<(1 << 28) ) return 4;
   340    343     return 5;
   341    344   }
   342    345   

Changes to ext/fts5/test/fts5ad.test.

     7      7   #    May you find forgiveness for yourself and forgive others.
     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #*************************************************************************
    11     11   # This file implements regression tests for SQLite library.  The
    12     12   # focus of this script is testing the FTS5 module.
    13     13   #
           14  +# More specifically, the focus is on testing prefix queries, both with and
           15  +# without prefix indexes.
    14     16   #
    15     17   
    16     18   source [file join [file dirname [info script]] fts5_common.tcl]
    17     19   set testprefix fts5ad
    18     20   
    19     21   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    20     22   ifcapable !fts5 {

Changes to ext/fts5/test/fts5config.test.

   239    239     5 {detail=',1'}
   240    240     6 {detail=''}
   241    241   } {
   242    242     set res [list 1 {malformed detail=... directive}]
   243    243     do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
   244    244   }
   245    245   
          246  +do_catchsql_test 12.1 {
          247  +  INSERT INTO t1(t1, rank) VALUES('rank', NULL);;
          248  +} {1 {SQL logic error or missing database}}
          249  +
   246    250   finish_test
   247    251   

Changes to ext/fts5/test/fts5detail.test.

    80     80   do_execsql_test 2.1 {
    81     81     INSERT INTO t2(t2) VALUES('integrity-check');
    82     82   }
    83     83   
    84     84   do_execsql_test 2.2 {
    85     85     SELECT fts5_test_poslist(t2) FROM t2('aa');
    86     86   } {0.0.0}
           87  +
           88  +do_execsql_test 2.3 {
           89  +  SELECT fts5_test_collist(t2) FROM t2('aa');
           90  +} {0.0}
    87     91   
    88     92   set ::pc 0
    89     93   #puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*]
    90     94   #exit
    91     95   
    92     96   #-------------------------------------------------------------------------
    93     97   # Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs

Changes to main.mk.

   462    462   sqlite3$(EXE):	$(TOP)/src/shell.c libsqlite3.a sqlite3.h
   463    463   	$(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) $(SHELL_OPT) \
   464    464   		$(TOP)/src/shell.c libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB)
   465    465   
   466    466   sqldiff$(EXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
   467    467   	$(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \
   468    468   		$(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB)
          469  +
          470  +srcck1$(EXE):	$(TOP)/tool/srcck1.c
          471  +	$(BCC) -o srcck1$(EXE) $(TOP)/tool/srcck1.c
          472  +
          473  +sourcetest:	srcck1$(EXE) sqlite3.c
          474  +	./srcck1 sqlite3.c
   469    475   
   470    476   fuzzershell$(EXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
   471    477   	$(TCCX) -o fuzzershell$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
   472    478   	  $(FUZZERSHELL_OPT) $(TOP)/tool/fuzzershell.c sqlite3.c \
   473    479   	  $(TLIBS) $(THREADLIB)
   474    480   
   475    481   fuzzcheck$(EXE):	$(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h
................................................................................
   751    757   #
   752    758   quicktest:	./testfixture$(EXE)
   753    759   	./testfixture$(EXE) $(TOP)/test/extraquick.test $(TESTOPTS)
   754    760   
   755    761   # The default test case.  Runs most of the faster standard TCL tests,
   756    762   # and fuzz tests, and sqlite3_analyzer and sqldiff tests.
   757    763   #
   758         -test:	$(TESTPROGS) fastfuzztest
          764  +test:	$(TESTPROGS) sourcetest fastfuzztest
   759    765   	./testfixture$(EXE) $(TOP)/test/veryquick.test $(TESTOPTS)
   760    766   
   761    767   # Run a test using valgrind.  This can take a really long time
   762    768   # because valgrind is so much slower than a native machine.
   763    769   #
   764    770   valgrindtest:	$(TESTPROGS) valgrindfuzz
   765    771   	OMIT_MISUSE=1 valgrind -v \
................................................................................
   876    882   	rm -f showdb showdb.exe
   877    883   	rm -f showjournal showjournal.exe
   878    884   	rm -f showstat4 showstat4.exe
   879    885   	rm -f showwal showwal.exe
   880    886   	rm -f speedtest1 speedtest1.exe
   881    887   	rm -f wordcount wordcount.exe
   882    888   	rm -f rbu rbu.exe
          889  +	rm -f srcck1 srcck1.exe
   883    890   	rm -f sqlite3.c sqlite3-*.c fts?amal.c tclsqlite3.c
   884    891   	rm -f sqlite3rc.h
   885    892   	rm -f shell.c sqlite3ext.h
   886    893   	rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c
   887    894   	rm -f sqlite-*-output.vsix
   888    895   	rm -f mptester mptester.exe
   889    896   	rm -f fuzzershell fuzzershell.exe
   890    897   	rm -f fuzzcheck fuzzcheck.exe
   891    898   	rm -f sqldiff sqldiff.exe
   892    899   	rm -f fts5.* fts5parse.*

Changes to src/alter.c.

   691    691     */
   692    692     if( pDflt ){
   693    693       sqlite3_value *pVal = 0;
   694    694       int rc;
   695    695       rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_BLOB, &pVal);
   696    696       assert( rc==SQLITE_OK || rc==SQLITE_NOMEM );
   697    697       if( rc!=SQLITE_OK ){
   698         -      assert( db->mallocFailed = 1 );
          698  +      assert( db->mallocFailed == 1 );
   699    699         return;
   700    700       }
   701    701       if( !pVal ){
   702    702         sqlite3ErrorMsg(pParse, "Cannot add a column with non-constant default");
   703    703         return;
   704    704       }
   705    705       sqlite3ValueFree(pVal);

Changes to src/btree.c.

  6171   6171     ** Use a call to btreeParseCellPtr() to verify that the values above
  6172   6172     ** were computed correctly.
  6173   6173     */
  6174   6174   #if SQLITE_DEBUG
  6175   6175     {
  6176   6176       CellInfo info;
  6177   6177       pPage->xParseCell(pPage, pCell, &info);
  6178         -    assert( nHeader=(int)(info.pPayload - pCell) );
         6178  +    assert( nHeader==(int)(info.pPayload - pCell) );
  6179   6179       assert( info.nKey==nKey );
  6180   6180       assert( *pnSize == info.nSize );
  6181   6181       assert( spaceLeft == info.nLocal );
  6182   6182     }
  6183   6183   #endif
  6184   6184   
  6185   6185     /* Write the payload into the local Cell and any extra into overflow pages */
................................................................................
  7830   7830   */
  7831   7831   static int balance(BtCursor *pCur){
  7832   7832     int rc = SQLITE_OK;
  7833   7833     const int nMin = pCur->pBt->usableSize * 2 / 3;
  7834   7834     u8 aBalanceQuickSpace[13];
  7835   7835     u8 *pFree = 0;
  7836   7836   
  7837         -  TESTONLY( int balance_quick_called = 0 );
  7838         -  TESTONLY( int balance_deeper_called = 0 );
         7837  +  VVA_ONLY( int balance_quick_called = 0 );
         7838  +  VVA_ONLY( int balance_deeper_called = 0 );
  7839   7839   
  7840   7840     do {
  7841   7841       int iPage = pCur->iPage;
  7842   7842       MemPage *pPage = pCur->apPage[iPage];
  7843   7843   
  7844   7844       if( iPage==0 ){
  7845   7845         if( pPage->nOverflow ){
  7846   7846           /* The root page of the b-tree is overfull. In this case call the
  7847   7847           ** balance_deeper() function to create a new child for the root-page
  7848   7848           ** and copy the current contents of the root-page to it. The
  7849   7849           ** next iteration of the do-loop will balance the child page.
  7850   7850           */ 
  7851         -        assert( (balance_deeper_called++)==0 );
         7851  +        assert( balance_deeper_called==0 );
         7852  +        VVA_ONLY( balance_deeper_called++ );
  7852   7853           rc = balance_deeper(pPage, &pCur->apPage[1]);
  7853   7854           if( rc==SQLITE_OK ){
  7854   7855             pCur->iPage = 1;
  7855   7856             pCur->aiIdx[0] = 0;
  7856   7857             pCur->aiIdx[1] = 0;
  7857   7858             assert( pCur->apPage[1]->nOverflow );
  7858   7859           }
................................................................................
  7883   7884             ** buffer. 
  7884   7885             **
  7885   7886             ** The purpose of the following assert() is to check that only a
  7886   7887             ** single call to balance_quick() is made for each call to this
  7887   7888             ** function. If this were not verified, a subtle bug involving reuse
  7888   7889             ** of the aBalanceQuickSpace[] might sneak in.
  7889   7890             */
  7890         -          assert( (balance_quick_called++)==0 );
         7891  +          assert( balance_quick_called==0 ); 
         7892  +          VVA_ONLY( balance_quick_called++ );
  7891   7893             rc = balance_quick(pParent, pPage, aBalanceQuickSpace);
  7892   7894           }else
  7893   7895   #endif
  7894   7896           {
  7895   7897             /* In this case, call balance_nonroot() to redistribute cells
  7896   7898             ** between pPage and up to 2 of its sibling pages. This involves
  7897   7899             ** modifying the contents of pParent, which may cause pParent to
................................................................................
  9350   9352     BtShared *pBt = p->pBt;
  9351   9353     int savedDbFlags = pBt->db->flags;
  9352   9354     char zErr[100];
  9353   9355     VVA_ONLY( int nRef );
  9354   9356   
  9355   9357     sqlite3BtreeEnter(p);
  9356   9358     assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE );
  9357         -  assert( (nRef = sqlite3PagerRefcount(pBt->pPager))>=0 );
         9359  +  VVA_ONLY( nRef = sqlite3PagerRefcount(pBt->pPager) );
         9360  +  assert( nRef>=0 );
  9358   9361     sCheck.pBt = pBt;
  9359   9362     sCheck.pPager = pBt->pPager;
  9360   9363     sCheck.nPage = btreePagecount(sCheck.pBt);
  9361   9364     sCheck.mxErr = mxErr;
  9362   9365     sCheck.nErr = 0;
  9363   9366     sCheck.mallocFailed = 0;
  9364   9367     sCheck.zPfx = 0;

Changes to src/build.c.

  1950   1950         sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb);
  1951   1951         sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG);
  1952   1952         pParse->nTab = 2;
  1953   1953         addrTop = sqlite3VdbeCurrentAddr(v) + 1;
  1954   1954         sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop);
  1955   1955         sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield);
  1956   1956         sqlite3Select(pParse, pSelect, &dest);
  1957         -      sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield);
         1957  +      sqlite3VdbeEndCoroutine(v, regYield);
  1958   1958         sqlite3VdbeJumpHere(v, addrTop - 1);
  1959   1959         if( pParse->nErr ) return;
  1960   1960         pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect);
  1961   1961         if( pSelTab==0 ) return;
  1962   1962         assert( p->aCol==0 );
  1963   1963         p->nCol = pSelTab->nCol;
  1964   1964         p->aCol = pSelTab->aCol;

Changes to src/insert.c.

   681    681       sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop);
   682    682       sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield);
   683    683       dest.iSdst = bIdListInOrder ? regData : 0;
   684    684       dest.nSdst = pTab->nCol;
   685    685       rc = sqlite3Select(pParse, pSelect, &dest);
   686    686       regFromSelect = dest.iSdst;
   687    687       if( rc || db->mallocFailed || pParse->nErr ) goto insert_cleanup;
   688         -    sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield);
          688  +    sqlite3VdbeEndCoroutine(v, regYield);
   689    689       sqlite3VdbeJumpHere(v, addrTop - 1);                       /* label B: */
   690    690       assert( pSelect->pEList );
   691    691       nColumn = pSelect->pEList->nExpr;
   692    692   
   693    693       /* Set useTempTable to TRUE if the result of the SELECT statement
   694    694       ** should be written into a temporary table (template 4).  Set to
   695    695       ** FALSE if each output row of the SELECT can be written directly into

Changes to src/main.c.

  3690   3690       ** assert() is disabled, then the return value is zero.  If X is
  3691   3691       ** false and assert() is enabled, then the assertion fires and the
  3692   3692       ** process aborts.  If X is false and assert() is disabled, then the
  3693   3693       ** return value is zero.
  3694   3694       */
  3695   3695       case SQLITE_TESTCTRL_ASSERT: {
  3696   3696         volatile int x = 0;
  3697         -      assert( (x = va_arg(ap,int))!=0 );
         3697  +      assert( /*side-effects-ok*/ (x = va_arg(ap,int))!=0 );
  3698   3698         rc = x;
  3699   3699         break;
  3700   3700       }
  3701   3701   
  3702   3702   
  3703   3703       /*
  3704   3704       **  sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, int X)

Changes to src/malloc.c.

   622    622     void *p;
   623    623     if( db ) return sqlite3DbMallocRawNN(db, n);
   624    624     p = sqlite3Malloc(n);
   625    625     sqlite3MemdebugSetType(p, MEMTYPE_HEAP);
   626    626     return p;
   627    627   }
   628    628   void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){
          629  +#ifndef SQLITE_OMIT_LOOKASIDE
          630  +  LookasideSlot *pBuf;
   629    631     assert( db!=0 );
   630    632     assert( sqlite3_mutex_held(db->mutex) );
   631    633     assert( db->pnBytesFreed==0 );
   632         -#ifndef SQLITE_OMIT_LOOKASIDE
   633         -  LookasideSlot *pBuf;
   634    634     if( db->lookaside.bDisable==0 ){
   635    635       assert( db->mallocFailed==0 );
   636    636       if( n>db->lookaside.sz ){
   637    637         db->lookaside.anStat[1]++;
   638    638       }else if( (pBuf = db->lookaside.pFree)==0 ){
   639    639         db->lookaside.anStat[2]++;
   640    640       }else{
................................................................................
   646    646         }
   647    647         return (void*)pBuf;
   648    648       }
   649    649     }else if( db->mallocFailed ){
   650    650       return 0;
   651    651     }
   652    652   #else
          653  +  assert( db!=0 );
          654  +  assert( sqlite3_mutex_held(db->mutex) );
          655  +  assert( db->pnBytesFreed==0 );
   653    656     if( db->mallocFailed ){
   654    657       return 0;
   655    658     }
   656    659   #endif
   657    660     return dbMallocRawFinish(db, n);
   658    661   }
   659    662   

Changes to src/select.c.

  2948   2948     */
  2949   2949     addrSelectA = sqlite3VdbeCurrentAddr(v) + 1;
  2950   2950     addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA);
  2951   2951     VdbeComment((v, "left SELECT"));
  2952   2952     pPrior->iLimit = regLimitA;
  2953   2953     explainSetInteger(iSub1, pParse->iNextSelectId);
  2954   2954     sqlite3Select(pParse, pPrior, &destA);
  2955         -  sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrA);
         2955  +  sqlite3VdbeEndCoroutine(v, regAddrA);
  2956   2956     sqlite3VdbeJumpHere(v, addr1);
  2957   2957   
  2958   2958     /* Generate a coroutine to evaluate the SELECT statement on 
  2959   2959     ** the right - the "B" select
  2960   2960     */
  2961   2961     addrSelectB = sqlite3VdbeCurrentAddr(v) + 1;
  2962   2962     addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB);
................................................................................
  2965   2965     savedOffset = p->iOffset;
  2966   2966     p->iLimit = regLimitB;
  2967   2967     p->iOffset = 0;  
  2968   2968     explainSetInteger(iSub2, pParse->iNextSelectId);
  2969   2969     sqlite3Select(pParse, p, &destB);
  2970   2970     p->iLimit = savedLimit;
  2971   2971     p->iOffset = savedOffset;
  2972         -  sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrB);
         2972  +  sqlite3VdbeEndCoroutine(v, regAddrB);
  2973   2973   
  2974   2974     /* Generate a subroutine that outputs the current row of the A
  2975   2975     ** select as the next output row of the compound select.
  2976   2976     */
  2977   2977     VdbeNoopComment((v, "Output routine for A"));
  2978   2978     addrOutA = generateOutputSubroutine(pParse,
  2979   2979                    p, &destA, pDest, regOutA,
................................................................................
  4986   4986         pItem->addrFillSub = addrTop;
  4987   4987         sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn);
  4988   4988         explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId);
  4989   4989         sqlite3Select(pParse, pSub, &dest);
  4990   4990         pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow);
  4991   4991         pItem->fg.viaCoroutine = 1;
  4992   4992         pItem->regResult = dest.iSdst;
  4993         -      sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn);
         4993  +      sqlite3VdbeEndCoroutine(v, pItem->regReturn);
  4994   4994         sqlite3VdbeJumpHere(v, addrTop-1);
  4995   4995         sqlite3ClearTempRegCache(pParse);
  4996   4996       }else{
  4997   4997         /* Generate a subroutine that will fill an ephemeral table with
  4998   4998         ** the content of this subquery.  pItem->addrFillSub will point
  4999   4999         ** to the address of the generated subroutine.  pItem->regReturn
  5000   5000         ** is a register allocated to hold the subroutine return address

Changes to src/test_config.c.

   365    365   #endif
   366    366   
   367    367   #ifdef SQLITE_ENABLE_FTS3
   368    368     Tcl_SetVar2(interp, "sqlite_options", "fts3", "1", TCL_GLOBAL_ONLY);
   369    369   #else
   370    370     Tcl_SetVar2(interp, "sqlite_options", "fts3", "0", TCL_GLOBAL_ONLY);
   371    371   #endif
          372  +
          373  +#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
          374  +  Tcl_SetVar2(interp, "sqlite_options", "fts3_tokenizer", "1", TCL_GLOBAL_ONLY);
          375  +#else
          376  +  Tcl_SetVar2(interp, "sqlite_options", "fts3_tokenizer", "0", TCL_GLOBAL_ONLY);
          377  +#endif
   372    378   
   373    379   #ifdef SQLITE_ENABLE_FTS5
   374    380     Tcl_SetVar2(interp, "sqlite_options", "fts5", "1", TCL_GLOBAL_ONLY);
   375    381   #else
   376    382     Tcl_SetVar2(interp, "sqlite_options", "fts5", "0", TCL_GLOBAL_ONLY);
   377    383   #endif
   378    384   

Changes to src/tokenize.c.

    14     14   ** This file contains C code that splits an SQL input string up into
    15     15   ** individual tokens and sends those tokens one-by-one over to the
    16     16   ** parser for analysis.
    17     17   */
    18     18   #include "sqliteInt.h"
    19     19   #include <stdlib.h>
    20     20   
           21  +/* Character classes for tokenizing
           22  +**
           23  +** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented
           24  +** using a lookup table, whereas a switch() directly on c uses a binary search.
           25  +** The lookup table is much faster.  To maximize speed, and to ensure that
           26  +** a lookup table is used, all of the classes need to be small integers and
           27  +** all of them need to be used within the switch.
           28  +*/
           29  +#define CC_X          0    /* The letter 'x', or start of BLOB literal */
           30  +#define CC_KYWD       1    /* Alphabetics or '_'.  Usable in a keyword */
           31  +#define CC_ID         2    /* unicode characters usable in IDs */
           32  +#define CC_DIGIT      3    /* Digits */
           33  +#define CC_DOLLAR     4    /* '$' */
           34  +#define CC_VARALPHA   5    /* '@', '#', ':'.  Alphabetic SQL variables */
           35  +#define CC_VARNUM     6    /* '?'.  Numeric SQL variables */
           36  +#define CC_SPACE      7    /* Space characters */
           37  +#define CC_QUOTE      8    /* '"', '\'', or '`'.  String literals, quoted ids */
           38  +#define CC_QUOTE2     9    /* '['.   [...] style quoted ids */
           39  +#define CC_PIPE      10    /* '|'.   Bitwise OR or concatenate */
           40  +#define CC_MINUS     11    /* '-'.  Minus or SQL-style comment */
           41  +#define CC_LT        12    /* '<'.  Part of < or <= or <> */
           42  +#define CC_GT        13    /* '>'.  Part of > or >= */
           43  +#define CC_EQ        14    /* '='.  Part of = or == */
           44  +#define CC_BANG      15    /* '!'.  Part of != */
           45  +#define CC_SLASH     16    /* '/'.  / or c-style comment */
           46  +#define CC_LP        17    /* '(' */
           47  +#define CC_RP        18    /* ')' */
           48  +#define CC_SEMI      19    /* ';' */
           49  +#define CC_PLUS      20    /* '+' */
           50  +#define CC_STAR      21    /* '*' */
           51  +#define CC_PERCENT   22    /* '%' */
           52  +#define CC_COMMA     23    /* ',' */
           53  +#define CC_AND       24    /* '&' */
           54  +#define CC_TILDA     25    /* '~' */
           55  +#define CC_DOT       26    /* '.' */
           56  +#define CC_ILLEGAL   27    /* Illegal character */
           57  +
           58  +static const unsigned char aiClass[] = {
           59  +#ifdef SQLITE_ASCII
           60  +/*         x0  x1  x2  x3  x4  x5  x6  x7  x8  x9  xa  xb  xc  xd  xe  xf */
           61  +/* 0x */   27, 27, 27, 27, 27, 27, 27, 27, 27,  7,  7, 27,  7,  7, 27, 27,
           62  +/* 1x */   27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
           63  +/* 2x */    7, 15,  8,  5,  4, 22, 24,  8, 17, 18, 21, 20, 23, 11, 26, 16,
           64  +/* 3x */    3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  5, 19, 12, 14, 13,  6,
           65  +/* 4x */    5,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
           66  +/* 5x */    1,  1,  1,  1,  1,  1,  1,  1,  0,  1,  1,  9, 27, 27, 27,  1,
           67  +/* 6x */    8,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
           68  +/* 7x */    1,  1,  1,  1,  1,  1,  1,  1,  0,  1,  1, 27, 10, 27, 25, 27,
           69  +/* 8x */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
           70  +/* 9x */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
           71  +/* Ax */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
           72  +/* Bx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
           73  +/* Cx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
           74  +/* Dx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
           75  +/* Ex */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
           76  +/* Fx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2
           77  +#endif
           78  +#ifdef SQLITE_EBCDIC
           79  +/*         x0  x1  x2  x3  x4  x5  x6  x7  x8  x9  xa  xb  xc  xd  xe  xf */
           80  +/* 0x */   27, 27, 27, 27, 27,  7, 27, 27, 27, 27, 27, 27,  7,  7, 27, 27,
           81  +/* 1x */   27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
           82  +/* 2x */   27, 27, 27, 27, 27,  7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
           83  +/* 3x */   27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
           84  +/* 4x */    7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 12, 17, 20, 10,
           85  +/* 5x */   24, 27, 27, 27, 27, 27, 27, 27, 27, 27, 15,  4, 21, 18, 19, 27,
           86  +/* 6x */   11, 16, 27, 27, 27, 27, 27, 27, 27, 27, 27, 23, 22,  1, 13,  7,
           87  +/* 7x */   27, 27, 27, 27, 27, 27, 27, 27, 27,  8,  5,  5,  5,  8, 14,  8,
           88  +/* 8x */   27,  1,  1,  1,  1,  1,  1,  1,  1,  1, 27, 27, 27, 27, 27, 27,
           89  +/* 9x */   27,  1,  1,  1,  1,  1,  1,  1,  1,  1, 27, 27, 27, 27, 27, 27,
           90  +/* 9x */   25,  1,  1,  1,  1,  1,  1,  0,  1,  1, 27, 27, 27, 27, 27, 27,
           91  +/* Bx */   27, 27, 27, 27, 27, 27, 27, 27, 27, 27,  9, 27, 27, 27, 27, 27,
           92  +/* Cx */   27,  1,  1,  1,  1,  1,  1,  1,  1,  1, 27, 27, 27, 27, 27, 27,
           93  +/* Dx */   27,  1,  1,  1,  1,  1,  1,  1,  1,  1, 27, 27, 27, 27, 27, 27,
           94  +/* Ex */   27, 27,  1,  1,  1,  1,  1,  0,  1,  1, 27, 27, 27, 27, 27, 27,
           95  +/* Fx */    3,  3,  3,  3,  3,  3,  3,  3,  3,  3, 27, 27, 27, 27, 27, 27,
           96  +#endif
           97  +};
           98  +
    21     99   /*
    22         -** The charMap() macro maps alphabetic characters into their
          100  +** The charMap() macro maps alphabetic characters (only) into their
    23    101   ** lower-case ASCII equivalent.  On ASCII machines, this is just
    24    102   ** an upper-to-lower case map.  On EBCDIC machines we also need
    25         -** to adjust the encoding.  Only alphabetic characters and underscores
    26         -** need to be translated.
          103  +** to adjust the encoding.  The mapping is only valid for alphabetics
          104  +** which are the only characters for which this feature is used. 
          105  +**
          106  +** Used by keywordhash.h
    27    107   */
    28    108   #ifdef SQLITE_ASCII
    29    109   # define charMap(X) sqlite3UpperToLower[(unsigned char)X]
    30    110   #endif
    31    111   #ifdef SQLITE_EBCDIC
    32    112   # define charMap(X) ebcdicToAscii[(unsigned char)X]
    33    113   const unsigned char ebcdicToAscii[] = {
................................................................................
    53    133   
    54    134   /*
    55    135   ** The sqlite3KeywordCode function looks up an identifier to determine if
    56    136   ** it is a keyword.  If it is a keyword, the token code of that keyword is 
    57    137   ** returned.  If the input is not a keyword, TK_ID is returned.
    58    138   **
    59    139   ** The implementation of this routine was generated by a program,
    60         -** mkkeywordhash.h, located in the tool subdirectory of the distribution.
          140  +** mkkeywordhash.c, located in the tool subdirectory of the distribution.
    61    141   ** The output of the mkkeywordhash.c program is written into a file
    62    142   ** named keywordhash.h and then included into this source file by
    63    143   ** the #include below.
    64    144   */
    65    145   #include "keywordhash.h"
    66    146   
    67    147   
................................................................................
   106    186   /* Make the IdChar function accessible from ctime.c */
   107    187   #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS
   108    188   int sqlite3IsIdChar(u8 c){ return IdChar(c); }
   109    189   #endif
   110    190   
   111    191   
   112    192   /*
   113         -** Return the length of the token that begins at z[0]. 
          193  +** Return the length (in bytes) of the token that begins at z[0]. 
   114    194   ** Store the token type in *tokenType before returning.
   115    195   */
   116    196   int sqlite3GetToken(const unsigned char *z, int *tokenType){
   117    197     int i, c;
   118         -  switch( *z ){
   119         -    case ' ': case '\t': case '\n': case '\f': case '\r': {
          198  +  switch( aiClass[*z] ){  /* Switch on the character-class of the first byte
          199  +                          ** of the token. See the comment on the CC_ defines
          200  +                          ** above. */
          201  +    case CC_SPACE: {
   120    202         testcase( z[0]==' ' );
   121    203         testcase( z[0]=='\t' );
   122    204         testcase( z[0]=='\n' );
   123    205         testcase( z[0]=='\f' );
   124    206         testcase( z[0]=='\r' );
   125    207         for(i=1; sqlite3Isspace(z[i]); i++){}
   126    208         *tokenType = TK_SPACE;
   127    209         return i;
   128    210       }
   129         -    case '-': {
          211  +    case CC_MINUS: {
   130    212         if( z[1]=='-' ){
   131    213           for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
   132    214           *tokenType = TK_SPACE;   /* IMP: R-22934-25134 */
   133    215           return i;
   134    216         }
   135    217         *tokenType = TK_MINUS;
   136    218         return 1;
   137    219       }
   138         -    case '(': {
          220  +    case CC_LP: {
   139    221         *tokenType = TK_LP;
   140    222         return 1;
   141    223       }
   142         -    case ')': {
          224  +    case CC_RP: {
   143    225         *tokenType = TK_RP;
   144    226         return 1;
   145    227       }
   146         -    case ';': {
          228  +    case CC_SEMI: {
   147    229         *tokenType = TK_SEMI;
   148    230         return 1;
   149    231       }
   150         -    case '+': {
          232  +    case CC_PLUS: {
   151    233         *tokenType = TK_PLUS;
   152    234         return 1;
   153    235       }
   154         -    case '*': {
          236  +    case CC_STAR: {
   155    237         *tokenType = TK_STAR;
   156    238         return 1;
   157    239       }
   158         -    case '/': {
          240  +    case CC_SLASH: {
   159    241         if( z[1]!='*' || z[2]==0 ){
   160    242           *tokenType = TK_SLASH;
   161    243           return 1;
   162    244         }
   163    245         for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
   164    246         if( c ) i++;
   165    247         *tokenType = TK_SPACE;   /* IMP: R-22934-25134 */
   166    248         return i;
   167    249       }
   168         -    case '%': {
          250  +    case CC_PERCENT: {
   169    251         *tokenType = TK_REM;
   170    252         return 1;
   171    253       }
   172         -    case '=': {
          254  +    case CC_EQ: {
   173    255         *tokenType = TK_EQ;
   174    256         return 1 + (z[1]=='=');
   175    257       }
   176         -    case '<': {
          258  +    case CC_LT: {
   177    259         if( (c=z[1])=='=' ){
   178    260           *tokenType = TK_LE;
   179    261           return 2;
   180    262         }else if( c=='>' ){
   181    263           *tokenType = TK_NE;
   182    264           return 2;
   183    265         }else if( c=='<' ){
................................................................................
   184    266           *tokenType = TK_LSHIFT;
   185    267           return 2;
   186    268         }else{
   187    269           *tokenType = TK_LT;
   188    270           return 1;
   189    271         }
   190    272       }
   191         -    case '>': {
          273  +    case CC_GT: {
   192    274         if( (c=z[1])=='=' ){
   193    275           *tokenType = TK_GE;
   194    276           return 2;
   195    277         }else if( c=='>' ){
   196    278           *tokenType = TK_RSHIFT;
   197    279           return 2;
   198    280         }else{
   199    281           *tokenType = TK_GT;
   200    282           return 1;
   201    283         }
   202    284       }
   203         -    case '!': {
          285  +    case CC_BANG: {
   204    286         if( z[1]!='=' ){
   205    287           *tokenType = TK_ILLEGAL;
   206    288           return 2;
   207    289         }else{
   208    290           *tokenType = TK_NE;
   209    291           return 2;
   210    292         }
   211    293       }
   212         -    case '|': {
          294  +    case CC_PIPE: {
   213    295         if( z[1]!='|' ){
   214    296           *tokenType = TK_BITOR;
   215    297           return 1;
   216    298         }else{
   217    299           *tokenType = TK_CONCAT;
   218    300           return 2;
   219    301         }
   220    302       }
   221         -    case ',': {
          303  +    case CC_COMMA: {
   222    304         *tokenType = TK_COMMA;
   223    305         return 1;
   224    306       }
   225         -    case '&': {
          307  +    case CC_AND: {
   226    308         *tokenType = TK_BITAND;
   227    309         return 1;
   228    310       }
   229         -    case '~': {
          311  +    case CC_TILDA: {
   230    312         *tokenType = TK_BITNOT;
   231    313         return 1;
   232    314       }
   233         -    case '`':
   234         -    case '\'':
   235         -    case '"': {
          315  +    case CC_QUOTE: {
   236    316         int delim = z[0];
   237    317         testcase( delim=='`' );
   238    318         testcase( delim=='\'' );
   239    319         testcase( delim=='"' );
   240    320         for(i=1; (c=z[i])!=0; i++){
   241    321           if( c==delim ){
   242    322             if( z[i+1]==delim ){
................................................................................
   253    333           *tokenType = TK_ID;
   254    334           return i+1;
   255    335         }else{
   256    336           *tokenType = TK_ILLEGAL;
   257    337           return i;
   258    338         }
   259    339       }
   260         -    case '.': {
          340  +    case CC_DOT: {
   261    341   #ifndef SQLITE_OMIT_FLOATING_POINT
   262    342         if( !sqlite3Isdigit(z[1]) )
   263    343   #endif
   264    344         {
   265    345           *tokenType = TK_DOT;
   266    346           return 1;
   267    347         }
   268    348         /* If the next character is a digit, this is a floating point
   269    349         ** number that begins with ".".  Fall thru into the next case */
   270    350       }
   271         -    case '0': case '1': case '2': case '3': case '4':
   272         -    case '5': case '6': case '7': case '8': case '9': {
          351  +    case CC_DIGIT: {
   273    352         testcase( z[0]=='0' );  testcase( z[0]=='1' );  testcase( z[0]=='2' );
   274    353         testcase( z[0]=='3' );  testcase( z[0]=='4' );  testcase( z[0]=='5' );
   275    354         testcase( z[0]=='6' );  testcase( z[0]=='7' );  testcase( z[0]=='8' );
   276    355         testcase( z[0]=='9' );
   277    356         *tokenType = TK_INTEGER;
   278    357   #ifndef SQLITE_OMIT_HEX_INTEGER
   279    358         if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){
................................................................................
   300    379   #endif
   301    380         while( IdChar(z[i]) ){
   302    381           *tokenType = TK_ILLEGAL;
   303    382           i++;
   304    383         }
   305    384         return i;
   306    385       }
   307         -    case '[': {
          386  +    case CC_QUOTE2: {
   308    387         for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
   309    388         *tokenType = c==']' ? TK_ID : TK_ILLEGAL;
   310    389         return i;
   311    390       }
   312         -    case '?': {
          391  +    case CC_VARNUM: {
   313    392         *tokenType = TK_VARIABLE;
   314    393         for(i=1; sqlite3Isdigit(z[i]); i++){}
   315    394         return i;
   316    395       }
   317         -#ifndef SQLITE_OMIT_TCL_VARIABLE
   318         -    case '$':
   319         -#endif
   320         -    case '@':  /* For compatibility with MS SQL Server */
   321         -    case '#':
   322         -    case ':': {
          396  +    case CC_DOLLAR:
          397  +    case CC_VARALPHA: {
   323    398         int n = 0;
   324    399         testcase( z[0]=='$' );  testcase( z[0]=='@' );
   325    400         testcase( z[0]==':' );  testcase( z[0]=='#' );
   326    401         *tokenType = TK_VARIABLE;
   327    402         for(i=1; (c=z[i])!=0; i++){
   328    403           if( IdChar(c) ){
   329    404             n++;
................................................................................
   344    419           }else{
   345    420             break;
   346    421           }
   347    422         }
   348    423         if( n==0 ) *tokenType = TK_ILLEGAL;
   349    424         return i;
   350    425       }
          426  +    case CC_KYWD: {
          427  +      for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
          428  +      if( IdChar(z[i]) ){
          429  +        /* This token started out using characters that can appear in keywords,
          430  +        ** but z[i] is a character not allowed within keywords, so this must
          431  +        ** be an identifier instead */
          432  +        i++;
          433  +        break;
          434  +      }
          435  +      *tokenType = TK_ID;
          436  +      return keywordCode((char*)z, i, tokenType);
          437  +    }
   351    438   #ifndef SQLITE_OMIT_BLOB_LITERAL
   352         -    case 'x': case 'X': {
          439  +    case CC_X: {
   353    440         testcase( z[0]=='x' ); testcase( z[0]=='X' );
   354    441         if( z[1]=='\'' ){
   355    442           *tokenType = TK_BLOB;
   356    443           for(i=2; sqlite3Isxdigit(z[i]); i++){}
   357    444           if( z[i]!='\'' || i%2 ){
   358    445             *tokenType = TK_ILLEGAL;
   359    446             while( z[i] && z[i]!='\'' ){ i++; }
   360    447           }
   361    448           if( z[i] ) i++;
   362    449           return i;
   363    450         }
   364         -      /* Otherwise fall through to the next case */
          451  +      /* If it is not a BLOB literal, then it must be an ID, since no
          452  +      ** SQL keywords start with the letter 'x'.  Fall through */
   365    453       }
   366    454   #endif
          455  +    case CC_ID: {
          456  +      i = 1;
          457  +      break;
          458  +    }
   367    459       default: {
   368         -      if( !IdChar(*z) ){
   369         -        break;
   370         -      }
   371         -      for(i=1; IdChar(z[i]); i++){}
   372         -      *tokenType = TK_ID;
   373         -      return keywordCode((char*)z, i, tokenType);
          460  +      *tokenType = TK_ILLEGAL;
          461  +      return 1;
   374    462       }
   375    463     }
   376         -  *tokenType = TK_ILLEGAL;
   377         -  return 1;
          464  +  while( IdChar(z[i]) ){ i++; }
          465  +  *tokenType = TK_ID;
          466  +  return i;
   378    467   }
   379    468   
   380    469   /*
   381    470   ** Run the parser on the given SQL string.  The parser structure is
   382    471   ** passed in.  An SQLITE_ status code is returned.  If an error occurs
   383    472   ** then an and attempt is made to write an error message into 
   384    473   ** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that

Changes to src/vdbe.h.

   176    176   int sqlite3VdbeGoto(Vdbe*,int);
   177    177   int sqlite3VdbeLoadString(Vdbe*,int,const char*);
   178    178   void sqlite3VdbeMultiLoad(Vdbe*,int,const char*,...);
   179    179   int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int);
   180    180   int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int);
   181    181   int sqlite3VdbeAddOp4Dup8(Vdbe*,int,int,int,int,const u8*,int);
   182    182   int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int);
          183  +void sqlite3VdbeEndCoroutine(Vdbe*,int);
   183    184   #if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS)
   184    185     void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N);
   185    186   #else
   186    187   # define sqlite3VdbeVerifyNoMallocRequired(A,B)
   187    188   #endif
   188    189   VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno);
   189    190   void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*);

Changes to src/vdbeaux.c.

   319    319     int p3,             /* The P3 operand */
   320    320     int p4              /* The P4 operand as an integer */
   321    321   ){
   322    322     int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3);
   323    323     sqlite3VdbeChangeP4(p, addr, SQLITE_INT_TO_PTR(p4), P4_INT32);
   324    324     return addr;
   325    325   }
          326  +
          327  +/* Insert the end of a co-routine
          328  +*/
          329  +void sqlite3VdbeEndCoroutine(Vdbe *v, int regYield){
          330  +  sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield);
          331  +
          332  +  /* Clear the temporary register cache, thereby ensuring that each
          333  +  ** co-routine has its own independent set of registers, because co-routines
          334  +  ** might expect their registers to be preserved across an OP_Yield, and
          335  +  ** that could cause problems if two or more co-routines are using the same
          336  +  ** temporary register.
          337  +  */
          338  +  v->pParse->nTempReg = 0;
          339  +  v->pParse->nRangeReg = 0;
          340  +}
   326    341   
   327    342   /*
   328    343   ** Create a new symbolic label for an instruction that has yet to be
   329    344   ** coded.  The symbolic label is really just a negative number.  The
   330    345   ** label can be used as the P2 value of an operation.  Later, when
   331    346   ** the label is resolved to a specific address, the VDBE will scan
   332    347   ** through its operation list and change all values of P2 which match

Changes to test/analyze9.test.

  1240   1240     SELECT * FROM t1 WHERE x='B' AND y>25 AND z=?;
  1241   1241   } {
  1242   1242     0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x=? AND y>?)}
  1243   1243   }
  1244   1244   
  1245   1245   
  1246   1246   finish_test
  1247         -
  1248         -
  1249         -

Changes to test/analyzeB.test.

   676    676       set val $i
   677    677       do_execsql_test 20.3.$i {
   678    678         SELECT count(*) FROM sqlite_stat3 WHERE sample=$val
   679    679       } {1}
   680    680   }
   681    681   
   682    682   finish_test
   683         -

Changes to test/analyzeD.test.

   110    110   do_eqp_test 1.8 {
   111    111     SELECT * FROM t1 WHERE a=13 AND c=150;
   112    112   } {
   113    113     0 0 0 {SEARCH TABLE t1 USING INDEX t1_c (c=?)}
   114    114   }
   115    115   
   116    116   finish_test
   117         -

Changes to test/cacheflush.test.

   317    317       SELECT a FROM ta;
   318    318       SELECT b FROM tb;
   319    319     }
   320    320   } {a b}
   321    321   
   322    322   test_restore_config_pagecache
   323    323   finish_test
   324         -

Changes to test/cffault.test.

   152    152   } -test {
   153    153     faultsim_test_result {0 {1 1 3 3 5 5 7 7 9 9}} {1 {disk I/O error}}
   154    154     catchsql ROLLBACK
   155    155     faultsim_integrity_check
   156    156   }
   157    157   
   158    158   finish_test
   159         -

Changes to test/corruptH.test.

   170    170         DELETE FROM t2 WHERE c=1;
   171    171       }
   172    172     }
   173    173     } msg] $msg
   174    174   } {1 {database disk image is malformed}}
   175    175   
   176    176   finish_test
   177         -

Changes to test/corruptI.test.

   202    202     INSERT INTO t1 VALUES(zeroblob(300));
   203    203     INSERT INTO t1 VALUES(zeroblob(600));
   204    204   } {}
   205    205   do_test 6.1 {
   206    206     db close
   207    207     hexio_write test.db 616 8FFFFFFF7F02
   208    208     sqlite3 db test.db
   209         -  breakpoint
   210    209     execsql { DELETE FROM t1 WHERE rowid=2 }
   211    210   } {}
   212    211   
   213    212   #-------------------------------------------------------------------------
   214    213   # See what happens if the sqlite_master entry associated with a PRIMARY
   215    214   # KEY or UNIQUE index is removed. 
   216    215   #

Changes to test/cost.test.

   283    283       SELECT rowid FROM t6 WHERE likelihood(a=0, 0.1) AND b='xyz' AND c=0
   284    284     } {
   285    285       0 0 0 {SEARCH TABLE t6 USING INDEX t6i1 (a=? AND b=?)}
   286    286     }
   287    287   }
   288    288   
   289    289   finish_test
   290         -
   291         -
   292         -

Changes to test/e_blobbytes.test.

    68     68   do_test 2.1 {
    69     69     sqlite3_blob_open db main q1 s 86 1 B
    70     70     list [catch { sqlite3_blob_write $B 86 "1" 1 } msg] $msg
    71     71   } {1 SQLITE_ERROR}
    72     72   sqlite3_blob_close $B
    73     73   
    74     74   finish_test
    75         -
    76         -

Changes to test/e_blobclose.test.

   164    164   # EVIDENCE-OF: R-25894-51060 Calling this routine with a null pointer
   165    165   # (such as would be returned by a failed call to sqlite3_blob_open()) is
   166    166   # a harmless no-op.
   167    167   #
   168    168   do_test 4.0 { sqlite3_blob_close 0 } {}
   169    169   
   170    170   finish_test
   171         -

Changes to test/e_blobopen.test.

   542    542   } [list \
   543    543       [string repeat [binary format c 1] 24] \
   544    544       [string repeat [binary format c 1] 45] \
   545    545   ]
   546    546   
   547    547   
   548    548   finish_test
   549         -

Changes to test/e_blobwrite.test.

   197    197   } {
   198    198     2 xyz ........................................
   199    199   }
   200    200   
   201    201   
   202    202   
   203    203   finish_test
   204         -

Changes to test/e_walckpt.test.

   747    747     db2 eval COMMIT
   748    748     wal_checkpoint_v2 db truncate
   749    749   } {0 0 0}
   750    750   
   751    751   
   752    752   
   753    753   finish_test
   754         -

Changes to test/fkey8.test.

    99     99       sqlite3_finalize $stmt
   100    100       set ret
   101    101     } $use_stmt
   102    102   }
   103    103   
   104    104   
   105    105   finish_test
   106         -

Changes to test/fordelete.test.

   203    203   } {1 2}
   204    204   do_execsql_test 5.3 {
   205    205     DELETE FROM t1 WHERE a = 2;
   206    206   } {}
   207    207   
   208    208   
   209    209   finish_test
   210         -

Changes to test/fts3atoken.test.

    20     20   
    21     21   # If SQLITE_ENABLE_FTS3 is defined, omit this file.
    22     22   ifcapable !fts3 {
    23     23     finish_test
    24     24     return
    25     25   }
    26     26   
    27         -set ::testprefix fts3token
           27  +set ::testprefix fts3atoken
    28     28   
    29     29   proc escape_string {str} {
    30     30     set out ""
    31     31     foreach char [split $str ""] {
    32     32       scan $char %c i
    33     33       if {$i<=127} {
    34     34         append out $char
................................................................................
    36     36         append out [format {\x%.4x} $i]
    37     37       }
    38     38     }
    39     39     set out
    40     40   }
    41     41   
    42     42   #--------------------------------------------------------------------------
    43         -# Test cases fts3token-1.* are the warm-body test for the SQL scalar
           43  +# Test cases fts3atoken-1.* are the warm-body test for the SQL scalar
    44     44   # function fts3_tokenizer(). The procedure is as follows:
    45     45   #
    46     46   #   1: Verify that there is no such fts3 tokenizer as 'blah'.
    47     47   #
    48     48   #   2: Query for the built-in tokenizer 'simple'. Insert a copy of the
    49     49   #      retrieved value as tokenizer 'blah'.
    50     50   #
................................................................................
    52     52   #      same as that retrieved for 'simple'.
    53     53   #
    54     54   #   4: Test that it is now possible to create an fts3 table using 
    55     55   #      tokenizer 'blah' (it was not possible in step 1).
    56     56   #
    57     57   #   5: Test that the table created to use tokenizer 'blah' is usable.
    58     58   #
    59         -do_test fts3token-1.1 {
    60         -  catchsql {
    61         -    CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
    62         -  }
    63         -} {1 {unknown tokenizer: blah}}
    64         -do_test fts3token-1.2 {
    65         -  execsql {
           59  +ifcapable fts3_tokenizer {
           60  +  do_test fts3atoken-1.1 {
           61  +    catchsql {
           62  +      CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
           63  +    }
           64  +  } {1 {unknown tokenizer: blah}}
           65  +  do_test fts3atoken-1.2 {
           66  +    execsql {
           67  +      SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL;
           68  +    }
           69  +  } {0}
           70  +  do_test fts3atoken-1.3 {
           71  +    execsql {
           72  +      SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple');
           73  +    }
           74  +  } {1}
           75  +  do_test fts3atoken-1.4 {
           76  +    catchsql {
           77  +      CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
           78  +    }
           79  +  } {0 {}}
           80  +  do_test fts3atoken-1.5 {
           81  +    execsql {
           82  +      INSERT INTO t1(content) VALUES('There was movement at the station');
           83  +      INSERT INTO t1(content) VALUES('For the word has passed around');
           84  +      INSERT INTO t1(content) VALUES('That the colt from ol regret had got');
           85  +      SELECT content FROM t1 WHERE content MATCH 'movement'
           86  +    }
           87  +  } {{There was movement at the station}}
           88  +} else {
           89  +  do_catchsql_test 1.6 {
    66     90       SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL;
    67         -  }
    68         -} {0}
    69         -do_test fts3token-1.3 {
    70         -  execsql {
    71         -    SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple');
    72         -  }
    73         -} {1}
    74         -do_test fts3token-1.4 {
    75         -  catchsql {
    76         -    CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
    77         -  }
    78         -} {0 {}}
    79         -do_test fts3token-1.5 {
    80         -  execsql {
    81         -    INSERT INTO t1(content) VALUES('There was movement at the station');
    82         -    INSERT INTO t1(content) VALUES('For the word has passed around');
    83         -    INSERT INTO t1(content) VALUES('That the colt from ol regret had got away');
    84         -    SELECT content FROM t1 WHERE content MATCH 'movement'
    85         -  }
    86         -} {{There was movement at the station}}
           91  +  } {1 {fts3tokenize: disabled - rebuild with -DSQLITE_ENABLE_FTS3_TOKENIZER}}
           92  +}
    87     93   
    88     94   #--------------------------------------------------------------------------
    89         -# Test cases fts3token-2.* test error cases in the scalar function based
           95  +# Test cases fts3atoken-2.* test error cases in the scalar function based
    90     96   # API for getting and setting tokenizers.
    91     97   #
    92         -do_test fts3token-2.1 {
           98  +do_test fts3atoken-2.1 {
    93     99     catchsql {
    94    100       SELECT fts3_tokenizer('nosuchtokenizer');
    95    101     }
    96    102   } {1 {unknown tokenizer: nosuchtokenizer}}
    97    103   
    98    104   #--------------------------------------------------------------------------
    99         -# Test cases fts3token-3.* test the three built-in tokenizers with a
          105  +# Test cases fts3atoken-3.* test the three built-in tokenizers with a
   100    106   # simple input string via the built-in test function. This is as much
   101    107   # to test the test function as the tokenizer implementations.
   102    108   #
   103         -do_test fts3token-3.1 {
          109  +do_test fts3atoken-3.1 {
   104    110     execsql {
   105    111       SELECT fts3_tokenizer_test('simple', 'I don''t see how');
   106    112     }
   107    113   } {{0 i I 1 don don 2 t t 3 see see 4 how how}}
   108         -do_test fts3token-3.2 {
          114  +do_test fts3atoken-3.2 {
   109    115     execsql {
   110    116       SELECT fts3_tokenizer_test('porter', 'I don''t see how');
   111    117     }
   112    118   } {{0 i I 1 don don 2 t t 3 see see 4 how how}}
   113    119   ifcapable icu {
   114         -  do_test fts3token-3.3 {
          120  +  do_test fts3atoken-3.3 {
   115    121       execsql {
   116    122         SELECT fts3_tokenizer_test('icu', 'I don''t see how');
   117    123       }
   118    124     } {{0 i I 1 don't don't 2 see see 3 how how}}
   119    125   }
   120    126   
   121    127   #--------------------------------------------------------------------------
   122         -# Test cases fts3token-4.* test the ICU tokenizer. In practice, this
          128  +# Test cases fts3atoken-4.* test the ICU tokenizer. In practice, this
   123    129   # tokenizer only has two modes - "thai" and "everybody else". Some other
   124    130   # Asian languages (Lao, Khmer etc.) require the same special treatment as 
   125    131   # Thai, but ICU doesn't support them yet.
   126    132   #
   127    133   ifcapable icu {
   128    134   
   129    135     proc do_icu_test {name locale input output} {
   130    136       set ::out [db eval { SELECT fts3_tokenizer_test('icu', $locale, $input) }]
   131    137       do_test $name {
   132    138         lindex $::out 0
   133    139       } $output
   134    140     }
   135    141     
   136         -  do_icu_test fts3token-4.1 en_US  {}   {}
   137         -  do_icu_test fts3token-4.2 en_US {Test cases fts3} [list \
          142  +  do_icu_test fts3atoken-4.1 en_US  {}   {}
          143  +  do_icu_test fts3atoken-4.2 en_US {Test cases fts3} [list \
   138    144       0 test Test 1 cases cases 2 fts3 fts3
   139    145     ]
   140    146   
   141    147     # The following test shows that ICU is smart enough to recognise
   142    148     # Thai chararacters, even when the locale is set to English/United 
   143    149     # States.
   144    150     #
   145    151     set input "\u0e2d\u0e30\u0e44\u0e23\u0e19\u0e30\u0e04\u0e23\u0e31\u0e1a"
   146    152     set output    "0 \u0e2d\u0e30\u0e44\u0e23 \u0e2d\u0e30\u0e44\u0e23 "
   147    153     append output "1 \u0e19\u0e30 \u0e19\u0e30 "
   148    154     append output "2 \u0e04\u0e23\u0e31\u0e1a \u0e04\u0e23\u0e31\u0e1a"
   149    155   
   150         -  do_icu_test fts3token-4.3 th_TH  $input $output
   151         -  do_icu_test fts3token-4.4 en_US  $input $output
          156  +  do_icu_test fts3atoken-4.3 th_TH  $input $output
          157  +  do_icu_test fts3atoken-4.4 en_US  $input $output
   152    158   
   153    159     # ICU handles an unknown locale by falling back to the default.
   154    160     # So this is not an error.
   155         -  do_icu_test fts3token-4.5 MiddleOfTheOcean  $input $output
          161  +  do_icu_test fts3atoken-4.5 MiddleOfTheOcean  $input $output
   156    162   
   157    163     set    longtoken "AReallyReallyLongTokenOneThatWillSurelyRequire"
   158    164     append longtoken "AReallocInTheIcuTokenizerCode"
   159    165   
   160    166     set    input "short tokens then "
   161    167     append input $longtoken
   162    168     set    output "0 short short "
   163    169     append output "1 tokens tokens "
   164    170     append output "2 then then "
   165    171     append output "3 [string tolower $longtoken] $longtoken"
   166    172   
   167         -  do_icu_test fts3token-4.6 MiddleOfTheOcean  $input $output
   168         -  do_icu_test fts3token-4.7 th_TH  $input $output
   169         -  do_icu_test fts3token-4.8 en_US  $input $output
          173  +  do_icu_test fts3atoken-4.6 MiddleOfTheOcean  $input $output
          174  +  do_icu_test fts3atoken-4.7 th_TH  $input $output
          175  +  do_icu_test fts3atoken-4.8 en_US  $input $output
   170    176   
   171    177     do_execsql_test 5.1 {
   172    178       CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US);
   173    179       insert into x1 (name) values (NULL);
   174    180       insert into x1 (name) values (NULL);
   175    181       delete from x1;
   176    182     }
................................................................................
   182    188   
   183    189     do_test 5.2 {
   184    190       set str [cp_to_str {19968 26085 32822 32645 27874 23433 20986}]
   185    191       execsql { INSERT INTO x1 VALUES($str) }
   186    192     } {}
   187    193   }
   188    194   
   189         -do_test fts3token-internal {
          195  +do_test fts3atoken-internal {
   190    196     execsql { SELECT fts3_tokenizer_internal_test() }
   191    197   } {ok}
   192    198   
   193    199   #-------------------------------------------------------------------------
   194    200   # Test empty tokenizer names.
   195    201   #
   196    202   do_catchsql_test 6.1.1 {
................................................................................
   202    208   do_catchsql_test 6.1.3 {
   203    209     CREATE VIRTUAL TABLE t3 USING fts4(tokenize="   ");
   204    210   } {1 {unknown tokenizer:    }}
   205    211   
   206    212   do_catchsql_test 6.2.1 {
   207    213     SELECT fts3_tokenizer(NULL);
   208    214   } {1 {unknown tokenizer: }}
   209         -do_catchsql_test 6.2.2 {
   210         -  SELECT fts3_tokenizer(NULL, X'1234567812345678');
   211         -} {1 {argument type mismatch}}
   212         -do_catchsql_test 6.2.3 {
   213         -  SELECT fts3_tokenizer(NULL, X'12345678');
   214         -} {1 {argument type mismatch}}
          215  +ifcapable fts3_tokenizer {
          216  +  do_catchsql_test 6.2.2 {
          217  +    SELECT fts3_tokenizer(NULL, X'1234567812345678');
          218  +  } {1 {argument type mismatch}}
          219  +  do_catchsql_test 6.2.3 {
          220  +    SELECT fts3_tokenizer(NULL, X'12345678');
          221  +  } {1 {argument type mismatch}}
          222  +}
   215    223   
   216    224   
   217    225   finish_test

Changes to test/fts3conf.test.

   208    208   }
   209    209   do_execsql_test 4.2.2 {
   210    210     SELECT * FROM t01 WHERE t01 MATCH 'b';
   211    211     INSERT INTO t01(t01) VALUES('integrity-check');
   212    212   } {}
   213    213   
   214    214   finish_test
   215         -

Changes to test/fts3expr4.test.

    75     75   do_simple_expr_test 3.8 { "abc"* } { PHRASE 3 0 abc }
    76     76   do_simple_expr_test 3.8 { "ab*c" } { PHRASE 3 0 ab+ c }
    77     77   
    78     78   do_icu_expr_test    3.9 { "ab*c" } { PHRASE 3 0 ab+ * c }
    79     79   do_icu_expr_test    3.10 { ab*c } { AND {PHRASE 3 0 ab+} {PHRASE 3 0 c}}
    80     80   
    81     81   finish_test
    82         -

Changes to test/fts3join.test.

    58     58   do_execsql_test 2.4 { SELECT * FROM ft3, ft2 WHERE y MATCH x; } {abc abc}
    59     59   
    60     60   do_catchsql_test 2.5 { 
    61     61     SELECT * FROM ft3, ft2 WHERE y MATCH x AND x MATCH y; 
    62     62   } {1 {unable to use function MATCH in the requested context}}
    63     63   
    64     64   finish_test
    65         -
    66         -

Changes to test/fts3matchinfo.test.

   520    520       }
   521    521       lappend r2 $M
   522    522     }
   523    523   
   524    524     do_execsql_test 11.1.$tn.2  {
   525    525       SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr
   526    526     } $r2
   527         -  breakpoint
   528    527   
   529    528     do_execsql_test 11.1.$tn.2  {
   530    529       SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr
   531    530     } $r2
   532    531   }
   533    532   set sqlite_fts3_enable_parentheses 0
   534    533   
................................................................................
   548    547   do_execsql_test 12.1 {
   549    548     INSERT INTO tt (rowid, c4, c45) VALUES(1, 'abc', 'abc');
   550    549     SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc';
   551    550   } [list [list [expr 1<<4] [expr 1<<(45-32)]]]
   552    551   
   553    552   set sqlite_fts3_enable_parentheses 0
   554    553   finish_test
   555         -

Changes to test/fts3offsets.test.

   117    117     2 {(A) x x x x x x x x x x x B} 
   118    118     1 {(A) (B) (C)}
   119    119   }
   120    120   
   121    121   
   122    122   set sqlite_fts3_enable_parentheses 0
   123    123   finish_test
   124         -

Changes to test/fts3snippet.test.

   555    555   } {64}
   556    556   
   557    557   
   558    558   
   559    559   
   560    560   set sqlite_fts3_enable_parentheses 0
   561    561   finish_test
   562         -

Changes to test/fts4check.test.

   206    206   
   207    207   do_execsql_test 5.4 {
   208    208     CREATE VIRTUAL TABLE t5 USING fts4(a, prefix="1,2,3");
   209    209     INSERT INTO t5(t5) VALUES('integrity-check');
   210    210   } {}
   211    211   
   212    212   finish_test
   213         -

Changes to test/fts4content.test.

   632    632   
   633    633   do_catchsql_test 11.1 {
   634    634     CREATE VIRTUAL TABLE x1 USING fts4(content=x1);
   635    635   } {1 {vtable constructor called recursively: x1}}
   636    636   
   637    637   
   638    638   finish_test
   639         -

Changes to test/fts4growth.test.

   430    430     SELECT sum(length(block)) FROM x6_segments 
   431    431     WHERE blockid BETWEEN 23695 AND 24147
   432    432   } {633507}
   433    433   
   434    434   
   435    435   
   436    436   finish_test
   437         -

Changes to test/fts4growth2.test.

    86     86       }
    87     87       execsql { SELECT max(level) FROM x1_segdir }
    88     88     } {1}
    89     89   }
    90     90   
    91     91   
    92     92   finish_test
    93         -

Changes to test/fts4langid.test.

   354    354     for {set i 0} {$i < 50} {incr i} {
   355    355       execsql { 
   356    356         INSERT INTO t4(docid, content, lid) VALUES($i, 'The Quick Brown Fox', $i) 
   357    357       }
   358    358     }
   359    359   }
   360    360   
   361         -do_test 4.1.0 {
   362         -  reset_db
   363         -  set ptr [fts3_test_tokenizer]
   364         -  execsql { SELECT fts3_tokenizer('testtokenizer', $ptr) }
   365         -  build_multilingual_db_2 db
   366         -} {}
   367         -do_execsql_test 4.1.1 {
   368         -  SELECT docid FROM t4 WHERE t4 MATCH 'quick';
   369         -} {0}
   370         -do_execsql_test 4.1.2 {
   371         -  SELECT docid FROM t4 WHERE t4 MATCH 'quick' AND lid=1;
   372         -} {}
   373         -do_execsql_test 4.1.3 {
   374         -  SELECT docid FROM t4 WHERE t4 MATCH 'Quick' AND lid=1;
   375         -} {1}
   376         -for {set i 0} {$i < 50} {incr i} {
   377         -  do_execsql_test 4.1.4.$i {
   378         -    SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i;
   379         -  } [expr 0==($i%2)]
   380         -}
   381         -do_catchsql_test 4.1.5 {
   382         -  INSERT INTO t4(content, lid) VALUES('hello world', 101)
   383         -} {1 {SQL logic error or missing database}}
          361  +ifcapable fts3_tokenizer {
          362  +  do_test 4.1.0 {
          363  +    reset_db
          364  +    set ptr [fts3_test_tokenizer]
          365  +    execsql { SELECT fts3_tokenizer('testtokenizer', $ptr) }
          366  +    build_multilingual_db_2 db
          367  +  } {}
          368  +  do_execsql_test 4.1.1 {
          369  +    SELECT docid FROM t4 WHERE t4 MATCH 'quick';
          370  +  } {0}
          371  +  do_execsql_test 4.1.2 {
          372  +    SELECT docid FROM t4 WHERE t4 MATCH 'quick' AND lid=1;
          373  +  } {}
          374  +  do_execsql_test 4.1.3 {
          375  +    SELECT docid FROM t4 WHERE t4 MATCH 'Quick' AND lid=1;
          376  +  } {1}
          377  +  for {set i 0} {$i < 50} {incr i} {
          378  +    do_execsql_test 4.1.4.$i {
          379  +      SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i;
          380  +    } [expr 0==($i%2)]
          381  +  }
          382  +  do_catchsql_test 4.1.5 {
          383  +    INSERT INTO t4(content, lid) VALUES('hello world', 101)
          384  +  } {1 {SQL logic error or missing database}}
          385  +}
   384    386   
   385    387   #-------------------------------------------------------------------------
   386    388   # Test cases 5.*
   387    389   #
   388    390   # The following test cases are designed to detect a 32-bit overflow bug
   389    391   # that existed at one point.
   390    392   #

Changes to test/fts4noti.test.

   224    224   
   225    225     SELECT count(*) FROM t2 WHERE t2 MATCH 'no';
   226    226     SELECT count(*) FROM t2 WHERE t2 MATCH 'yes';
   227    227     SELECT count(*) FROM t2 WHERE t2 MATCH 'yep';
   228    228   } {0 1 1 0 1 1}
   229    229   
   230    230   finish_test
   231         -
   232         -
   233         -

Changes to test/fts4onepass.test.

   140    140         INSERT INTO ft2(ft2) VALUES('integrity-check');
   141    141       }
   142    142     }
   143    143     eval $tcl2
   144    144   }
   145    145   
   146    146   finish_test
   147         -

Changes to test/fuzz3.test.

   169    169     do_test fuzz3-$ii.$iNew.[incr iTest] {
   170    170       db_checksum
   171    171     } $::cksum
   172    172   }
   173    173   
   174    174   test_restore_config_pagecache
   175    175   finish_test
   176         -

Changes to test/incrcorrupt.test.

   120    120   do_test 2.14 { sqlite3_errmsg db } {not an error}
   121    121   
   122    122   do_test 2.15 { sqlite3_finalize $stmt } {SQLITE_CORRUPT}
   123    123   do_test 2.16 { sqlite3_errcode db } {SQLITE_CORRUPT}
   124    124   do_test 2.17 { sqlite3_errmsg db } {database disk image is malformed}
   125    125   
   126    126   finish_test
   127         -

Changes to test/mallocK.test.

   168    168     execsql { SELECT * FROM x2 WHERE x = str('19') AND y = str('4') }
   169    169   } -test {
   170    170     faultsim_test_result [list 0 {}]
   171    171   }
   172    172   
   173    173   
   174    174   finish_test
   175         -

Changes to test/mallocL.test.

    36     36     } -test {
    37     37       faultsim_test_result [list 0 [lrange $::vals 0 $::j]]
    38     38     }
    39     39   }
    40     40   
    41     41   
    42     42   finish_test
    43         -

Changes to test/ovfl.test.

    41     41   } {}
    42     42   
    43     43   do_execsql_test 1.2 {
    44     44     SELECT sum(length(c2)) FROM t1;
    45     45   } [expr 2000 * 2000]
    46     46   
    47     47   finish_test
    48         -
    49         -

Changes to test/pragma2.test.

   251    251     UPDATE t2 SET c=c-1;
   252    252     PRAGMA lock_status;
   253    253   } {main unlocked temp unknown aux1 exclusive}
   254    254   db close
   255    255   forcedelete test.db
   256    256   sqlite3 db test.db
   257    257   
   258         -breakpoint
   259    258   do_execsql_test pragma2-5.1 {
   260    259     PRAGMA page_size=16384;
   261    260     CREATE TABLE t1(x);
   262    261     PRAGMA cache_size=2;
   263    262     PRAGMA cache_spill=YES;
   264    263     PRAGMA cache_spill;
   265    264   } {2}

Changes to test/quota.test.

    17     17   ifcapable !curdir {
    18     18     finish_test
    19     19     return
    20     20   }
    21     21   
    22     22   source $testdir/malloc_common.tcl
    23     23   
           24  +forcedelete bak.db
    24     25   unset -nocomplain defaultVfs
    25     26   set defaultVfs [file_control_vfsname db]
    26     27   db close
    27     28   
    28     29   do_test quota-1.1 { sqlite3_quota_initialize nosuchvfs 1 } {SQLITE_ERROR}
    29     30   do_test quota-1.2 { sqlite3_quota_initialize "" 1 }        {SQLITE_OK}
    30     31   do_test quota-1.3 { sqlite3_quota_initialize "" 1 }        {SQLITE_MISUSE}

Changes to test/rollback2.test.

   150    150   } -select {
   151    151     SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h ASC;
   152    152   } -result {
   153    153     2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40
   154    154   }
   155    155   
   156    156   finish_test
   157         -

Changes to test/rollbackfault.test.

    76     76         error "statements don't look right"
    77     77       }
    78     78     }
    79     79   }
    80     80   
    81     81   
    82     82   finish_test
    83         -
    84         -

Changes to test/select4.test.

   911    911   } {123 456}
   912    912   do_execsql_test select4-14.16 {
   913    913     VALUES(1),(2),(3),(4) UNION ALL SELECT 5 LIMIT 99;
   914    914   } {1 2 3 4 5}
   915    915   do_execsql_test select4-14.17 {
   916    916     VALUES(1),(2),(3),(4) UNION ALL SELECT 5 LIMIT 3;
   917    917   } {1 2 3}
          918  +
          919  +# Ticket https://www.sqlite.org/src/info/d06a25c84454a372
          920  +# Incorrect answer due to two co-routines using the same registers and expecting
          921  +# those register values to be preserved across a Yield.
          922  +#
          923  +do_execsql_test select4-15.1 {
          924  +  DROP TABLE IF EXISTS tx;
          925  +  CREATE TABLE tx(id INTEGER PRIMARY KEY, a, b);
          926  +  INSERT INTO tx(a,b) VALUES(33,456);
          927  +  INSERT INTO tx(a,b) VALUES(33,789);
          928  +
          929  +  SELECT DISTINCT t0.id, t0.a, t0.b
          930  +    FROM tx AS t0, tx AS t1
          931  +   WHERE t0.a=t1.a AND t1.a=33 AND t0.b=456
          932  +  UNION
          933  +  SELECT DISTINCT t0.id, t0.a, t0.b
          934  +    FROM tx AS t0, tx AS t1
          935  +   WHERE t0.a=t1.a AND t1.a=33 AND t0.b=789
          936  +   ORDER BY 1;
          937  +} {1 33 456 2 33 789}
   918    938   
   919    939   finish_test

Changes to test/select7.test.

   216    216   do_catchsql_test 8.2 {
   217    217     CREATE VIEW v0 as SELECT x, y FROM t01 UNION SELECT x FROM t02;
   218    218     EXPLAIN QUERY PLAN SELECT * FROM v0 WHERE x='0' OR y;
   219    219   } {1 {SELECTs to the left and right of UNION do not have the same number of result columns}}
   220    220   
   221    221   
   222    222   finish_test
   223         -
   224         -

Changes to test/shared3.test.

   135    135   } {1}
   136    136   do_test 3.5 {
   137    137     execsql { COMMIT }
   138    138   } {}
   139    139   
   140    140   sqlite3_enable_shared_cache $::enable_shared_cache
   141    141   finish_test
   142         -

Changes to test/snapshot.test.

    51     51   # block of tests above.
    52     52   #
    53     53   do_execsql_test 2.1.0 {
    54     54     BEGIN;
    55     55       SELECT * FROM t1;
    56     56   } {1 2 3 4 5 6 7 8}
    57     57   
    58         -breakpoint
    59     58   do_test 2.1.1 {
    60     59     set snapshot [sqlite3_snapshot_get db main]
    61     60     execsql {
    62     61       COMMIT;
    63     62       INSERT INTO t1 VALUES(9, 10);
    64     63       SELECT * FROM t1;
    65     64     }

Changes to test/sort3.test.

   107    107     440000 440000000 
   108    108     440000 440000000 
   109    109     440000 440000000 
   110    110     440000 440000000
   111    111   }
   112    112   
   113    113   finish_test
   114         -

Changes to test/sort5.test.

    38     38   do_execsql_test 1.2 {
    39     39     CREATE INDEX i1 ON t1(b);
    40     40   }
    41     41   
    42     42   db close
    43     43   tvfs delete
    44     44   finish_test
    45         -

Changes to test/spellfix.test.

   399    399     do_catchsql_test 7.5.2.$tn.1 $sql $err
   400    400     do_execsql_test 7.5.2.$tn.2 { SELECT rowid, word FROM t4 } $res
   401    401     do_test 7.5.2.$tn.3 { sqlite3_get_autocommit db } $bRollback
   402    402     catchsql ROLLBACK
   403    403   }
   404    404   
   405    405   finish_test
   406         -

Changes to test/sqllog.test.

   106    106   
   107    107   catch { db close }
   108    108   sqlite3_shutdown
   109    109   unset ::env(SQLITE_SQLLOG_DIR)
   110    110   unset ::env(SQLITE_SQLLOG_CONDITIONAL)
   111    111   sqlite3_config_sqllog
   112    112   sqlite3_initialize
   113         -breakpoint
   114    113   finish_test
   115         -
   116         -

Changes to test/tkt-9f2eb3abac.test.

    72     72   } -body {
    73     73     execsql { SELECT * FROM t1,t2 WHERE a=? AND b=? AND c=? AND d=? AND e=? }
    74     74   } -test {
    75     75     faultsim_test_result {0 {}} 
    76     76   }
    77     77   
    78     78   finish_test
    79         -

Changes to test/tkt-ba7cbfaedc.test.

    57     57     select * from t1 group by id order by id asc;
    58     58     select * from t1 group by id order by id desc;
    59     59   } {
    60     60     1 2 3 4 5   1 2 3 4 5   5 4 3 2 1
    61     61   }
    62     62   
    63     63   finish_test
    64         -
    65         -

Changes to test/triggerE.test.

   104    104     INSERT INTO t2 VALUES(NULL, 'z');
   105    105     INSERT INTO t3 VALUES(1, 2);
   106    106     SELECT * FROM t3;
   107    107     SELECT * FROM t2;
   108    108   } {1 2 x y z z}
   109    109   
   110    110   finish_test
   111         -
   112         -

Changes to test/vtab_shared.test.

   272    272       db close
   273    273     } {}
   274    274     db2 close
   275    275   }
   276    276   
   277    277   sqlite3_enable_shared_cache 0
   278    278   finish_test
   279         -

Changes to test/wal6.test.

   234    234   } {0 {1 2}}
   235    235   do_test 4.4.2 { 
   236    236     catchsql { SELECT * FROM t2 } db2 
   237    237   } {1 {database disk image is malformed}}
   238    238   
   239    239   
   240    240   finish_test
   241         -

Changes to test/waloverwrite.test.

   127    127   
   128    128         execsql {SAVEPOINT abc}
   129    129         for {set i 0} {$i < 5} {incr i} {
   130    130           foreach x [db eval {SELECT x FROM t1}] {
   131    131             execsql { UPDATE t1 SET y = randomblob(797) WHERE x=$x }
   132    132           }
   133    133         }
   134         -      breakpoint
   135    134         execsql {ROLLBACK TO abc}
   136    135   
   137    136       }
   138    137   
   139    138       set nPg [wal_frame_count test.db-wal 1024]
   140    139       expr $nPg>55 && $nPg<75
   141    140     } {1}
................................................................................
   157    156     do_test 1.$tn.10 {
   158    157       execsql { PRAGMA integrity_check } db2
   159    158     } ok
   160    159     db2 close
   161    160   }
   162    161   
   163    162   finish_test
   164         -

Changes to test/whereI.test.

    85     85   
    86     86     SELECT c||'.'||b FROM t3 WHERE a='t' OR d='t'
    87     87   } {
    88     88     2.1 2.2 1.2
    89     89   }
    90     90   
    91     91   finish_test
    92         -

Changes to test/withM.test.

    68     68     }
    69     69   } -test {
    70     70     faultsim_test_result {0 {1 1 2 4 3 9 4 16 5 25}}
    71     71     db close
    72     72   }
    73     73   
    74     74   finish_test
    75         -
    76         -
    77         -

Changes to tool/mkkeywordhash.c.

   273    273     { "WHEN",             "TK_WHEN",         ALWAYS                 },
   274    274     { "WHERE",            "TK_WHERE",        ALWAYS                 },
   275    275   };
   276    276   
   277    277   /* Number of keywords */
   278    278   static int nKeyword = (sizeof(aKeywordTable)/sizeof(aKeywordTable[0]));
   279    279   
   280         -/* Map all alphabetic characters into the same case */
          280  +/* Map all alphabetic characters into lower-case for hashing.  This is
          281  +** only valid for alphabetics.  In particular it does not work for '_'
          282  +** and so the hash cannot be on a keyword position that might be an '_'.
          283  +*/
   281    284   #define charMap(X)   (0x20|(X))
   282    285   
   283    286   /*
   284    287   ** Comparision function for two Keyword records
   285    288   */
   286    289   static int keywordCompare1(const void *a, const void *b){
   287    290     const Keyword *pA = (Keyword*)a;
................................................................................
   561    564       if( j>=5 ){
   562    565         printf("\n");
   563    566         j = 0;
   564    567       }
   565    568     }
   566    569     printf("%s  };\n", j==0 ? "" : "\n");
   567    570   
   568         -  printf("  int h, i;\n");
          571  +  printf("  int i, j;\n");
          572  +  printf("  const char *zKW;\n");
   569    573     printf("  if( n>=2 ){\n");
   570         -  printf("    h = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n) %% %d;\n",
          574  +  printf("    i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n) %% %d;\n",
   571    575             bestSize);
   572         -  printf("    for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){\n");
   573         -  printf("      if( aLen[i]==n &&"
   574         -                     " sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){\n");
          576  +  printf("    for(i=((int)aHash[i])-1; i>=0; i=((int)aNext[i])-1){\n");
          577  +  printf("      if( aLen[i]!=n ) continue;\n");
          578  +  printf("      j = 0;\n");
          579  +  printf("      zKW = &zText[aOffset[i]];\n");
          580  +  printf("#ifdef SQLITE_ASCII\n");
          581  +  printf("      while( j<n && (z[j]&~0x20)==zKW[j] ){ j++; }\n");
          582  +  printf("#endif\n");
          583  +  printf("#ifdef SQLITE_EBCDIC\n");
          584  +  printf("      while( j<n && toupper(z[j])==zKW[j] ){ j++; }\n");
          585  +  printf("#endif\n");
          586  +  printf("      if( j<n ) continue;\n");
   575    587     for(i=0; i<nKeyword; i++){
   576         -    printf("        testcase( i==%d ); /* %s */\n",
          588  +    printf("      testcase( i==%d ); /* %s */\n",
   577    589              i, aKeywordTable[i].zOrigName);
   578    590     }
   579         -  printf("        *pType = aCode[i];\n");
   580         -  printf("        break;\n");
   581         -  printf("      }\n");
          591  +  printf("      *pType = aCode[i];\n");
          592  +  printf("      break;\n");
   582    593     printf("    }\n");
   583    594     printf("  }\n");
   584    595     printf("  return n;\n");
   585    596     printf("}\n");
   586    597     printf("int sqlite3KeywordCode(const unsigned char *z, int n){\n");
   587    598     printf("  int id = TK_ID;\n");
   588    599     printf("  keywordCode((char*)z, n, &id);\n");
   589    600     printf("  return id;\n");
   590    601     printf("}\n");
   591    602     printf("#define SQLITE_N_KEYWORD %d\n", nKeyword);
   592    603   
   593    604     return 0;
   594    605   }

Added tool/srcck1.c.

            1  +/*
            2  +** The program does some simple static analysis of the sqlite3.c source
            3  +** file looking for mistakes.
            4  +**
            5  +** Usage:
            6  +**
            7  +**      ./srcck1 sqlite3.c
            8  +**
            9  +** This program looks for instances of assert(), ALWAYS(), NEVER() or
           10  +** testcase() that contain side-effects and reports errors if any such
           11  +** instances are found.
           12  +**
           13  +** The aim of this utility is to prevent recurrences of errors such
           14  +** as the one fixed at:
           15  +**
           16  +**   https://www.sqlite.org/src/info/a2952231ac7abe16
           17  +**
           18  +** Note that another similar error was found by this utility when it was
           19  +** first written.  That other error was fixed by the same check-in that
           20  +** committed the first version of this utility program.
           21  +*/
           22  +#include <stdlib.h>
           23  +#include <ctype.h>
           24  +#include <stdio.h>
           25  +#include <string.h>
           26  +
           27  +/* Read the complete text of a file into memory.  Return a pointer to
           28  +** the result.  Panic if unable to read the file or allocate memory.
           29  +*/
           30  +static char *readFile(const char *zFilename){
           31  +  FILE *in;
           32  +  char *z;
           33  +  long n;
           34  +  size_t got;
           35  +
           36  +  in = fopen(zFilename, "rb");
           37  +  if( in==0 ){
           38  +    fprintf(stderr, "unable to open '%s' for reading\n", zFilename);
           39  +    exit(1);
           40  +  }
           41  +  fseek(in, 0, SEEK_END);
           42  +  n = ftell(in);
           43  +  rewind(in);
           44  +  z = malloc( n+1 );
           45  +  if( z==0 ){
           46  +    fprintf(stderr, "cannot allocate %d bytes to store '%s'\n", 
           47  +            (int)(n+1), zFilename);
           48  +    exit(1);
           49  +  }
           50  +  got = fread(z, 1, n, in);
           51  +  fclose(in);
           52  +  if( got!=(size_t)n ){
           53  +    fprintf(stderr, "only read %d of %d bytes from '%s'\n",
           54  +           (int)got, (int)n, zFilename);
           55  +    exit(1);
           56  +  }
           57  +  z[n] = 0;
           58  +  return z;
           59  +}
           60  +
           61  +/* Change the C code in the argument to see if it might have
           62  +** side effects.  The only accurate way to know this is to do a full
           63  +** parse of the C code, which this routine does not do.  This routine
           64  +** uses a simple heuristic of looking for:
           65  +**
           66  +**    *  '=' not immediately after '>', '<', '!', or '='.
           67  +**    *  '++'
           68  +**    *  '--'
           69  +**
           70  +** If the code contains the phrase "side-effects-ok" is inside a 
           71  +** comment, then always return false.  This is used to disable checking
           72  +** for assert()s with deliberate side-effects, such as used by
           73  +** SQLITE_TESTCTRL_ASSERT - a facility that allows applications to
           74  +** determine at runtime whether or not assert()s are enabled.  
           75  +** Obviously, that determination cannot be made unless the assert()
           76  +** has some side-effect.
           77  +**
           78  +** Return true if a side effect is seen.  Return false if not.
           79  +*/
           80  +static int hasSideEffect(const char *z, unsigned int n){
           81  +  unsigned int i;
           82  +  for(i=0; i<n; i++){
           83  +    if( z[i]=='/' && strncmp(&z[i], "/*side-effects-ok*/", 19)==0 ) return 0;
           84  +    if( z[i]=='=' && i>0 && z[i-1]!='=' && z[i-1]!='>'
           85  +           && z[i-1]!='<' && z[i-1]!='!' && z[i+1]!='=' ) return 1;
           86  +    if( z[i]=='+' && z[i+1]=='+' ) return 1;
           87  +    if( z[i]=='-' && z[i+1]=='-' ) return 1;
           88  +  }
           89  +  return 0;
           90  +}
           91  +
           92  +/* Return the number of bytes in string z[] prior to the first unmatched ')'
           93  +** character.
           94  +*/
           95  +static unsigned int findCloseParen(const char *z){
           96  +  unsigned int nOpen = 0;
           97  +  unsigned i;
           98  +  for(i=0; z[i]; i++){
           99  +    if( z[i]=='(' ) nOpen++;
          100  +    if( z[i]==')' ){
          101  +      if( nOpen==0 ) break;
          102  +      nOpen--;
          103  +    }
          104  +  }
          105  +  return i;
          106  +}
          107  +
          108  +/* Search for instances of assert(...), ALWAYS(...), NEVER(...), and/or
          109  +** testcase(...) where the argument contains side effects.
          110  +**
          111  +** Print error messages whenever a side effect is found.  Return the number
          112  +** of problems seen.
          113  +*/
          114  +static unsigned int findAllSideEffects(const char *z){
          115  +  unsigned int lineno = 1;   /* Line number */
          116  +  unsigned int i;
          117  +  unsigned int nErr = 0;
          118  +  char c, prevC = 0;
          119  +  for(i=0; (c = z[i])!=0; prevC=c, i++){
          120  +    if( c=='\n' ){ lineno++; continue; }
          121  +    if( isalpha(c) && !isalpha(prevC) ){
          122  +      if( strncmp(&z[i],"assert(",7)==0
          123  +       || strncmp(&z[i],"ALWAYS(",7)==0
          124  +       || strncmp(&z[i],"NEVER(",6)==0
          125  +       || strncmp(&z[i],"testcase(",9)==0
          126  +      ){
          127  +        unsigned int n;
          128  +        const char *z2 = &z[i+5];
          129  +        while( z2[0]!='(' ){ z2++; }
          130  +        z2++;
          131  +        n = findCloseParen(z2);
          132  +        if( hasSideEffect(z2, n) ){
          133  +          nErr++;
          134  +          fprintf(stderr, "side-effect line %u: %.*s\n", lineno,
          135  +                  (int)(&z2[n+1] - &z[i]), &z[i]);
          136  +        }
          137  +      }
          138  +    }
          139  +  }
          140  +  return nErr;
          141  +}
          142  +
          143  +int main(int argc, char **argv){
          144  +  char *z;
          145  +  unsigned int nErr = 0;
          146  +  if( argc!=2 ){
          147  +    fprintf(stderr, "Usage: %s FILENAME\n", argv[0]);
          148  +    return 1;
          149  +  }
          150  +  z = readFile(argv[1]);
          151  +  nErr = findAllSideEffects(z);
          152  +  free(z);
          153  +  if( nErr ){
          154  +    fprintf(stderr, "Found %u undesirable side-effects\n", nErr);
          155  +    return 1;
          156  +  }
          157  +  return 0; 
          158  +}