/ Check-in [4b365167]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge latest trunk changes with this branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 4b3651677e7132c4c45605bc1f216fc08ef31198
User & Date: dan 2015-01-01 18:03:49
Context
2015-01-02
14:55
Allow the rank column to be remapped on a per-query basis by including a term similar to "rank match 'bm25(10,2)'" in a where clause. check-in: 1cd15a17 user: dan tags: fts5
2015-01-01
18:03
Merge latest trunk changes with this branch. check-in: 4b365167 user: dan tags: fts5
16:46
Add a version of the unicode61 tokenizer to fts5. check-in: d09f7800 user: dan tags: fts5
14:13
Fix two test cases in memsubsys1 so that they work with the mmap permutation. check-in: 66269d0d user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   173    173            icu.lo insert.lo journal.lo legacy.lo loadext.lo \
   174    174            main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
   175    175            memjournal.lo \
   176    176            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
   177    177            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
   178    178            pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
   179    179            random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \
   180         -         table.lo tokenize.lo trigger.lo \
          180  +         table.lo threads.lo tokenize.lo trigger.lo \
   181    181            update.lo util.lo vacuum.lo \
   182    182            vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
   183    183            vdbetrace.lo wal.lo walker.lo where.lo utf.lo vtab.lo
   184    184   
   185    185   # Object files for the amalgamation.
   186    186   #
   187    187   LIBOBJS1 = sqlite3.lo
................................................................................
   259    259     $(TOP)/src/status.c \
   260    260     $(TOP)/src/shell.c \
   261    261     $(TOP)/src/sqlite.h.in \
   262    262     $(TOP)/src/sqlite3ext.h \
   263    263     $(TOP)/src/sqliteInt.h \
   264    264     $(TOP)/src/sqliteLimit.h \
   265    265     $(TOP)/src/table.c \
          266  +  $(TOP)/src/threads.c \
   266    267     $(TOP)/src/tclsqlite.c \
   267    268     $(TOP)/src/tokenize.c \
   268    269     $(TOP)/src/trigger.c \
   269    270     $(TOP)/src/utf.c \
   270    271     $(TOP)/src/update.c \
   271    272     $(TOP)/src/util.c \
   272    273     $(TOP)/src/vacuum.c \
................................................................................
   354    355     $(TOP)/src/test6.c \
   355    356     $(TOP)/src/test7.c \
   356    357     $(TOP)/src/test8.c \
   357    358     $(TOP)/src/test9.c \
   358    359     $(TOP)/src/test_autoext.c \
   359    360     $(TOP)/src/test_async.c \
   360    361     $(TOP)/src/test_backup.c \
          362  +  $(TOP)/src/test_blob.c \
   361    363     $(TOP)/src/test_btree.c \
   362    364     $(TOP)/src/test_config.c \
   363    365     $(TOP)/src/test_demovfs.c \
   364    366     $(TOP)/src/test_devsym.c \
   365    367     $(TOP)/src/test_fs.c \
   366    368     $(TOP)/src/test_func.c \
   367    369     $(TOP)/src/test_hexio.c \
................................................................................
   389    391     $(TOP)/ext/fts3/fts3_test.c 
   390    392   
   391    393   # Statically linked extensions
   392    394   #
   393    395   TESTSRC += \
   394    396     $(TOP)/ext/misc/amatch.c \
   395    397     $(TOP)/ext/misc/closure.c \
          398  +  $(TOP)/ext/misc/eval.c \
   396    399     $(TOP)/ext/misc/fileio.c \
   397    400     $(TOP)/ext/misc/fuzzer.c \
   398    401     $(TOP)/ext/misc/ieee754.c \
   399    402     $(TOP)/ext/misc/nextchar.c \
   400    403     $(TOP)/ext/misc/percentile.c \
   401    404     $(TOP)/ext/misc/regexp.c \
   402    405     $(TOP)/ext/misc/spellfix.c \
................................................................................
   523    526   		-o $@ $(TOP)/src/shell.c libsqlite3.la \
   524    527   		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
   525    528   
   526    529   mptester$(EXE):	sqlite3.c $(TOP)/mptest/mptest.c
   527    530   	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \
   528    531   		$(TLIBS) -rpath "$(libdir)"
   529    532   
          533  +mptest:	mptester$(EXE)
          534  +	rm -f mptest1.db
          535  +	./mptester$(EXE) mptest1.db $(TOP)/mptest/crash01.test
          536  +	rm -f mptest2.db
          537  +	./mptester$(EXE) mptest2.db $(TOP)/mptest/multiwrite01.test
   530    538   
   531    539   # This target creates a directory named "tsrc" and fills it with
   532    540   # copies of all of the C source code and header files needed to
   533    541   # build on the target system.  Some of the C source code and header
   534    542   # files are automatically generated.  This target takes care of
   535    543   # all that automatic generation.
   536    544   #
................................................................................
   733    741   
   734    742   status.lo:	$(TOP)/src/status.c $(HDR)
   735    743   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/status.c
   736    744   
   737    745   table.lo:	$(TOP)/src/table.c $(HDR)
   738    746   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/table.c
   739    747   
          748  +threads.lo:	$(TOP)/src/threads.c $(HDR)
          749  +	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/threads.c
          750  +
   740    751   tokenize.lo:	$(TOP)/src/tokenize.c keywordhash.h $(HDR)
   741    752   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/tokenize.c
   742    753   
   743    754   trigger.lo:	$(TOP)/src/trigger.c $(HDR)
   744    755   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/trigger.c
   745    756   
   746    757   update.lo:	$(TOP)/src/update.c $(HDR)
................................................................................
   935    946   	echo "static const char *zMainloop = " >> $@
   936    947   	$(NAWK) -f $(TOP)/tool/tostr.awk $(TOP)/tool/spaceanal.tcl >> $@
   937    948   	echo "; return zMainloop; }" >> $@
   938    949   
   939    950   sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
   940    951   	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
   941    952   
   942         -showdb$(TEXE):	$(TOP)/tool/showdb.c sqlite3.c
   943         -	$(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.c $(TLIBS)
          953  +showdb$(TEXE):	$(TOP)/tool/showdb.c sqlite3.lo
          954  +	$(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS)
          955  +
          956  +showstat4$(TEXE):	$(TOP)/tool/showstat4.c sqlite3.lo
          957  +	$(LTLINK) -o $@ $(TOP)/tool/showstat4.c sqlite3.lo $(TLIBS)
          958  +
          959  +showjournal$(TEXE):	$(TOP)/tool/showjournal.c sqlite3.lo
          960  +	$(LTLINK) -o $@ $(TOP)/tool/showjournal.c sqlite3.lo $(TLIBS)
          961  +
          962  +showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
          963  +	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)
          964  +
          965  +rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo
          966  +	$(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS)
   944    967   
   945    968   LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
   946    969   	$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c
   947    970   
   948    971   wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.c
   949    972   	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS)
   950    973   
   951    974   speedtest1$(TEXE):	$(TOP)/test/wordcount.c sqlite3.lo
   952    975   	$(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS)
          976  +
          977  +# This target will fail if the SQLite amalgamation contains any exported
          978  +# symbols that do not begin with "sqlite3_". It is run as part of the
          979  +# releasetest.tcl script.
          980  +#
          981  +checksymbols: sqlite3.lo
          982  +	nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0
          983  +	echo '0 errors out of 1 tests'
          984  +
          985  +# The next two rules are used to support the "threadtest" target. Building
          986  +# threadtest runs a few thread-safety tests that are implemented in C. This
          987  +# target is invoked by the releasetest.tcl script.
          988  +# 
          989  +THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
          990  +                  $(TOP)/test/tt3_checkpoint.c \
          991  +                  $(TOP)/test/tt3_index.c      \
          992  +                  $(TOP)/test/tt3_vacuum.c      \
          993  +                  $(TOP)/test/tt3_stress.c      \
          994  +                  $(TOP)/test/tt3_lookaside1.c
          995  +
          996  +threadtest3$(TEXE): sqlite3.lo $(THREADTEST3_SRC)
          997  +	$(LTLINK) $(TOP)/test/threadtest3.c sqlite3.lo -o $@ $(TLIBS)
          998  +
          999  +threadtest: threadtest3$(TEXE)
         1000  +	./threadtest3$(TEXE)
         1001  +
         1002  +releasetest:	
         1003  +	$(TCLSH_CMD) $(TOP)/test/releasetest.tcl
   953   1004   
   954   1005   # Standard install and cleanup targets
   955   1006   #
   956   1007   lib_install:	libsqlite3.la
   957   1008   	$(INSTALL) -d $(DESTDIR)$(libdir)
   958   1009   	$(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir)
   959   1010   	
................................................................................
   981   1032   	rm -f lemon$(BEXE) lempar.c parse.* sqlite*.tar.gz
   982   1033   	rm -f mkkeywordhash$(BEXE) keywordhash.h
   983   1034   	rm -f *.da *.bb *.bbg gmon.out
   984   1035   	rm -rf quota2a quota2b quota2c
   985   1036   	rm -rf tsrc .target_source
   986   1037   	rm -f tclsqlite3$(TEXE)
   987   1038   	rm -f testfixture$(TEXE) test.db
         1039  +	rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE)
         1040  +	rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE)
         1041  +	rm -f wordcount$(TEXE)
   988   1042   	rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def
   989   1043   	rm -f sqlite3.c
   990   1044   	rm -f sqlite3rc.h
   991   1045   	rm -f shell.c sqlite3ext.h
   992   1046   	rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c
   993   1047   	rm -f sqlite-*-output.vsix
   994   1048   	rm -f mptester mptester.exe

Changes to Makefile.msc.

   104    104   !ENDIF
   105    105   
   106    106   # Set this to one of the following values to enable various debugging
   107    107   # features.  Each level includes the debugging options from the previous
   108    108   # levels.  Currently, the recognized values for DEBUG are:
   109    109   #
   110    110   # 0 == NDEBUG: Disables assert() and other runtime diagnostics.
   111         -# 1 == Disables NDEBUG and all optimizations and then enables PDBs.
   112         -# 2 == SQLITE_DEBUG: Enables various diagnostics messages and code.
   113         -# 3 == SQLITE_WIN32_MALLOC_VALIDATE: Validate the Win32 native heap per call.
   114         -# 4 == SQLITE_DEBUG_OS_TRACE: Enables output from the OSTRACE() macros.
   115         -# 5 == SQLITE_ENABLE_IOTRACE: Enables output from the IOTRACE() macros.
          111  +# 1 == SQLITE_ENABLE_API_ARMOR: extra attempts to detect misuse of the API.
          112  +# 2 == Disables NDEBUG and all optimizations and then enables PDBs.
          113  +# 3 == SQLITE_DEBUG: Enables various diagnostics messages and code.
          114  +# 4 == SQLITE_WIN32_MALLOC_VALIDATE: Validate the Win32 native heap per call.
          115  +# 5 == SQLITE_DEBUG_OS_TRACE: Enables output from the OSTRACE() macros.
          116  +# 6 == SQLITE_ENABLE_IOTRACE: Enables output from the IOTRACE() macros.
   116    117   #
   117    118   !IFNDEF DEBUG
   118    119   DEBUG = 0
   119    120   !ENDIF
   120    121   
   121    122   # Enable use of available compiler optimizations?  Normally, this should be
   122    123   # non-zero.  Setting this to zero, thus disabling all compiler optimizations,
................................................................................
   276    277   
   277    278   # Also, we need to dynamically link to the correct MSVC runtime
   278    279   # when compiling for WinRT (e.g. debug or release) OR if the
   279    280   # USE_CRT_DLL option is set to force dynamically linking to the
   280    281   # MSVC runtime library.
   281    282   #
   282    283   !IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0
   283         -!IF $(DEBUG)>0
          284  +!IF $(DEBUG)>1
   284    285   TCC = $(TCC) -MDd
   285    286   BCC = $(BCC) -MDd
   286    287   !ELSE
   287    288   TCC = $(TCC) -MD
   288    289   BCC = $(BCC) -MD
   289    290   !ENDIF
   290    291   !ELSE
   291         -!IF $(DEBUG)>0
          292  +!IF $(DEBUG)>1
   292    293   TCC = $(TCC) -MTd
   293    294   BCC = $(BCC) -MTd
   294    295   !ELSE
   295    296   TCC = $(TCC) -MT
   296    297   BCC = $(BCC) -MT
   297    298   !ENDIF
   298    299   !ENDIF
................................................................................
   309    310   !ENDIF
   310    311   
   311    312   # The mksqlite3c.tcl script accepts some options on the command
   312    313   # line.  When compiling with debugging enabled, some of these
   313    314   # options are necessary in order to allow debugging symbols to
   314    315   # work correctly with Visual Studio when using the amalgamation.
   315    316   #
   316         -!IF $(DEBUG)>0
          317  +!IF $(DEBUG)>1
   317    318   MKSQLITE3C_ARGS = --linemacros
   318    319   !ELSE
   319    320   MKSQLITE3C_ARGS =
   320    321   !ENDIF
   321    322   
   322    323   # Define -DNDEBUG to compile without debugging (i.e., for production usage)
   323    324   # Omitting the define will cause extra debugging code to be inserted and
................................................................................
   325    326   #
   326    327   !IF $(DEBUG)==0
   327    328   TCC = $(TCC) -DNDEBUG
   328    329   BCC = $(BCC) -DNDEBUG
   329    330   RCC = $(RCC) -DNDEBUG
   330    331   !ENDIF
   331    332   
   332         -!IF $(DEBUG)>1
          333  +!IF $(DEBUG)>0
          334  +TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR
          335  +RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR
          336  +!ENDIF
          337  +
          338  +!IF $(DEBUG)>2
   333    339   TCC = $(TCC) -DSQLITE_DEBUG
   334    340   RCC = $(RCC) -DSQLITE_DEBUG
   335    341   !ENDIF
   336    342   
   337         -!IF $(DEBUG)>3
          343  +!IF $(DEBUG)>4
   338    344   TCC = $(TCC) -DSQLITE_DEBUG_OS_TRACE=1
   339    345   RCC = $(RCC) -DSQLITE_DEBUG_OS_TRACE=1
   340    346   !ENDIF
   341    347   
   342         -!IF $(DEBUG)>4
          348  +!IF $(DEBUG)>5
   343    349   TCC = $(TCC) -DSQLITE_ENABLE_IOTRACE
   344    350   RCC = $(RCC) -DSQLITE_ENABLE_IOTRACE
   345    351   !ENDIF
   346    352   
   347    353   # Prevent warnings about "insecure" MSVC runtime library functions
   348    354   # being used.
   349    355   #
................................................................................
   367    373   #
   368    374   !ELSEIF $(WIN32HEAP)!=0
   369    375   TCC = $(TCC) -DSQLITE_WIN32_MALLOC=1
   370    376   RCC = $(RCC) -DSQLITE_WIN32_MALLOC=1
   371    377   
   372    378   # Validate the heap on every call into the native Win32 heap subsystem?
   373    379   #
   374         -!IF $(DEBUG)>2
          380  +!IF $(DEBUG)>3
   375    381   TCC = $(TCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
   376    382   RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
   377    383   !ENDIF
   378    384   !ENDIF
   379    385   
   380    386   # The locations of the Tcl header and library files.  Also, the library that
   381    387   # non-stubs enabled programs using Tcl must link against.  These variables
................................................................................
   478    484   # nmake /f Makefile.msc all "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1"
   479    485   #
   480    486   TCC = $(TCC) $(OPTS)
   481    487   RCC = $(RCC) $(OPTS)
   482    488   
   483    489   # If compiling for debugging, add some defines.
   484    490   #
   485         -!IF $(DEBUG)>0
          491  +!IF $(DEBUG)>1
   486    492   TCC = $(TCC) -D_DEBUG
   487    493   BCC = $(BCC) -D_DEBUG
   488    494   RCC = $(RCC) -D_DEBUG
   489    495   !ENDIF
   490    496   
   491    497   # If optimizations are enabled or disabled (either implicitly or
   492    498   # explicitly), add the necessary flags.
   493    499   #
   494         -!IF $(DEBUG)>0 || $(OPTIMIZATIONS)==0
          500  +!IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0
   495    501   TCC = $(TCC) -Od
   496    502   BCC = $(BCC) -Od
   497    503   !ELSEIF $(OPTIMIZATIONS)>=3
   498    504   TCC = $(TCC) -Ox
   499    505   BCC = $(BCC) -Ox
   500    506   !ELSEIF $(OPTIMIZATIONS)==2
   501    507   TCC = $(TCC) -O2
................................................................................
   503    509   !ELSEIF $(OPTIMIZATIONS)==1
   504    510   TCC = $(TCC) -O1
   505    511   BCC = $(BCC) -O1
   506    512   !ENDIF
   507    513   
   508    514   # If symbols are enabled (or compiling for debugging), enable PDBs.
   509    515   #
   510         -!IF $(DEBUG)>0 || $(SYMBOLS)!=0
          516  +!IF $(DEBUG)>1 || $(SYMBOLS)!=0
   511    517   TCC = $(TCC) -Zi
   512    518   BCC = $(BCC) -Zi
   513    519   !ENDIF
   514    520   
   515    521   # If ICU support is enabled, add the compiler options for it.
   516    522   #
   517    523   !IF $(USE_ICU)!=0
................................................................................
   588    594   LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE
   589    595   LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib
   590    596   LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib
   591    597   !ENDIF
   592    598   
   593    599   # If either debugging or symbols are enabled, enable PDBs.
   594    600   #
   595         -!IF $(DEBUG)>0 || $(SYMBOLS)!=0
          601  +!IF $(DEBUG)>1 || $(SYMBOLS)!=0
   596    602   LDFLAGS = /DEBUG
   597    603   !ENDIF
   598    604   
   599    605   # Start with the Tcl related linker options.
   600    606   #
   601    607   !IF $(NO_TCL)==0
   602    608   LTLIBPATHS = /LIBPATH:$(TCLLIBDIR)
................................................................................
   632    638            icu.lo insert.lo journal.lo legacy.lo loadext.lo \
   633    639            main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
   634    640            memjournal.lo \
   635    641            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
   636    642            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
   637    643            pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
   638    644            random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \
   639         -         table.lo tokenize.lo trigger.lo \
          645  +         table.lo threads.lo tokenize.lo trigger.lo \
   640    646            update.lo util.lo vacuum.lo \
   641    647            vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
   642    648            vdbetrace.lo wal.lo walker.lo where.lo utf.lo vtab.lo
   643    649   
   644    650   # Object files for the amalgamation.
   645    651   #
   646    652   LIBOBJS1 = sqlite3.lo
................................................................................
   729    735     $(TOP)\src\status.c \
   730    736     $(TOP)\src\shell.c \
   731    737     $(TOP)\src\sqlite.h.in \
   732    738     $(TOP)\src\sqlite3ext.h \
   733    739     $(TOP)\src\sqliteInt.h \
   734    740     $(TOP)\src\sqliteLimit.h \
   735    741     $(TOP)\src\table.c \
          742  +  $(TOP)\src\threads.c \
   736    743     $(TOP)\src\tclsqlite.c \
   737    744     $(TOP)\src\tokenize.c \
   738    745     $(TOP)\src\trigger.c \
   739    746     $(TOP)\src\utf.c \
   740    747     $(TOP)\src\update.c \
   741    748     $(TOP)\src\util.c \
   742    749     $(TOP)\src\vacuum.c \
................................................................................
   823    830     $(TOP)\src\test6.c \
   824    831     $(TOP)\src\test7.c \
   825    832     $(TOP)\src\test8.c \
   826    833     $(TOP)\src\test9.c \
   827    834     $(TOP)\src\test_autoext.c \
   828    835     $(TOP)\src\test_async.c \
   829    836     $(TOP)\src\test_backup.c \
          837  +  $(TOP)\src\test_blob.c \
   830    838     $(TOP)\src\test_btree.c \
   831    839     $(TOP)\src\test_config.c \
   832    840     $(TOP)\src\test_demovfs.c \
   833    841     $(TOP)\src\test_devsym.c \
   834    842     $(TOP)\src\test_fs.c \
   835    843     $(TOP)\src\test_func.c \
   836    844     $(TOP)\src\test_hexio.c \
................................................................................
   858    866     $(TOP)\ext\fts3\fts3_test.c
   859    867   
   860    868   # Statically linked extensions
   861    869   #
   862    870   TESTEXT = \
   863    871     $(TOP)\ext\misc\amatch.c \
   864    872     $(TOP)\ext\misc\closure.c \
          873  +  $(TOP)\ext\misc\eval.c \
   865    874     $(TOP)\ext\misc\fileio.c \
   866    875     $(TOP)\ext\misc\fuzzer.c \
   867    876     $(TOP)\ext\misc\ieee754.c \
   868    877     $(TOP)\ext\misc\nextchar.c \
   869    878     $(TOP)\ext\misc\percentile.c \
   870    879     $(TOP)\ext\misc\regexp.c \
   871    880     $(TOP)\ext\misc\spellfix.c \
................................................................................
  1211   1220   	$(LTCOMPILE) -c $(TOP)\src\select.c
  1212   1221   
  1213   1222   status.lo:	$(TOP)\src\status.c $(HDR)
  1214   1223   	$(LTCOMPILE) -c $(TOP)\src\status.c
  1215   1224   
  1216   1225   table.lo:	$(TOP)\src\table.c $(HDR)
  1217   1226   	$(LTCOMPILE) -c $(TOP)\src\table.c
         1227  +
         1228  +threads.lo:	$(TOP)\src\threads.c $(HDR)
         1229  +	$(LTCOMPILE) -c $(TOP)\src\threads.c
  1218   1230   
  1219   1231   tokenize.lo:	$(TOP)\src\tokenize.c keywordhash.h $(HDR)
  1220   1232   	$(LTCOMPILE) -c $(TOP)\src\tokenize.c
  1221   1233   
  1222   1234   trigger.lo:	$(TOP)\src\trigger.c $(HDR)
  1223   1235   	$(LTCOMPILE) -c $(TOP)\src\trigger.c
  1224   1236   
................................................................................
  1430   1442   
  1431   1443   testloadext.dll: testloadext.lo
  1432   1444   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo
  1433   1445   
  1434   1446   showdb.exe:	$(TOP)\tool\showdb.c $(SQLITE3C)
  1435   1447   	$(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1436   1448   		$(TOP)\tool\showdb.c $(SQLITE3C)
         1449  +
         1450  +showstat4.exe:	$(TOP)\tool\showstat4.c $(SQLITE3C)
         1451  +	$(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         1452  +		$(TOP)\tool\showstat4.c $(SQLITE3C)
         1453  +
         1454  +showjournal.exe:	$(TOP)\tool\showjournal.c $(SQLITE3C)
         1455  +	$(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         1456  +		$(TOP)\tool\showjournal.c $(SQLITE3C)
         1457  +
         1458  +showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C)
         1459  +	$(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         1460  +		$(TOP)\tool\showwal.c $(SQLITE3C)
         1461  +
         1462  +fts3view.exe:	$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C)
         1463  +	$(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         1464  +		$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C)
         1465  +
         1466  +rollback-test.exe:	$(TOP)\tool\rollback-test.c $(SQLITE3C)
         1467  +	$(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         1468  +		$(TOP)\tool\rollback-test.c $(SQLITE3C)
  1437   1469   
  1438   1470   LogEst.exe:	$(TOP)\tool\logest.c sqlite3.h
  1439   1471   	$(LTLINK) -Fe$@ $(TOP)\tool\LogEst.c
  1440   1472   
  1441   1473   wordcount.exe:	$(TOP)\test\wordcount.c $(SQLITE3C)
  1442   1474   	$(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1443   1475   		$(TOP)\test\wordcount.c $(SQLITE3C)
................................................................................
  1459   1491   	-rmdir /Q/S quota2b
  1460   1492   	-rmdir /Q/S quota2c
  1461   1493   	-rmdir /Q/S tsrc
  1462   1494   	del /Q .target_source
  1463   1495   	del /Q tclsqlite3.exe tclsqlite3.exp
  1464   1496   	del /Q testloadext.dll testloadext.exp
  1465   1497   	del /Q testfixture.exe testfixture.exp test.db
         1498  +	del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe
         1499  +	del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe
         1500  +	del /Q wordcount.exe
  1466   1501   	del /Q sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def
  1467   1502   	del /Q sqlite3.c sqlite3-*.c
  1468   1503   	del /Q sqlite3rc.h
  1469   1504   	del /Q shell.c sqlite3ext.h
  1470   1505   	del /Q sqlite3_analyzer.exe sqlite3_analyzer.exp sqlite3_analyzer.c
  1471   1506   	del /Q sqlite-*-output.vsix
  1472   1507   	del /Q mptester.exe

Changes to README.md.

    49     49   
    50     50   There are several build options that can be set via the NMAKE command
    51     51   line.  For example, to build for WinRT, simply add "FOR_WINRT=1" argument
    52     52   to the "sqlite3.dll" command line above.  When debugging into the SQLite
    53     53   code, adding the "DEBUG=1" argument to one of the above command lines is
    54     54   recommended.
    55     55   
    56         -SQLite does not require Tcl to run, but a Tcl installation is required
    57         -by the makefiles (including those for MSVC).  SQLite contains a lot of
    58         -generated code and Tcl is used to do much of that code generation.  The
    59         -makefiles also require AWK.
           56  +SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation
           57  +is required by the makefiles (including those for MSVC).  SQLite contains
           58  +a lot of generated code and Tcl is used to do much of that code generation.
           59  +The makefiles also require AWK.
    60     60   
    61     61   ## Source Code Tour
    62     62   
    63     63   Most of the core source files are in the **src/** subdirectory.  But
    64     64   src/ also contains files used to build the "testfixture" test harness;
    65     65   those file all begin with "test".  And src/ contains the "shell.c" file
    66     66   which is the main program for the "sqlite3.exe" command-line shell and
................................................................................
    91     91   of the automatically-generated files, simply run "make target_source".
    92     92   The "target_source" make target will create a subdirectory "tsrc/" and
    93     93   fill it with all the source files needed to build SQLite, both
    94     94   manually-edited files and automatically-generated files.
    95     95   
    96     96   The SQLite interface is defined by the **sqlite3.h** header file, which is
    97     97   generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION.  The
    98         -Tcl script at tool/mksqlite3h.tcl does the conversion.  The manifest.uuid
    99         -file contains the SHA1 hash of the particular check-in and is used to generate
   100         -the SQLITE_SOURCE_ID macro.  The VERSION file contains the current SQLite
   101         -version number.  The sqlite3.h header is really just a copy of src/sqlite.h.in
   102         -with the source-id and version number inserted at just the right spots.
   103         -Note that comment text in the sqlite3.h file is used to generate much of
   104         -the SQLite API documentation.  The Tcl scripts used to generate that
   105         -documentation are in a separate source repository.
           98  +[Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
           99  +The manifest.uuid file contains the SHA1 hash of the particular check-in
          100  +and is used to generate the SQLITE\_SOURCE\_ID macro.  The VERSION file
          101  +contains the current SQLite version number.  The sqlite3.h header is really
          102  +just a copy of src/sqlite.h.in with the source-id and version number inserted
          103  +at just the right spots. Note that comment text in the sqlite3.h file is
          104  +used to generate much of the SQLite API documentation.  The Tcl scripts
          105  +used to generate that documentation are in a separate source repository.
   106    106   
   107    107   The SQL language parser is **parse.c** which is generate from a grammar in
   108    108   the src/parse.y file.  The conversion of "parse.y" into "parse.c" is done
   109    109   by the [lemon](./doc/lemon.html) LALR(1) parser generator.  The source code
   110    110   for lemon is at tool/lemon.c.  Lemon uses a
   111    111   template for generating its parser.  A generic template is in tool/lempar.c,
   112    112   but SQLite uses a slightly modified template found in src/lempar.c.

Changes to VERSION.

     1         -3.8.5
            1  +3.8.8

Changes to autoconf/tea/Makefile.in.

    69     69   
    70     70   srcdir		= @srcdir@
    71     71   prefix		= @prefix@
    72     72   exec_prefix	= @exec_prefix@
    73     73   
    74     74   bindir		= @bindir@
    75     75   libdir		= @libdir@
           76  +datarootdir	= @datarootdir@
    76     77   datadir		= @datadir@
    77     78   mandir		= @mandir@
    78     79   includedir	= @includedir@
    79     80   
    80     81   DESTDIR		=
    81     82   
    82     83   PKG_DIR		= $(PACKAGE_NAME)$(PACKAGE_VERSION)

Changes to autoconf/tea/configure.in.

   162    162   AC_DEFINE(USE_TCL_STUBS, 1, [Use Tcl stubs])
   163    163   #AC_DEFINE(USE_TK_STUBS, 1, [Use Tk stubs])
   164    164   
   165    165   
   166    166   #--------------------------------------------------------------------
   167    167   # Redefine fdatasync as fsync on systems that lack fdatasync
   168    168   #--------------------------------------------------------------------
   169         -
   170         -AC_CHECK_FUNC(fdatasync, , AC_DEFINE(fdatasync, fsync))
          169  +#
          170  +#AC_CHECK_FUNC(fdatasync, , AC_DEFINE(fdatasync, fsync))
          171  +# Check for library functions that SQLite can optionally use.
          172  +AC_CHECK_FUNCS([fdatasync usleep fullfsync localtime_r gmtime_r])
   171    173   
   172    174   AC_FUNC_STRERROR_R
   173    175   
   174    176   
   175    177   #--------------------------------------------------------------------
   176    178   # This macro generates a line to use when building a library.  It
   177    179   # depends on values set by the TEA_ENABLE_SHARED, TEA_ENABLE_SYMBOLS,

Changes to autoconf/tea/tclconfig/tcl.m4.

  1637   1637   	    ])
  1638   1638   	    ;;
  1639   1639   	FreeBSD-*)
  1640   1640   	    # This configuration from FreeBSD Ports.
  1641   1641   	    SHLIB_CFLAGS="-fPIC"
  1642   1642   	    SHLIB_LD="${CC} -shared"
  1643   1643   	    TCL_SHLIB_LD_EXTRAS="-Wl,-soname=\$[@]"
         1644  +	    TK_SHLIB_LD_EXTRAS="-Wl,-soname,\$[@]"
  1644   1645   	    SHLIB_SUFFIX=".so"
  1645   1646   	    LDFLAGS=""
  1646   1647   	    AS_IF([test $doRpath = yes], [
  1647   1648   		CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}'
  1648   1649   		LD_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}'])
  1649   1650   	    AS_IF([test "${TCL_THREADS}" = "1"], [
  1650   1651   		# The -pthread needs to go in the LDFLAGS, not LIBS
  1651   1652   		LIBS=`echo $LIBS | sed s/-pthread//`
  1652   1653   		CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
  1653   1654   		LDFLAGS="$LDFLAGS $PTHREAD_LIBS"])
  1654         -	    # Version numbers are dot-stripped by system policy.
  1655         -	    TCL_TRIM_DOTS=`echo ${PACKAGE_VERSION} | tr -d .`
  1656         -	    UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a'
  1657         -	    SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}\$\{DBGX\}.so.1'
  1658         -	    TCL_LIB_VERSIONS_OK=nodots
         1655  +	    case $system in
         1656  +	    FreeBSD-3.*)
         1657  +		# Version numbers are dot-stripped by system policy.
         1658  +		TCL_TRIM_DOTS=`echo ${VERSION} | tr -d .`
         1659  +		UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a'
         1660  +		SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so'
         1661  +		TCL_LIB_VERSIONS_OK=nodots
         1662  +		;;
         1663  +	    esac
  1659   1664   	    ;;
  1660   1665   	Darwin-*)
  1661   1666   	    CFLAGS_OPTIMIZE="-Os"
  1662   1667   	    SHLIB_CFLAGS="-fno-common"
  1663   1668   	    # To avoid discrepancies between what headers configure sees during
  1664   1669   	    # preprocessing tests and compiling tests, move any -isysroot and
  1665   1670   	    # -mmacosx-version-min flags from CFLAGS to CPPFLAGS:
................................................................................
  1822   1827   	    LD_SEARCH_FLAGS=""
  1823   1828   	    ;;
  1824   1829   	SCO_SV-3.2*)
  1825   1830   	    AS_IF([test "$GCC" = yes], [
  1826   1831   		SHLIB_CFLAGS="-fPIC -melf"
  1827   1832   		LDFLAGS="$LDFLAGS -melf -Wl,-Bexport"
  1828   1833   	    ], [
  1829         -	       SHLIB_CFLAGS="-Kpic -belf"
  1830         -	       LDFLAGS="$LDFLAGS -belf -Wl,-Bexport"
         1834  +		SHLIB_CFLAGS="-Kpic -belf"
         1835  +		LDFLAGS="$LDFLAGS -belf -Wl,-Bexport"
  1831   1836   	    ])
  1832   1837   	    SHLIB_LD="ld -G"
  1833   1838   	    SHLIB_LD_LIBS=""
  1834   1839   	    SHLIB_SUFFIX=".so"
  1835   1840   	    CC_SEARCH_FLAGS=""
  1836   1841   	    LD_SEARCH_FLAGS=""
  1837   1842   	    ;;
................................................................................
  4154   4159   	    no_celib=
  4155   4160   	    CELIB_DIR=${ac_cv_c_celibconfig}
  4156   4161   	    CELIB_DIR=`echo "$CELIB_DIR" | sed -e 's!\\\!/!g'`
  4157   4162   	    AC_MSG_RESULT([found $CELIB_DIR])
  4158   4163   	fi
  4159   4164       fi
  4160   4165   ])
  4161         -
  4162         -
  4163   4166   # Local Variables:
  4164   4167   # mode: autoconf
  4165   4168   # End:

Changes to configure.

     1      1   #! /bin/sh
     2      2   # Guess values for system-dependent variables and create Makefiles.
     3         -# Generated by GNU Autoconf 2.62 for sqlite 3.8.5.
            3  +# Generated by GNU Autoconf 2.62 for sqlite 3.8.8.
     4      4   #
     5      5   # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
     6      6   # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
     7      7   # This configure script is free software; the Free Software Foundation
     8      8   # gives unlimited permission to copy, distribute and modify it.
     9      9   ## --------------------- ##
    10     10   ## M4sh Initialization.  ##
................................................................................
   739    739   MFLAGS=
   740    740   MAKEFLAGS=
   741    741   SHELL=${CONFIG_SHELL-/bin/sh}
   742    742   
   743    743   # Identity of this package.
   744    744   PACKAGE_NAME='sqlite'
   745    745   PACKAGE_TARNAME='sqlite'
   746         -PACKAGE_VERSION='3.8.5'
   747         -PACKAGE_STRING='sqlite 3.8.5'
          746  +PACKAGE_VERSION='3.8.8'
          747  +PACKAGE_STRING='sqlite 3.8.8'
   748    748   PACKAGE_BUGREPORT=''
   749    749   
   750    750   # Factoring default headers for most tests.
   751    751   ac_includes_default="\
   752    752   #include <stdio.h>
   753    753   #ifdef HAVE_SYS_TYPES_H
   754    754   # include <sys/types.h>
................................................................................
  1479   1479   #
  1480   1480   # Report the --help message.
  1481   1481   #
  1482   1482   if test "$ac_init_help" = "long"; then
  1483   1483     # Omit some internal or obsolete options to make the list less imposing.
  1484   1484     # This message is too long to be a string in the A/UX 3.1 sh.
  1485   1485     cat <<_ACEOF
  1486         -\`configure' configures sqlite 3.8.5 to adapt to many kinds of systems.
         1486  +\`configure' configures sqlite 3.8.8 to adapt to many kinds of systems.
  1487   1487   
  1488   1488   Usage: $0 [OPTION]... [VAR=VALUE]...
  1489   1489   
  1490   1490   To assign environment variables (e.g., CC, CFLAGS...), specify them as
  1491   1491   VAR=VALUE.  See below for descriptions of some of the useful variables.
  1492   1492   
  1493   1493   Defaults for the options are specified in brackets.
................................................................................
  1544   1544     --build=BUILD     configure for building on BUILD [guessed]
  1545   1545     --host=HOST       cross-compile to build programs to run on HOST [BUILD]
  1546   1546   _ACEOF
  1547   1547   fi
  1548   1548   
  1549   1549   if test -n "$ac_init_help"; then
  1550   1550     case $ac_init_help in
  1551         -     short | recursive ) echo "Configuration of sqlite 3.8.5:";;
         1551  +     short | recursive ) echo "Configuration of sqlite 3.8.8:";;
  1552   1552      esac
  1553   1553     cat <<\_ACEOF
  1554   1554   
  1555   1555   Optional Features:
  1556   1556     --disable-option-checking  ignore unrecognized --enable/--with options
  1557   1557     --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  1558   1558     --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
  1660   1660       cd "$ac_pwd" || { ac_status=$?; break; }
  1661   1661     done
  1662   1662   fi
  1663   1663   
  1664   1664   test -n "$ac_init_help" && exit $ac_status
  1665   1665   if $ac_init_version; then
  1666   1666     cat <<\_ACEOF
  1667         -sqlite configure 3.8.5
         1667  +sqlite configure 3.8.8
  1668   1668   generated by GNU Autoconf 2.62
  1669   1669   
  1670   1670   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
  1671   1671   2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
  1672   1672   This configure script is free software; the Free Software Foundation
  1673   1673   gives unlimited permission to copy, distribute and modify it.
  1674   1674   _ACEOF
  1675   1675     exit
  1676   1676   fi
  1677   1677   cat >config.log <<_ACEOF
  1678   1678   This file contains any messages produced by compilers while
  1679   1679   running configure, to aid debugging if configure makes a mistake.
  1680   1680   
  1681         -It was created by sqlite $as_me 3.8.5, which was
         1681  +It was created by sqlite $as_me 3.8.8, which was
  1682   1682   generated by GNU Autoconf 2.62.  Invocation command line was
  1683   1683   
  1684   1684     $ $0 $@
  1685   1685   
  1686   1686   _ACEOF
  1687   1687   exec 5>>config.log
  1688   1688   {
................................................................................
 14017  14017   
 14018  14018   exec 6>&1
 14019  14019   
 14020  14020   # Save the log message, to keep $[0] and so on meaningful, and to
 14021  14021   # report actual input values of CONFIG_FILES etc. instead of their
 14022  14022   # values after options handling.
 14023  14023   ac_log="
 14024         -This file was extended by sqlite $as_me 3.8.5, which was
        14024  +This file was extended by sqlite $as_me 3.8.8, which was
 14025  14025   generated by GNU Autoconf 2.62.  Invocation command line was
 14026  14026   
 14027  14027     CONFIG_FILES    = $CONFIG_FILES
 14028  14028     CONFIG_HEADERS  = $CONFIG_HEADERS
 14029  14029     CONFIG_LINKS    = $CONFIG_LINKS
 14030  14030     CONFIG_COMMANDS = $CONFIG_COMMANDS
 14031  14031     $ $0 $@
................................................................................
 14070  14070   $config_commands
 14071  14071   
 14072  14072   Report bugs to <bug-autoconf@gnu.org>."
 14073  14073   
 14074  14074   _ACEOF
 14075  14075   cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 14076  14076   ac_cs_version="\\
 14077         -sqlite config.status 3.8.5
        14077  +sqlite config.status 3.8.8
 14078  14078   configured by $0, generated by GNU Autoconf 2.62,
 14079  14079     with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\"
 14080  14080   
 14081  14081   Copyright (C) 2008 Free Software Foundation, Inc.
 14082  14082   This config.status script is free software; the Free Software Foundation
 14083  14083   gives unlimited permission to copy, distribute and modify it."
 14084  14084   

Changes to ext/fts3/fts3.c.

  3112   3112     if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++];
  3113   3113     if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++];
  3114   3114     assert( iIdx==nVal );
  3115   3115   
  3116   3116     /* In case the cursor has been used before, clear it now. */
  3117   3117     sqlite3_finalize(pCsr->pStmt);
  3118   3118     sqlite3_free(pCsr->aDoclist);
         3119  +  sqlite3_free(pCsr->aMatchinfo);
  3119   3120     sqlite3Fts3ExprFree(pCsr->pExpr);
  3120   3121     memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
  3121   3122   
  3122   3123     /* Set the lower and upper bounds on docids to return */
  3123   3124     pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64);
  3124   3125     pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64);
  3125   3126   
................................................................................
  3746   3747   **
  3747   3748   ** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed
  3748   3749   ** to by the argument to point to the "simple" tokenizer implementation.
  3749   3750   ** And so on.
  3750   3751   */
  3751   3752   void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
  3752   3753   void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
  3753         -#ifdef SQLITE_ENABLE_FTS4_UNICODE61
         3754  +#ifndef SQLITE_DISABLE_FTS3_UNICODE
  3754   3755   void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule);
  3755   3756   #endif
  3756   3757   #ifdef SQLITE_ENABLE_ICU
  3757   3758   void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
  3758   3759   #endif
  3759   3760   
  3760   3761   /*
................................................................................
  3764   3765   ** function is called by the sqlite3_extension_init() entry point.
  3765   3766   */
  3766   3767   int sqlite3Fts3Init(sqlite3 *db){
  3767   3768     int rc = SQLITE_OK;
  3768   3769     Fts3Hash *pHash = 0;
  3769   3770     const sqlite3_tokenizer_module *pSimple = 0;
  3770   3771     const sqlite3_tokenizer_module *pPorter = 0;
  3771         -#ifdef SQLITE_ENABLE_FTS4_UNICODE61
         3772  +#ifndef SQLITE_DISABLE_FTS3_UNICODE
  3772   3773     const sqlite3_tokenizer_module *pUnicode = 0;
  3773   3774   #endif
  3774   3775   
  3775   3776   #ifdef SQLITE_ENABLE_ICU
  3776   3777     const sqlite3_tokenizer_module *pIcu = 0;
  3777   3778     sqlite3Fts3IcuTokenizerModule(&pIcu);
  3778   3779   #endif
  3779   3780   
  3780         -#ifdef SQLITE_ENABLE_FTS4_UNICODE61
         3781  +#ifndef SQLITE_DISABLE_FTS3_UNICODE
  3781   3782     sqlite3Fts3UnicodeTokenizer(&pUnicode);
  3782   3783   #endif
  3783   3784   
  3784   3785   #ifdef SQLITE_TEST
  3785   3786     rc = sqlite3Fts3InitTerm(db);
  3786   3787     if( rc!=SQLITE_OK ) return rc;
  3787   3788   #endif
................................................................................
  3801   3802     }
  3802   3803   
  3803   3804     /* Load the built-in tokenizers into the hash table */
  3804   3805     if( rc==SQLITE_OK ){
  3805   3806       if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
  3806   3807        || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) 
  3807   3808   
  3808         -#ifdef SQLITE_ENABLE_FTS4_UNICODE61
         3809  +#ifndef SQLITE_DISABLE_FTS3_UNICODE
  3809   3810        || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) 
  3810   3811   #endif
  3811   3812   #ifdef SQLITE_ENABLE_ICU
  3812   3813        || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
  3813   3814   #endif
  3814   3815       ){
  3815   3816         rc = SQLITE_NOMEM;
................................................................................
  4422   4423         for(i=0; rc==SQLITE_OK && i<p->nToken && bEof==0; i++){
  4423   4424           rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof);
  4424   4425           if( a[i].bIgnore==0 && (bMaxSet==0 || DOCID_CMP(iMax, a[i].iDocid)<0) ){
  4425   4426             iMax = a[i].iDocid;
  4426   4427             bMaxSet = 1;
  4427   4428           }
  4428   4429         }
  4429         -      assert( rc!=SQLITE_OK || a[p->nToken-1].bIgnore==0 );
         4430  +      assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) );
  4430   4431         assert( rc!=SQLITE_OK || bMaxSet );
  4431   4432   
  4432   4433         /* Keep advancing iterators until they all point to the same document */
  4433   4434         for(i=0; i<p->nToken; i++){
  4434   4435           while( rc==SQLITE_OK && bEof==0 
  4435   4436               && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 
  4436   4437           ){

Changes to ext/fts3/fts3Int.h.

   581    581   int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
   582    582   int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
   583    583   
   584    584   /* fts3_tokenize_vtab.c */
   585    585   int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *);
   586    586   
   587    587   /* fts3_unicode2.c (functions generated by parsing unicode text files) */
   588         -#ifdef SQLITE_ENABLE_FTS4_UNICODE61
          588  +#ifndef SQLITE_DISABLE_FTS3_UNICODE
   589    589   int sqlite3FtsUnicodeFold(int, int);
   590    590   int sqlite3FtsUnicodeIsalnum(int);
   591    591   int sqlite3FtsUnicodeIsdiacritic(int);
   592    592   #endif
   593    593   
   594    594   #endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
   595    595   #endif /* _FTSINT_H */

Changes to ext/fts3/fts3_expr.c.

   186    186     sqlite3_tokenizer_cursor *pCursor;
   187    187     Fts3Expr *pRet = 0;
   188    188     int i = 0;
   189    189   
   190    190     /* Set variable i to the maximum number of bytes of input to tokenize. */
   191    191     for(i=0; i<n; i++){
   192    192       if( sqlite3_fts3_enable_parentheses && (z[i]=='(' || z[i]==')') ) break;
   193         -    if( z[i]=='*' || z[i]=='"' ) break;
          193  +    if( z[i]=='"' ) break;
   194    194     }
   195    195   
   196    196     *pnConsumed = i;
   197    197     rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, i, &pCursor);
   198    198     if( rc==SQLITE_OK ){
   199    199       const char *zToken;
   200    200       int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0;

Changes to ext/fts3/fts3_porter.c.

   179    179   ** of m for the first i bytes of a word.
   180    180   **
   181    181   ** Return true if the m-value for z is 1 or more.  In other words,
   182    182   ** return true if z contains at least one vowel that is followed
   183    183   ** by a consonant.
   184    184   **
   185    185   ** In this routine z[] is in reverse order.  So we are really looking
   186         -** for an instance of of a consonant followed by a vowel.
          186  +** for an instance of a consonant followed by a vowel.
   187    187   */
   188    188   static int m_gt_0(const char *z){
   189    189     while( isVowel(z) ){ z++; }
   190    190     if( *z==0 ) return 0;
   191    191     while( isConsonant(z) ){ z++; }
   192    192     return *z!=0;
   193    193   }

Changes to ext/fts3/fts3_unicode.c.

     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   ******************************************************************************
    12     12   **
    13     13   ** Implementation of the "unicode" full-text-search tokenizer.
    14     14   */
    15     15   
    16         -#ifdef SQLITE_ENABLE_FTS4_UNICODE61
           16  +#ifndef SQLITE_DISABLE_FTS3_UNICODE
    17     17   
    18     18   #include "fts3Int.h"
    19     19   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    20     20   
    21     21   #include <assert.h>
    22     22   #include <stdlib.h>
    23     23   #include <stdio.h>
................................................................................
   227    227     pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
   228    228     if( pNew==NULL ) return SQLITE_NOMEM;
   229    229     memset(pNew, 0, sizeof(unicode_tokenizer));
   230    230     pNew->bRemoveDiacritic = 1;
   231    231   
   232    232     for(i=0; rc==SQLITE_OK && i<nArg; i++){
   233    233       const char *z = azArg[i];
   234         -    int n = strlen(z);
          234  +    int n = (int)strlen(z);
   235    235   
   236    236       if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
   237    237         pNew->bRemoveDiacritic = 1;
   238    238       }
   239    239       else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
   240    240         pNew->bRemoveDiacritic = 0;
   241    241       }
................................................................................
   314    314     int *pnToken,                   /* OUT: Number of bytes at *paToken */
   315    315     int *piStart,                   /* OUT: Starting offset of token */
   316    316     int *piEnd,                     /* OUT: Ending offset of token */
   317    317     int *piPos                      /* OUT: Position integer of token */
   318    318   ){
   319    319     unicode_cursor *pCsr = (unicode_cursor *)pC;
   320    320     unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer);
   321         -  int iCode;
          321  +  int iCode = 0;
   322    322     char *zOut;
   323    323     const unsigned char *z = &pCsr->aInput[pCsr->iOff];
   324    324     const unsigned char *zStart = z;
   325    325     const unsigned char *zEnd;
   326    326     const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput];
   327    327   
   328    328     /* Scan past any delimiter characters before the start of the next token.
................................................................................
   359    359       if( z>=zTerm ) break;
   360    360       READ_UTF8(z, zTerm, iCode);
   361    361     }while( unicodeIsAlnum(p, iCode) 
   362    362          || sqlite3FtsUnicodeIsdiacritic(iCode)
   363    363     );
   364    364   
   365    365     /* Set the output variables and return. */
   366         -  pCsr->iOff = (z - pCsr->aInput);
          366  +  pCsr->iOff = (int)(z - pCsr->aInput);
   367    367     *paToken = pCsr->zToken;
   368         -  *pnToken = zOut - pCsr->zToken;
   369         -  *piStart = (zStart - pCsr->aInput);
   370         -  *piEnd = (zEnd - pCsr->aInput);
          368  +  *pnToken = (int)(zOut - pCsr->zToken);
          369  +  *piStart = (int)(zStart - pCsr->aInput);
          370  +  *piEnd = (int)(zEnd - pCsr->aInput);
   371    371     *piPos = pCsr->iToken++;
   372    372     return SQLITE_OK;
   373    373   }
   374    374   
   375    375   /*
   376    376   ** Set *ppModule to a pointer to the sqlite3_tokenizer_module 
   377    377   ** structure for the unicode tokenizer.
................................................................................
   386    386       unicodeNext,
   387    387       0,
   388    388     };
   389    389     *ppModule = &module;
   390    390   }
   391    391   
   392    392   #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
   393         -#endif /* ifndef SQLITE_ENABLE_FTS4_UNICODE61 */
          393  +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */

Changes to ext/fts3/fts3_unicode2.c.

    11     11   ******************************************************************************
    12     12   */
    13     13   
    14     14   /*
    15     15   ** DO NOT EDIT THIS MACHINE GENERATED FILE.
    16     16   */
    17     17   
    18         -#if defined(SQLITE_ENABLE_FTS4_UNICODE61)
           18  +#ifndef SQLITE_DISABLE_FTS3_UNICODE
    19     19   #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
    20     20   
    21     21   #include <assert.h>
    22     22   
    23     23   /*
    24     24   ** Return true if the argument corresponds to a unicode codepoint
    25     25   ** classified as either a letter or a number. Otherwise false.
................................................................................
    35     35     ** The most significant 22 bits in each 32-bit value contain the first 
    36     36     ** codepoint in the range. The least significant 10 bits are used to store
    37     37     ** the size of the range (always at least 1). In other words, the value 
    38     38     ** ((C<<22) + N) represents a range of N codepoints starting with codepoint 
    39     39     ** C. It is not possible to represent a range larger than 1023 codepoints 
    40     40     ** using this format.
    41     41     */
    42         -  const static unsigned int aEntry[] = {
           42  +  static const unsigned int aEntry[] = {
    43     43       0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07,
    44     44       0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01,
    45     45       0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401,
    46     46       0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01,
    47     47       0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01,
    48     48       0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802,
    49     49       0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F,
................................................................................
   127    127       0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
   128    128     };
   129    129   
   130    130     if( c<128 ){
   131    131       return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
   132    132     }else if( c<(1<<22) ){
   133    133       unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
   134         -    int iRes;
          134  +    int iRes = 0;
   135    135       int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
   136    136       int iLo = 0;
   137    137       while( iHi>=iLo ){
   138    138         int iTest = (iHi + iLo) / 2;
   139    139         if( key >= aEntry[iTest] ){
   140    140           iRes = iTest;
   141    141           iLo = iTest+1;
................................................................................
   198    198         iLo = iTest+1;
   199    199       }else{
   200    200         iHi = iTest-1;
   201    201       }
   202    202     }
   203    203     assert( key>=aDia[iRes] );
   204    204     return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
   205         -};
          205  +}
   206    206   
   207    207   
   208    208   /*
   209    209   ** Return true if the argument interpreted as a unicode codepoint
   210    210   ** is a diacritical modifier character.
   211    211   */
   212    212   int sqlite3FtsUnicodeIsdiacritic(int c){
................................................................................
   358    358     else if( c>=66560 && c<66600 ){
   359    359       ret = c + 40;
   360    360     }
   361    361   
   362    362     return ret;
   363    363   }
   364    364   #endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */
   365         -#endif /* !defined(SQLITE_ENABLE_FTS4_UNICODE61) */
          365  +#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */

Changes to ext/fts3/tool/fts3view.c.

   372    372     n = 0;
   373    373     while( sqlite3_step(pStmt)==SQLITE_ROW ){
   374    374       n = sqlite3_column_int(pStmt, 0);
   375    375     }
   376    376     sqlite3_finalize(pStmt);
   377    377     nLeaf = nSeg - nIdx;
   378    378     printf("Leaf segments larger than %5d bytes.... %9d   %5.2f%%\n",
   379         -         pgsz-45, n, n*100.0/nLeaf);
          379  +         pgsz-45, n, nLeaf>0 ? n*100.0/nLeaf : 0.0);
   380    380   
   381    381     pStmt = prepare(db, "SELECT max(level%%1024) FROM '%q_segdir'", zTab);
   382    382     mxLevel = 0;
   383    383     while( sqlite3_step(pStmt)==SQLITE_ROW ){
   384    384       mxLevel = sqlite3_column_int(pStmt, 0);
   385    385     }
   386    386     sqlite3_finalize(pStmt);
................................................................................
   550    550   ){
   551    551     sqlite3_int64 iChild;
   552    552     sqlite3_int64 iPrefix;
   553    553     sqlite3_int64 nTerm;
   554    554     sqlite3_int64 n;
   555    555     sqlite3_int64 iDocsz;
   556    556     int iHeight;
   557         -  int i = 0;
          557  +  sqlite3_int64 i = 0;
   558    558     int cnt = 0;
   559    559     char zTerm[1000];
   560    560   
   561    561     i += getVarint(aData, &n);
   562    562     iHeight = (int)n;
   563    563     printf("height: %d\n", iHeight);
   564    564     if( iHeight>0 ){
................................................................................
   572    572         iPrefix = 0;
   573    573       }
   574    574       i += getVarint(aData+i, &nTerm);
   575    575       if( iPrefix+nTerm+1 >= sizeof(zTerm) ){
   576    576         fprintf(stderr, "term to long\n");
   577    577         exit(1);
   578    578       }
   579         -    memcpy(zTerm+iPrefix, aData+i, nTerm);
          579  +    memcpy(zTerm+iPrefix, aData+i, (size_t)nTerm);
   580    580       zTerm[iPrefix+nTerm] = 0;
   581    581       i += nTerm;
   582    582       if( iHeight==0 ){
   583    583         i += getVarint(aData+i, &iDocsz);
   584         -      printf("term: %-25s doclist %7lld bytes offset %d\n", zTerm, iDocsz, i);
          584  +      printf("term: %-25s doclist %7lld bytes offset %lld\n", zTerm, iDocsz, i);
   585    585         i += iDocsz;
   586    586       }else{
   587    587         printf("term: %-25s child %lld\n", zTerm, ++iChild);
   588    588       }
   589    589     }
   590    590   }
   591    591     
................................................................................
   745    745   ** is azExtra[2].
   746    746   **
   747    747   ** If the --raw option is present in azExtra, then a hex dump is provided.
   748    748   ** Otherwise a decoding is shown.
   749    749   */
   750    750   static void showDoclist(sqlite3 *db, const char *zTab){
   751    751     const unsigned char *aData;
   752         -  sqlite3_int64 offset, nData;
          752  +  sqlite3_int64 offset;
          753  +  int nData;
   753    754     sqlite3_stmt *pStmt;
   754    755   
   755    756     offset = atoi64(azExtra[1]);
   756         -  nData = atoi64(azExtra[2]);
          757  +  nData = atoi(azExtra[2]);
   757    758     pStmt = prepareToGetSegment(db, zTab, azExtra[0]);
   758    759     if( sqlite3_step(pStmt)!=SQLITE_ROW ){
   759    760       sqlite3_finalize(pStmt);
   760    761       return;
   761    762     }
   762    763     aData = sqlite3_column_blob(pStmt, 0);
   763         -  printf("Doclist at %s offset %lld of size %lld bytes:\n",
          764  +  printf("Doclist at %s offset %lld of size %d bytes:\n",
   764    765            azExtra[0], offset, nData);
   765    766     if( findOption("raw", 0, 0)!=0 ){
   766    767       printBlob(aData+offset, nData);
   767    768     }else{
   768    769       decodeDoclist(aData+offset, nData);
   769    770     }
   770    771     sqlite3_finalize(pStmt);

Changes to ext/fts3/unicode/mkunicode.tcl.

   156    156         iLo = iTest+1;
   157    157       }else{
   158    158         iHi = iTest-1;
   159    159       }
   160    160     }
   161    161     assert( key>=aDia[iRes] );
   162    162     return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);}
   163         -  puts "\};"
          163  +  puts "\}"
   164    164   }
   165    165   
   166    166   proc print_isdiacritic {zFunc map} {
   167    167   
   168    168     set lCode [list]
   169    169     foreach m $map {
   170    170       foreach {code char} $m {}
................................................................................
   294    294     ** codepoint in the range. The least significant 10 bits are used to store
   295    295     ** the size of the range (always at least 1). In other words, the value 
   296    296     ** ((C<<22) + N) represents a range of N codepoints starting with codepoint 
   297    297     ** C. It is not possible to represent a range larger than 1023 codepoints 
   298    298     ** using this format.
   299    299     */
   300    300     }]
   301         -  puts -nonewline "  const static unsigned int aEntry\[\] = \{"
          301  +  puts -nonewline "  static const unsigned int aEntry\[\] = \{"
   302    302     set i 0
   303    303     foreach range $lRange {
   304    304       foreach {iFirst nRange} $range {}
   305    305       set u32 [format "0x%08X" [expr ($iFirst<<10) + $nRange]]
   306    306   
   307    307       if {($i % 5)==0} {puts "" ; puts -nonewline "   "}
   308    308       puts -nonewline " $u32,"
................................................................................
   345    345     an_print_range_array $lRange
   346    346     an_print_ascii_bitmap $lRange
   347    347     puts {
   348    348     if( c<128 ){
   349    349       return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
   350    350     }else if( c<(1<<22) ){
   351    351       unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
   352         -    int iRes;
          352  +    int iRes = 0;
   353    353       int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
   354    354       int iLo = 0;
   355    355       while( iHi>=iLo ){
   356    356         int iTest = (iHi + iLo) / 2;
   357    357         if( key >= aEntry[iTest] ){
   358    358           iRes = iTest;
   359    359           iLo = iTest+1;
................................................................................
   731    731   ** DO NOT EDIT THIS MACHINE GENERATED FILE.
   732    732   */
   733    733     }]
   734    734     puts ""
   735    735     if {$::generate_fts5_code} {
   736    736       puts "#if defined(SQLITE_ENABLE_FTS5)"
   737    737     } else {
   738         -    puts "#if defined(SQLITE_ENABLE_FTS4_UNICODE61)"
          738  +    puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE"
   739    739       puts "#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)"
   740    740     }
   741    741     puts ""
   742    742     puts "#include <assert.h>"
   743    743     puts ""
   744    744   }
   745    745   
................................................................................
   827    827     print_test_main 
   828    828   }
   829    829   
   830    830   if {$generate_fts5_code} {
   831    831     puts "#endif /* defined(SQLITE_ENABLE_FTS5) */"
   832    832   } else {
   833    833     puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */"
   834         -  puts "#endif /* !defined(SQLITE_ENABLE_FTS4_UNICODE61) */"
          834  +  puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */"
   835    835   }

Added ext/misc/eval.c.

            1  +/*
            2  +** 2014-11-10
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** This SQLite extension implements SQL function eval() which runs
           14  +** SQL statements recursively.
           15  +*/
           16  +#include "sqlite3ext.h"
           17  +SQLITE_EXTENSION_INIT1
           18  +#include <string.h>
           19  +
           20  +/*
           21  +** Structure used to accumulate the output
           22  +*/
           23  +struct EvalResult {
           24  +  char *z;               /* Accumulated output */
           25  +  const char *zSep;      /* Separator */
           26  +  int szSep;             /* Size of the separator string */
           27  +  sqlite3_int64 nAlloc;  /* Number of bytes allocated for z[] */
           28  +  sqlite3_int64 nUsed;   /* Number of bytes of z[] actually used */
           29  +};
           30  +
           31  +/*
           32  +** Callback from sqlite_exec() for the eval() function.
           33  +*/
           34  +static int callback(void *pCtx, int argc, char **argv, char **colnames){
           35  +  struct EvalResult *p = (struct EvalResult*)pCtx;
           36  +  int i; 
           37  +  for(i=0; i<argc; i++){
           38  +    const char *z = argv[i] ? argv[i] : "";
           39  +    size_t sz = strlen(z);
           40  +    if( (sqlite3_int64)sz+p->nUsed+p->szSep+1 > p->nAlloc ){
           41  +      char *zNew;
           42  +      p->nAlloc = p->nAlloc*2 + sz + p->szSep + 1;
           43  +      /* Using sqlite3_realloc64() would be better, but it is a recent
           44  +      ** addition and will cause a segfault if loaded by an older version
           45  +      ** of SQLite.  */
           46  +      zNew = p->nAlloc<=0x7fffffff ? sqlite3_realloc(p->z, (int)p->nAlloc) : 0;
           47  +      if( zNew==0 ){
           48  +        sqlite3_free(p->z);
           49  +        memset(p, 0, sizeof(*p));
           50  +        return 1;
           51  +      }
           52  +      p->z = zNew;
           53  +    }
           54  +    if( p->nUsed>0 ){
           55  +      memcpy(&p->z[p->nUsed], p->zSep, p->szSep);
           56  +      p->nUsed += p->szSep;
           57  +    }
           58  +    memcpy(&p->z[p->nUsed], z, sz);
           59  +    p->nUsed += sz;
           60  +  }
           61  +  return 0;
           62  +}
           63  +
           64  +/*
           65  +** Implementation of the eval(X) and eval(X,Y) SQL functions.
           66  +**
           67  +** Evaluate the SQL text in X.  Return the results, using string
           68  +** Y as the separator.  If Y is omitted, use a single space character.
           69  +*/
           70  +static void sqlEvalFunc(
           71  +  sqlite3_context *context,
           72  +  int argc,
           73  +  sqlite3_value **argv
           74  +){
           75  +  const char *zSql;
           76  +  sqlite3 *db;
           77  +  char *zErr = 0;
           78  +  int rc;
           79  +  struct EvalResult x;
           80  +
           81  +  memset(&x, 0, sizeof(x));
           82  +  x.zSep = " ";
           83  +  zSql = (const char*)sqlite3_value_text(argv[0]);
           84  +  if( zSql==0 ) return;
           85  +  if( argc>1 ){
           86  +    x.zSep = (const char*)sqlite3_value_text(argv[1]);
           87  +    if( x.zSep==0 ) return;
           88  +  }
           89  +  x.szSep = (int)strlen(x.zSep);
           90  +  db = sqlite3_context_db_handle(context);
           91  +  rc = sqlite3_exec(db, zSql, callback, &x, &zErr);
           92  +  if( rc!=SQLITE_OK ){
           93  +    sqlite3_result_error(context, zErr, -1);
           94  +    sqlite3_free(zErr);
           95  +  }else if( x.zSep==0 ){
           96  +    sqlite3_result_error_nomem(context);
           97  +    sqlite3_free(x.z);
           98  +  }else{
           99  +    sqlite3_result_text(context, x.z, (int)x.nUsed, sqlite3_free);
          100  +  }
          101  +}
          102  +
          103  +
          104  +#ifdef _WIN32
          105  +__declspec(dllexport)
          106  +#endif
          107  +int sqlite3_eval_init(
          108  +  sqlite3 *db, 
          109  +  char **pzErrMsg, 
          110  +  const sqlite3_api_routines *pApi
          111  +){
          112  +  int rc = SQLITE_OK;
          113  +  SQLITE_EXTENSION_INIT2(pApi);
          114  +  (void)pzErrMsg;  /* Unused parameter */
          115  +  rc = sqlite3_create_function(db, "eval", 1, SQLITE_UTF8, 0,
          116  +                               sqlEvalFunc, 0, 0);
          117  +  if( rc==SQLITE_OK ){
          118  +    rc = sqlite3_create_function(db, "eval", 2, SQLITE_UTF8, 0,
          119  +                                 sqlEvalFunc, 0, 0);
          120  +  }
          121  +  return rc;
          122  +}

Changes to ext/misc/fileio.c.

    57     57   static void writefileFunc(
    58     58     sqlite3_context *context,
    59     59     int argc,
    60     60     sqlite3_value **argv
    61     61   ){
    62     62     FILE *out;
    63     63     const char *z;
    64         -  int n;
    65     64     sqlite3_int64 rc;
    66     65     const char *zFile;
    67     66   
    68     67     zFile = (const char*)sqlite3_value_text(argv[0]);
    69     68     if( zFile==0 ) return;
    70     69     out = fopen(zFile, "wb");
    71     70     if( out==0 ) return;
    72     71     z = (const char*)sqlite3_value_blob(argv[1]);
    73     72     if( z==0 ){
    74         -    n = 0;
    75     73       rc = 0;
    76     74     }else{
    77         -    n = sqlite3_value_bytes(argv[1]);
    78         -    rc = fwrite(z, 1, n, out);
           75  +    rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out);
    79     76     }
    80     77     fclose(out);
    81     78     sqlite3_result_int64(context, rc);
    82     79   }
    83     80   
    84     81   
    85     82   #ifdef _WIN32

Added ext/misc/showauth.c.

            1  +/*
            2  +** 2014-09-21
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** This SQLite extension adds a debug "authorizer" callback to the database
           14  +** connection.  The callback merely writes the authorization request to
           15  +** standard output and returns SQLITE_OK.
           16  +**
           17  +** This extension can be used (for example) in the command-line shell to
           18  +** trace the operation of the authorizer.
           19  +*/
           20  +#include "sqlite3ext.h"
           21  +SQLITE_EXTENSION_INIT1
           22  +#include <stdio.h>
           23  +
           24  +/*
           25  +** Display the authorization request
           26  +*/
           27  +static int authCallback(
           28  +  void *pClientData,
           29  +  int op,
           30  +  const char *z1,
           31  +  const char *z2,
           32  +  const char *z3,
           33  +  const char *z4
           34  +){
           35  +  const char *zOp;
           36  +  char zOpSpace[50];
           37  +  switch( op ){
           38  +    case SQLITE_CREATE_INDEX:        zOp = "CREATE_INDEX";        break;
           39  +    case SQLITE_CREATE_TABLE:        zOp = "CREATE_TABLE";        break;
           40  +    case SQLITE_CREATE_TEMP_INDEX:   zOp = "CREATE_TEMP_INDEX";   break;
           41  +    case SQLITE_CREATE_TEMP_TABLE:   zOp = "CREATE_TEMP_TABLE";   break;
           42  +    case SQLITE_CREATE_TEMP_TRIGGER: zOp = "CREATE_TEMP_TRIGGER"; break;
           43  +    case SQLITE_CREATE_TEMP_VIEW:    zOp = "CREATE_TEMP_VIEW";    break;
           44  +    case SQLITE_CREATE_TRIGGER:      zOp = "CREATE_TRIGGER";      break;
           45  +    case SQLITE_CREATE_VIEW:         zOp = "CREATE_VIEW";         break;
           46  +    case SQLITE_DELETE:              zOp = "DELETE";              break;
           47  +    case SQLITE_DROP_INDEX:          zOp = "DROP_INDEX";          break;
           48  +    case SQLITE_DROP_TABLE:          zOp = "DROP_TABLE";          break;
           49  +    case SQLITE_DROP_TEMP_INDEX:     zOp = "DROP_TEMP_INDEX";     break;
           50  +    case SQLITE_DROP_TEMP_TABLE:     zOp = "DROP_TEMP_TABLE";     break;
           51  +    case SQLITE_DROP_TEMP_TRIGGER:   zOp = "DROP_TEMP_TRIGGER";   break;
           52  +    case SQLITE_DROP_TEMP_VIEW:      zOp = "DROP_TEMP_VIEW";      break;
           53  +    case SQLITE_DROP_TRIGGER:        zOp = "DROP_TRIGGER";        break;
           54  +    case SQLITE_DROP_VIEW:           zOp = "DROP_VIEW";           break;
           55  +    case SQLITE_INSERT:              zOp = "INSERT";              break;
           56  +    case SQLITE_PRAGMA:              zOp = "PRAGMA";              break;
           57  +    case SQLITE_READ:                zOp = "READ";                break;
           58  +    case SQLITE_SELECT:              zOp = "SELECT";              break;
           59  +    case SQLITE_TRANSACTION:         zOp = "TRANSACTION";         break;
           60  +    case SQLITE_UPDATE:              zOp = "UPDATE";              break;
           61  +    case SQLITE_ATTACH:              zOp = "ATTACH";              break;
           62  +    case SQLITE_DETACH:              zOp = "DETACH";              break;
           63  +    case SQLITE_ALTER_TABLE:         zOp = "ALTER_TABLE";         break;
           64  +    case SQLITE_REINDEX:             zOp = "REINDEX";             break;
           65  +    case SQLITE_ANALYZE:             zOp = "ANALYZE";             break;
           66  +    case SQLITE_CREATE_VTABLE:       zOp = "CREATE_VTABLE";       break;
           67  +    case SQLITE_DROP_VTABLE:         zOp = "DROP_VTABLE";         break;
           68  +    case SQLITE_FUNCTION:            zOp = "FUNCTION";            break;
           69  +    case SQLITE_SAVEPOINT:           zOp = "SAVEPOINT";           break;
           70  +    case SQLITE_COPY:                zOp = "COPY";                break;
           71  +    case SQLITE_RECURSIVE:           zOp = "RECURSIVE";           break;
           72  +
           73  +
           74  +    default: {
           75  +      sqlite3_snprintf(sizeof(zOpSpace), zOpSpace, "%d", op);
           76  +      zOp = zOpSpace;
           77  +      break;
           78  +    }
           79  +  }
           80  +  if( z1==0 ) z1 = "NULL";
           81  +  if( z2==0 ) z2 = "NULL";
           82  +  if( z3==0 ) z3 = "NULL";
           83  +  if( z4==0 ) z4 = "NULL";
           84  +  printf("AUTH: %s,%s,%s,%s,%s\n", zOp, z1, z2, z3, z4);
           85  +  return SQLITE_OK;
           86  +}
           87  +
           88  +
           89  +
           90  +#ifdef _WIN32
           91  +__declspec(dllexport)
           92  +#endif
           93  +int sqlite3_showauth_init(
           94  +  sqlite3 *db, 
           95  +  char **pzErrMsg, 
           96  +  const sqlite3_api_routines *pApi
           97  +){
           98  +  int rc = SQLITE_OK;
           99  +  SQLITE_EXTENSION_INIT2(pApi);
          100  +  (void)pzErrMsg;  /* Unused parameter */
          101  +  rc = sqlite3_set_authorizer(db, authCallback, 0);
          102  +  return rc;
          103  +}

Changes to ext/misc/spellfix.c.

  1889   1889     sqlite3 *db,
  1890   1890     void *pAux,
  1891   1891     int argc, const char *const*argv,
  1892   1892     sqlite3_vtab **ppVTab,
  1893   1893     char **pzErr
  1894   1894   ){
  1895   1895     spellfix1_vtab *pNew = 0;
  1896         -  const char *zModule = argv[0];
         1896  +  /* const char *zModule = argv[0]; // not used */
  1897   1897     const char *zDbName = argv[1];
  1898   1898     const char *zTableName = argv[2];
  1899   1899     int nDbName;
  1900   1900     int rc = SQLITE_OK;
  1901   1901     int i;
  1902   1902   
  1903   1903     nDbName = (int)strlen(zDbName);
................................................................................
  1943   1943            "  k2 TEXT\n"
  1944   1944            ");\n",
  1945   1945            zDbName, zTableName
  1946   1946         );
  1947   1947         spellfix1DbExec(&rc, db,
  1948   1948            "CREATE INDEX IF NOT EXISTS \"%w\".\"%w_vocab_index_langid_k2\" "
  1949   1949               "ON \"%w_vocab\"(langid,k2);",
  1950         -         zDbName, zModule, zTableName
         1950  +         zDbName, zTableName, zTableName
  1951   1951         );
  1952   1952       }
  1953   1953       for(i=3; rc==SQLITE_OK && i<argc; i++){
  1954   1954         if( strncmp(argv[i],"edit_cost_table=",16)==0 && pNew->zCostTable==0 ){
  1955   1955           pNew->zCostTable = spellfix1Dequote(&argv[i][16]);
  1956   1956           if( pNew->zCostTable==0 ) rc = SQLITE_NOMEM;
  1957   1957           continue;
................................................................................
  2732   2732       }
  2733   2733       zK2 = (char*)phoneticHash((const unsigned char*)zK1, i);
  2734   2734       if( zK2==0 ){
  2735   2735         sqlite3_free(zK1);
  2736   2736         return SQLITE_NOMEM;
  2737   2737       }
  2738   2738       if( sqlite3_value_type(argv[0])==SQLITE_NULL ){
  2739         -      spellfix1DbExec(&rc, db,
  2740         -             "INSERT INTO \"%w\".\"%w_vocab\"(rank,langid,word,k1,k2) "
  2741         -             "VALUES(%d,%d,%Q,%Q,%Q)",
  2742         -             p->zDbName, p->zTableName,
  2743         -             iRank, iLang, zWord, zK1, zK2
  2744         -      );
         2739  +      if( sqlite3_value_type(argv[1])==SQLITE_NULL ){
         2740  +        spellfix1DbExec(&rc, db,
         2741  +               "INSERT INTO \"%w\".\"%w_vocab\"(rank,langid,word,k1,k2) "
         2742  +               "VALUES(%d,%d,%Q,%Q,%Q)",
         2743  +               p->zDbName, p->zTableName,
         2744  +               iRank, iLang, zWord, zK1, zK2
         2745  +        );
         2746  +      }else{
         2747  +        newRowid = sqlite3_value_int64(argv[1]);
         2748  +        spellfix1DbExec(&rc, db,
         2749  +               "INSERT INTO \"%w\".\"%w_vocab\"(id,rank,langid,word,k1,k2) "
         2750  +               "VALUES(%lld,%d,%d,%Q,%Q,%Q)",
         2751  +               p->zDbName, p->zTableName,
         2752  +               newRowid, iRank, iLang, zWord, zK1, zK2
         2753  +        );
         2754  +      }
  2745   2755         *pRowid = sqlite3_last_insert_rowid(db);
  2746   2756       }else{
  2747   2757         rowid = sqlite3_value_int64(argv[0]);
  2748   2758         newRowid = *pRowid = sqlite3_value_int64(argv[1]);
  2749   2759         spellfix1DbExec(&rc, db,
  2750   2760                "UPDATE \"%w\".\"%w_vocab\" SET id=%lld, rank=%d, langid=%d,"
  2751   2761                " word=%Q, k1=%Q, k2=%Q WHERE id=%lld",

Changes to ext/rtree/rtree.c.

  1529   1529     RtreeNode *pRoot = 0;
  1530   1530     int ii;
  1531   1531     int rc = SQLITE_OK;
  1532   1532     int iCell = 0;
  1533   1533   
  1534   1534     rtreeReference(pRtree);
  1535   1535   
         1536  +  /* Reset the cursor to the same state as rtreeOpen() leaves it in. */
  1536   1537     freeCursorConstraints(pCsr);
         1538  +  sqlite3_free(pCsr->aPoint);
         1539  +  memset(pCsr, 0, sizeof(RtreeCursor));
         1540  +  pCsr->base.pVtab = (sqlite3_vtab*)pRtree;
         1541  +
  1537   1542     pCsr->iStrategy = idxNum;
  1538         -
  1539   1543     if( idxNum==1 ){
  1540   1544       /* Special case - lookup by rowid. */
  1541   1545       RtreeNode *pLeaf;        /* Leaf on which the required cell resides */
  1542   1546       RtreeSearchPoint *p;     /* Search point for the the leaf */
  1543   1547       i64 iRowid = sqlite3_value_int64(argv[0]);
  1544   1548       i64 iNode = 0;
  1545   1549       rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode);

Changes to ext/rtree/rtree1.test.

    29     29   #   rtree-4.*: Test INSERT
    30     30   #   rtree-5.*: Test DELETE
    31     31   #   rtree-6.*: Test UPDATE
    32     32   #   rtree-7.*: Test renaming an r-tree table.
    33     33   #   rtree-8.*: Test constrained scans of r-tree data.
    34     34   #
    35     35   #   rtree-12.*: Test that on-conflict clauses are supported.
           36  +#   rtree-13.*: Test that bug [d2889096e7bdeac6d] has been fixed.
    36     37   #
    37     38   
    38     39   ifcapable !rtree {
    39     40     finish_test
    40     41     return
    41     42   }
    42     43   
................................................................................
   509    510       do_test $testname.2 [list sql_uses_stmt db $sql] $uses
   510    511       do_execsql_test $testname.3 { SELECT * FROM t1 ORDER BY idx } $data
   511    512   
   512    513       do_test $testname.4 { rtree_check db t1 } 0
   513    514       db close
   514    515     }
   515    516   }
          517  +
          518  +#-------------------------------------------------------------------------
          519  +# Test that bug [d2889096e7bdeac6d] has been fixed.
          520  +#
          521  +reset_db
          522  +do_execsql_test 13.1 {
          523  +  CREATE VIRTUAL TABLE t9 USING rtree(id, xmin, xmax);
          524  +  INSERT INTO t9 VALUES(1,0,0);            
          525  +  INSERT INTO t9 VALUES(2,0,0);
          526  +  SELECT * FROM t9 WHERE id IN (1, 2);
          527  +} {1 0.0 0.0 2 0.0 0.0}
          528  +
          529  +do_execsql_test 13.2 {
          530  +  WITH r(x) AS (
          531  +    SELECT 1 UNION ALL
          532  +    SELECT 2 UNION ALL
          533  +    SELECT 3
          534  +  )
          535  +  SELECT * FROM r CROSS JOIN t9 WHERE id=x;
          536  +} {1 1 0.0 0.0 2 2 0.0 0.0}
          537  +
   516    538   finish_test

Changes to ext/rtree/rtree6.test.

    98     98     0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1} 
    99     99     0 1 1 {SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)}
   100    100   }
   101    101   do_eqp_test rtree6.2.4.2 {
   102    102     SELECT * FROM t1,t2 WHERE v=10 and x1<10 and x2>10
   103    103   } {
   104    104     0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1} 
   105         -  0 1 1 {SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)}
          105  +  0 1 1 {SEARCH TABLE t2 USING AUTOMATIC PARTIAL COVERING INDEX (v=?)}
   106    106   }
   107    107   
   108    108   do_eqp_test rtree6.2.5 {
   109    109     SELECT * FROM t1,t2 WHERE k=ii AND x1<v
   110    110   } {
   111    111     0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:} 
   112    112     0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)}

Added ext/rtree/rtreeF.test.

            1  +# 2014-08-21
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file contains tests for the r-tree module.
           12  +#
           13  +# This file contains test cases for the ticket
           14  +# [369d57fb8e5ccdff06f197a37147a88f9de95cda] (2014-08-21)
           15  +#
           16  +#  The following SQL causes an assertion fault while running
           17  +#  sqlite3_prepare() on the DELETE statement:
           18  +#
           19  +#     CREATE TABLE t1(x);
           20  +#     CREATE TABLE t2(y);
           21  +#     CREATE VIRTUAL TABLE t3 USING rtree(a,b,c);
           22  +#     CREATE TRIGGER t2del AFTER DELETE ON t2 WHEN (SELECT 1 from t1) BEGIN 
           23  +#       DELETE FROM t3 WHERE a=old.y; 
           24  +#     END;
           25  +#     DELETE FROM t2 WHERE y=1;
           26  +# 
           27  +
           28  +if {![info exists testdir]} {
           29  +  set testdir [file join [file dirname [info script]] .. .. test]
           30  +} 
           31  +source $testdir/tester.tcl
           32  +ifcapable !rtree { finish_test ; return }
           33  +
           34  +do_execsql_test rtreeF-1.1 {
           35  +  CREATE TABLE t1(x);
           36  +  CREATE TABLE t2(y);
           37  +  CREATE VIRTUAL TABLE t3 USING rtree(a,b,c);
           38  +  CREATE TRIGGER t2dwl AFTER DELETE ON t2 WHEN (SELECT 1 from t1) BEGIN 
           39  +    DELETE FROM t3 WHERE a=old.y; 
           40  +  END;
           41  +
           42  +  INSERT INTO t1(x) VALUES(999);
           43  +  INSERT INTO t2(y) VALUES(1),(2),(3),(4),(5);
           44  +  INSERT INTO t3(a,b,c) VALUES(1,2,3),(2,3,4),(3,4,5),(4,5,6),(5,6,7);
           45  +
           46  +  SELECT a FROM t3 ORDER BY a;
           47  +  SELECT '|';
           48  +  SELECT y FROM t2 ORDER BY y;
           49  +} {1 2 3 4 5 | 1 2 3 4 5}
           50  +do_execsql_test rtreeF-1.2 {
           51  +  DELETE FROM t2 WHERE y=3;
           52  +
           53  +  SELECT a FROM t3 ORDER BY a;
           54  +  SELECT '|';
           55  +  SELECT y FROM t2 ORDER BY y;
           56  +} {1 2 4 5 | 1 2 4 5}
           57  +do_execsql_test rtreeF-1.3 {
           58  +  DELETE FROM t1;
           59  +  DELETE FROM t2 WHERE y=5;
           60  +
           61  +  SELECT a FROM t3 ORDER BY a;
           62  +  SELECT '|';
           63  +  SELECT y FROM t2 ORDER BY y;
           64  +} {1 2 4 5 | 1 2 4}
           65  +do_execsql_test rtreeF-1.4 {
           66  +  INSERT INTO t1 DEFAULT VALUES;
           67  +  DELETE FROM t2 WHERE y=5;
           68  +
           69  +  SELECT a FROM t3 ORDER BY a;
           70  +  SELECT '|';
           71  +  SELECT y FROM t2 ORDER BY y;
           72  +} {1 2 4 5 | 1 2 4}
           73  +do_execsql_test rtreeF-1.5 {
           74  +  DELETE FROM t2 WHERE y=2;
           75  +
           76  +  SELECT a FROM t3 ORDER BY a;
           77  +  SELECT '|';
           78  +  SELECT y FROM t2 ORDER BY y;
           79  +} {1 4 5 | 1 4}
           80  +
           81  +finish_test

Added ext/userauth/sqlite3userauth.h.

            1  +/*
            2  +** 2014-09-08
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +**
           13  +** This file contains the application interface definitions for the
           14  +** user-authentication extension feature.
           15  +**
           16  +** To compile with the user-authentication feature, append this file to
           17  +** end of an SQLite amalgamation header file ("sqlite3.h"), then add
           18  +** the SQLITE_USER_AUTHENTICATION compile-time option.  See the
           19  +** user-auth.txt file in the same source directory as this file for
           20  +** additional information.
           21  +*/
           22  +#ifdef SQLITE_USER_AUTHENTICATION
           23  +
           24  +/*
           25  +** If a database contains the SQLITE_USER table, then the
           26  +** sqlite3_user_authenticate() interface must be invoked with an
           27  +** appropriate username and password prior to enable read and write
           28  +** access to the database.
           29  +**
           30  +** Return SQLITE_OK on success or SQLITE_ERROR if the username/password
           31  +** combination is incorrect or unknown.
           32  +**
           33  +** If the SQLITE_USER table is not present in the database file, then
           34  +** this interface is a harmless no-op returnning SQLITE_OK.
           35  +*/
           36  +int sqlite3_user_authenticate(
           37  +  sqlite3 *db,           /* The database connection */
           38  +  const char *zUsername, /* Username */
           39  +  const char *aPW,       /* Password or credentials */
           40  +  int nPW                /* Number of bytes in aPW[] */
           41  +);
           42  +
           43  +/*
           44  +** The sqlite3_user_add() interface can be used (by an admin user only)
           45  +** to create a new user.  When called on a no-authentication-required
           46  +** database, this routine converts the database into an authentication-
           47  +** required database, automatically makes the added user an
           48  +** administrator, and logs in the current connection as that user.
           49  +** The sqlite3_user_add() interface only works for the "main" database, not
           50  +** for any ATTACH-ed databases.  Any call to sqlite3_user_add() by a
           51  +** non-admin user results in an error.
           52  +*/
           53  +int sqlite3_user_add(
           54  +  sqlite3 *db,           /* Database connection */
           55  +  const char *zUsername, /* Username to be added */
           56  +  const char *aPW,       /* Password or credentials */
           57  +  int nPW,               /* Number of bytes in aPW[] */
           58  +  int isAdmin            /* True to give new user admin privilege */
           59  +);
           60  +
           61  +/*
           62  +** The sqlite3_user_change() interface can be used to change a users
           63  +** login credentials or admin privilege.  Any user can change their own
           64  +** login credentials.  Only an admin user can change another users login
           65  +** credentials or admin privilege setting.  No user may change their own 
           66  +** admin privilege setting.
           67  +*/
           68  +int sqlite3_user_change(
           69  +  sqlite3 *db,           /* Database connection */
           70  +  const char *zUsername, /* Username to change */
           71  +  const char *aPW,       /* New password or credentials */
           72  +  int nPW,               /* Number of bytes in aPW[] */
           73  +  int isAdmin            /* Modified admin privilege for the user */
           74  +);
           75  +
           76  +/*
           77  +** The sqlite3_user_delete() interface can be used (by an admin user only)
           78  +** to delete a user.  The currently logged-in user cannot be deleted,
           79  +** which guarantees that there is always an admin user and hence that
           80  +** the database cannot be converted into a no-authentication-required
           81  +** database.
           82  +*/
           83  +int sqlite3_user_delete(
           84  +  sqlite3 *db,           /* Database connection */
           85  +  const char *zUsername  /* Username to remove */
           86  +);
           87  +
           88  +#endif /* SQLITE_USER_AUTHENTICATION */

Added ext/userauth/user-auth.txt.

            1  +Activate the user authentication logic by including the
            2  +ext/userauth/userauth.c source code file in the build and
            3  +adding the -DSQLITE_USER_AUTHENTICATION compile-time option.
            4  +The ext/userauth/sqlite3userauth.h header file is available to
            5  +applications to define the interface.
            6  +
            7  +When using the SQLite amalgamation, it is sufficient to append
            8  +the ext/userauth/userauth.c source file onto the end of the
            9  +amalgamation.
           10  +
           11  +The following new APIs are available when user authentication is
           12  +activated:
           13  +
           14  +   int sqlite3_user_authenticate(
           15  +     sqlite3 *db,           /* The database connection */
           16  +     const char *zUsername, /* Username */
           17  +     const char *aPW,       /* Password or credentials */
           18  +     int nPW                /* Number of bytes in aPW[] */
           19  +   );
           20  +   
           21  +   int sqlite3_user_add(
           22  +     sqlite3 *db,           /* Database connection */
           23  +     const char *zUsername, /* Username to be added */
           24  +     const char *aPW,       /* Password or credentials */
           25  +     int nPW,               /* Number of bytes in aPW[] */
           26  +     int isAdmin            /* True to give new user admin privilege */
           27  +   );
           28  +   
           29  +   int sqlite3_user_change(
           30  +     sqlite3 *db,           /* Database connection */
           31  +     const char *zUsername, /* Username to change */
           32  +     const void *aPW,       /* Modified password or credentials */
           33  +     int nPW,               /* Number of bytes in aPW[] */
           34  +     int isAdmin            /* Modified admin privilege for the user */
           35  +   );
           36  +   
           37  +   int sqlite3_user_delete(
           38  +     sqlite3 *db,           /* Database connection */
           39  +     const char *zUsername  /* Username to remove */
           40  +   );
           41  +
           42  +With this extension, a database can be marked as requiring authentication.
           43  +By default a database does not require authentication.
           44  +
           45  +The sqlite3_open(), sqlite3_open16(), and sqlite3_open_v2() interfaces
           46  +work as before: they open a new database connection.  However, if the
           47  +database being opened requires authentication, then attempts to read
           48  +or write from the database will fail with an SQLITE_AUTH error until 
           49  +after sqlite3_user_authenticate() has been called successfully.  The 
           50  +sqlite3_user_authenticate() call will return SQLITE_OK if the 
           51  +authentication credentials are accepted and SQLITE_ERROR if not.
           52  +
           53  +Calling sqlite3_user_authenticate() on a no-authentication-required
           54  +database connection is a harmless no-op.  
           55  +
           56  +If the database is encrypted, then sqlite3_key_v2() must be called first,
           57  +with the correct decryption key, prior to invoking sqlite3_user_authenticate().
           58  +
           59  +To recapitulate: When opening an existing unencrypted authentication-
           60  +required database, the call sequence is:
           61  +
           62  +    sqlite3_open_v2()
           63  +    sqlite3_user_authenticate();
           64  +    /* Database is now usable */
           65  +
           66  +To open an existing, encrypted, authentication-required database, the
           67  +call sequence is:
           68  +
           69  +    sqlite3_open_v2();
           70  +    sqlite3_key_v2();
           71  +    sqlite3_user_authenticate();
           72  +    /* Database is now usable */
           73  +
           74  +When opening a no-authentication-required database, the database
           75  +connection is treated as if it was authenticated as an admin user.
           76  +
           77  +When ATTACH-ing new database files to a connection, each newly attached
           78  +database that is an authentication-required database is checked using
           79  +the same username and password as supplied to the main database.  If that
           80  +check fails, then the ATTACH command fails with an SQLITE_AUTH error.
           81  +
           82  +The sqlite3_user_add() interface can be used (by an admin user only)
           83  +to create a new user.  When called on a no-authentication-required
           84  +database and when A is true, the sqlite3_user_add(D,U,P,N,A) routine
           85  +converts the database into an authentication-required database and
           86  +logs in the database connection D as user U with password P,N.
           87  +To convert a no-authentication-required database into an authentication-
           88  +required database, the isAdmin parameter must be true.  If
           89  +sqlite3_user_add(D,U,P,N,A) is called on a no-authentication-required
           90  +database and A is false, then the call fails with an SQLITE_AUTH error.
           91  +
           92  +Any call to sqlite3_user_add() by a non-admin user results in an error.
           93  +
           94  +Hence, to create a new, unencrypted, authentication-required database,
           95  +the call sequence is:
           96  +
           97  +    sqlite3_open_v2();
           98  +    sqlite3_user_add();
           99  +
          100  +And to create a new, encrypted, authentication-required database, the call
          101  +sequence is:
          102  +
          103  +    sqlite3_open_v2();
          104  +    sqlite3_key_v2();
          105  +    sqlite3_user_add();
          106  +
          107  +The sqlite3_user_delete() interface can be used (by an admin user only)
          108  +to delete a user.  The currently logged-in user cannot be deleted,
          109  +which guarantees that there is always an admin user and hence that
          110  +the database cannot be converted into a no-authentication-required
          111  +database.
          112  +
          113  +The sqlite3_user_change() interface can be used to change a users
          114  +login credentials or admin privilege.  Any user can change their own
          115  +password.  Only an admin user can change another users login
          116  +credentials or admin privilege setting.  No user may change their own 
          117  +admin privilege setting.
          118  +
          119  +The sqlite3_set_authorizer() callback is modified to take a 7th parameter
          120  +which is the username of the currently logged in user, or NULL for a
          121  +no-authentication-required database.
          122  +
          123  +-----------------------------------------------------------------------------
          124  +Implementation notes:
          125  +
          126  +An authentication-required database is identified by the presence of a
          127  +new table:
          128  +
          129  +    CREATE TABLE sqlite_user(
          130  +      uname TEXT PRIMARY KEY,
          131  +      isAdmin BOOLEAN,
          132  +      pw BLOB
          133  +    ) WITHOUT ROWID;
          134  +
          135  +The sqlite_user table is inaccessible (unreadable and unwriteable) to
          136  +non-admin users and is read-only for admin users.  However, if the same
          137  +database file is opened by a version of SQLite that omits
          138  +the -DSQLITE_USER_AUTHENTICATION compile-time option, then the sqlite_user
          139  +table will be readable by anybody and writeable by anybody if
          140  +the "PRAGMA writable_schema=ON" statement is run first.
          141  +
          142  +The sqlite_user.pw field is encoded by a built-in SQL function
          143  +"sqlite_crypt(X,Y)".  The two arguments are both BLOBs.  The first argument
          144  +is the plaintext password supplied to the sqlite3_user_authenticate()
          145  +interface.  The second argument is the sqlite_user.pw value and is supplied
          146  +so that the function can extract the "salt" used by the password encoder.
          147  +The result of sqlite_crypt(X,Y) is another blob which is the value that
          148  +ends up being stored in sqlite_user.pw.  To verify credentials X supplied
          149  +by the sqlite3_user_authenticate() routine, SQLite runs:
          150  +
          151  +    sqlite_user.pw == sqlite_crypt(X, sqlite_user.pw)
          152  +
          153  +To compute an appropriate sqlite_user.pw value from a new or modified
          154  +password X, sqlite_crypt(X,NULL) is run.  A new random salt is selected
          155  +when the second argument is NULL.
          156  +
          157  +The built-in version of of sqlite_crypt() uses a simple Ceasar-cypher
          158  +which prevents passwords from being revealed by searching the raw database
          159  +for ASCII text, but is otherwise trivally broken.  For better password
          160  +security, the database should be encrypted using the SQLite Encryption
          161  +Extension or similar technology.  Or, the application can use the
          162  +sqlite3_create_function() interface to provide an alternative
          163  +implementation of sqlite_crypt() that computes a stronger password hash,
          164  +perhaps using a cryptographic hash function like SHA1.

Added ext/userauth/userauth.c.

            1  +/*
            2  +** 2014-09-08
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +**
           13  +** This file contains the bulk of the implementation of the
           14  +** user-authentication extension feature.  Some parts of the user-
           15  +** authentication code are contained within the SQLite core (in the
           16  +** src/ subdirectory of the main source code tree) but those parts
           17  +** that could reasonable be separated out are moved into this file.
           18  +**
           19  +** To compile with the user-authentication feature, append this file to
           20  +** end of an SQLite amalgamation, then add the SQLITE_USER_AUTHENTICATION
           21  +** compile-time option.  See the user-auth.txt file in the same source
           22  +** directory as this file for additional information.
           23  +*/
           24  +#ifdef SQLITE_USER_AUTHENTICATION
           25  +#ifndef _SQLITEINT_H_
           26  +# include "sqliteInt.h"
           27  +#endif
           28  +
           29  +/*
           30  +** Prepare an SQL statement for use by the user authentication logic.
           31  +** Return a pointer to the prepared statement on success.  Return a
           32  +** NULL pointer if there is an error of any kind.
           33  +*/
           34  +static sqlite3_stmt *sqlite3UserAuthPrepare(
           35  +  sqlite3 *db,
           36  +  const char *zFormat,
           37  +  ...
           38  +){
           39  +  sqlite3_stmt *pStmt;
           40  +  char *zSql;
           41  +  int rc;
           42  +  va_list ap;
           43  +  int savedFlags = db->flags;
           44  +
           45  +  va_start(ap, zFormat);
           46  +  zSql = sqlite3_vmprintf(zFormat, ap);
           47  +  va_end(ap);
           48  +  if( zSql==0 ) return 0;
           49  +  db->flags |= SQLITE_WriteSchema;
           50  +  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
           51  +  db->flags = savedFlags;
           52  +  sqlite3_free(zSql);
           53  +  if( rc ){
           54  +    sqlite3_finalize(pStmt);
           55  +    pStmt = 0;
           56  +  }
           57  +  return pStmt;
           58  +}
           59  +
           60  +/*
           61  +** Check to see if the sqlite_user table exists in database zDb.
           62  +*/
           63  +static int userTableExists(sqlite3 *db, const char *zDb){
           64  +  int rc;
           65  +  sqlite3_mutex_enter(db->mutex);
           66  +  sqlite3BtreeEnterAll(db);
           67  +  if( db->init.busy==0 ){
           68  +    char *zErr = 0;
           69  +    sqlite3Init(db, &zErr);
           70  +    sqlite3DbFree(db, zErr);
           71  +  }
           72  +  rc = sqlite3FindTable(db, "sqlite_user", zDb)!=0;
           73  +  sqlite3BtreeLeaveAll(db);
           74  +  sqlite3_mutex_leave(db->mutex);
           75  +  return rc;
           76  +}
           77  +
           78  +/*
           79  +** Check to see if database zDb has a "sqlite_user" table and if it does
           80  +** whether that table can authenticate zUser with nPw,zPw.  Write one of
           81  +** the UAUTH_* user authorization level codes into *peAuth and return a
           82  +** result code.
           83  +*/
           84  +static int userAuthCheckLogin(
           85  +  sqlite3 *db,               /* The database connection to check */
           86  +  const char *zDb,           /* Name of specific database to check */
           87  +  u8 *peAuth                 /* OUT: One of UAUTH_* constants */
           88  +){
           89  +  sqlite3_stmt *pStmt;
           90  +  int rc;
           91  +
           92  +  *peAuth = UAUTH_Unknown;
           93  +  if( !userTableExists(db, "main") ){
           94  +    *peAuth = UAUTH_Admin;  /* No sqlite_user table.  Everybody is admin. */
           95  +    return SQLITE_OK;
           96  +  }
           97  +  if( db->auth.zAuthUser==0 ){
           98  +    *peAuth = UAUTH_Fail;
           99  +    return SQLITE_OK;
          100  +  }
          101  +  pStmt = sqlite3UserAuthPrepare(db,
          102  +            "SELECT pw=sqlite_crypt(?1,pw), isAdmin FROM \"%w\".sqlite_user"
          103  +            " WHERE uname=?2", zDb);
          104  +  if( pStmt==0 ) return SQLITE_NOMEM;
          105  +  sqlite3_bind_blob(pStmt, 1, db->auth.zAuthPW, db->auth.nAuthPW,SQLITE_STATIC);
          106  +  sqlite3_bind_text(pStmt, 2, db->auth.zAuthUser, -1, SQLITE_STATIC);
          107  +  rc = sqlite3_step(pStmt);
          108  +  if( rc==SQLITE_ROW && sqlite3_column_int(pStmt,0) ){
          109  +    *peAuth = sqlite3_column_int(pStmt, 1) + UAUTH_User;
          110  +  }else{
          111  +    *peAuth = UAUTH_Fail;
          112  +  }
          113  +  return sqlite3_finalize(pStmt);
          114  +}
          115  +int sqlite3UserAuthCheckLogin(
          116  +  sqlite3 *db,               /* The database connection to check */
          117  +  const char *zDb,           /* Name of specific database to check */
          118  +  u8 *peAuth                 /* OUT: One of UAUTH_* constants */
          119  +){
          120  +  int rc;
          121  +  u8 savedAuthLevel;
          122  +  assert( zDb!=0 );
          123  +  assert( peAuth!=0 );
          124  +  savedAuthLevel = db->auth.authLevel;
          125  +  db->auth.authLevel = UAUTH_Admin;
          126  +  rc = userAuthCheckLogin(db, zDb, peAuth);
          127  +  db->auth.authLevel = savedAuthLevel;
          128  +  return rc;
          129  +}
          130  +
          131  +/*
          132  +** If the current authLevel is UAUTH_Unknown, the take actions to figure
          133  +** out what authLevel should be
          134  +*/
          135  +void sqlite3UserAuthInit(sqlite3 *db){
          136  +  if( db->auth.authLevel==UAUTH_Unknown ){
          137  +    u8 authLevel = UAUTH_Fail;
          138  +    sqlite3UserAuthCheckLogin(db, "main", &authLevel);
          139  +    db->auth.authLevel = authLevel;
          140  +    if( authLevel<UAUTH_Admin ) db->flags &= ~SQLITE_WriteSchema;
          141  +  }
          142  +}
          143  +
          144  +/*
          145  +** Implementation of the sqlite_crypt(X,Y) function.
          146  +**
          147  +** If Y is NULL then generate a new hash for password X and return that
          148  +** hash.  If Y is not null, then generate a hash for password X using the
          149  +** same salt as the previous hash Y and return the new hash.
          150  +*/
          151  +void sqlite3CryptFunc(
          152  +  sqlite3_context *context,
          153  +  int NotUsed,
          154  +  sqlite3_value **argv
          155  +){
          156  +  const char *zIn;
          157  +  int nIn, ii;
          158  +  u8 *zOut;
          159  +  char zSalt[8];
          160  +  zIn = sqlite3_value_blob(argv[0]);
          161  +  nIn = sqlite3_value_bytes(argv[0]);
          162  +  if( sqlite3_value_type(argv[1])==SQLITE_BLOB
          163  +   && sqlite3_value_bytes(argv[1])==nIn+sizeof(zSalt)
          164  +  ){
          165  +    memcpy(zSalt, sqlite3_value_blob(argv[1]), sizeof(zSalt));
          166  +  }else{
          167  +    sqlite3_randomness(sizeof(zSalt), zSalt);
          168  +  }
          169  +  zOut = sqlite3_malloc( nIn+sizeof(zSalt) );
          170  +  if( zOut==0 ){
          171  +    sqlite3_result_error_nomem(context);
          172  +  }else{
          173  +    memcpy(zOut, zSalt, sizeof(zSalt));
          174  +    for(ii=0; ii<nIn; ii++){
          175  +      zOut[ii+sizeof(zSalt)] = zIn[ii]^zSalt[ii&0x7];
          176  +    }
          177  +    sqlite3_result_blob(context, zOut, nIn+sizeof(zSalt), sqlite3_free);
          178  +  }
          179  +}
          180  +
          181  +/*
          182  +** If a database contains the SQLITE_USER table, then the
          183  +** sqlite3_user_authenticate() interface must be invoked with an
          184  +** appropriate username and password prior to enable read and write
          185  +** access to the database.
          186  +**
          187  +** Return SQLITE_OK on success or SQLITE_ERROR if the username/password
          188  +** combination is incorrect or unknown.
          189  +**
          190  +** If the SQLITE_USER table is not present in the database file, then
          191  +** this interface is a harmless no-op returnning SQLITE_OK.
          192  +*/
          193  +int sqlite3_user_authenticate(
          194  +  sqlite3 *db,           /* The database connection */
          195  +  const char *zUsername, /* Username */
          196  +  const char *zPW,       /* Password or credentials */
          197  +  int nPW                /* Number of bytes in aPW[] */
          198  +){
          199  +  int rc;
          200  +  u8 authLevel = UAUTH_Fail;
          201  +  db->auth.authLevel = UAUTH_Unknown;
          202  +  sqlite3_free(db->auth.zAuthUser);
          203  +  sqlite3_free(db->auth.zAuthPW);
          204  +  memset(&db->auth, 0, sizeof(db->auth));
          205  +  db->auth.zAuthUser = sqlite3_mprintf("%s", zUsername);
          206  +  if( db->auth.zAuthUser==0 ) return SQLITE_NOMEM;
          207  +  db->auth.zAuthPW = sqlite3_malloc( nPW+1 );
          208  +  if( db->auth.zAuthPW==0 ) return SQLITE_NOMEM;
          209  +  memcpy(db->auth.zAuthPW,zPW,nPW);
          210  +  db->auth.nAuthPW = nPW;
          211  +  rc = sqlite3UserAuthCheckLogin(db, "main", &authLevel);
          212  +  db->auth.authLevel = authLevel;
          213  +  sqlite3ExpirePreparedStatements(db);
          214  +  if( rc ){
          215  +    return rc;           /* OOM error, I/O error, etc. */
          216  +  }
          217  +  if( authLevel<UAUTH_User ){
          218  +    return SQLITE_AUTH;  /* Incorrect username and/or password */
          219  +  }
          220  +  return SQLITE_OK;      /* Successful login */
          221  +}
          222  +
          223  +/*
          224  +** The sqlite3_user_add() interface can be used (by an admin user only)
          225  +** to create a new user.  When called on a no-authentication-required
          226  +** database, this routine converts the database into an authentication-
          227  +** required database, automatically makes the added user an
          228  +** administrator, and logs in the current connection as that user.
          229  +** The sqlite3_user_add() interface only works for the "main" database, not
          230  +** for any ATTACH-ed databases.  Any call to sqlite3_user_add() by a
          231  +** non-admin user results in an error.
          232  +*/
          233  +int sqlite3_user_add(
          234  +  sqlite3 *db,           /* Database connection */
          235  +  const char *zUsername, /* Username to be added */
          236  +  const char *aPW,       /* Password or credentials */
          237  +  int nPW,               /* Number of bytes in aPW[] */
          238  +  int isAdmin            /* True to give new user admin privilege */
          239  +){
          240  +  sqlite3_stmt *pStmt;
          241  +  int rc;
          242  +  sqlite3UserAuthInit(db);
          243  +  if( db->auth.authLevel<UAUTH_Admin ) return SQLITE_AUTH;
          244  +  if( !userTableExists(db, "main") ){
          245  +    if( !isAdmin ) return SQLITE_AUTH;
          246  +    pStmt = sqlite3UserAuthPrepare(db, 
          247  +              "CREATE TABLE sqlite_user(\n"
          248  +              "  uname TEXT PRIMARY KEY,\n"
          249  +              "  isAdmin BOOLEAN,\n"
          250  +              "  pw BLOB\n"
          251  +              ") WITHOUT ROWID;");
          252  +    if( pStmt==0 ) return SQLITE_NOMEM;
          253  +    sqlite3_step(pStmt);
          254  +    rc = sqlite3_finalize(pStmt);
          255  +    if( rc ) return rc;
          256  +  }
          257  +  pStmt = sqlite3UserAuthPrepare(db, 
          258  +            "INSERT INTO sqlite_user(uname,isAdmin,pw)"
          259  +            " VALUES(%Q,%d,sqlite_crypt(?1,NULL))",
          260  +            zUsername, isAdmin!=0);
          261  +  if( pStmt==0 ) return SQLITE_NOMEM;
          262  +  sqlite3_bind_blob(pStmt, 1, aPW, nPW, SQLITE_STATIC);
          263  +  sqlite3_step(pStmt);
          264  +  rc = sqlite3_finalize(pStmt);
          265  +  if( rc ) return rc;
          266  +  if( db->auth.zAuthUser==0 ){
          267  +    assert( isAdmin!=0 );
          268  +    sqlite3_user_authenticate(db, zUsername, aPW, nPW);
          269  +  }
          270  +  return SQLITE_OK;
          271  +}
          272  +
          273  +/*
          274  +** The sqlite3_user_change() interface can be used to change a users
          275  +** login credentials or admin privilege.  Any user can change their own
          276  +** login credentials.  Only an admin user can change another users login
          277  +** credentials or admin privilege setting.  No user may change their own 
          278  +** admin privilege setting.
          279  +*/
          280  +int sqlite3_user_change(
          281  +  sqlite3 *db,           /* Database connection */
          282  +  const char *zUsername, /* Username to change */
          283  +  const char *aPW,       /* Modified password or credentials */
          284  +  int nPW,               /* Number of bytes in aPW[] */
          285  +  int isAdmin            /* Modified admin privilege for the user */
          286  +){
          287  +  sqlite3_stmt *pStmt;
          288  +  int rc;
          289  +  u8 authLevel;
          290  +
          291  +  authLevel = db->auth.authLevel;
          292  +  if( authLevel<UAUTH_User ){
          293  +    /* Must be logged in to make a change */
          294  +    return SQLITE_AUTH;
          295  +  }
          296  +  if( strcmp(db->auth.zAuthUser, zUsername)!=0 ){
          297  +    if( db->auth.authLevel<UAUTH_Admin ){
          298  +      /* Must be an administrator to change a different user */
          299  +      return SQLITE_AUTH;
          300  +    }
          301  +  }else if( isAdmin!=(authLevel==UAUTH_Admin) ){
          302  +    /* Cannot change the isAdmin setting for self */
          303  +    return SQLITE_AUTH;
          304  +  }
          305  +  db->auth.authLevel = UAUTH_Admin;
          306  +  if( !userTableExists(db, "main") ){
          307  +    /* This routine is a no-op if the user to be modified does not exist */
          308  +  }else{
          309  +    pStmt = sqlite3UserAuthPrepare(db,
          310  +              "UPDATE sqlite_user SET isAdmin=%d, pw=sqlite_crypt(?1,NULL)"
          311  +              " WHERE uname=%Q", isAdmin, zUsername);
          312  +    if( pStmt==0 ){
          313  +      rc = SQLITE_NOMEM;
          314  +    }else{
          315  +      sqlite3_bind_blob(pStmt, 1, aPW, nPW, SQLITE_STATIC);
          316  +      sqlite3_step(pStmt);
          317  +      rc = sqlite3_finalize(pStmt);
          318  +    }
          319  +  }
          320  +  db->auth.authLevel = authLevel;
          321  +  return rc;
          322  +}
          323  +
          324  +/*
          325  +** The sqlite3_user_delete() interface can be used (by an admin user only)
          326  +** to delete a user.  The currently logged-in user cannot be deleted,
          327  +** which guarantees that there is always an admin user and hence that
          328  +** the database cannot be converted into a no-authentication-required
          329  +** database.
          330  +*/
          331  +int sqlite3_user_delete(
          332  +  sqlite3 *db,           /* Database connection */
          333  +  const char *zUsername  /* Username to remove */
          334  +){
          335  +  sqlite3_stmt *pStmt;
          336  +  if( db->auth.authLevel<UAUTH_Admin ){
          337  +    /* Must be an administrator to delete a user */
          338  +    return SQLITE_AUTH;
          339  +  }
          340  +  if( strcmp(db->auth.zAuthUser, zUsername)==0 ){
          341  +    /* Cannot delete self */
          342  +    return SQLITE_AUTH;
          343  +  }
          344  +  if( !userTableExists(db, "main") ){
          345  +    /* This routine is a no-op if the user to be deleted does not exist */
          346  +    return SQLITE_OK;
          347  +  }
          348  +  pStmt = sqlite3UserAuthPrepare(db,
          349  +              "DELETE FROM sqlite_user WHERE uname=%Q", zUsername);
          350  +  if( pStmt==0 ) return SQLITE_NOMEM;
          351  +  sqlite3_step(pStmt);
          352  +  return sqlite3_finalize(pStmt);
          353  +}
          354  +
          355  +#endif /* SQLITE_USER_AUTHENTICATION */

Changes to main.mk.

    42     42   # build the SQLite library and testing tools.
    43     43   ################################################################################
    44     44   
    45     45   # This is how we compile
    46     46   #
    47     47   TCCX =  $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) 
    48     48   TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3
    49         -TCCX += -I$(TOP)/ext/async
           49  +TCCX += -I$(TOP)/ext/async -I$(TOP)/ext/userauth
    50     50   TCCX += -I$(TOP)/ext/fts5
    51     51   
    52     52   # Object files for the SQLite library.
    53     53   #
    54     54   LIBOBJ+= vdbe.o parse.o \
    55     55            alter.o analyze.o attach.o auth.o \
    56     56            backup.o bitvec.o btmutex.o btree.o build.o \
................................................................................
    63     63            icu.o insert.o journal.o legacy.o loadext.o \
    64     64            main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \
    65     65            memjournal.o \
    66     66            mutex.o mutex_noop.o mutex_unix.o mutex_w32.o \
    67     67            notify.o opcodes.o os.o os_unix.o os_win.o \
    68     68            pager.o pcache.o pcache1.o pragma.o prepare.o printf.o \
    69     69            random.o resolve.o rowset.o rtree.o select.o status.o \
    70         -         table.o tokenize.o trigger.o \
    71         -         update.o util.o vacuum.o \
           70  +         table.o threads.o tokenize.o trigger.o \
           71  +         update.o userauth.o util.o vacuum.o \
    72     72            vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbesort.o \
    73     73   	 vdbetrace.o wal.o walker.o where.o utf.o vtab.o
    74     74   
    75     75   LIBOBJ += fts5.o
    76     76   LIBOBJ += fts5_aux.o
    77     77   LIBOBJ += fts5_buffer.o
    78     78   LIBOBJ += fts5_config.o
................................................................................
   155    155     $(TOP)/src/shell.c \
   156    156     $(TOP)/src/sqlite.h.in \
   157    157     $(TOP)/src/sqlite3ext.h \
   158    158     $(TOP)/src/sqliteInt.h \
   159    159     $(TOP)/src/sqliteLimit.h \
   160    160     $(TOP)/src/table.c \
   161    161     $(TOP)/src/tclsqlite.c \
          162  +  $(TOP)/src/threads.c \
   162    163     $(TOP)/src/tokenize.c \
   163    164     $(TOP)/src/trigger.c \
   164    165     $(TOP)/src/utf.c \
   165    166     $(TOP)/src/update.c \
   166    167     $(TOP)/src/util.c \
   167    168     $(TOP)/src/vacuum.c \
   168    169     $(TOP)/src/vdbe.c \
................................................................................
   222    223   SRC += \
   223    224     $(TOP)/ext/icu/sqliteicu.h \
   224    225     $(TOP)/ext/icu/icu.c
   225    226   SRC += \
   226    227     $(TOP)/ext/rtree/sqlite3rtree.h \
   227    228     $(TOP)/ext/rtree/rtree.h \
   228    229     $(TOP)/ext/rtree/rtree.c
   229         -
          230  +SRC += \
          231  +  $(TOP)/ext/userauth/userauth.c \
          232  +  $(TOP)/ext/userauth/sqlite3userauth.h
   230    233   SRC += \
   231    234      $(TOP)/ext/fts5/fts5.h \
   232    235      $(TOP)/ext/fts5/fts5Int.h \
   233    236      $(TOP)/ext/fts5/fts5_aux.c \
   234    237      $(TOP)/ext/fts5/fts5_buffer.c \
   235    238      $(TOP)/ext/fts5/fts5.c \
   236    239      $(TOP)/ext/fts5/fts5_config.c \
................................................................................
   266    269     $(TOP)/src/test6.c \
   267    270     $(TOP)/src/test7.c \
   268    271     $(TOP)/src/test8.c \
   269    272     $(TOP)/src/test9.c \
   270    273     $(TOP)/src/test_autoext.c \
   271    274     $(TOP)/src/test_async.c \
   272    275     $(TOP)/src/test_backup.c \
          276  +  $(TOP)/src/test_blob.c \
   273    277     $(TOP)/src/test_btree.c \
   274    278     $(TOP)/src/test_config.c \
   275    279     $(TOP)/src/test_demovfs.c \
   276    280     $(TOP)/src/test_devsym.c \
   277    281     $(TOP)/src/test_fs.c \
   278    282     $(TOP)/src/test_func.c \
   279    283     $(TOP)/src/test_hexio.c \
................................................................................
   300    304     $(TOP)/src/test_wsd.c
   301    305   
   302    306   # Extensions to be statically loaded.
   303    307   #
   304    308   TESTSRC += \
   305    309     $(TOP)/ext/misc/amatch.c \
   306    310     $(TOP)/ext/misc/closure.c \
          311  +  $(TOP)/ext/misc/eval.c \
   307    312     $(TOP)/ext/misc/fileio.c \
   308    313     $(TOP)/ext/misc/fuzzer.c \
   309    314     $(TOP)/ext/misc/ieee754.c \
   310    315     $(TOP)/ext/misc/nextchar.c \
   311    316     $(TOP)/ext/misc/percentile.c \
   312    317     $(TOP)/ext/misc/regexp.c \
   313    318     $(TOP)/ext/misc/spellfix.c \
................................................................................
   339    344     $(TOP)/src/pragma.c \
   340    345     $(TOP)/src/prepare.c \
   341    346     $(TOP)/src/printf.c \
   342    347     $(TOP)/src/random.c \
   343    348     $(TOP)/src/pcache.c \
   344    349     $(TOP)/src/pcache1.c \
   345    350     $(TOP)/src/select.c \
          351  +  $(TOP)/src/threads.c \
   346    352     $(TOP)/src/tokenize.c \
   347    353     $(TOP)/src/utf.c \
   348    354     $(TOP)/src/util.c \
   349    355     $(TOP)/src/vdbeapi.c \
   350    356     $(TOP)/src/vdbeaux.c \
   351    357     $(TOP)/src/vdbe.c \
   352    358     $(TOP)/src/vdbemem.c \
................................................................................
   402    408   EXTHDR += \
   403    409     $(TOP)/ext/rtree/rtree.h
   404    410   EXTHDR += \
   405    411     $(TOP)/ext/icu/sqliteicu.h
   406    412   EXTHDR += \
   407    413     $(TOP)/ext/fts5/fts5Int.h  \
   408    414     $(TOP)/ext/fts5/fts5.h 
          415  +EXTHDR += \
          416  +  $(TOP)/ext/userauth/sqlite3userauth.h
   409    417   
   410    418   # This is the default Makefile target.  The objects listed here
   411    419   # are what get build when you type just "make" with no arguments.
   412    420   #
   413    421   all:	sqlite3.h libsqlite3.a sqlite3$(EXE)
   414    422   
   415    423   libsqlite3.a:	$(LIBOBJ)
................................................................................
   586    594   
   587    595   fts5.o:	$(TOP)/ext/fts5/fts5.c $(HDR) $(EXTHDR)
   588    596   	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5.c
   589    597   
   590    598   rtree.o:	$(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
   591    599   	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
   592    600   
   593         -
   594    601   # FTS5 things
   595    602   #
   596    603   fts5_aux.o:	$(TOP)/ext/fts5/fts5_aux.c $(HDR) $(EXTHDR)
   597    604   	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_aux.c
   598    605   
   599    606   fts5_buffer.o:	$(TOP)/ext/fts5/fts5_buffer.c $(HDR) $(EXTHDR)
   600    607   	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_buffer.c
................................................................................
   623    630   fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
   624    631   	cp $(TOP)/ext/fts5/fts5parse.y .
   625    632   	rm -f fts5parse.h
   626    633   	./lemon $(OPTS) fts5parse.y
   627    634   	mv fts5parse.c fts5parse.c.orig
   628    635   	cat fts5parse.c.orig | sed 's/yy/fts5yy/g' | sed 's/YY/fts5YY/g' > fts5parse.c
   629    636   
          637  +
          638  +userauth.o:	$(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
          639  +	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c
          640  +
   630    641   
   631    642   # Rules for building test programs and for running tests
   632    643   #
   633    644   tclsqlite3:	$(TOP)/src/tclsqlite.c libsqlite3.a
   634    645   	$(TCCX) $(TCL_FLAGS) -DTCLSH=1 -o tclsqlite3 \
   635    646   		$(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB)
   636    647   
................................................................................
   681    692   test:	testfixture$(EXE) sqlite3$(EXE)
   682    693   	./testfixture$(EXE) $(TOP)/test/veryquick.test
   683    694   
   684    695   # The next two rules are used to support the "threadtest" target. Building
   685    696   # threadtest runs a few thread-safety tests that are implemented in C. This
   686    697   # target is invoked by the releasetest.tcl script.
   687    698   # 
   688         -threadtest3$(EXE): sqlite3.o $(TOP)/test/threadtest3.c $(TOP)/test/tt3_checkpoint.c
   689         -	$(TCCX) -O2 sqlite3.o $(TOP)/test/threadtest3.c \
   690         -		-o threadtest3$(EXE) $(THREADLIB)
          699  +THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
          700  +                  $(TOP)/test/tt3_checkpoint.c \
          701  +                  $(TOP)/test/tt3_index.c      \
          702  +                  $(TOP)/test/tt3_vacuum.c      \
          703  +                  $(TOP)/test/tt3_stress.c      \
          704  +                  $(TOP)/test/tt3_lookaside1.c
          705  +
          706  +threadtest3$(EXE): sqlite3.o $(THREADTEST3_SRC)
          707  +	$(TCCX) $(TOP)/test/threadtest3.c sqlite3.o -o $@ $(THREADLIB)
   691    708   
   692    709   threadtest: threadtest3$(EXE)
   693    710   	./threadtest3$(EXE)
   694    711   
   695    712   TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO)
   696    713   $(TEST_EXTENSION): $(TOP)/src/test_loadext.c
   697    714   	$(MKSHLIB) $(TOP)/src/test_loadext.c -o $(TEST_EXTENSION)
   698    715   
   699    716   extensiontest: testfixture$(EXE) $(TEST_EXTENSION)
   700    717   	./testfixture$(EXE) $(TOP)/test/loadext.test
   701    718   
   702         -showdb$(EXE):	$(TOP)/tool/showdb.c sqlite3.c
          719  +showdb$(EXE):	$(TOP)/tool/showdb.c sqlite3.o
   703    720   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showdb$(EXE) \
   704         -		$(TOP)/tool/showdb.c sqlite3.c
          721  +		$(TOP)/tool/showdb.c sqlite3.o $(THREADLIB)
          722  +
          723  +showstat4$(EXE):	$(TOP)/tool/showstat4.c sqlite3.o
          724  +	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showstat4$(EXE) \
          725  +		$(TOP)/tool/showstat4.c sqlite3.o $(THREADLIB)
          726  +
          727  +showjournal$(EXE):	$(TOP)/tool/showjournal.c sqlite3.o
          728  +	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showjournal$(EXE) \
          729  +		$(TOP)/tool/showjournal.c sqlite3.o $(THREADLIB)
          730  +
          731  +showwal$(EXE):	$(TOP)/tool/showwal.c sqlite3.o
          732  +	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \
          733  +		$(TOP)/tool/showwal.c sqlite3.o $(THREADLIB)
          734  +
          735  +fts3view$(EXE):	$(TOP)/ext/fts3/tool/fts3view.c sqlite3.o
          736  +	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o fts3view$(EXE) \
          737  +		$(TOP)/ext/fts3/tool/fts3view.c sqlite3.o $(THREADLIB)
          738  +
          739  +rollback-test$(EXE):	$(TOP)/tool/rollback-test.c sqlite3.o
          740  +	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o rollback-test$(EXE) \
          741  +		$(TOP)/tool/rollback-test.c sqlite3.o $(THREADLIB)
   705    742   
   706    743   LogEst$(EXE):	$(TOP)/tool/logest.c sqlite3.h
   707    744   	$(TCC) -o LogEst$(EXE) $(TOP)/tool/logest.c
   708    745   
   709    746   wordcount$(EXE):	$(TOP)/test/wordcount.c sqlite3.c
   710    747   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o wordcount$(EXE) \
   711    748   		$(TOP)/test/wordcount.c sqlite3.c
................................................................................
   745    782   	rm -rf quota2a quota2b quota2c
   746    783   	rm -rf tsrc target_source
   747    784   	rm -f testloadext.dll libtestloadext.so
   748    785   	rm -f amalgamation-testfixture amalgamation-testfixture.exe
   749    786   	rm -f fts3-testfixture fts3-testfixture.exe
   750    787   	rm -f testfixture testfixture.exe
   751    788   	rm -f threadtest3 threadtest3.exe
          789  +	rm -f LogEst LogEst.exe
          790  +	rm -f fts3view fts3view.exe
          791  +	rm -f rollback-test rollback-test.exe
          792  +	rm -f showdb showdb.exe
          793  +	rm -f showjournal showjournal.exe
          794  +	rm -f showstat4 showstat4.exe
          795  +	rm -f showwal showwal.exe
          796  +	rm -f speedtest1 speedtest1.exe
          797  +	rm -f wordcount wordcount.exe
   752    798   	rm -f sqlite3.c sqlite3-*.c fts?amal.c tclsqlite3.c
   753    799   	rm -f sqlite3rc.h
   754    800   	rm -f shell.c sqlite3ext.h
   755    801   	rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c
   756    802   	rm -f sqlite-*-output.vsix
   757    803   	rm -f mptester mptester.exe
   758         -	rm -f showdb

Changes to mptest/mptest.c.

  1391   1391       }
  1392   1392       sqlite3_finalize(pStmt);
  1393   1393     }
  1394   1394     sqlite3_close(g.db);  
  1395   1395     maybeClose(g.pLog);
  1396   1396     maybeClose(g.pErrLog);
  1397   1397     if( iClient==0 ){
  1398         -    printf("Summary: %d errors in %d tests\n", g.nError, g.nTest);
         1398  +    printf("Summary: %d errors out of %d tests\n", g.nError, g.nTest);
  1399   1399     }
  1400   1400     return g.nError>0;
  1401   1401   }

Changes to sqlite3.1.

     1      1   .\"                                      Hey, EMACS: -*- nroff -*-
     2      2   .\" First parameter, NAME, should be all caps
     3      3   .\" Second parameter, SECTION, should be 1-8, maybe w/ subsection
     4      4   .\" other parameters are allowed: see man(7), man(1)
     5         -.TH SQLITE3 1 "Mon Jan 31 11:14:00 2014"
            5  +.TH SQLITE3 1 "Fri Oct 31 10:41:31 EDT 2014"
     6      6   .\" Please adjust this date whenever revising the manpage.
     7      7   .\"
     8      8   .\" Some roff macros, for reference:
     9      9   .\" .nh        disable hyphenation
    10     10   .\" .hy        enable hyphenation
    11     11   .\" .ad l      left justify
    12     12   .\" .ad b      justify to both left and right margins
................................................................................
    45     45   
    46     46   For example, to create a new database file named "mydata.db", create
    47     47   a table named "memos" and insert a couple of records into that table:
    48     48   .sp
    49     49   $ 
    50     50   .B sqlite3 mydata.db
    51     51   .br
    52         -SQLite version 3.8.3
           52  +SQLite version 3.8.8
    53     53   .br
    54     54   Enter ".help" for instructions
    55     55   .br
    56     56   sqlite>
    57     57   .B create table memos(text, priority INTEGER);
    58     58   .br
    59     59   sqlite>
................................................................................
   103    103   
   104    104   A list of available meta-commands can be viewed at any time by issuing
   105    105   the '.help' command.  For example:
   106    106   .sp
   107    107   sqlite>
   108    108   .B .help
   109    109   .nf
   110         -.cc |
   111         -.backup ?DB? FILE      Backup DB (default "main") to FILE
   112         -.bail ON|OFF           Stop after hitting an error.  Default OFF
   113         -.databases             List names and files of attached databases
   114         -.dump ?TABLE? ...      Dump the database in an SQL text format
          110  +.tr %.
          111  +%backup ?DB? FILE      Backup DB (default "main") to FILE
          112  +%bail on|off           Stop after hitting an error.  Default OFF
          113  +%clone NEWDB           Clone data into NEWDB from the existing database
          114  +%databases             List names and files of attached databases
          115  +%dump ?TABLE? ...      Dump the database in an SQL text format
   115    116                            If TABLE specified, only dump tables matching
   116    117                            LIKE pattern TABLE.
   117         -.echo ON|OFF           Turn command echo on or off
   118         -.exit                  Exit this program
   119         -.explain ?ON|OFF?      Turn output mode suitable for EXPLAIN on or off.
          118  +%echo on|off           Turn command echo on or off
          119  +%eqp on|off            Enable or disable automatic EXPLAIN QUERY PLAN
          120  +%exit                  Exit this program
          121  +%explain ?on|off?      Turn output mode suitable for EXPLAIN on or off.
   120    122                            With no args, it turns EXPLAIN on.
   121         -.header(s) ON|OFF      Turn display of headers on or off
   122         -.help                  Show this message
   123         -.import FILE TABLE     Import data from FILE into TABLE
   124         -.indices ?TABLE?       Show names of all indices
          123  +%fullschema            Show schema and the content of sqlite_stat tables
          124  +%headers on|off        Turn display of headers on or off
          125  +%help                  Show this message
          126  +%import FILE TABLE     Import data from FILE into TABLE
          127  +%indices ?TABLE?       Show names of all indices
   125    128                            If TABLE specified, only show indices for tables
   126    129                            matching LIKE pattern TABLE.
   127         -.load FILE ?ENTRY?     Load an extension library
   128         -.log FILE|off          Turn logging on or off.  FILE can be stderr/stdout
   129         -.mode MODE ?TABLE?     Set output mode where MODE is one of:
          130  +%load FILE ?ENTRY?     Load an extension library
          131  +%log FILE|off          Turn logging on or off.  FILE can be stderr/stdout
          132  +%mode MODE ?TABLE?     Set output mode where MODE is one of:
   130    133                            csv      Comma-separated values
   131    134                            column   Left-aligned columns.  (See .width)
   132    135                            html     HTML <table> code
   133    136                            insert   SQL insert statements for TABLE
   134    137                            line     One value per line
   135    138                            list     Values delimited by .separator string
   136    139                            tabs     Tab-separated values
   137    140                            tcl      TCL list elements
   138         -.nullvalue STRING      Use STRING in place of NULL values
   139         -.open ?FILENAME?       Close existing database and reopen FILENAME
   140         -.output FILENAME       Send output to FILENAME
   141         -.output stdout         Send output to the screen
   142         -.print STRING...       Print literal STRING
   143         -.prompt MAIN CONTINUE  Replace the standard prompts
   144         -.quit                  Exit this program
   145         -.read FILENAME         Execute SQL in FILENAME
   146         -.restore ?DB? FILE     Restore content of DB (default "main") from FILE
   147         -.schema ?TABLE?        Show the CREATE statements
          141  +%nullvalue STRING      Use STRING in place of NULL values
          142  +%once FILENAME         Output for the next SQL command only to FILENAME
          143  +%open ?FILENAME?       Close existing database and reopen FILENAME
          144  +%output ?FILENAME?     Send output to FILENAME or stdout
          145  +%print STRING...       Print literal STRING
          146  +%prompt MAIN CONTINUE  Replace the standard prompts
          147  +%quit                  Exit this program
          148  +%read FILENAME         Execute SQL in FILENAME
          149  +%restore ?DB? FILE     Restore content of DB (default "main") from FILE
          150  +%save FILE             Write in-memory database into FILE
          151  +%schema ?TABLE?        Show the CREATE statements
   148    152                            If TABLE specified, only show tables matching
   149    153                            LIKE pattern TABLE.
   150         -.separator STRING      Change separator used by output mode and .import
   151         -.show                  Show the current values for various settings
   152         -.stats ON|OFF          Turn stats on or off
   153         -.tables ?TABLE?        List names of tables
          154  +%separator STRING ?NL? Change separator used by output mode and .import
          155  +                         NL is the end-of-line mark for CSV
          156  +%shell CMD ARGS...     Run CMD ARGS... in a system shell
          157  +%show                  Show the current values for various settings
          158  +%stats on|off          Turn stats on or off
          159  +%system CMD ARGS...    Run CMD ARGS... in a system shell
          160  +%tables ?TABLE?        List names of tables
   154    161                            If TABLE specified, only list tables matching
   155    162                            LIKE pattern TABLE.
   156         -.timeout MS            Try opening locked tables for MS milliseconds
   157         -.trace FILE|off        Output each SQL statement as it is run
   158         -.vfsname ?AUX?         Print the name of the VFS stack
   159         -.width NUM1 NUM2 ...   Set column widths for "column" mode
   160         -.timer ON|OFF          Turn the CPU timer measurement on or off
          163  +%timeout MS            Try opening locked tables for MS milliseconds
          164  +%timer on|off          Turn SQL timer on or off
          165  +%trace FILE|off        Output each SQL statement as it is run
          166  +%vfsname ?AUX?         Print the name of the VFS stack
          167  +%width NUM1 NUM2 ...   Set column widths for "column" mode
          168  +                         Negative values right-justify
   161    169   sqlite>
   162         -|cc .
   163    170   .sp
   164    171   .fi
   165    172   .SH OPTIONS
   166    173   .B sqlite3
   167    174   has the following options:
   168    175   .TP
   169    176   .B \-bail
................................................................................
   265    272   read and processed.  It should generally only contain meta-commands.
   266    273   
   267    274   o If the -init option is present, the specified file is processed.
   268    275   
   269    276   o All other command line options are processed.
   270    277   
   271    278   .SH SEE ALSO
   272         -http://www.sqlite.org/
          279  +http://www.sqlite.org/cli.html
   273    280   .br
   274    281   The sqlite3-doc package.
   275    282   .SH AUTHOR
   276    283   This manual page was originally written by Andreas Rottmann
   277    284   <rotty@debian.org>, for the Debian GNU/Linux system (but may be used
   278    285   by others). It was subsequently revised by Bill Bumgarner <bbum@mac.com> and
   279    286   further updated by Laszlo Boszormenyi <gcs@debian.hu> .

Changes to src/alter.c.

   170    170     int len = 0;
   171    171     char *zRet;
   172    172     sqlite3 *db = sqlite3_context_db_handle(context);
   173    173   
   174    174     UNUSED_PARAMETER(NotUsed);
   175    175   
   176    176     /* The principle used to locate the table name in the CREATE TRIGGER 
   177         -  ** statement is that the table name is the first token that is immediatedly
   178         -  ** preceded by either TK_ON or TK_DOT and immediatedly followed by one
          177  +  ** statement is that the table name is the first token that is immediately
          178  +  ** preceded by either TK_ON or TK_DOT and immediately followed by one
   179    179     ** of TK_WHEN, TK_BEGIN or TK_FOR.
   180    180     */
   181    181     if( zSql ){
   182    182       do {
   183    183   
   184    184         if( !*zCsr ){
   185    185           /* Ran out of input before finding the table name. Return NULL. */

Changes to src/analyze.c.

    31     31   ** SQLITE_ENABLE_STAT3 defined.  The functionality of sqlite_stat3
    32     32   ** is a superset of sqlite_stat2.  The sqlite_stat4 is an enhanced
    33     33   ** version of sqlite_stat3 and is only available when compiled with
    34     34   ** SQLITE_ENABLE_STAT4 and in SQLite versions 3.8.1 and later.  It is
    35     35   ** not possible to enable both STAT3 and STAT4 at the same time.  If they
    36     36   ** are both enabled, then STAT4 takes precedence.
    37     37   **
    38         -** For most applications, sqlite_stat1 provides all the statisics required
           38  +** For most applications, sqlite_stat1 provides all the statistics required
    39     39   ** for the query planner to make good choices.
    40     40   **
    41     41   ** Format of sqlite_stat1:
    42     42   **
    43     43   ** There is normally one row per index, with the index identified by the
    44     44   ** name in the idx column.  The tbl column is the name of the table to
    45     45   ** which the index belongs.  In each such row, the stat column will be
................................................................................
   242    242     }
   243    243   
   244    244     /* Open the sqlite_stat[134] tables for writing. */
   245    245     for(i=0; aTable[i].zCols; i++){
   246    246       assert( i<ArraySize(aTable) );
   247    247       sqlite3VdbeAddOp4Int(v, OP_OpenWrite, iStatCur+i, aRoot[i], iDb, 3);
   248    248       sqlite3VdbeChangeP5(v, aCreateTbl[i]);
          249  +    VdbeComment((v, aTable[i].zName));
   249    250     }
   250    251   }
   251    252   
   252    253   /*
   253    254   ** Recommended number of samples for sqlite_stat4
   254    255   */
   255    256   #ifndef SQLITE_STAT4_SAMPLES
................................................................................
   277    278     int iCol;                       /* If !isPSample, the reason for inclusion */
   278    279     u32 iHash;                      /* Tiebreaker hash */
   279    280   #endif
   280    281   };                                                    
   281    282   struct Stat4Accum {
   282    283     tRowcnt nRow;             /* Number of rows in the entire table */
   283    284     tRowcnt nPSample;         /* How often to do a periodic sample */
   284         -  int nCol;                 /* Number of columns in index + rowid */
          285  +  int nCol;                 /* Number of columns in index + pk/rowid */
          286  +  int nKeyCol;              /* Number of index columns w/o the pk/rowid */
   285    287     int mxSample;             /* Maximum number of samples to accumulate */
   286    288     Stat4Sample current;      /* Current row as a Stat4Sample */
   287    289     u32 iPrn;                 /* Pseudo-random number used for sampling */
   288    290     Stat4Sample *aBest;       /* Array of nCol best samples */
   289    291     int iMin;                 /* Index in a[] of entry with minimum score */
   290    292     int nSample;              /* Current number of samples */
   291    293     int iGet;                 /* Index of current sample accessed by stat_get() */
................................................................................
   363    365     for(i=0; i<p->mxSample; i++) sampleClear(p->db, p->a+i);
   364    366     sampleClear(p->db, &p->current);
   365    367   #endif
   366    368     sqlite3DbFree(p->db, p);
   367    369   }
   368    370   
   369    371   /*
   370         -** Implementation of the stat_init(N,C) SQL function. The two parameters
   371         -** are the number of rows in the table or index (C) and the number of columns
   372         -** in the index (N).  The second argument (C) is only used for STAT3 and STAT4.
          372  +** Implementation of the stat_init(N,K,C) SQL function. The three parameters
          373  +** are:
          374  +**     N:    The number of columns in the index including the rowid/pk (note 1)
          375  +**     K:    The number of columns in the index excluding the rowid/pk.
          376  +**     C:    The number of rows in the index (note 2)
          377  +**
          378  +** Note 1:  In the special case of the covering index that implements a
          379  +** WITHOUT ROWID table, N is the number of PRIMARY KEY columns, not the
          380  +** total number of columns in the table.
          381  +**
          382  +** Note 2:  C is only used for STAT3 and STAT4.
          383  +**
          384  +** For indexes on ordinary rowid tables, N==K+1.  But for indexes on
          385  +** WITHOUT ROWID tables, N=K+P where P is the number of columns in the
          386  +** PRIMARY KEY of the table.  The covering index that implements the
          387  +** original WITHOUT ROWID table as N==K as a special case.
   373    388   **
   374    389   ** This routine allocates the Stat4Accum object in heap memory. The return 
   375         -** value is a pointer to the the Stat4Accum object encoded as a blob (i.e. 
   376         -** the size of the blob is sizeof(void*) bytes). 
          390  +** value is a pointer to the Stat4Accum object.  The datatype of the
          391  +** return value is BLOB, but it is really just a pointer to the Stat4Accum
          392  +** object.
   377    393   */
   378    394   static void statInit(
   379    395     sqlite3_context *context,
   380    396     int argc,
   381    397     sqlite3_value **argv
   382    398   ){
   383    399     Stat4Accum *p;
   384    400     int nCol;                       /* Number of columns in index being sampled */
          401  +  int nKeyCol;                    /* Number of key columns */
   385    402     int nColUp;                     /* nCol rounded up for alignment */
   386    403     int n;                          /* Bytes of space to allocate */
   387    404     sqlite3 *db;                    /* Database connection */
   388    405   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
   389    406     int mxSample = SQLITE_STAT4_SAMPLES;
   390    407   #endif
   391    408   
   392    409     /* Decode the three function arguments */
   393    410     UNUSED_PARAMETER(argc);
   394    411     nCol = sqlite3_value_int(argv[0]);
   395         -  assert( nCol>1 );               /* >1 because it includes the rowid column */
          412  +  assert( nCol>0 );
   396    413     nColUp = sizeof(tRowcnt)<8 ? (nCol+1)&~1 : nCol;
          414  +  nKeyCol = sqlite3_value_int(argv[1]);
          415  +  assert( nKeyCol<=nCol );
          416  +  assert( nKeyCol>0 );
   397    417   
   398    418     /* Allocate the space required for the Stat4Accum object */
   399    419     n = sizeof(*p) 
   400    420       + sizeof(tRowcnt)*nColUp                  /* Stat4Accum.anEq */
   401    421       + sizeof(tRowcnt)*nColUp                  /* Stat4Accum.anDLt */
   402    422   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
   403    423       + sizeof(tRowcnt)*nColUp                  /* Stat4Accum.anLt */
................................................................................
   411    431       sqlite3_result_error_nomem(context);
   412    432       return;
   413    433     }
   414    434   
   415    435     p->db = db;
   416    436     p->nRow = 0;
   417    437     p->nCol = nCol;
          438  +  p->nKeyCol = nKeyCol;
   418    439     p->current.anDLt = (tRowcnt*)&p[1];
   419    440     p->current.anEq = &p->current.anDLt[nColUp];
   420    441   
   421    442   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
   422    443     {
   423    444       u8 *pSpace;                     /* Allocated space not yet assigned */
   424    445       int i;                          /* Used to iterate through p->aSample[] */
   425    446   
   426    447       p->iGet = -1;
   427    448       p->mxSample = mxSample;
   428         -    p->nPSample = (tRowcnt)(sqlite3_value_int64(argv[1])/(mxSample/3+1) + 1);
          449  +    p->nPSample = (tRowcnt)(sqlite3_value_int64(argv[2])/(mxSample/3+1) + 1);
   429    450       p->current.anLt = &p->current.anEq[nColUp];
   430         -    p->iPrn = nCol*0x689e962d ^ sqlite3_value_int(argv[1])*0xd0944565;
          451  +    p->iPrn = nCol*0x689e962d ^ sqlite3_value_int(argv[2])*0xd0944565;
   431    452     
   432    453       /* Set up the Stat4Accum.a[] and aBest[] arrays */
   433    454       p->a = (struct Stat4Sample*)&p->current.anLt[nColUp];
   434    455       p->aBest = &p->a[mxSample];
   435    456       pSpace = (u8*)(&p->a[mxSample+nCol]);
   436    457       for(i=0; i<(mxSample+nCol); i++){
   437    458         p->a[i].anEq = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp);
................................................................................
   442    463     
   443    464       for(i=0; i<nCol; i++){
   444    465         p->aBest[i].iCol = i;
   445    466       }
   446    467     }
   447    468   #endif
   448    469   
   449         -  /* Return a pointer to the allocated object to the caller */
   450         -  sqlite3_result_blob(context, p, sizeof(p), stat4Destructor);
          470  +  /* Return a pointer to the allocated object to the caller.  Note that
          471  +  ** only the pointer (the 2nd parameter) matters.  The size of the object
          472  +  ** (given by the 3rd parameter) is never used and can be any positive
          473  +  ** value. */
          474  +  sqlite3_result_blob(context, p, sizeof(*p), stat4Destructor);
   451    475   }
   452    476   static const FuncDef statInitFuncdef = {
   453         -  1+IsStat34,      /* nArg */
          477  +  2+IsStat34,      /* nArg */
   454    478     SQLITE_UTF8,     /* funcFlags */
   455    479     0,               /* pUserData */
   456    480     0,               /* pNext */
   457    481     statInit,        /* xFunc */
   458    482     0,               /* xStep */
   459    483     0,               /* xFinalize */
   460    484     "stat_init",     /* zName */
................................................................................
   670    694   ** Arguments:
   671    695   **
   672    696   **    P     Pointer to the Stat4Accum object created by stat_init()
   673    697   **    C     Index of left-most column to differ from previous row
   674    698   **    R     Rowid for the current row.  Might be a key record for
   675    699   **          WITHOUT ROWID tables.
   676    700   **
   677         -** The SQL function always returns NULL.
          701  +** This SQL function always returns NULL.  It's purpose it to accumulate
          702  +** statistical data and/or samples in the Stat4Accum object about the
          703  +** index being analyzed.  The stat_get() SQL function will later be used to
          704  +** extract relevant information for constructing the sqlite_statN tables.
   678    705   **
   679    706   ** The R parameter is only used for STAT3 and STAT4
   680    707   */
   681    708   static void statPush(
   682    709     sqlite3_context *context,
   683    710     int argc,
   684    711     sqlite3_value **argv
................................................................................
   687    714   
   688    715     /* The three function arguments */
   689    716     Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]);
   690    717     int iChng = sqlite3_value_int(argv[1]);
   691    718   
   692    719     UNUSED_PARAMETER( argc );
   693    720     UNUSED_PARAMETER( context );
   694         -  assert( p->nCol>1 );        /* Includes rowid field */
          721  +  assert( p->nCol>0 );
   695    722     assert( iChng<p->nCol );
   696    723   
   697    724     if( p->nRow==0 ){
   698    725       /* This is the first call to this function. Do initialization. */
   699    726       for(i=0; i<p->nCol; i++) p->current.anEq[i] = 1;
   700    727     }else{
   701    728       /* Second and subsequent calls get processed here */
................................................................................
   764    791   #define STAT_GET_ROWID 1          /* "rowid" column of stat[34] entry */
   765    792   #define STAT_GET_NEQ   2          /* "neq" column of stat[34] entry */
   766    793   #define STAT_GET_NLT   3          /* "nlt" column of stat[34] entry */
   767    794   #define STAT_GET_NDLT  4          /* "ndlt" column of stat[34] entry */
   768    795   
   769    796   /*
   770    797   ** Implementation of the stat_get(P,J) SQL function.  This routine is
   771         -** used to query the results.  Content is returned for parameter J
          798  +** used to query statistical information that has been gathered into
          799  +** the Stat4Accum object by prior calls to stat_push().  The P parameter
          800  +** has type BLOB but it is really just a pointer to the Stat4Accum object.
          801  +** The content to returned is determined by the parameter J
   772    802   ** which is one of the STAT_GET_xxxx values defined above.
   773    803   **
   774    804   ** If neither STAT3 nor STAT4 are enabled, then J is always
   775    805   ** STAT_GET_STAT1 and is hence omitted and this routine becomes
   776    806   ** a one-parameter function, stat_get(P), that always returns the
   777    807   ** stat1 table entry information.
   778    808   */
................................................................................
   815    845       ** rows, then each estimate is computed as:
   816    846       **
   817    847       **        I = (K+D-1)/D
   818    848       */
   819    849       char *z;
   820    850       int i;
   821    851   
   822         -    char *zRet = sqlite3MallocZero(p->nCol * 25);
          852  +    char *zRet = sqlite3MallocZero( (p->nKeyCol+1)*25 );
   823    853       if( zRet==0 ){
   824    854         sqlite3_result_error_nomem(context);
   825    855         return;
   826    856       }
   827    857   
   828    858       sqlite3_snprintf(24, zRet, "%llu", (u64)p->nRow);
   829    859       z = zRet + sqlite3Strlen30(zRet);
   830         -    for(i=0; i<(p->nCol-1); i++){
          860  +    for(i=0; i<p->nKeyCol; i++){
   831    861         u64 nDistinct = p->current.anDLt[i] + 1;
   832    862         u64 iVal = (p->nRow + nDistinct - 1) / nDistinct;
   833    863         sqlite3_snprintf(24, z, " %llu", iVal);
   834    864         z += sqlite3Strlen30(z);
   835    865         assert( p->current.anEq[i] );
   836    866       }
   837    867       assert( z[0]=='\0' && z>zRet );
................................................................................
   983   1013     iTabCur = iTab++;
   984   1014     iIdxCur = iTab++;
   985   1015     pParse->nTab = MAX(pParse->nTab, iTab);
   986   1016     sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead);
   987   1017     sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0);
   988   1018   
   989   1019     for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
   990         -    int nCol;                     /* Number of columns indexed by pIdx */
   991         -    int *aGotoChng;               /* Array of jump instruction addresses */
         1020  +    int nCol;                     /* Number of columns in pIdx. "N" */
   992   1021       int addrRewind;               /* Address of "OP_Rewind iIdxCur" */
   993         -    int addrGotoChng0;            /* Address of "Goto addr_chng_0" */
   994   1022       int addrNextRow;              /* Address of "next_row:" */
   995   1023       const char *zIdxName;         /* Name of the index */
         1024  +    int nColTest;                 /* Number of columns to test for changes */
   996   1025   
   997   1026       if( pOnlyIdx && pOnlyIdx!=pIdx ) continue;
   998   1027       if( pIdx->pPartIdxWhere==0 ) needTableCnt = 0;
   999         -    VdbeNoopComment((v, "Begin analysis of %s", pIdx->zName));
  1000         -    nCol = pIdx->nKeyCol;
  1001         -    aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*(nCol+1));
  1002         -    if( aGotoChng==0 ) continue;
         1028  +    if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIdx) ){
         1029  +      nCol = pIdx->nKeyCol;
         1030  +      zIdxName = pTab->zName;
         1031  +      nColTest = nCol - 1;
         1032  +    }else{
         1033  +      nCol = pIdx->nColumn;
         1034  +      zIdxName = pIdx->zName;
         1035  +      nColTest = pIdx->uniqNotNull ? pIdx->nKeyCol-1 : nCol-1;
         1036  +    }
  1003   1037   
  1004   1038       /* Populate the register containing the index name. */
  1005         -    if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){
  1006         -      zIdxName = pTab->zName;
  1007         -    }else{
  1008         -      zIdxName = pIdx->zName;
  1009         -    }
  1010   1039       sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, zIdxName, 0);
         1040  +    VdbeComment((v, "Analysis for %s.%s", pTab->zName, zIdxName));
  1011   1041   
  1012   1042       /*
  1013   1043       ** Pseudo-code for loop that calls stat_push():
  1014   1044       **
  1015   1045       **   Rewind csr
  1016   1046       **   if eof(csr) goto end_of_scan;
  1017   1047       **   regChng = 0
................................................................................
  1028   1058       **
  1029   1059       **  chng_addr_0:
  1030   1060       **   regPrev(0) = idx(0)
  1031   1061       **  chng_addr_1:
  1032   1062       **   regPrev(1) = idx(1)
  1033   1063       **  ...
  1034   1064       **
  1035         -    **  chng_addr_N:
         1065  +    **  endDistinctTest:
  1036   1066       **   regRowid = idx(rowid)
  1037   1067       **   stat_push(P, regChng, regRowid)
  1038   1068       **   Next csr
  1039   1069       **   if !eof(csr) goto next_row;
  1040   1070       **
  1041   1071       **  end_of_scan:
  1042   1072       */
  1043   1073   
  1044   1074       /* Make sure there are enough memory cells allocated to accommodate 
  1045   1075       ** the regPrev array and a trailing rowid (the rowid slot is required
  1046   1076       ** when building a record to insert into the sample column of 
  1047   1077       ** the sqlite_stat4 table.  */
  1048         -    pParse->nMem = MAX(pParse->nMem, regPrev+nCol);
         1078  +    pParse->nMem = MAX(pParse->nMem, regPrev+nColTest);
  1049   1079   
  1050   1080       /* Open a read-only cursor on the index being analyzed. */
  1051   1081       assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) );
  1052   1082       sqlite3VdbeAddOp3(v, OP_OpenRead, iIdxCur, pIdx->tnum, iDb);
  1053   1083       sqlite3VdbeSetP4KeyInfo(pParse, pIdx);
  1054   1084       VdbeComment((v, "%s", pIdx->zName));
  1055   1085   
  1056   1086       /* Invoke the stat_init() function. The arguments are:
  1057   1087       ** 
  1058         -    **    (1) the number of columns in the index including the rowid,
  1059         -    **    (2) the number of rows in the index,
         1088  +    **    (1) the number of columns in the index including the rowid
         1089  +    **        (or for a WITHOUT ROWID table, the number of PK columns),
         1090  +    **    (2) the number of columns in the key without the rowid/pk
         1091  +    **    (3) the number of rows in the index,
         1092  +    **
  1060   1093       **
  1061         -    ** The second argument is only used for STAT3 and STAT4
         1094  +    ** The third argument is only used for STAT3 and STAT4
  1062   1095       */
  1063   1096   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  1064         -    sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+2);
         1097  +    sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3);
  1065   1098   #endif
  1066         -    sqlite3VdbeAddOp2(v, OP_Integer, nCol+1, regStat4+1);
         1099  +    sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1);
         1100  +    sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2);
  1067   1101       sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4+1, regStat4);
  1068   1102       sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF);
  1069         -    sqlite3VdbeChangeP5(v, 1+IsStat34);
         1103  +    sqlite3VdbeChangeP5(v, 2+IsStat34);
  1070   1104   
  1071   1105       /* Implementation of the following:
  1072   1106       **
  1073   1107       **   Rewind csr
  1074   1108       **   if eof(csr) goto end_of_scan;
  1075   1109       **   regChng = 0
  1076   1110       **   goto next_push_0;
  1077   1111       **
  1078   1112       */
  1079   1113       addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur);
  1080   1114       VdbeCoverage(v);
  1081   1115       sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng);
  1082         -    addrGotoChng0 = sqlite3VdbeAddOp0(v, OP_Goto);
  1083         -
  1084         -    /*
  1085         -    **  next_row:
  1086         -    **   regChng = 0
  1087         -    **   if( idx(0) != regPrev(0) ) goto chng_addr_0
  1088         -    **   regChng = 1
  1089         -    **   if( idx(1) != regPrev(1) ) goto chng_addr_1
  1090         -    **   ...
  1091         -    **   regChng = N
  1092         -    **   goto chng_addr_N
  1093         -    */
  1094   1116       addrNextRow = sqlite3VdbeCurrentAddr(v);
  1095         -    for(i=0; i<nCol; i++){
  1096         -      char *pColl = (char*)sqlite3LocateCollSeq(pParse, pIdx->azColl[i]);
  1097         -      sqlite3VdbeAddOp2(v, OP_Integer, i, regChng);
  1098         -      sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp);
  1099         -      aGotoChng[i] = 
  1100         -      sqlite3VdbeAddOp4(v, OP_Ne, regTemp, 0, regPrev+i, pColl, P4_COLLSEQ);
  1101         -      sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
  1102         -      VdbeCoverage(v);
  1103         -    }
  1104         -    sqlite3VdbeAddOp2(v, OP_Integer, nCol, regChng);
  1105         -    aGotoChng[nCol] = sqlite3VdbeAddOp0(v, OP_Goto);
  1106         -
  1107         -    /*
  1108         -    **  chng_addr_0:
  1109         -    **   regPrev(0) = idx(0)
  1110         -    **  chng_addr_1:
  1111         -    **   regPrev(1) = idx(1)
  1112         -    **  ...
  1113         -    */
  1114         -    sqlite3VdbeJumpHere(v, addrGotoChng0);
  1115         -    for(i=0; i<nCol; i++){
  1116         -      sqlite3VdbeJumpHere(v, aGotoChng[i]);
  1117         -      sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regPrev+i);
  1118         -    }
  1119         -
         1117  +
         1118  +    if( nColTest>0 ){
         1119  +      int endDistinctTest = sqlite3VdbeMakeLabel(v);
         1120  +      int *aGotoChng;               /* Array of jump instruction addresses */
         1121  +      aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*nColTest);
         1122  +      if( aGotoChng==0 ) continue;
         1123  +
         1124  +      /*
         1125  +      **  next_row:
         1126  +      **   regChng = 0
         1127  +      **   if( idx(0) != regPrev(0) ) goto chng_addr_0
         1128  +      **   regChng = 1
         1129  +      **   if( idx(1) != regPrev(1) ) goto chng_addr_1
         1130  +      **   ...
         1131  +      **   regChng = N
         1132  +      **   goto endDistinctTest
         1133  +      */
         1134  +      sqlite3VdbeAddOp0(v, OP_Goto);
         1135  +      addrNextRow = sqlite3VdbeCurrentAddr(v);
         1136  +      if( nColTest==1 && pIdx->nKeyCol==1 && IsUniqueIndex(pIdx) ){
         1137  +        /* For a single-column UNIQUE index, once we have found a non-NULL
         1138  +        ** row, we know that all the rest will be distinct, so skip 
         1139  +        ** subsequent distinctness tests. */
         1140  +        sqlite3VdbeAddOp2(v, OP_NotNull, regPrev, endDistinctTest);
         1141  +        VdbeCoverage(v);
         1142  +      }
         1143  +      for(i=0; i<nColTest; i++){
         1144  +        char *pColl = (char*)sqlite3LocateCollSeq(pParse, pIdx->azColl[i]);
         1145  +        sqlite3VdbeAddOp2(v, OP_Integer, i, regChng);
         1146  +        sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp);
         1147  +        aGotoChng[i] = 
         1148  +        sqlite3VdbeAddOp4(v, OP_Ne, regTemp, 0, regPrev+i, pColl, P4_COLLSEQ);
         1149  +        sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
         1150  +        VdbeCoverage(v);
         1151  +      }
         1152  +      sqlite3VdbeAddOp2(v, OP_Integer, nColTest, regChng);
         1153  +      sqlite3VdbeAddOp2(v, OP_Goto, 0, endDistinctTest);
         1154  +  
         1155  +  
         1156  +      /*
         1157  +      **  chng_addr_0:
         1158  +      **   regPrev(0) = idx(0)
         1159  +      **  chng_addr_1:
         1160  +      **   regPrev(1) = idx(1)
         1161  +      **  ...
         1162  +      */
         1163  +      sqlite3VdbeJumpHere(v, addrNextRow-1);
         1164  +      for(i=0; i<nColTest; i++){
         1165  +        sqlite3VdbeJumpHere(v, aGotoChng[i]);
         1166  +        sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regPrev+i);
         1167  +      }
         1168  +      sqlite3VdbeResolveLabel(v, endDistinctTest);
         1169  +      sqlite3DbFree(db, aGotoChng);
         1170  +    }
         1171  +  
  1120   1172       /*
  1121   1173       **  chng_addr_N:
  1122   1174       **   regRowid = idx(rowid)            // STAT34 only
  1123   1175       **   stat_push(P, regChng, regRowid)  // 3rd parameter STAT34 only
  1124   1176       **   Next csr
  1125   1177       **   if !eof(csr) goto next_row;
  1126   1178       */
  1127         -    sqlite3VdbeJumpHere(v, aGotoChng[nCol]);
  1128   1179   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  1129   1180       assert( regRowid==(regStat4+2) );
  1130   1181       if( HasRowid(pTab) ){
  1131   1182         sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, regRowid);
  1132   1183       }else{
  1133   1184         Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable);
  1134   1185         int j, k, regKey;
................................................................................
  1146   1197       sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regTemp);
  1147   1198       sqlite3VdbeChangeP4(v, -1, (char*)&statPushFuncdef, P4_FUNCDEF);
  1148   1199       sqlite3VdbeChangeP5(v, 2+IsStat34);
  1149   1200       sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v);
  1150   1201   
  1151   1202       /* Add the entry to the stat1 table. */
  1152   1203       callStatGet(v, regStat4, STAT_GET_STAT1, regStat1);
  1153         -    sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "aaa", 0);
         1204  +    assert( "BBB"[0]==SQLITE_AFF_TEXT );
         1205  +    sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0);
  1154   1206       sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid);
  1155   1207       sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid);
  1156   1208       sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
  1157   1209   
  1158   1210       /* Add the entries to the stat3 or stat4 table. */
  1159   1211   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  1160   1212       {
................................................................................
  1164   1216         int regSample = regStat1+3;
  1165   1217         int regCol = regStat1+4;
  1166   1218         int regSampleRowid = regCol + nCol;
  1167   1219         int addrNext;
  1168   1220         int addrIsNull;
  1169   1221         u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound;
  1170   1222   
  1171         -      pParse->nMem = MAX(pParse->nMem, regCol+nCol+1);
         1223  +      pParse->nMem = MAX(pParse->nMem, regCol+nCol);
  1172   1224   
  1173   1225         addrNext = sqlite3VdbeCurrentAddr(v);
  1174   1226         callStatGet(v, regStat4, STAT_GET_ROWID, regSampleRowid);
  1175   1227         addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, regSampleRowid);
  1176   1228         VdbeCoverage(v);
  1177   1229         callStatGet(v, regStat4, STAT_GET_NEQ, regEq);
  1178   1230         callStatGet(v, regStat4, STAT_GET_NLT, regLt);
................................................................................
  1186   1238         sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, 
  1187   1239                                         pIdx->aiColumn[0], regSample);
  1188   1240   #else
  1189   1241         for(i=0; i<nCol; i++){
  1190   1242           i16 iCol = pIdx->aiColumn[i];
  1191   1243           sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i);
  1192   1244         }
  1193         -      sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol+1, regSample);
         1245  +      sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol, regSample);
  1194   1246   #endif
  1195   1247         sqlite3VdbeAddOp3(v, OP_MakeRecord, regTabname, 6, regTemp);
  1196   1248         sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regNewRowid);
  1197   1249         sqlite3VdbeAddOp3(v, OP_Insert, iStatCur+1, regTemp, regNewRowid);
  1198   1250         sqlite3VdbeAddOp2(v, OP_Goto, 1, addrNext); /* P1==1 for end-of-loop */
  1199   1251         sqlite3VdbeJumpHere(v, addrIsNull);
  1200   1252       }
  1201   1253   #endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */
  1202   1254   
  1203   1255       /* End of analysis */
  1204   1256       sqlite3VdbeJumpHere(v, addrRewind);
  1205         -    sqlite3DbFree(db, aGotoChng);
  1206   1257     }
  1207   1258   
  1208   1259   
  1209   1260     /* Create a single sqlite_stat1 entry containing NULL as the index
  1210   1261     ** name and the row count as the content.
  1211   1262     */
  1212   1263     if( pOnlyIdx==0 && needTableCnt ){
  1213   1264       VdbeComment((v, "%s", pTab->zName));
  1214   1265       sqlite3VdbeAddOp2(v, OP_Count, iTabCur, regStat1);
  1215   1266       jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); VdbeCoverage(v);
  1216   1267       sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname);
  1217         -    sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "aaa", 0);
         1268  +    assert( "BBB"[0]==SQLITE_AFF_TEXT );
         1269  +    sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0);
  1218   1270       sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid);
  1219   1271       sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid);
  1220   1272       sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
  1221   1273       sqlite3VdbeJumpHere(v, jZeroRows);
  1222   1274     }
  1223   1275   }
  1224   1276   
................................................................................
  1299   1351     sqlite3 *db = pParse->db;
  1300   1352     int iDb;
  1301   1353     int i;
  1302   1354     char *z, *zDb;
  1303   1355     Table *pTab;
  1304   1356     Index *pIdx;
  1305   1357     Token *pTableName;
         1358  +  Vdbe *v;
  1306   1359   
  1307   1360     /* Read the database schema. If an error occurs, leave an error message
  1308   1361     ** and code in pParse and return NULL. */
  1309   1362     assert( sqlite3BtreeHoldsAllMutexes(pParse->db) );
  1310   1363     if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){
  1311   1364       return;
  1312   1365     }
................................................................................
  1346   1399           }else if( (pTab = sqlite3LocateTable(pParse, 0, z, zDb))!=0 ){
  1347   1400             analyzeTable(pParse, pTab, 0);
  1348   1401           }
  1349   1402           sqlite3DbFree(db, z);
  1350   1403         }
  1351   1404       }   
  1352   1405     }
         1406  +  v = sqlite3GetVdbe(pParse);
         1407  +  if( v ) sqlite3VdbeAddOp0(v, OP_Expire);
  1353   1408   }
  1354   1409   
  1355   1410   /*
  1356   1411   ** Used to pass information from the analyzer reader through to the
  1357   1412   ** callback routine.
  1358   1413   */
  1359   1414   typedef struct analysisInfo analysisInfo;
................................................................................
  1378   1433     int c;
  1379   1434     int i;
  1380   1435     tRowcnt v;
  1381   1436   
  1382   1437   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  1383   1438     if( z==0 ) z = "";
  1384   1439   #else
  1385         -  if( NEVER(z==0) ) z = "";
         1440  +  assert( z!=0 );
  1386   1441   #endif
  1387   1442     for(i=0; *z && i<nOut; i++){
  1388   1443       v = 0;
  1389   1444       while( (c=z[0])>='0' && c<='9' ){
  1390   1445         v = v*10 + c - '0';
  1391   1446         z++;
  1392   1447       }
  1393   1448   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  1394         -    if( aOut ){
  1395         -      aOut[i] = v;
  1396         -    }else
         1449  +    if( aOut ) aOut[i] = v;
         1450  +    if( aLog ) aLog[i] = sqlite3LogEst(v);
  1397   1451   #else
  1398   1452       assert( aOut==0 );
  1399   1453       UNUSED_PARAMETER(aOut);
         1454  +    assert( aLog!=0 );
         1455  +    aLog[i] = sqlite3LogEst(v);
  1400   1456   #endif
  1401         -    {
  1402         -      aLog[i] = sqlite3LogEst(v);
  1403         -    }
  1404   1457       if( *z==' ' ) z++;
  1405   1458     }
  1406   1459   #ifndef SQLITE_ENABLE_STAT3_OR_STAT4
  1407         -  assert( pIndex!=0 );
         1460  +  assert( pIndex!=0 ); {
  1408   1461   #else
  1409         -  if( pIndex )
         1462  +  if( pIndex ){
         1463  +#endif
         1464  +    pIndex->bUnordered = 0;
         1465  +    pIndex->noSkipScan = 0;
         1466  +    while( z[0] ){
         1467  +      if( sqlite3_strglob("unordered*", z)==0 ){
         1468  +        pIndex->bUnordered = 1;
         1469  +      }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){
         1470  +        pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3));
         1471  +      }else if( sqlite3_strglob("noskipscan*", z)==0 ){
         1472  +        pIndex->noSkipScan = 1;
         1473  +      }
         1474  +#ifdef SQLITE_ENABLE_COSTMULT
         1475  +      else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){
         1476  +        pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9));
         1477  +      }
  1410   1478   #endif
  1411         -  {
  1412         -    if( strcmp(z, "unordered")==0 ){
  1413         -      pIndex->bUnordered = 1;
  1414         -    }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){
  1415         -      int v32 = 0;
  1416         -      sqlite3GetInt32(z+3, &v32);
  1417         -      pIndex->szIdxRow = sqlite3LogEst(v32);
         1479  +      while( z[0]!=0 && z[0]!=' ' ) z++;
         1480  +      while( z[0]==' ' ) z++;
  1418   1481       }
  1419   1482     }
  1420   1483   }
  1421   1484   
  1422   1485   /*
  1423   1486   ** This callback is invoked once for each index when reading the
  1424   1487   ** sqlite_stat1 table.  
................................................................................
  1452   1515       pIndex = sqlite3PrimaryKeyIndex(pTable);
  1453   1516     }else{
  1454   1517       pIndex = sqlite3FindIndex(pInfo->db, argv[1], pInfo->zDatabase);
  1455   1518     }
  1456   1519     z = argv[2];
  1457   1520   
  1458   1521     if( pIndex ){
  1459         -    decodeIntArray((char*)z, pIndex->nKeyCol+1, 0, pIndex->aiRowLogEst, pIndex);
         1522  +    int nCol = pIndex->nKeyCol+1;
         1523  +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4
         1524  +    tRowcnt * const aiRowEst = pIndex->aiRowEst = (tRowcnt*)sqlite3MallocZero(
         1525  +        sizeof(tRowcnt) * nCol
         1526  +    );
         1527  +    if( aiRowEst==0 ) pInfo->db->mallocFailed = 1;
         1528  +#else
         1529  +    tRowcnt * const aiRowEst = 0;
         1530  +#endif
         1531  +    pIndex->bUnordered = 0;
         1532  +    decodeIntArray((char*)z, nCol, aiRowEst, pIndex->aiRowLogEst, pIndex);
  1460   1533       if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0];
  1461   1534     }else{
  1462   1535       Index fakeIdx;
  1463   1536       fakeIdx.szIdxRow = pTable->szTabRow;
         1537  +#ifdef SQLITE_ENABLE_COSTMULT
         1538  +    fakeIdx.pTable = pTable;
         1539  +#endif
  1464   1540       decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx);
  1465   1541       pTable->szTabRow = fakeIdx.szIdxRow;
  1466   1542     }
  1467   1543   
  1468   1544     return 0;
  1469   1545   }
  1470   1546   
................................................................................
  1498   1574   ** stored in pIdx->aSample[]. 
  1499   1575   */
  1500   1576   static void initAvgEq(Index *pIdx){
  1501   1577     if( pIdx ){
  1502   1578       IndexSample *aSample = pIdx->aSample;
  1503   1579       IndexSample *pFinal = &aSample[pIdx->nSample-1];
  1504   1580       int iCol;
  1505         -    for(iCol=0; iCol<pIdx->nKeyCol; iCol++){
         1581  +    int nCol = 1;
         1582  +    if( pIdx->nSampleCol>1 ){
         1583  +      /* If this is stat4 data, then calculate aAvgEq[] values for all
         1584  +      ** sample columns except the last. The last is always set to 1, as
         1585  +      ** once the trailing PK fields are considered all index keys are
         1586  +      ** unique.  */
         1587  +      nCol = pIdx->nSampleCol-1;
         1588  +      pIdx->aAvgEq[nCol] = 1;
         1589  +    }
         1590  +    for(iCol=0; iCol<nCol; iCol++){
         1591  +      int nSample = pIdx->nSample;
  1506   1592         int i;                    /* Used to iterate through samples */
  1507   1593         tRowcnt sumEq = 0;        /* Sum of the nEq values */
  1508         -      tRowcnt nSum = 0;         /* Number of terms contributing to sumEq */
  1509   1594         tRowcnt avgEq = 0;
  1510         -      tRowcnt nDLt = pFinal->anDLt[iCol];
         1595  +      tRowcnt nRow;             /* Number of rows in index */
         1596  +      i64 nSum100 = 0;          /* Number of terms contributing to sumEq */
         1597  +      i64 nDist100;             /* Number of distinct values in index */
         1598  +
         1599  +      if( !pIdx->aiRowEst || iCol>=pIdx->nKeyCol || pIdx->aiRowEst[iCol+1]==0 ){
         1600  +        nRow = pFinal->anLt[iCol];
         1601  +        nDist100 = (i64)100 * pFinal->anDLt[iCol];
         1602  +        nSample--;
         1603  +      }else{
         1604  +        nRow = pIdx->aiRowEst[0];
         1605  +        nDist100 = ((i64)100 * pIdx->aiRowEst[0]) / pIdx->aiRowEst[iCol+1];
         1606  +      }
         1607  +      pIdx->nRowEst0 = nRow;
  1511   1608   
  1512   1609         /* Set nSum to the number of distinct (iCol+1) field prefixes that
  1513         -      ** occur in the stat4 table for this index before pFinal. Set
  1514         -      ** sumEq to the sum of the nEq values for column iCol for the same
  1515         -      ** set (adding the value only once where there exist dupicate 
  1516         -      ** prefixes).  */
  1517         -      for(i=0; i<(pIdx->nSample-1); i++){
  1518         -        if( aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] ){
         1610  +      ** occur in the stat4 table for this index. Set sumEq to the sum of 
         1611  +      ** the nEq values for column iCol for the same set (adding the value 
         1612  +      ** only once where there exist duplicate prefixes).  */
         1613  +      for(i=0; i<nSample; i++){
         1614  +        if( i==(pIdx->nSample-1)
         1615  +         || aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] 
         1616  +        ){
  1519   1617             sumEq += aSample[i].anEq[iCol];
  1520         -          nSum++;
         1618  +          nSum100 += 100;
  1521   1619           }
  1522   1620         }
  1523         -      if( nDLt>nSum ){
  1524         -        avgEq = (pFinal->anLt[iCol] - sumEq)/(nDLt - nSum);
         1621  +
         1622  +      if( nDist100>nSum100 ){
         1623  +        avgEq = ((i64)100 * (nRow - sumEq))/(nDist100 - nSum100);
  1525   1624         }
  1526   1625         if( avgEq==0 ) avgEq = 1;
  1527   1626         pIdx->aAvgEq[iCol] = avgEq;
  1528         -      if( pIdx->nSampleCol==1 ) break;
  1529   1627       }
  1530   1628     }
  1531   1629   }
  1532   1630   
  1533   1631   /*
  1534   1632   ** Look up an index by name.  Or, if the name of a WITHOUT ROWID table
  1535   1633   ** is supplied instead, find the PRIMARY KEY index for that table.
................................................................................
  1580   1678     }
  1581   1679     rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
  1582   1680     sqlite3DbFree(db, zSql);
  1583   1681     if( rc ) return rc;
  1584   1682   
  1585   1683     while( sqlite3_step(pStmt)==SQLITE_ROW ){
  1586   1684       int nIdxCol = 1;              /* Number of columns in stat4 records */
  1587         -    int nAvgCol = 1;              /* Number of entries in Index.aAvgEq */
  1588   1685   
  1589   1686       char *zIndex;   /* Index name */
  1590   1687       Index *pIdx;    /* Pointer to the index object */
  1591   1688       int nSample;    /* Number of samples */
  1592   1689       int nByte;      /* Bytes of space required */
  1593   1690       int i;          /* Bytes of space required */
  1594   1691       tRowcnt *pSpace;
................................................................................
  1598   1695       nSample = sqlite3_column_int(pStmt, 1);
  1599   1696       pIdx = findIndexOrPrimaryKey(db, zIndex, zDb);
  1600   1697       assert( pIdx==0 || bStat3 || pIdx->nSample==0 );
  1601   1698       /* Index.nSample is non-zero at this point if data has already been
  1602   1699       ** loaded from the stat4 table. In this case ignore stat3 data.  */
  1603   1700       if( pIdx==0 || pIdx->nSample ) continue;
  1604   1701       if( bStat3==0 ){
  1605         -      nIdxCol = pIdx->nKeyCol+1;
  1606         -      nAvgCol = pIdx->nKeyCol;
         1702  +      assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 );
         1703  +      if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
         1704  +        nIdxCol = pIdx->nKeyCol;
         1705  +      }else{
         1706  +        nIdxCol = pIdx->nColumn;
         1707  +      }
  1607   1708       }
  1608   1709       pIdx->nSampleCol = nIdxCol;
  1609   1710       nByte = sizeof(IndexSample) * nSample;
  1610   1711       nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample;
  1611         -    nByte += nAvgCol * sizeof(tRowcnt);     /* Space for Index.aAvgEq[] */
         1712  +    nByte += nIdxCol * sizeof(tRowcnt);     /* Space for Index.aAvgEq[] */
  1612   1713   
  1613   1714       pIdx->aSample = sqlite3DbMallocZero(db, nByte);
  1614   1715       if( pIdx->aSample==0 ){
  1615   1716         sqlite3_finalize(pStmt);
  1616   1717         return SQLITE_NOMEM;
  1617   1718       }
  1618   1719       pSpace = (tRowcnt*)&pIdx->aSample[nSample];
  1619         -    pIdx->aAvgEq = pSpace; pSpace += nAvgCol;
         1720  +    pIdx->aAvgEq = pSpace; pSpace += nIdxCol;
  1620   1721       for(i=0; i<nSample; i++){
  1621   1722         pIdx->aSample[i].anEq = pSpace; pSpace += nIdxCol;
  1622   1723         pIdx->aSample[i].anLt = pSpace; pSpace += nIdxCol;
  1623   1724         pIdx->aSample[i].anDLt = pSpace; pSpace += nIdxCol;
  1624   1725       }
  1625   1726       assert( ((u8*)pSpace)-nByte==(u8*)(pIdx->aSample) );
  1626   1727     }
................................................................................
  1761   1862       rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0);
  1762   1863       sqlite3DbFree(db, zSql);
  1763   1864     }
  1764   1865   
  1765   1866   
  1766   1867     /* Load the statistics from the sqlite_stat4 table. */
  1767   1868   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  1768         -  if( rc==SQLITE_OK ){
         1869  +  if( rc==SQLITE_OK && OptimizationEnabled(db, SQLITE_Stat34) ){
  1769   1870       int lookasideEnabled = db->lookaside.bEnabled;
  1770   1871       db->lookaside.bEnabled = 0;
  1771   1872       rc = loadStat4(db, sInfo.zDatabase);
  1772   1873       db->lookaside.bEnabled = lookasideEnabled;
         1874  +  }
         1875  +  for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){
         1876  +    Index *pIdx = sqliteHashData(i);
         1877  +    sqlite3_free(pIdx->aiRowEst);
         1878  +    pIdx->aiRowEst = 0;
  1773   1879     }
  1774   1880   #endif
  1775   1881   
  1776   1882     if( rc==SQLITE_NOMEM ){
  1777   1883       db->mallocFailed = 1;
  1778   1884     }
  1779   1885     return rc;
  1780   1886   }
  1781   1887   
  1782   1888   
  1783   1889   #endif /* SQLITE_OMIT_ANALYZE */

Changes to src/attach.c.

   146    146       if( !aNew->pSchema ){
   147    147         rc = SQLITE_NOMEM;
   148    148       }else if( aNew->pSchema->file_format && aNew->pSchema->enc!=ENC(db) ){
   149    149         zErrDyn = sqlite3MPrintf(db, 
   150    150           "attached databases must use the same text encoding as main database");
   151    151         rc = SQLITE_ERROR;
   152    152       }
          153  +    sqlite3BtreeEnter(aNew->pBt);
   153    154       pPager = sqlite3BtreePager(aNew->pBt);
   154    155       sqlite3PagerLockingMode(pPager, db->dfltLockMode);
   155    156       sqlite3BtreeSecureDelete(aNew->pBt,
   156    157                                sqlite3BtreeSecureDelete(db->aDb[0].pBt,-1) );
   157    158   #ifndef SQLITE_OMIT_PAGER_PRAGMAS
   158    159       sqlite3BtreeSetPagerFlags(aNew->pBt, 3 | (db->flags & PAGER_FLAGS_MASK));
   159    160   #endif
          161  +    sqlite3BtreeLeave(aNew->pBt);
   160    162     }
   161    163     aNew->safety_level = 3;
   162    164     aNew->zName = sqlite3DbStrDup(db, zName);
   163    165     if( rc==SQLITE_OK && aNew->zName==0 ){
   164    166       rc = SQLITE_NOMEM;
   165    167     }
   166    168   
................................................................................
   203    205     ** we found it.
   204    206     */
   205    207     if( rc==SQLITE_OK ){
   206    208       sqlite3BtreeEnterAll(db);
   207    209       rc = sqlite3Init(db, &zErrDyn);
   208    210       sqlite3BtreeLeaveAll(db);
   209    211     }
          212  +#ifdef SQLITE_USER_AUTHENTICATION
          213  +  if( rc==SQLITE_OK ){
          214  +    u8 newAuth = 0;
          215  +    rc = sqlite3UserAuthCheckLogin(db, zName, &newAuth);
          216  +    if( newAuth<db->auth.authLevel ){
          217  +      rc = SQLITE_AUTH_USER;
          218  +    }
          219  +  }
          220  +#endif
   210    221     if( rc ){
   211    222       int iDb = db->nDb - 1;
   212    223       assert( iDb>=2 );
   213    224       if( db->aDb[iDb].pBt ){
   214    225         sqlite3BtreeClose(db->aDb[iDb].pBt);
   215    226         db->aDb[iDb].pBt = 0;
   216    227         db->aDb[iDb].pSchema = 0;

Changes to src/auth.c.

    68     68   ** setting of the auth function is NULL.
    69     69   */
    70     70   int sqlite3_set_authorizer(
    71     71     sqlite3 *db,
    72     72     int (*xAuth)(void*,int,const char*,const char*,const char*,const char*),
    73     73     void *pArg
    74     74   ){
           75  +#ifdef SQLITE_ENABLE_API_ARMOR
           76  +  if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
           77  +#endif
    75     78     sqlite3_mutex_enter(db->mutex);
    76         -  db->xAuth = xAuth;
           79  +  db->xAuth = (sqlite3_xauth)xAuth;
    77     80     db->pAuthArg = pArg;
    78     81     sqlite3ExpirePreparedStatements(db);
    79     82     sqlite3_mutex_leave(db->mutex);
    80     83     return SQLITE_OK;
    81     84   }
    82     85   
    83     86   /*
................................................................................
   104    107     const char *zCol,               /* Column name */
   105    108     int iDb                         /* Index of containing database. */
   106    109   ){
   107    110     sqlite3 *db = pParse->db;       /* Database handle */
   108    111     char *zDb = db->aDb[iDb].zName; /* Name of attached database */
   109    112     int rc;                         /* Auth callback return code */
   110    113   
   111         -  rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext);
          114  +  rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext
          115  +#ifdef SQLITE_USER_AUTHENTICATION
          116  +                 ,db->auth.zAuthUser
          117  +#endif
          118  +                );
   112    119     if( rc==SQLITE_DENY ){
   113    120       if( db->nDb>2 || iDb!=0 ){
   114    121         sqlite3ErrorMsg(pParse, "access to %s.%s.%s is prohibited",zDb,zTab,zCol);
   115    122       }else{
   116    123         sqlite3ErrorMsg(pParse, "access to %s.%s is prohibited", zTab, zCol);
   117    124       }
   118    125       pParse->rc = SQLITE_AUTH;
................................................................................
   204    211     if( db->init.busy || IN_DECLARE_VTAB ){
   205    212       return SQLITE_OK;
   206    213     }
   207    214   
   208    215     if( db->xAuth==0 ){
   209    216       return SQLITE_OK;
   210    217     }
   211         -  rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext);
          218  +  rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext
          219  +#ifdef SQLITE_USER_AUTHENTICATION
          220  +                 ,db->auth.zAuthUser
          221  +#endif
          222  +                );
   212    223     if( rc==SQLITE_DENY ){
   213    224       sqlite3ErrorMsg(pParse, "not authorized");
   214    225       pParse->rc = SQLITE_AUTH;
   215    226     }else if( rc!=SQLITE_OK && rc!=SQLITE_IGNORE ){
   216    227       rc = SQLITE_DENY;
   217    228       sqliteAuthBadReturnCode(pParse);
   218    229     }

Changes to src/backup.c.

    83     83     int i = sqlite3FindDbName(pDb, zDb);
    84     84   
    85     85     if( i==1 ){
    86     86       Parse *pParse;
    87     87       int rc = 0;
    88     88       pParse = sqlite3StackAllocZero(pErrorDb, sizeof(*pParse));
    89     89       if( pParse==0 ){
    90         -      sqlite3Error(pErrorDb, SQLITE_NOMEM, "out of memory");
           90  +      sqlite3ErrorWithMsg(pErrorDb, SQLITE_NOMEM, "out of memory");
    91     91         rc = SQLITE_NOMEM;
    92     92       }else{
    93     93         pParse->db = pDb;
    94     94         if( sqlite3OpenTempDatabase(pParse) ){
    95         -        sqlite3Error(pErrorDb, pParse->rc, "%s", pParse->zErrMsg);
           95  +        sqlite3ErrorWithMsg(pErrorDb, pParse->rc, "%s", pParse->zErrMsg);
    96     96           rc = SQLITE_ERROR;
    97     97         }
    98     98         sqlite3DbFree(pErrorDb, pParse->zErrMsg);
    99     99         sqlite3ParserReset(pParse);
   100    100         sqlite3StackFree(pErrorDb, pParse);
   101    101       }
   102    102       if( rc ){
   103    103         return 0;
   104    104       }
   105    105     }
   106    106   
   107    107     if( i<0 ){
   108         -    sqlite3Error(pErrorDb, SQLITE_ERROR, "unknown database %s", zDb);
          108  +    sqlite3ErrorWithMsg(pErrorDb, SQLITE_ERROR, "unknown database %s", zDb);
   109    109       return 0;
   110    110     }
   111    111   
   112    112     return pDb->aDb[i].pBt;
   113    113   }
   114    114   
   115    115   /*
................................................................................
   117    117   ** of the source.
   118    118   */
   119    119   static int setDestPgsz(sqlite3_backup *p){
   120    120     int rc;
   121    121     rc = sqlite3BtreeSetPageSize(p->pDest,sqlite3BtreeGetPageSize(p->pSrc),-1,0);
   122    122     return rc;
   123    123   }
          124  +
          125  +/*
          126  +** Check that there is no open read-transaction on the b-tree passed as the
          127  +** second argument. If there is not, return SQLITE_OK. Otherwise, if there
          128  +** is an open read-transaction, return SQLITE_ERROR and leave an error 
          129  +** message in database handle db.
          130  +*/
          131  +static int checkReadTransaction(sqlite3 *db, Btree *p){
          132  +  if( sqlite3BtreeIsInReadTrans(p) ){
          133  +    sqlite3ErrorWithMsg(db, SQLITE_ERROR, "destination database is in use");
          134  +    return SQLITE_ERROR;
          135  +  }
          136  +  return SQLITE_OK;
          137  +}
   124    138   
   125    139   /*
   126    140   ** Create an sqlite3_backup process to copy the contents of zSrcDb from
   127    141   ** connection handle pSrcDb to zDestDb in pDestDb. If successful, return
   128    142   ** a pointer to the new sqlite3_backup object.
   129    143   **
   130    144   ** If an error occurs, NULL is returned and an error code and error message
................................................................................
   133    147   sqlite3_backup *sqlite3_backup_init(
   134    148     sqlite3* pDestDb,                     /* Database to write to */
   135    149     const char *zDestDb,                  /* Name of database within pDestDb */
   136    150     sqlite3* pSrcDb,                      /* Database connection to read from */
   137    151     const char *zSrcDb                    /* Name of database within pSrcDb */
   138    152   ){
   139    153     sqlite3_backup *p;                    /* Value to return */
          154  +
          155  +#ifdef SQLITE_ENABLE_API_ARMOR
          156  +  if( !sqlite3SafetyCheckOk(pSrcDb)||!sqlite3SafetyCheckOk(pDestDb) ){
          157  +    (void)SQLITE_MISUSE_BKPT;
          158  +    return 0;
          159  +  }
          160  +#endif
   140    161   
   141    162     /* Lock the source database handle. The destination database
   142    163     ** handle is not locked in this routine, but it is locked in
   143    164     ** sqlite3_backup_step(). The user is required to ensure that no
   144    165     ** other thread accesses the destination handle for the duration
   145    166     ** of the backup operation.  Any attempt to use the destination
   146    167     ** database connection while a backup is in progress may cause
   147    168     ** a malfunction or a deadlock.
   148    169     */
   149    170     sqlite3_mutex_enter(pSrcDb->mutex);
   150    171     sqlite3_mutex_enter(pDestDb->mutex);
   151    172   
   152    173     if( pSrcDb==pDestDb ){
   153         -    sqlite3Error(
          174  +    sqlite3ErrorWithMsg(
   154    175           pDestDb, SQLITE_ERROR, "source and destination must be distinct"
   155    176       );
   156    177       p = 0;
   157    178     }else {
   158    179       /* Allocate space for a new sqlite3_backup object...
   159    180       ** EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a
   160    181       ** call to sqlite3_backup_init() and is destroyed by a call to
   161    182       ** sqlite3_backup_finish(). */
   162    183       p = (sqlite3_backup *)sqlite3MallocZero(sizeof(sqlite3_backup));
   163    184       if( !p ){
   164         -      sqlite3Error(pDestDb, SQLITE_NOMEM, 0);
          185  +      sqlite3Error(pDestDb, SQLITE_NOMEM);
   165    186       }
   166    187     }
   167    188   
   168    189     /* If the allocation succeeded, populate the new object. */
   169    190     if( p ){
   170    191       p->pSrc = findBtree(pDestDb, pSrcDb, zSrcDb);
   171    192       p->pDest = findBtree(pDestDb, pDestDb, zDestDb);
   172    193       p->pDestDb = pDestDb;
   173    194       p->pSrcDb = pSrcDb;
   174    195       p->iNext = 1;
   175    196       p->isAttached = 0;
   176    197   
   177         -    if( 0==p->pSrc || 0==p->pDest || setDestPgsz(p)==SQLITE_NOMEM ){
          198  +    if( 0==p->pSrc || 0==p->pDest 
          199  +     || setDestPgsz(p)==SQLITE_NOMEM 
          200  +     || checkReadTransaction(pDestDb, p->pDest)!=SQLITE_OK 
          201  +     ){
   178    202         /* One (or both) of the named databases did not exist or an OOM
   179         -      ** error was hit.  The error has already been written into the
   180         -      ** pDestDb handle.  All that is left to do here is free the
   181         -      ** sqlite3_backup structure.
   182         -      */
          203  +      ** error was hit. Or there is a transaction open on the destination
          204  +      ** database. The error has already been written into the pDestDb 
          205  +      ** handle. All that is left to do here is free the sqlite3_backup 
          206  +      ** structure.  */
   183    207         sqlite3_free(p);
   184    208         p = 0;
   185    209       }
   186    210     }
   187    211     if( p ){
   188    212       p->pSrc->nBackup++;
   189    213     }
................................................................................
   330    354   */
   331    355   int sqlite3_backup_step(sqlite3_backup *p, int nPage){
   332    356     int rc;
   333    357     int destMode;       /* Destination journal mode */
   334    358     int pgszSrc = 0;    /* Source page size */
   335    359     int pgszDest = 0;   /* Destination page size */
   336    360   
          361  +#ifdef SQLITE_ENABLE_API_ARMOR
          362  +  if( p==0 ) return SQLITE_MISUSE_BKPT;
          363  +#endif
   337    364     sqlite3_mutex_enter(p->pSrcDb->mutex);
   338    365     sqlite3BtreeEnter(p->pSrc);
   339    366     if( p->pDestDb ){
   340    367       sqlite3_mutex_enter(p->pDestDb->mutex);
   341    368     }
   342    369   
   343    370     rc = p->rc;
................................................................................
   593    620       while( *pp!=p ){
   594    621         pp = &(*pp)->pNext;
   595    622       }
   596    623       *pp = p->pNext;
   597    624     }
   598    625   
   599    626     /* If a transaction is still open on the Btree, roll it back. */
   600         -  sqlite3BtreeRollback(p->pDest, SQLITE_OK);
          627  +  sqlite3BtreeRollback(p->pDest, SQLITE_OK, 0);
   601    628   
   602    629     /* Set the error code of the destination database handle. */
   603    630     rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc;
   604    631     if( p->pDestDb ){
   605         -    sqlite3Error(p->pDestDb, rc, 0);
          632  +    sqlite3Error(p->pDestDb, rc);
   606    633   
   607    634       /* Exit the mutexes and free the backup context structure. */
   608    635       sqlite3LeaveMutexAndCloseZombie(p->pDestDb);
   609    636     }
   610    637     sqlite3BtreeLeave(p->pSrc);
   611    638     if( p->pDestDb ){
   612    639       /* EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a
................................................................................
   619    646   }
   620    647   
   621    648   /*
   622    649   ** Return the number of pages still to be backed up as of the most recent
   623    650   ** call to sqlite3_backup_step().
   624    651   */
   625    652   int sqlite3_backup_remaining(sqlite3_backup *p){
          653  +#ifdef SQLITE_ENABLE_API_ARMOR
          654  +  if( p==0 ){
          655  +    (void)SQLITE_MISUSE_BKPT;
          656  +    return 0;
          657  +  }
          658  +#endif
   626    659     return p->nRemaining;
   627    660   }
   628    661   
   629    662   /*
   630    663   ** Return the total number of pages in the source database as of the most 
   631    664   ** recent call to sqlite3_backup_step().
   632    665   */
   633    666   int sqlite3_backup_pagecount(sqlite3_backup *p){
          667  +#ifdef SQLITE_ENABLE_API_ARMOR
          668  +  if( p==0 ){
          669  +    (void)SQLITE_MISUSE_BKPT;
          670  +    return 0;
          671  +  }
          672  +#endif
   634    673     return p->nPagecount;
   635    674   }
   636    675   
   637    676   /*
   638    677   ** This function is called after the contents of page iPage of the
   639    678   ** source database have been modified. If page iPage has already been 
   640    679   ** copied into the destination database, then the data written to the

Changes to src/btmutex.c.

    34     34     p->locked = 1;
    35     35   }
    36     36   
    37     37   /*
    38     38   ** Release the BtShared mutex associated with B-Tree handle p and
    39     39   ** clear the p->locked boolean.
    40     40   */
    41         -static void unlockBtreeMutex(Btree *p){
           41  +static void SQLITE_NOINLINE unlockBtreeMutex(Btree *p){
    42     42     BtShared *pBt = p->pBt;
    43     43     assert( p->locked==1 );
    44     44     assert( sqlite3_mutex_held(pBt->mutex) );
    45     45     assert( sqlite3_mutex_held(p->db->mutex) );
    46     46     assert( p->db==pBt->db );
    47     47   
    48     48     sqlite3_mutex_leave(pBt->mutex);
    49     49     p->locked = 0;
    50     50   }
    51     51   
           52  +/* Forward reference */
           53  +static void SQLITE_NOINLINE btreeLockCarefully(Btree *p);
           54  +
    52     55   /*
    53     56   ** Enter a mutex on the given BTree object.
    54     57   **
    55     58   ** If the object is not sharable, then no mutex is ever required
    56     59   ** and this routine is a no-op.  The underlying mutex is non-recursive.
    57     60   ** But we keep a reference count in Btree.wantToLock so the behavior
    58     61   ** of this interface is recursive.
................................................................................
    62     65   ** Btrees belonging to the same database connection as the p Btree
    63     66   ** which need to be locked after p.  If we cannot get a lock on
    64     67   ** p, then first unlock all of the others on p->pNext, then wait
    65     68   ** for the lock to become available on p, then relock all of the
    66     69   ** subsequent Btrees that desire a lock.
    67     70   */
    68     71   void sqlite3BtreeEnter(Btree *p){
    69         -  Btree *pLater;
    70         -
    71     72     /* Some basic sanity checking on the Btree.  The list of Btrees
    72     73     ** connected by pNext and pPrev should be in sorted order by
    73     74     ** Btree.pBt value. All elements of the list should belong to
    74     75     ** the same connection. Only shared Btrees are on the list. */
    75     76     assert( p->pNext==0 || p->pNext->pBt>p->pBt );
    76     77     assert( p->pPrev==0 || p->pPrev->pBt<p->pBt );
    77     78     assert( p->pNext==0 || p->pNext->db==p->db );
................................................................................
    88     89     /* Unless the database is sharable and unlocked, then BtShared.db
    89     90     ** should already be set correctly. */
    90     91     assert( (p->locked==0 && p->sharable) || p->pBt->db==p->db );
    91     92   
    92     93     if( !p->sharable ) return;
    93     94     p->wantToLock++;
    94     95     if( p->locked ) return;
           96  +  btreeLockCarefully(p);
           97  +}
           98  +
           99  +/* This is a helper function for sqlite3BtreeLock(). By moving
          100  +** complex, but seldom used logic, out of sqlite3BtreeLock() and
          101  +** into this routine, we avoid unnecessary stack pointer changes
          102  +** and thus help the sqlite3BtreeLock() routine to run much faster
          103  +** in the common case.
          104  +*/
          105  +static void SQLITE_NOINLINE btreeLockCarefully(Btree *p){
          106  +  Btree *pLater;
    95    107   
    96    108     /* In most cases, we should be able to acquire the lock we
    97         -  ** want without having to go throught the ascending lock
          109  +  ** want without having to go through the ascending lock
    98    110     ** procedure that follows.  Just be sure not to block.
    99    111     */
   100    112     if( sqlite3_mutex_try(p->pBt->mutex)==SQLITE_OK ){
   101    113       p->pBt->db = p->db;
   102    114       p->locked = 1;
   103    115       return;
   104    116     }
................................................................................
   119    131     lockBtreeMutex(p);
   120    132     for(pLater=p->pNext; pLater; pLater=pLater->pNext){
   121    133       if( pLater->wantToLock ){
   122    134         lockBtreeMutex(pLater);
   123    135       }
   124    136     }
   125    137   }
          138  +
   126    139   
   127    140   /*
   128    141   ** Exit the recursive mutex on a Btree.
   129    142   */
   130    143   void sqlite3BtreeLeave(Btree *p){
   131    144     if( p->sharable ){
   132    145       assert( p->wantToLock>0 );

Changes to src/btree.c.

     5      5   ** a legal notice, here is a blessing:
     6      6   **
     7      7   **    May you do good and not evil.
     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   *************************************************************************
    12         -** This file implements a external (disk-based) database using BTrees.
           12  +** This file implements an external (disk-based) database using BTrees.
    13     13   ** See the header comment on "btreeInt.h" for additional information.
    14     14   ** Including a description of file format and an overview of operation.
    15     15   */
    16     16   #include "btreeInt.h"
    17     17   
    18     18   /*
    19     19   ** The header string that appears at the beginning of every
................................................................................
   158    158     }
   159    159   
   160    160     /* If the client is reading  or writing an index and the schema is
   161    161     ** not loaded, then it is too difficult to actually check to see if
   162    162     ** the correct locks are held.  So do not bother - just return true.
   163    163     ** This case does not come up very often anyhow.
   164    164     */
   165         -  if( isIndex && (!pSchema || (pSchema->flags&DB_SchemaLoaded)==0) ){
          165  +  if( isIndex && (!pSchema || (pSchema->schemaFlags&DB_SchemaLoaded)==0) ){
   166    166       return 1;
   167    167     }
   168    168   
   169    169     /* Figure out the root-page that the lock should be held on. For table
   170    170     ** b-trees, this is just the root page of the b-tree being read or
   171    171     ** written. For index b-trees, it is the root page of the associated
   172    172     ** table.  */
................................................................................
   483    483     i64 iRow,               /* The rowid that might be changing */
   484    484     int isClearTable        /* True if all rows are being deleted */
   485    485   ){
   486    486     BtCursor *p;
   487    487     BtShared *pBt = pBtree->pBt;
   488    488     assert( sqlite3BtreeHoldsMutex(pBtree) );
   489    489     for(p=pBt->pCursor; p; p=p->pNext){
   490         -    if( (p->curFlags & BTCF_Incrblob)!=0 && (isClearTable || p->info.nKey==iRow) ){
          490  +    if( (p->curFlags & BTCF_Incrblob)!=0
          491  +     && (isClearTable || p->info.nKey==iRow)
          492  +    ){
   491    493         p->eState = CURSOR_INVALID;
   492    494       }
   493    495     }
   494    496   }
   495    497   
   496    498   #else
   497    499     /* Stub function when INCRBLOB is omitted */
................................................................................
   602    604     /* If this is an intKey table, then the above call to BtreeKeySize()
   603    605     ** stores the integer key in pCur->nKey. In this case this value is
   604    606     ** all that is required. Otherwise, if pCur is not open on an intKey
   605    607     ** table, then malloc space for and store the pCur->nKey bytes of key 
   606    608     ** data.
   607    609     */
   608    610     if( 0==pCur->apPage[0]->intKey ){
   609         -    void *pKey = sqlite3Malloc( (int)pCur->nKey );
          611  +    void *pKey = sqlite3Malloc( pCur->nKey );
   610    612       if( pKey ){
   611    613         rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey);
   612    614         if( rc==SQLITE_OK ){
   613    615           pCur->pKey = pKey;
   614    616         }else{
   615    617           sqlite3_free(pKey);
   616    618         }
................................................................................
   625    627       pCur->eState = CURSOR_REQUIRESEEK;
   626    628     }
   627    629   
   628    630     invalidateOverflowCache(pCur);
   629    631     return rc;
   630    632   }
   631    633   
          634  +/* Forward reference */
          635  +static int SQLITE_NOINLINE saveCursorsOnList(BtCursor*,Pgno,BtCursor*);
          636  +
   632    637   /*
   633    638   ** Save the positions of all cursors (except pExcept) that are open on
   634         -** the table  with root-page iRoot. Usually, this is called just before cursor
   635         -** pExcept is used to modify the table (BtreeDelete() or BtreeInsert()).
          639  +** the table with root-page iRoot.  "Saving the cursor position" means that
          640  +** the location in the btree is remembered in such a way that it can be
          641  +** moved back to the same spot after the btree has been modified.  This
          642  +** routine is called just before cursor pExcept is used to modify the
          643  +** table, for example in BtreeDelete() or BtreeInsert().
          644  +**
          645  +** Implementation note:  This routine merely checks to see if any cursors
          646  +** need to be saved.  It calls out to saveCursorsOnList() in the (unusual)
          647  +** event that cursors are in need to being saved.
   636    648   */
   637    649   static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){
   638    650     BtCursor *p;
   639    651     assert( sqlite3_mutex_held(pBt->mutex) );
   640    652     assert( pExcept==0 || pExcept->pBt==pBt );
   641    653     for(p=pBt->pCursor; p; p=p->pNext){
          654  +    if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ) break;
          655  +  }
          656  +  return p ? saveCursorsOnList(p, iRoot, pExcept) : SQLITE_OK;
          657  +}
          658  +
          659  +/* This helper routine to saveAllCursors does the actual work of saving
          660  +** the cursors if and when a cursor is found that actually requires saving.
          661  +** The common case is that no cursors need to be saved, so this routine is
          662  +** broken out from its caller to avoid unnecessary stack pointer movement.
          663  +*/
          664  +static int SQLITE_NOINLINE saveCursorsOnList(
          665  +  BtCursor *p,         /* The first cursor that needs saving */
          666  +  Pgno iRoot,          /* Only save cursor with this iRoot. Save all if zero */
          667  +  BtCursor *pExcept    /* Do not save this cursor */
          668  +){
          669  +  do{
   642    670       if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){
   643    671         if( p->eState==CURSOR_VALID ){
   644    672           int rc = saveCursorPosition(p);
   645    673           if( SQLITE_OK!=rc ){
   646    674             return rc;
   647    675           }
   648    676         }else{
   649    677           testcase( p->iPage>0 );
   650    678           btreeReleaseAllCursorPages(p);
   651    679         }
   652    680       }
   653         -  }
          681  +    p = p->pNext;
          682  +  }while( p );
   654    683     return SQLITE_OK;
   655    684   }
   656    685   
   657    686   /*
   658    687   ** Clear the current cursor position.
   659    688   */
   660    689   void sqlite3BtreeClearCursor(BtCursor *pCur){
................................................................................
   731    760   
   732    761   #define restoreCursorPosition(p) \
   733    762     (p->eState>=CURSOR_REQUIRESEEK ? \
   734    763            btreeRestoreCursorPosition(p) : \
   735    764            SQLITE_OK)
   736    765   
   737    766   /*
   738         -** Determine whether or not a cursor has moved from the position it
   739         -** was last placed at.  Cursors can move when the row they are pointing
   740         -** at is deleted out from under them.
   741         -**
   742         -** This routine returns an error code if something goes wrong.  The
   743         -** integer *pHasMoved is set as follows:
   744         -**
   745         -**    0:   The cursor is unchanged
   746         -**    1:   The cursor is still pointing at the same row, but the pointers
   747         -**         returned by sqlite3BtreeKeyFetch() or sqlite3BtreeDataFetch()
   748         -**         might now be invalid because of a balance() or other change to the
   749         -**         b-tree.
   750         -**    2:   The cursor is no longer pointing to the row.  The row might have
   751         -**         been deleted out from under the cursor.
   752         -*/
   753         -int sqlite3BtreeCursorHasMoved(BtCursor *pCur, int *pHasMoved){
   754         -  int rc;
   755         -
   756         -  if( pCur->eState==CURSOR_VALID ){
   757         -    *pHasMoved = 0;
   758         -    return SQLITE_OK;
   759         -  }
          767  +** Determine whether or not a cursor has moved from the position where
          768  +** it was last placed, or has been invalidated for any other reason.
          769  +** Cursors can move when the row they are pointing at is deleted out
          770  +** from under them, for example.  Cursor might also move if a btree
          771  +** is rebalanced.
          772  +**
          773  +** Calling this routine with a NULL cursor pointer returns false.
          774  +**
          775  +** Use the separate sqlite3BtreeCursorRestore() routine to restore a cursor
          776  +** back to where it ought to be if this routine returns true.
          777  +*/
          778  +int sqlite3BtreeCursorHasMoved(BtCursor *pCur){
          779  +  return pCur->eState!=CURSOR_VALID;
          780  +}
          781  +
          782  +/*
          783  +** This routine restores a cursor back to its original position after it
          784  +** has been moved by some outside activity (such as a btree rebalance or
          785  +** a row having been deleted out from under the cursor).  
          786  +**
          787  +** On success, the *pDifferentRow parameter is false if the cursor is left
          788  +** pointing at exactly the same row.  *pDifferntRow is the row the cursor
          789  +** was pointing to has been deleted, forcing the cursor to point to some
          790  +** nearby row.
          791  +**
          792  +** This routine should only be called for a cursor that just returned
          793  +** TRUE from sqlite3BtreeCursorHasMoved().
          794  +*/
          795  +int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow){
          796  +  int rc;
          797  +
          798  +  assert( pCur!=0 );
          799  +  assert( pCur->eState!=CURSOR_VALID );
   760    800     rc = restoreCursorPosition(pCur);
   761    801     if( rc ){
   762         -    *pHasMoved = 2;
          802  +    *pDifferentRow = 1;
   763    803       return rc;
   764    804     }
   765    805     if( pCur->eState!=CURSOR_VALID || NEVER(pCur->skipNext!=0) ){
   766         -    *pHasMoved = 2;
          806  +    *pDifferentRow = 1;
   767    807     }else{
   768         -    *pHasMoved = 1;
          808  +    *pDifferentRow = 0;
   769    809     }
   770    810     return SQLITE_OK;
   771    811   }
   772    812   
   773    813   #ifndef SQLITE_OMIT_AUTOVACUUM
   774    814   /*
   775    815   ** Given a page number of a regular database page, return the page
................................................................................
   926    966   }
   927    967   
   928    968   /*
   929    969   ** Parse a cell content block and fill in the CellInfo structure.  There
   930    970   ** are two versions of this function.  btreeParseCell() takes a 
   931    971   ** cell index as the second argument and btreeParseCellPtr() 
   932    972   ** takes a pointer to the body of the cell as its second argument.
   933         -**
   934         -** Within this file, the parseCell() macro can be called instead of
   935         -** btreeParseCellPtr(). Using some compilers, this will be faster.
   936    973   */
   937    974   static void btreeParseCellPtr(
   938    975     MemPage *pPage,         /* Page containing the cell */
   939    976     u8 *pCell,              /* Pointer to the cell text. */
   940    977     CellInfo *pInfo         /* Fill in this structure */
   941    978   ){
   942         -  u16 n;                  /* Number bytes in cell content header */
          979  +  u8 *pIter;              /* For scanning through pCell */
   943    980     u32 nPayload;           /* Number of bytes of cell payload */
   944    981   
   945    982     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
   946         -
   947         -  pInfo->pCell = pCell;
   948    983     assert( pPage->leaf==0 || pPage->leaf==1 );
   949         -  n = pPage->childPtrSize;
   950         -  assert( n==4-4*pPage->leaf );
   951         -  if( pPage->intKey ){
   952         -    if( pPage->hasData ){
   953         -      assert( n==0 );
   954         -      n = getVarint32(pCell, nPayload);
   955         -    }else{
   956         -      nPayload = 0;
   957         -    }
   958         -    n += getVarint(&pCell[n], (u64*)&pInfo->nKey);
   959         -    pInfo->nData = nPayload;
          984  +  if( pPage->intKeyLeaf ){
          985  +    assert( pPage->childPtrSize==0 );
          986  +    pIter = pCell + getVarint32(pCell, nPayload);
          987  +    pIter += getVarint(pIter, (u64*)&pInfo->nKey);
          988  +  }else if( pPage->noPayload ){
          989  +    assert( pPage->childPtrSize==4 );
          990  +    pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey);
          991  +    pInfo->nPayload = 0;
          992  +    pInfo->nLocal = 0;
          993  +    pInfo->iOverflow = 0;
          994  +    pInfo->pPayload = 0;
          995  +    return;
   960    996     }else{
   961         -    pInfo->nData = 0;
   962         -    n += getVarint32(&pCell[n], nPayload);
          997  +    pIter = pCell + pPage->childPtrSize;
          998  +    pIter += getVarint32(pIter, nPayload);
   963    999       pInfo->nKey = nPayload;
   964   1000     }
   965   1001     pInfo->nPayload = nPayload;
   966         -  pInfo->nHeader = n;
         1002  +  pInfo->pPayload = pIter;
   967   1003     testcase( nPayload==pPage->maxLocal );
   968   1004     testcase( nPayload==pPage->maxLocal+1 );
   969         -  if( likely(nPayload<=pPage->maxLocal) ){
         1005  +  if( nPayload<=pPage->maxLocal ){
   970   1006       /* This is the (easy) common case where the entire payload fits
   971   1007       ** on the local page.  No overflow is required.
   972   1008       */
   973         -    if( (pInfo->nSize = (u16)(n+nPayload))<4 ) pInfo->nSize = 4;
         1009  +    pInfo->nSize = nPayload + (u16)(pIter - pCell);
         1010  +    if( pInfo->nSize<4 ) pInfo->nSize = 4;
   974   1011       pInfo->nLocal = (u16)nPayload;
   975   1012       pInfo->iOverflow = 0;
   976   1013     }else{
   977   1014       /* If the payload will not fit completely on the local page, we have
   978   1015       ** to decide how much to store locally and how much to spill onto
   979   1016       ** overflow pages.  The strategy is to minimize the amount of unused
   980   1017       ** space on overflow pages while keeping the amount of local storage
................................................................................
   993   1030       testcase( surplus==maxLocal );
   994   1031       testcase( surplus==maxLocal+1 );
   995   1032       if( surplus <= maxLocal ){
   996   1033         pInfo->nLocal = (u16)surplus;
   997   1034       }else{
   998   1035         pInfo->nLocal = (u16)minLocal;
   999   1036       }
  1000         -    pInfo->iOverflow = (u16)(pInfo->nLocal + n);
         1037  +    pInfo->iOverflow = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell);
  1001   1038       pInfo->nSize = pInfo->iOverflow + 4;
  1002   1039     }
  1003   1040   }
  1004         -#define parseCell(pPage, iCell, pInfo) \
  1005         -  btreeParseCellPtr((pPage), findCell((pPage), (iCell)), (pInfo))
  1006   1041   static void btreeParseCell(
  1007   1042     MemPage *pPage,         /* Page containing the cell */
  1008   1043     int iCell,              /* The cell index.  First cell is 0 */
  1009   1044     CellInfo *pInfo         /* Fill in this structure */
  1010   1045   ){
  1011         -  parseCell(pPage, iCell, pInfo);
         1046  +  btreeParseCellPtr(pPage, findCell(pPage, iCell), pInfo);
  1012   1047   }
  1013   1048   
  1014   1049   /*
  1015   1050   ** Compute the total number of bytes that a Cell needs in the cell
  1016   1051   ** data area of the btree-page.  The return number includes the cell
  1017   1052   ** data header and the local payload, but not any overflow page or
  1018   1053   ** the space used by the cell pointer.
  1019   1054   */
  1020   1055   static u16 cellSizePtr(MemPage *pPage, u8 *pCell){
  1021         -  u8 *pIter = &pCell[pPage->childPtrSize];
  1022         -  u32 nSize;
         1056  +  u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */
         1057  +  u8 *pEnd;                                /* End mark for a varint */
         1058  +  u32 nSize;                               /* Size value to return */
  1023   1059   
  1024   1060   #ifdef SQLITE_DEBUG
  1025   1061     /* The value returned by this function should always be the same as
  1026   1062     ** the (CellInfo.nSize) value found by doing a full parse of the
  1027   1063     ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of
  1028   1064     ** this function verifies that this invariant is not violated. */
  1029   1065     CellInfo debuginfo;
  1030   1066     btreeParseCellPtr(pPage, pCell, &debuginfo);
  1031   1067   #endif
  1032   1068   
         1069  +  if( pPage->noPayload ){
         1070  +    pEnd = &pIter[9];
         1071  +    while( (*pIter++)&0x80 && pIter<pEnd );
         1072  +    assert( pPage->childPtrSize==4 );
         1073  +    return (u16)(pIter - pCell);
         1074  +  }
         1075  +  nSize = *pIter;
         1076  +  if( nSize>=0x80 ){
         1077  +    pEnd = &pIter[9];
         1078  +    nSize &= 0x7f;
         1079  +    do{
         1080  +      nSize = (nSize<<7) | (*++pIter & 0x7f);
         1081  +    }while( *(pIter)>=0x80 && pIter<pEnd );
         1082  +  }
         1083  +  pIter++;
  1033   1084     if( pPage->intKey ){
  1034         -    u8 *pEnd;
  1035         -    if( pPage->hasData ){
  1036         -      pIter += getVarint32(pIter, nSize);
  1037         -    }else{
  1038         -      nSize = 0;
  1039         -    }
  1040         -
  1041   1085       /* pIter now points at the 64-bit integer key value, a variable length 
  1042   1086       ** integer. The following block moves pIter to point at the first byte
  1043   1087       ** past the end of the key value. */
  1044   1088       pEnd = &pIter[9];
  1045   1089       while( (*pIter++)&0x80 && pIter<pEnd );
  1046         -  }else{
  1047         -    pIter += getVarint32(pIter, nSize);
  1048   1090     }
  1049         -
  1050   1091     testcase( nSize==pPage->maxLocal );
  1051   1092     testcase( nSize==pPage->maxLocal+1 );
  1052         -  if( nSize>pPage->maxLocal ){
         1093  +  if( nSize<=pPage->maxLocal ){
         1094  +    nSize += (u32)(pIter - pCell);
         1095  +    if( nSize<4 ) nSize = 4;
         1096  +  }else{
  1053   1097       int minLocal = pPage->minLocal;
  1054   1098       nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4);
  1055   1099       testcase( nSize==pPage->maxLocal );
  1056   1100       testcase( nSize==pPage->maxLocal+1 );
  1057   1101       if( nSize>pPage->maxLocal ){
  1058   1102         nSize = minLocal;
  1059   1103       }
  1060         -    nSize += 4;
         1104  +    nSize += 4 + (u16)(pIter - pCell);
  1061   1105     }
  1062         -  nSize += (u32)(pIter - pCell);
  1063         -
  1064         -  /* The minimum size of any cell is 4 bytes. */
  1065         -  if( nSize<4 ){
  1066         -    nSize = 4;
  1067         -  }
  1068         -
  1069         -  assert( nSize==debuginfo.nSize );
         1106  +  assert( nSize==debuginfo.nSize || CORRUPT_DB );
  1070   1107     return (u16)nSize;
  1071   1108   }
  1072   1109   
  1073   1110   #ifdef SQLITE_DEBUG
  1074   1111   /* This variation on cellSizePtr() is used inside of assert() statements
  1075   1112   ** only. */
  1076   1113   static u16 cellSize(MemPage *pPage, int iCell){
................................................................................
  1085   1122   ** for the overflow page.
  1086   1123   */
  1087   1124   static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){
  1088   1125     CellInfo info;
  1089   1126     if( *pRC ) return;
  1090   1127     assert( pCell!=0 );
  1091   1128     btreeParseCellPtr(pPage, pCell, &info);
  1092         -  assert( (info.nData+(pPage->intKey?0:info.nKey))==info.nPayload );
  1093   1129     if( info.iOverflow ){
  1094   1130       Pgno ovfl = get4byte(&pCell[info.iOverflow]);
  1095   1131       ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC);
  1096   1132     }
  1097   1133   }
  1098   1134   #endif
  1099   1135   
  1100   1136   
  1101   1137   /*
  1102   1138   ** Defragment the page given.  All Cells are moved to the
  1103   1139   ** end of the page and all free space is collected into one
  1104   1140   ** big FreeBlk that occurs in between the header and cell
  1105   1141   ** pointer array and the cell content area.
         1142  +**
         1143  +** EVIDENCE-OF: R-44582-60138 SQLite may from time to time reorganize a
         1144  +** b-tree page so that there are no freeblocks or fragment bytes, all
         1145  +** unused bytes are contained in the unallocated space region, and all
         1146  +** cells are packed tightly at the end of the page.
  1106   1147   */
  1107   1148   static int defragmentPage(MemPage *pPage){
  1108   1149     int i;                     /* Loop counter */
  1109         -  int pc;                    /* Address of a i-th cell */
         1150  +  int pc;                    /* Address of the i-th cell */
  1110   1151     int hdr;                   /* Offset to the page header */
  1111   1152     int size;                  /* Size of a cell */
  1112   1153     int usableSize;            /* Number of usable bytes on a page */
  1113   1154     int cellOffset;            /* Offset to the cell pointer array */
  1114   1155     int cbrk;                  /* Offset to the cell content area */
  1115   1156     int nCell;                 /* Number of cells on the page */
  1116   1157     unsigned char *data;       /* The page data */
  1117   1158     unsigned char *temp;       /* Temp area for cell content */
         1159  +  unsigned char *src;        /* Source of content */
  1118   1160     int iCellFirst;            /* First allowable cell index */
  1119   1161     int iCellLast;             /* Last possible cell index */
  1120   1162   
  1121   1163   
  1122   1164     assert( sqlite3PagerIswriteable(pPage->pDbPage) );
  1123   1165     assert( pPage->pBt!=0 );
  1124   1166     assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE );
  1125   1167     assert( pPage->nOverflow==0 );
  1126   1168     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  1127         -  temp = sqlite3PagerTempSpace(pPage->pBt->pPager);
  1128         -  data = pPage->aData;
         1169  +  temp = 0;
         1170  +  src = data = pPage->aData;
  1129   1171     hdr = pPage->hdrOffset;
  1130   1172     cellOffset = pPage->cellOffset;
  1131   1173     nCell = pPage->nCell;
  1132   1174     assert( nCell==get2byte(&data[hdr+3]) );
  1133   1175     usableSize = pPage->pBt->usableSize;
  1134         -  cbrk = get2byte(&data[hdr+5]);
  1135         -  memcpy(&temp[cbrk], &data[cbrk], usableSize - cbrk);
  1136   1176     cbrk = usableSize;
  1137   1177     iCellFirst = cellOffset + 2*nCell;
  1138   1178     iCellLast = usableSize - 4;
  1139   1179     for(i=0; i<nCell; i++){
  1140   1180       u8 *pAddr;     /* The i-th cell pointer */
  1141   1181       pAddr = &data[cellOffset + i*2];
  1142   1182       pc = get2byte(pAddr);
................................................................................
  1147   1187       ** if SQLITE_ENABLE_OVERSIZE_CELL_CHECK is defined 
  1148   1188       */
  1149   1189       if( pc<iCellFirst || pc>iCellLast ){
  1150   1190         return SQLITE_CORRUPT_BKPT;
  1151   1191       }
  1152   1192   #endif
  1153   1193       assert( pc>=iCellFirst && pc<=iCellLast );
  1154         -    size = cellSizePtr(pPage, &temp[pc]);
         1194  +    size = cellSizePtr(pPage, &src[pc]);
  1155   1195       cbrk -= size;
  1156   1196   #if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK)
  1157   1197       if( cbrk<iCellFirst ){
  1158   1198         return SQLITE_CORRUPT_BKPT;
  1159   1199       }
  1160   1200   #else
  1161   1201       if( cbrk<iCellFirst || pc+size>usableSize ){
  1162   1202         return SQLITE_CORRUPT_BKPT;
  1163   1203       }
  1164   1204   #endif
  1165   1205       assert( cbrk+size<=usableSize && cbrk>=iCellFirst );
  1166   1206       testcase( cbrk+size==usableSize );
  1167   1207       testcase( pc+size==usableSize );
  1168         -    memcpy(&data[cbrk], &temp[pc], size);
  1169   1208       put2byte(pAddr, cbrk);
         1209  +    if( temp==0 ){
         1210  +      int x;
         1211  +      if( cbrk==pc ) continue;
         1212  +      temp = sqlite3PagerTempSpace(pPage->pBt->pPager);
         1213  +      x = get2byte(&data[hdr+5]);
         1214  +      memcpy(&temp[x], &data[x], (cbrk+size) - x);
         1215  +      src = temp;
         1216  +    }
         1217  +    memcpy(&data[cbrk], &src[pc], size);
  1170   1218     }
  1171   1219     assert( cbrk>=iCellFirst );
  1172   1220     put2byte(&data[hdr+5], cbrk);
  1173   1221     data[hdr+1] = 0;
  1174   1222     data[hdr+2] = 0;
  1175   1223     data[hdr+7] = 0;
  1176   1224     memset(&data[iCellFirst], 0, cbrk-iCellFirst);
  1177   1225     assert( sqlite3PagerIswriteable(pPage->pDbPage) );
  1178   1226     if( cbrk-iCellFirst!=pPage->nFree ){
  1179   1227       return SQLITE_CORRUPT_BKPT;
  1180   1228     }
  1181   1229     return SQLITE_OK;
  1182   1230   }
         1231  +
         1232  +/*
         1233  +** Search the free-list on page pPg for space to store a cell nByte bytes in
         1234  +** size. If one can be found, return a pointer to the space and remove it
         1235  +** from the free-list.
         1236  +**
         1237  +** If no suitable space can be found on the free-list, return NULL.
         1238  +**
         1239  +** This function may detect corruption within pPg.  If corruption is
         1240  +** detected then *pRc is set to SQLITE_CORRUPT and NULL is returned.
         1241  +**
         1242  +** If a slot of at least nByte bytes is found but cannot be used because 
         1243  +** there are already at least 60 fragmented bytes on the page, return NULL.
         1244  +** In this case, if pbDefrag parameter is not NULL, set *pbDefrag to true.
         1245  +*/
         1246  +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){
         1247  +  const int hdr = pPg->hdrOffset;
         1248  +  u8 * const aData = pPg->aData;
         1249  +  int iAddr;
         1250  +  int pc;
         1251  +  int usableSize = pPg->pBt->usableSize;
         1252  +
         1253  +  for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){
         1254  +    int size;            /* Size of the free slot */
         1255  +    /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of
         1256  +    ** increasing offset. */
         1257  +    if( pc>usableSize-4 || pc<iAddr+4 ){
         1258  +      *pRc = SQLITE_CORRUPT_BKPT;
         1259  +      return 0;
         1260  +    }
         1261  +    /* EVIDENCE-OF: R-22710-53328 The third and fourth bytes of each
         1262  +    ** freeblock form a big-endian integer which is the size of the freeblock
         1263  +    ** in bytes, including the 4-byte header. */
         1264  +    size = get2byte(&aData[pc+2]);
         1265  +    if( size>=nByte ){
         1266  +      int x = size - nByte;
         1267  +      testcase( x==4 );
         1268  +      testcase( x==3 );
         1269  +      if( x<4 ){
         1270  +        /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total
         1271  +        ** number of bytes in fragments may not exceed 60. */
         1272  +        if( aData[hdr+7]>=60 ){
         1273  +          if( pbDefrag ) *pbDefrag = 1;
         1274  +          return 0;
         1275  +        }
         1276  +        /* Remove the slot from the free-list. Update the number of
         1277  +        ** fragmented bytes within the page. */
         1278  +        memcpy(&aData[iAddr], &aData[pc], 2);
         1279  +        aData[hdr+7] += (u8)x;
         1280  +      }else if( size+pc > usableSize ){
         1281  +        *pRc = SQLITE_CORRUPT_BKPT;
         1282  +        return 0;
         1283  +      }else{
         1284  +        /* The slot remains on the free-list. Reduce its size to account
         1285  +         ** for the portion used by the new allocation. */
         1286  +        put2byte(&aData[pc+2], x);
         1287  +      }
         1288  +      return &aData[pc + x];
         1289  +    }
         1290  +  }
         1291  +
         1292  +  return 0;
         1293  +}
  1183   1294   
  1184   1295   /*
  1185   1296   ** Allocate nByte bytes of space from within the B-Tree page passed
  1186   1297   ** as the first argument. Write into *pIdx the index into pPage->aData[]
  1187   1298   ** of the first byte of allocated space. Return either SQLITE_OK or
  1188   1299   ** an error code (usually SQLITE_CORRUPT).
  1189   1300   **
................................................................................
  1193   1304   ** the first two bytes past the cell pointer area since presumably this
  1194   1305   ** allocation is being made in order to insert a new cell, so we will
  1195   1306   ** also end up needing a new cell pointer.
  1196   1307   */
  1197   1308   static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){
  1198   1309     const int hdr = pPage->hdrOffset;    /* Local cache of pPage->hdrOffset */
  1199   1310     u8 * const data = pPage->aData;      /* Local cache of pPage->aData */
  1200         -  int nFrag;                           /* Number of fragmented bytes on pPage */
  1201   1311     int top;                             /* First byte of cell content area */
         1312  +  int rc = SQLITE_OK;                  /* Integer return code */
  1202   1313     int gap;        /* First byte of gap between cell pointers and cell content */
  1203         -  int rc;         /* Integer return code */
  1204         -  int usableSize; /* Usable size of the page */
  1205   1314     
  1206   1315     assert( sqlite3PagerIswriteable(pPage->pDbPage) );
  1207   1316     assert( pPage->pBt );
  1208   1317     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  1209   1318     assert( nByte>=0 );  /* Minimum cell size is 4 */
  1210   1319     assert( pPage->nFree>=nByte );
  1211   1320     assert( pPage->nOverflow==0 );
  1212         -  usableSize = pPage->pBt->usableSize;
  1213         -  assert( nByte < usableSize-8 );
         1321  +  assert( nByte < (int)(pPage->pBt->usableSize-8) );
  1214   1322   
  1215         -  nFrag = data[hdr+7];
  1216   1323     assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf );
  1217   1324     gap = pPage->cellOffset + 2*pPage->nCell;
         1325  +  assert( gap<=65536 );
         1326  +  /* EVIDENCE-OF: R-29356-02391 If the database uses a 65536-byte page size
         1327  +  ** and the reserved space is zero (the usual value for reserved space)
         1328  +  ** then the cell content offset of an empty page wants to be 65536.
         1329  +  ** However, that integer is too large to be stored in a 2-byte unsigned
         1330  +  ** integer, so a value of 0 is used in its place. */
  1218   1331     top = get2byteNotZero(&data[hdr+5]);
  1219   1332     if( gap>top ) return SQLITE_CORRUPT_BKPT;
         1333  +
         1334  +  /* If there is enough space between gap and top for one more cell pointer
         1335  +  ** array entry offset, and if the freelist is not empty, then search the
         1336  +  ** freelist looking for a free slot big enough to satisfy the request.
         1337  +  */
  1220   1338     testcase( gap+2==top );
  1221   1339     testcase( gap+1==top );
  1222   1340     testcase( gap==top );
  1223         -
  1224         -  if( nFrag>=60 ){
  1225         -    /* Always defragment highly fragmented pages */
  1226         -    rc = defragmentPage(pPage);
         1341  +  if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){
         1342  +    int bDefrag = 0;
         1343  +    u8 *pSpace = pageFindSlot(pPage, nByte, &rc, &bDefrag);
  1227   1344       if( rc ) return rc;
  1228         -    top = get2byteNotZero(&data[hdr+5]);
  1229         -  }else if( gap+2<=top ){
  1230         -    /* Search the freelist looking for a free slot big enough to satisfy 
  1231         -    ** the request. The allocation is made from the first free slot in 
  1232         -    ** the list that is large enough to accommodate it.
  1233         -    */
  1234         -    int pc, addr;
  1235         -    for(addr=hdr+1; (pc = get2byte(&data[addr]))>0; addr=pc){
  1236         -      int size;            /* Size of the free slot */
  1237         -      if( pc>usableSize-4 || pc<addr+4 ){
  1238         -        return SQLITE_CORRUPT_BKPT;
  1239         -      }
  1240         -      size = get2byte(&data[pc+2]);
  1241         -      if( size>=nByte ){
  1242         -        int x = size - nByte;
  1243         -        testcase( x==4 );
  1244         -        testcase( x==3 );
  1245         -        if( x<4 ){
  1246         -          /* Remove the slot from the free-list. Update the number of
  1247         -          ** fragmented bytes within the page. */
  1248         -          memcpy(&data[addr], &data[pc], 2);
  1249         -          data[hdr+7] = (u8)(nFrag + x);
  1250         -        }else if( size+pc > usableSize ){
  1251         -          return SQLITE_CORRUPT_BKPT;
  1252         -        }else{
  1253         -          /* The slot remains on the free-list. Reduce its size to account
  1254         -          ** for the portion used by the new allocation. */
  1255         -          put2byte(&data[pc+2], x);
  1256         -        }
  1257         -        *pIdx = pc + x;
  1258         -        return SQLITE_OK;
  1259         -      }
         1345  +    if( bDefrag ) goto defragment_page;
         1346  +    if( pSpace ){
         1347  +      assert( pSpace>=data && (pSpace - data)<65536 );
         1348  +      *pIdx = (int)(pSpace - data);
         1349  +      return SQLITE_OK;
  1260   1350       }
  1261   1351     }
  1262   1352   
  1263         -  /* Check to make sure there is enough space in the gap to satisfy
  1264         -  ** the allocation.  If not, defragment.
         1353  +  /* The request could not be fulfilled using a freelist slot.  Check
         1354  +  ** to see if defragmentation is necessary.
  1265   1355     */
  1266   1356     testcase( gap+2+nByte==top );
  1267   1357     if( gap+2+nByte>top ){
         1358  + defragment_page:
         1359  +    assert( pPage->nCell>0 || CORRUPT_DB );
  1268   1360       rc = defragmentPage(pPage);
  1269   1361       if( rc ) return rc;
  1270   1362       top = get2byteNotZero(&data[hdr+5]);
  1271   1363       assert( gap+nByte<=top );
  1272   1364     }
  1273   1365   
  1274   1366   
................................................................................
  1283   1375     assert( top+nByte <= (int)pPage->pBt->usableSize );
  1284   1376     *pIdx = top;
  1285   1377     return SQLITE_OK;
  1286   1378   }
  1287   1379   
  1288   1380   /*
  1289   1381   ** Return a section of the pPage->aData to the freelist.
  1290         -** The first byte of the new free block is pPage->aDisk[start]
  1291         -** and the size of the block is "size" bytes.
         1382  +** The first byte of the new free block is pPage->aData[iStart]
         1383  +** and the size of the block is iSize bytes.
  1292   1384   **
  1293         -** Most of the effort here is involved in coalesing adjacent
  1294         -** free blocks into a single big free block.
         1385  +** Adjacent freeblocks are coalesced.
         1386  +**
         1387  +** Note that even though the freeblock list was checked by btreeInitPage(),
         1388  +** that routine will not detect overlap between cells or freeblocks.  Nor
         1389  +** does it detect cells or freeblocks that encrouch into the reserved bytes
         1390  +** at the end of the page.  So do additional corruption checks inside this
         1391  +** routine and return SQLITE_CORRUPT if any problems are found.
  1295   1392   */
  1296         -static int freeSpace(MemPage *pPage, int start, int size){
  1297         -  int addr, pbegin, hdr;
  1298         -  int iLast;                        /* Largest possible freeblock offset */
  1299         -  unsigned char *data = pPage->aData;
         1393  +static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
         1394  +  u16 iPtr;                             /* Address of ptr to next freeblock */
         1395  +  u16 iFreeBlk;                         /* Address of the next freeblock */
         1396  +  u8 hdr;                               /* Page header size.  0 or 100 */
         1397  +  u8 nFrag = 0;                         /* Reduction in fragmentation */
         1398  +  u16 iOrigSize = iSize;                /* Original value of iSize */
         1399  +  u32 iLast = pPage->pBt->usableSize-4; /* Largest possible freeblock offset */
         1400  +  u32 iEnd = iStart + iSize;            /* First byte past the iStart buffer */
         1401  +  unsigned char *data = pPage->aData;   /* Page content */
  1300   1402   
  1301   1403     assert( pPage->pBt!=0 );
  1302   1404     assert( sqlite3PagerIswriteable(pPage->pDbPage) );
  1303         -  assert( start>=pPage->hdrOffset+6+pPage->childPtrSize );
  1304         -  assert( (start + size) <= (int)pPage->pBt->usableSize );
         1405  +  assert( iStart>=pPage->hdrOffset+6+pPage->childPtrSize );
         1406  +  assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize );
  1305   1407     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  1306         -  assert( size>=0 );   /* Minimum cell size is 4 */
         1408  +  assert( iSize>=4 );   /* Minimum cell size is 4 */
         1409  +  assert( iStart<=iLast );
  1307   1410   
         1411  +  /* Overwrite deleted information with zeros when the secure_delete
         1412  +  ** option is enabled */
  1308   1413     if( pPage->pBt->btsFlags & BTS_SECURE_DELETE ){
  1309         -    /* Overwrite deleted information with zeros when the secure_delete
  1310         -    ** option is enabled */
  1311         -    memset(&data[start], 0, size);
         1414  +    memset(&data[iStart], 0, iSize);
  1312   1415     }
  1313   1416   
  1314         -  /* Add the space back into the linked list of freeblocks.  Note that
  1315         -  ** even though the freeblock list was checked by btreeInitPage(),
  1316         -  ** btreeInitPage() did not detect overlapping cells or
  1317         -  ** freeblocks that overlapped cells.   Nor does it detect when the
  1318         -  ** cell content area exceeds the value in the page header.  If these
  1319         -  ** situations arise, then subsequent insert operations might corrupt
  1320         -  ** the freelist.  So we do need to check for corruption while scanning
  1321         -  ** the freelist.
         1417  +  /* The list of freeblocks must be in ascending order.  Find the 
         1418  +  ** spot on the list where iStart should be inserted.
  1322   1419     */
  1323   1420     hdr = pPage->hdrOffset;
  1324         -  addr = hdr + 1;
  1325         -  iLast = pPage->pBt->usableSize - 4;
  1326         -  assert( start<=iLast );
  1327         -  while( (pbegin = get2byte(&data[addr]))<start && pbegin>0 ){
  1328         -    if( pbegin<addr+4 ){
  1329         -      return SQLITE_CORRUPT_BKPT;
  1330         -    }
  1331         -    addr = pbegin;
  1332         -  }
  1333         -  if( pbegin>iLast ){
  1334         -    return SQLITE_CORRUPT_BKPT;
  1335         -  }
  1336         -  assert( pbegin>addr || pbegin==0 );
  1337         -  put2byte(&data[addr], start);
  1338         -  put2byte(&data[start], pbegin);
  1339         -  put2byte(&data[start+2], size);
  1340         -  pPage->nFree = pPage->nFree + (u16)size;
  1341         -
  1342         -  /* Coalesce adjacent free blocks */
  1343         -  addr = hdr + 1;
  1344         -  while( (pbegin = get2byte(&data[addr]))>0 ){
  1345         -    int pnext, psize, x;
  1346         -    assert( pbegin>addr );
  1347         -    assert( pbegin <= (int)pPage->pBt->usableSize-4 );
  1348         -    pnext = get2byte(&data[pbegin]);
  1349         -    psize = get2byte(&data[pbegin+2]);
  1350         -    if( pbegin + psize + 3 >= pnext && pnext>0 ){
  1351         -      int frag = pnext - (pbegin+psize);
  1352         -      if( (frag<0) || (frag>(int)data[hdr+7]) ){
  1353         -        return SQLITE_CORRUPT_BKPT;
  1354         -      }
  1355         -      data[hdr+7] -= (u8)frag;
  1356         -      x = get2byte(&data[pnext]);
  1357         -      put2byte(&data[pbegin], x);
  1358         -      x = pnext + get2byte(&data[pnext+2]) - pbegin;
  1359         -      put2byte(&data[pbegin+2], x);
  1360         -    }else{
  1361         -      addr = pbegin;
  1362         -    }
  1363         -  }
  1364         -
  1365         -  /* If the cell content area begins with a freeblock, remove it. */
  1366         -  if( data[hdr+1]==data[hdr+5] && data[hdr+2]==data[hdr+6] ){
  1367         -    int top;
  1368         -    pbegin = get2byte(&data[hdr+1]);
  1369         -    memcpy(&data[hdr+1], &data[pbegin], 2);
  1370         -    top = get2byte(&data[hdr+5]) + get2byte(&data[pbegin+2]);
  1371         -    put2byte(&data[hdr+5], top);
  1372         -  }
  1373         -  assert( sqlite3PagerIswriteable(pPage->pDbPage) );
         1421  +  iPtr = hdr + 1;
         1422  +  if( data[iPtr+1]==0 && data[iPtr]==0 ){
         1423  +    iFreeBlk = 0;  /* Shortcut for the case when the freelist is empty */
         1424  +  }else{
         1425  +    while( (iFreeBlk = get2byte(&data[iPtr]))>0 && iFreeBlk<iStart ){
         1426  +      if( iFreeBlk<iPtr+4 ) return SQLITE_CORRUPT_BKPT;
         1427  +      iPtr = iFreeBlk;
         1428  +    }
         1429  +    if( iFreeBlk>iLast ) return SQLITE_CORRUPT_BKPT;
         1430  +    assert( iFreeBlk>iPtr || iFreeBlk==0 );
         1431  +  
         1432  +    /* At this point:
         1433  +    **    iFreeBlk:   First freeblock after iStart, or zero if none
         1434  +    **    iPtr:       The address of a pointer iFreeBlk
         1435  +    **
         1436  +    ** Check to see if iFreeBlk should be coalesced onto the end of iStart.
         1437  +    */
         1438  +    if( iFreeBlk && iEnd+3>=iFreeBlk ){
         1439  +      nFrag = iFreeBlk - iEnd;
         1440  +      if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_BKPT;
         1441  +      iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]);
         1442  +      iSize = iEnd - iStart;
         1443  +      iFreeBlk = get2byte(&data[iFreeBlk]);
         1444  +    }
         1445  +  
         1446  +    /* If iPtr is another freeblock (that is, if iPtr is not the freelist
         1447  +    ** pointer in the page header) then check to see if iStart should be
         1448  +    ** coalesced onto the end of iPtr.
         1449  +    */
         1450  +    if( iPtr>hdr+1 ){
         1451  +      int iPtrEnd = iPtr + get2byte(&data[iPtr+2]);
         1452  +      if( iPtrEnd+3>=iStart ){
         1453  +        if( iPtrEnd>iStart ) return SQLITE_CORRUPT_BKPT;
         1454  +        nFrag += iStart - iPtrEnd;
         1455  +        iSize = iEnd - iPtr;
         1456  +        iStart = iPtr;
         1457  +      }
         1458  +    }
         1459  +    if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_BKPT;
         1460  +    data[hdr+7] -= nFrag;
         1461  +  }
         1462  +  if( iStart==get2byte(&data[hdr+5]) ){
         1463  +    /* The new freeblock is at the beginning of the cell content area,
         1464  +    ** so just extend the cell content area rather than create another
         1465  +    ** freelist entry */
         1466  +    if( iPtr!=hdr+1 ) return SQLITE_CORRUPT_BKPT;
         1467  +    put2byte(&data[hdr+1], iFreeBlk);
         1468  +    put2byte(&data[hdr+5], iEnd);
         1469  +  }else{
         1470  +    /* Insert the new freeblock into the freelist */
         1471  +    put2byte(&data[iPtr], iStart);
         1472  +    put2byte(&data[iStart], iFreeBlk);
         1473  +    put2byte(&data[iStart+2], iSize);
         1474  +  }
         1475  +  pPage->nFree += iOrigSize;
  1374   1476     return SQLITE_OK;
  1375   1477   }
  1376   1478   
  1377   1479   /*
  1378   1480   ** Decode the flags byte (the first byte of the header) for a page
  1379   1481   ** and initialize fields of the MemPage structure accordingly.
  1380   1482   **
................................................................................
  1392   1494     assert( pPage->hdrOffset==(pPage->pgno==1 ? 100 : 0) );
  1393   1495     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  1394   1496     pPage->leaf = (u8)(flagByte>>3);  assert( PTF_LEAF == 1<<3 );
  1395   1497     flagByte &= ~PTF_LEAF;
  1396   1498     pPage->childPtrSize = 4-4*pPage->leaf;
  1397   1499     pBt = pPage->pBt;
  1398   1500     if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){
         1501  +    /* EVIDENCE-OF: R-03640-13415 A value of 5 means the page is an interior
         1502  +    ** table b-tree page. */
         1503  +    assert( (PTF_LEAFDATA|PTF_INTKEY)==5 );
         1504  +    /* EVIDENCE-OF: R-20501-61796 A value of 13 means the page is a leaf
         1505  +    ** table b-tree page. */
         1506  +    assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 );
  1399   1507       pPage->intKey = 1;
  1400         -    pPage->hasData = pPage->leaf;
         1508  +    pPage->intKeyLeaf = pPage->leaf;
         1509  +    pPage->noPayload = !pPage->leaf;
  1401   1510       pPage->maxLocal = pBt->maxLeaf;
  1402   1511       pPage->minLocal = pBt->minLeaf;
  1403   1512     }else if( flagByte==PTF_ZERODATA ){
         1513  +    /* EVIDENCE-OF: R-27225-53936 A value of 2 means the page is an interior
         1514  +    ** index b-tree page. */
         1515  +    assert( (PTF_ZERODATA)==2 );
         1516  +    /* EVIDENCE-OF: R-16571-11615 A value of 10 means the page is a leaf
         1517  +    ** index b-tree page. */
         1518  +    assert( (PTF_ZERODATA|PTF_LEAF)==10 );
  1404   1519       pPage->intKey = 0;
  1405         -    pPage->hasData = 0;
         1520  +    pPage->intKeyLeaf = 0;
         1521  +    pPage->noPayload = 0;
  1406   1522       pPage->maxLocal = pBt->maxLocal;
  1407   1523       pPage->minLocal = pBt->minLocal;
  1408   1524     }else{
         1525  +    /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is
         1526  +    ** an error. */
  1409   1527       return SQLITE_CORRUPT_BKPT;
  1410   1528     }
  1411   1529     pPage->max1bytePayload = pBt->max1bytePayload;
  1412   1530     return SQLITE_OK;
  1413   1531   }
  1414   1532   
  1415   1533   /*
................................................................................
  1441   1559       int iCellFirst;    /* First allowable cell or freeblock offset */
  1442   1560       int iCellLast;     /* Last possible cell or freeblock offset */
  1443   1561   
  1444   1562       pBt = pPage->pBt;
  1445   1563   
  1446   1564       hdr = pPage->hdrOffset;
  1447   1565       data = pPage->aData;
         1566  +    /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating
         1567  +    ** the b-tree page type. */
  1448   1568       if( decodeFlags(pPage, data[hdr]) ) return SQLITE_CORRUPT_BKPT;
  1449   1569       assert( pBt->pageSize>=512 && pBt->pageSize<=65536 );
  1450   1570       pPage->maskPage = (u16)(pBt->pageSize - 1);
  1451   1571       pPage->nOverflow = 0;
  1452   1572       usableSize = pBt->usableSize;
  1453         -    pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf;
         1573  +    pPage->cellOffset = cellOffset = hdr + 8 + pPage->childPtrSize;
  1454   1574       pPage->aDataEnd = &data[usableSize];
  1455   1575       pPage->aCellIdx = &data[cellOffset];
         1576  +    /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates
         1577  +    ** the start of the cell content area. A zero value for this integer is
         1578  +    ** interpreted as 65536. */
  1456   1579       top = get2byteNotZero(&data[hdr+5]);
         1580  +    /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the
         1581  +    ** number of cells on the page. */
  1457   1582       pPage->nCell = get2byte(&data[hdr+3]);
  1458   1583       if( pPage->nCell>MX_CELL(pBt) ){
  1459   1584         /* To many cells for a single page.  The page must be corrupt */
  1460   1585         return SQLITE_CORRUPT_BKPT;
  1461   1586       }
  1462   1587       testcase( pPage->nCell==MX_CELL(pBt) );
         1588  +    /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only
         1589  +    ** possible for a root page of a table that contains no rows) then the
         1590  +    ** offset to the cell content area will equal the page size minus the
         1591  +    ** bytes of reserved space. */
         1592  +    assert( pPage->nCell>0 || top==usableSize || CORRUPT_DB );
  1463   1593   
  1464   1594       /* A malformed database page might cause us to read past the end
  1465   1595       ** of page when parsing a cell.  
  1466   1596       **
  1467   1597       ** The following block of code checks early to see if a cell extends
  1468   1598       ** past the end of a page boundary and causes SQLITE_CORRUPT to be 
  1469   1599       ** returned if it does.
................................................................................
  1489   1619             return SQLITE_CORRUPT_BKPT;
  1490   1620           }
  1491   1621         }
  1492   1622         if( !pPage->leaf ) iCellLast++;
  1493   1623       }  
  1494   1624   #endif
  1495   1625   
  1496         -    /* Compute the total free space on the page */
         1626  +    /* Compute the total free space on the page
         1627  +    ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the
         1628  +    ** start of the first freeblock on the page, or is zero if there are no
         1629  +    ** freeblocks. */
  1497   1630       pc = get2byte(&data[hdr+1]);
  1498         -    nFree = data[hdr+7] + top;
         1631  +    nFree = data[hdr+7] + top;  /* Init nFree to non-freeblock free space */
  1499   1632       while( pc>0 ){
  1500   1633         u16 next, size;
  1501   1634         if( pc<iCellFirst || pc>iCellLast ){
  1502         -        /* Start of free block is off the page */
         1635  +        /* EVIDENCE-OF: R-55530-52930 In a well-formed b-tree page, there will
         1636  +        ** always be at least one cell before the first freeblock.
         1637  +        **
         1638  +        ** Or, the freeblock is off the end of the page
         1639  +        */
  1503   1640           return SQLITE_CORRUPT_BKPT; 
  1504   1641         }
  1505   1642         next = get2byte(&data[pc]);
  1506   1643         size = get2byte(&data[pc+2]);
  1507   1644         if( (next>0 && next<=pc+size+3) || pc+size>usableSize ){
  1508   1645           /* Free blocks must be in ascending order. And the last byte of
  1509   1646           ** the free-block must lie on the database page.  */
................................................................................
  1628   1765   */
  1629   1766   static Pgno btreePagecount(BtShared *pBt){
  1630   1767     return pBt->nPage;
  1631   1768   }
  1632   1769   u32 sqlite3BtreeLastPage(Btree *p){
  1633   1770     assert( sqlite3BtreeHoldsMutex(p) );
  1634   1771     assert( ((p->pBt->nPage)&0x8000000)==0 );
  1635         -  return (int)btreePagecount(p->pBt);
         1772  +  return btreePagecount(p->pBt);
  1636   1773   }
  1637   1774   
  1638   1775   /*
  1639   1776   ** Get a page from the pager and initialize it.  This routine is just a
  1640   1777   ** convenience wrapper around separate calls to btreeGetPage() and 
  1641   1778   ** btreeInitPage().
  1642   1779   **
................................................................................
  1901   2038     
  1902   2039       pBt->pCursor = 0;
  1903   2040       pBt->pPage1 = 0;
  1904   2041       if( sqlite3PagerIsreadonly(pBt->pPager) ) pBt->btsFlags |= BTS_READ_ONLY;
  1905   2042   #ifdef SQLITE_SECURE_DELETE
  1906   2043       pBt->btsFlags |= BTS_SECURE_DELETE;
  1907   2044   #endif
         2045  +    /* EVIDENCE-OF: R-51873-39618 The page size for a database file is
         2046  +    ** determined by the 2-byte integer located at an offset of 16 bytes from
         2047  +    ** the beginning of the database file. */
  1908   2048       pBt->pageSize = (zDbHeader[16]<<8) | (zDbHeader[17]<<16);
  1909   2049       if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE
  1910   2050            || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){
  1911   2051         pBt->pageSize = 0;
  1912   2052   #ifndef SQLITE_OMIT_AUTOVACUUM
  1913   2053         /* If the magic name ":memory:" will create an in-memory database, then
  1914   2054         ** leave the autoVacuum mode at 0 (do not auto-vacuum), even if
................................................................................
  1919   2059         if( zFilename && !isMemdb ){
  1920   2060           pBt->autoVacuum = (SQLITE_DEFAULT_AUTOVACUUM ? 1 : 0);
  1921   2061           pBt->incrVacuum = (SQLITE_DEFAULT_AUTOVACUUM==2 ? 1 : 0);
  1922   2062         }
  1923   2063   #endif
  1924   2064         nReserve = 0;
  1925   2065       }else{
         2066  +      /* EVIDENCE-OF: R-37497-42412 The size of the reserved region is
         2067  +      ** determined by the one-byte unsigned integer found at an offset of 20
         2068  +      ** into the database file header. */
  1926   2069         nReserve = zDbHeader[20];
  1927   2070         pBt->btsFlags |= BTS_PAGESIZE_FIXED;
  1928   2071   #ifndef SQLITE_OMIT_AUTOVACUUM
  1929   2072         pBt->autoVacuum = (get4byte(&zDbHeader[36 + 4*4])?1:0);
  1930   2073         pBt->incrVacuum = (get4byte(&zDbHeader[36 + 7*4])?1:0);
  1931   2074   #endif
  1932   2075       }
................................................................................
  2053   2196   #else
  2054   2197     return 1;
  2055   2198   #endif
  2056   2199   }
  2057   2200   
  2058   2201   /*
  2059   2202   ** Make sure pBt->pTmpSpace points to an allocation of 
  2060         -** MX_CELL_SIZE(pBt) bytes.
         2203  +** MX_CELL_SIZE(pBt) bytes with a 4-byte prefix for a left-child
         2204  +** pointer.
  2061   2205   */
  2062   2206   static void allocateTempSpace(BtShared *pBt){
  2063   2207     if( !pBt->pTmpSpace ){
  2064   2208       pBt->pTmpSpace = sqlite3PageMalloc( pBt->pageSize );
  2065   2209   
  2066   2210       /* One of the uses of pBt->pTmpSpace is to format cells before
  2067   2211       ** inserting them into a leaf page (function fillInCell()). If
................................................................................
  2068   2212       ** a cell is less than 4 bytes in size, it is rounded up to 4 bytes
  2069   2213       ** by the various routines that manipulate binary cells. Which
  2070   2214       ** can mean that fillInCell() only initializes the first 2 or 3
  2071   2215       ** bytes of pTmpSpace, but that the first 4 bytes are copied from
  2072   2216       ** it into a database page. This is not actually a problem, but it
  2073   2217       ** does cause a valgrind error when the 1 or 2 bytes of unitialized 
  2074   2218       ** data is passed to system call write(). So to avoid this error,
  2075         -    ** zero the first 4 bytes of temp space here.  */
  2076         -    if( pBt->pTmpSpace ) memset(pBt->pTmpSpace, 0, 4);
         2219  +    ** zero the first 4 bytes of temp space here.
         2220  +    **
         2221  +    ** Also:  Provide four bytes of initialized space before the
         2222  +    ** beginning of pTmpSpace as an area available to prepend the
         2223  +    ** left-child pointer to the beginning of a cell.
         2224  +    */
         2225  +    if( pBt->pTmpSpace ){
         2226  +      memset(pBt->pTmpSpace, 0, 8);
         2227  +      pBt->pTmpSpace += 4;
         2228  +    }
  2077   2229     }
  2078   2230   }
  2079   2231   
  2080   2232   /*
  2081   2233   ** Free the pBt->pTmpSpace allocation
  2082   2234   */
  2083   2235   static void freeTempSpace(BtShared *pBt){
  2084         -  sqlite3PageFree( pBt->pTmpSpace);
  2085         -  pBt->pTmpSpace = 0;
         2236  +  if( pBt->pTmpSpace ){
         2237  +    pBt->pTmpSpace -= 4;
         2238  +    sqlite3PageFree(pBt->pTmpSpace);
         2239  +    pBt->pTmpSpace = 0;
         2240  +  }
  2086   2241   }
  2087   2242   
  2088   2243   /*
  2089   2244   ** Close an open database and invalidate all cursors.
  2090   2245   */
  2091   2246   int sqlite3BtreeClose(Btree *p){
  2092   2247     BtShared *pBt = p->pBt;
................................................................................
  2104   2259       }
  2105   2260     }
  2106   2261   
  2107   2262     /* Rollback any active transaction and free the handle structure.
  2108   2263     ** The call to sqlite3BtreeRollback() drops any table-locks held by
  2109   2264     ** this handle.
  2110   2265     */
  2111         -  sqlite3BtreeRollback(p, SQLITE_OK);
         2266  +  sqlite3BtreeRollback(p, SQLITE_OK, 0);
  2112   2267     sqlite3BtreeLeave(p);
  2113   2268   
  2114   2269     /* If there are still other outstanding references to the shared-btree
  2115   2270     ** structure, return now. The remainder of this procedure cleans 
  2116   2271     ** up the shared-btree.
  2117   2272     */
  2118   2273     assert( p->wantToLock==0 && p->locked==0 );
................................................................................
  2416   2571       nPage = nPageFile;
  2417   2572     }
  2418   2573     if( nPage>0 ){
  2419   2574       u32 pageSize;
  2420   2575       u32 usableSize;
  2421   2576       u8 *page1 = pPage1->aData;
  2422   2577       rc = SQLITE_NOTADB;
         2578  +    /* EVIDENCE-OF: R-43737-39999 Every valid SQLite database file begins
         2579  +    ** with the following 16 bytes (in hex): 53 51 4c 69 74 65 20 66 6f 72 6d
         2580  +    ** 61 74 20 33 00. */
  2423   2581       if( memcmp(page1, zMagicHeader, 16)!=0 ){
  2424   2582         goto page1_init_failed;
  2425   2583       }
  2426   2584   
  2427   2585   #ifdef SQLITE_OMIT_WAL
  2428   2586       if( page1[18]>1 ){
  2429   2587         pBt->btsFlags |= BTS_READ_ONLY;
................................................................................
  2456   2614           releasePage(pPage1);
  2457   2615           return SQLITE_OK;
  2458   2616         }
  2459   2617         rc = SQLITE_NOTADB;
  2460   2618       }
  2461   2619   #endif
  2462   2620   
  2463         -    /* The maximum embedded fraction must be exactly 25%.  And the minimum
  2464         -    ** embedded fraction must be 12.5% for both leaf-data and non-leaf-data.
         2621  +    /* EVIDENCE-OF: R-15465-20813 The maximum and minimum embedded payload
         2622  +    ** fractions and the leaf payload fraction values must be 64, 32, and 32.
         2623  +    **
  2465   2624       ** The original design allowed these amounts to vary, but as of
  2466   2625       ** version 3.6.0, we require them to be fixed.
  2467   2626       */
  2468   2627       if( memcmp(&page1[21], "\100\040\040",3)!=0 ){
  2469   2628         goto page1_init_failed;
  2470   2629       }
         2630  +    /* EVIDENCE-OF: R-51873-39618 The page size for a database file is
         2631  +    ** determined by the 2-byte integer located at an offset of 16 bytes from
         2632  +    ** the beginning of the database file. */
  2471   2633       pageSize = (page1[16]<<8) | (page1[17]<<16);
         2634  +    /* EVIDENCE-OF: R-25008-21688 The size of a page is a power of two
         2635  +    ** between 512 and 65536 inclusive. */
  2472   2636       if( ((pageSize-1)&pageSize)!=0
  2473   2637        || pageSize>SQLITE_MAX_PAGE_SIZE 
  2474   2638        || pageSize<=256 
  2475   2639       ){
  2476   2640         goto page1_init_failed;
  2477   2641       }
  2478   2642       assert( (pageSize & 7)==0 );
         2643  +    /* EVIDENCE-OF: R-59310-51205 The "reserved space" size in the 1-byte
         2644  +    ** integer at offset 20 is the number of bytes of space at the end of
         2645  +    ** each page to reserve for extensions. 
         2646  +    **
         2647  +    ** EVIDENCE-OF: R-37497-42412 The size of the reserved region is
         2648  +    ** determined by the one-byte unsigned integer found at an offset of 20
         2649  +    ** into the database file header. */
  2479   2650       usableSize = pageSize - page1[20];
  2480   2651       if( (u32)pageSize!=pBt->pageSize ){
  2481   2652         /* After reading the first page of the database assuming a page size
  2482   2653         ** of BtShared.pageSize, we have discovered that the page-size is
  2483   2654         ** actually pageSize. Unlock the database, leave pBt->pPage1 at
  2484   2655         ** zero and return SQLITE_OK. The caller will call this function
  2485   2656         ** again with the correct page-size.
................................................................................
  2492   2663                                      pageSize-usableSize);
  2493   2664         return rc;
  2494   2665       }
  2495   2666       if( (pBt->db->flags & SQLITE_RecoveryMode)==0 && nPage>nPageFile ){
  2496   2667         rc = SQLITE_CORRUPT_BKPT;
  2497   2668         goto page1_init_failed;
  2498   2669       }
         2670  +    /* EVIDENCE-OF: R-28312-64704 However, the usable size is not allowed to
         2671  +    ** be less than 480. In other words, if the page size is 512, then the
         2672  +    ** reserved space size cannot exceed 32. */
  2499   2673       if( usableSize<480 ){
  2500   2674         goto page1_init_failed;
  2501   2675       }
  2502   2676       pBt->pageSize = pageSize;
  2503   2677       pBt->usableSize = usableSize;
  2504   2678   #ifndef SQLITE_OMIT_AUTOVACUUM
  2505   2679       pBt->autoVacuum = (get4byte(&page1[36 + 4*4])?1:0);
................................................................................
  2546   2720   ** in assert() expressions, so it is only compiled if NDEBUG is not
  2547   2721   ** defined.
  2548   2722   **
  2549   2723   ** Only write cursors are counted if wrOnly is true.  If wrOnly is
  2550   2724   ** false then all cursors are counted.
  2551   2725   **
  2552   2726   ** For the purposes of this routine, a cursor is any cursor that
  2553         -** is capable of reading or writing to the databse.  Cursors that
         2727  +** is capable of reading or writing to the database.  Cursors that
  2554   2728   ** have been tripped into the CURSOR_FAULT state are not counted.
  2555   2729   */
  2556   2730   static int countValidCursors(BtShared *pBt, int wrOnly){
  2557   2731     BtCursor *pCur;
  2558   2732     int r = 0;
  2559   2733     for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){
  2560   2734       if( (wrOnly==0 || (pCur->curFlags & BTCF_WriteFlag)!=0)
................................................................................
  2572   2746   **
  2573   2747   ** If there is a transaction in progress, this routine is a no-op.
  2574   2748   */
  2575   2749   static void unlockBtreeIfUnused(BtShared *pBt){
  2576   2750     assert( sqlite3_mutex_held(pBt->mutex) );
  2577   2751     assert( countValidCursors(pBt,0)==0 || pBt->inTransaction>TRANS_NONE );
  2578   2752     if( pBt->inTransaction==TRANS_NONE && pBt->pPage1!=0 ){
  2579         -    assert( pBt->pPage1->aData );
         2753  +    MemPage *pPage1 = pBt->pPage1;
         2754  +    assert( pPage1->aData );
  2580   2755       assert( sqlite3PagerRefcount(pBt->pPager)==1 );
  2581         -    assert( pBt->pPage1->aData );
  2582         -    releasePage(pBt->pPage1);
  2583   2756       pBt->pPage1 = 0;
         2757  +    releasePage(pPage1);
  2584   2758     }
  2585   2759   }
  2586   2760   
  2587   2761   /*
  2588   2762   ** If pBt points to an empty file then convert that empty file
  2589   2763   ** into a new empty database by initializing the first page of
  2590   2764   ** the database.
................................................................................
  3010   3184   
  3011   3185   /*
  3012   3186   ** Perform a single step of an incremental-vacuum. If successful, return
  3013   3187   ** SQLITE_OK. If there is no work to do (and therefore no point in 
  3014   3188   ** calling this function again), return SQLITE_DONE. Or, if an error 
  3015   3189   ** occurs, return some other error code.
  3016   3190   **
  3017         -** More specificly, this function attempts to re-organize the database so 
         3191  +** More specifically, this function attempts to re-organize the database so 
  3018   3192   ** that the last page of the file currently in use is no longer in use.
  3019   3193   **
  3020   3194   ** Parameter nFin is the number of pages that this database would contain
  3021   3195   ** were this function called until it returns SQLITE_DONE.
  3022   3196   **
  3023   3197   ** If the bCommit parameter is non-zero, this function assumes that the 
  3024   3198   ** caller will keep calling incrVacuumStep() until it returns SQLITE_DONE 
  3025         -** or an error. bCommit is passed true for an auto-vacuum-on-commmit 
         3199  +** or an error. bCommit is passed true for an auto-vacuum-on-commit 
  3026   3200   ** operation, or false for an incremental vacuum.
  3027   3201   */
  3028   3202   static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){
  3029   3203     Pgno nFreeList;           /* Number of pages still on the free-list */
  3030   3204     int rc;
  3031   3205   
  3032   3206     assert( sqlite3_mutex_held(pBt->mutex) );
................................................................................
  3372   3546       assert( pBt->inTransaction==TRANS_WRITE );
  3373   3547       assert( pBt->nTransaction>0 );
  3374   3548       rc = sqlite3PagerCommitPhaseTwo(pBt->pPager);
  3375   3549       if( rc!=SQLITE_OK && bCleanup==0 ){
  3376   3550         sqlite3BtreeLeave(p);
  3377   3551         return rc;
  3378   3552       }
         3553  +    p->iDataVersion--;  /* Compensate for pPager->iDataVersion++; */
  3379   3554       pBt->inTransaction = TRANS_READ;
  3380   3555       btreeClearHasContent(pBt);
  3381   3556     }
  3382   3557   
  3383   3558     btreeEndTransaction(p);
  3384   3559     sqlite3BtreeLeave(p);
  3385   3560     return SQLITE_OK;
................................................................................
  3397   3572     }
  3398   3573     sqlite3BtreeLeave(p);
  3399   3574     return rc;
  3400   3575   }
  3401   3576   
  3402   3577   /*
  3403   3578   ** This routine sets the state to CURSOR_FAULT and the error
  3404         -** code to errCode for every cursor on BtShared that pBtree
  3405         -** references.
  3406         -**
  3407         -** Every cursor is tripped, including cursors that belong
  3408         -** to other database connections that happen to be sharing
  3409         -** the cache with pBtree.
  3410         -**
  3411         -** This routine gets called when a rollback occurs.
  3412         -** All cursors using the same cache must be tripped
  3413         -** to prevent them from trying to use the btree after
  3414         -** the rollback.  The rollback may have deleted tables
  3415         -** or moved root pages, so it is not sufficient to
  3416         -** save the state of the cursor.  The cursor must be
  3417         -** invalidated.
  3418         -*/
  3419         -void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){
         3579  +** code to errCode for every cursor on any BtShared that pBtree
         3580  +** references.  Or if the writeOnly flag is set to 1, then only
         3581  +** trip write cursors and leave read cursors unchanged.
         3582  +**
         3583  +** Every cursor is a candidate to be tripped, including cursors
         3584  +** that belong to other database connections that happen to be
         3585  +** sharing the cache with pBtree.
         3586  +**
         3587  +** This routine gets called when a rollback occurs. If the writeOnly
         3588  +** flag is true, then only write-cursors need be tripped - read-only
         3589  +** cursors save their current positions so that they may continue 
         3590  +** following the rollback. Or, if writeOnly is false, all cursors are 
         3591  +** tripped. In general, writeOnly is false if the transaction being
         3592  +** rolled back modified the database schema. In this case b-tree root
         3593  +** pages may be moved or deleted from the database altogether, making
         3594  +** it unsafe for read cursors to continue.
         3595  +**
         3596  +** If the writeOnly flag is true and an error is encountered while 
         3597  +** saving the current position of a read-only cursor, all cursors, 
         3598  +** including all read-cursors are tripped.
         3599  +**
         3600  +** SQLITE_OK is returned if successful, or if an error occurs while
         3601  +** saving a cursor position, an SQLite error code.
         3602  +*/
         3603  +int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){
  3420   3604     BtCursor *p;
  3421         -  if( pBtree==0 ) return;
  3422         -  sqlite3BtreeEnter(pBtree);
  3423         -  for(p=pBtree->pBt->pCursor; p; p=p->pNext){
  3424         -    int i;
  3425         -    sqlite3BtreeClearCursor(p);
  3426         -    p->eState = CURSOR_FAULT;
  3427         -    p->skipNext = errCode;
  3428         -    for(i=0; i<=p->iPage; i++){
  3429         -      releasePage(p->apPage[i]);
  3430         -      p->apPage[i] = 0;
  3431         -    }
  3432         -  }
  3433         -  sqlite3BtreeLeave(pBtree);
  3434         -}
  3435         -
  3436         -/*
  3437         -** Rollback the transaction in progress.  All cursors will be
  3438         -** invalided by this operation.  Any attempt to use a cursor
  3439         -** that was open at the beginning of this operation will result
  3440         -** in an error.
         3605  +  int rc = SQLITE_OK;
         3606  +
         3607  +  assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 );
         3608  +  if( pBtree ){
         3609  +    sqlite3BtreeEnter(pBtree);
         3610  +    for(p=pBtree->pBt->pCursor; p; p=p->pNext){
         3611  +      int i;
         3612  +      if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){
         3613  +        if( p->eState==CURSOR_VALID ){
         3614  +          rc = saveCursorPosition(p);
         3615  +          if( rc!=SQLITE_OK ){
         3616  +            (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0);
         3617  +            break;
         3618  +          }
         3619  +        }
         3620  +      }else{
         3621  +        sqlite3BtreeClearCursor(p);
         3622  +        p->eState = CURSOR_FAULT;
         3623  +        p->skipNext = errCode;
         3624  +      }
         3625  +      for(i=0; i<=p->iPage; i++){
         3626  +        releasePage(p->apPage[i]);
         3627  +        p->apPage[i] = 0;
         3628  +      }
         3629  +    }
         3630  +    sqlite3BtreeLeave(pBtree);
         3631  +  }
         3632  +  return rc;
         3633  +}
         3634  +
         3635  +/*
         3636  +** Rollback the transaction in progress.
         3637  +**
         3638  +** If tripCode is not SQLITE_OK then cursors will be invalidated (tripped).
         3639  +** Only write cursors are tripped if writeOnly is true but all cursors are
         3640  +** tripped if writeOnly is false.  Any attempt to use
         3641  +** a tripped cursor will result in an error.
  3441   3642   **
  3442   3643   ** This will release the write lock on the database file.  If there
  3443   3644   ** are no active cursors, it also releases the read lock.
  3444   3645   */
  3445         -int sqlite3BtreeRollback(Btree *p, int tripCode){
         3646  +int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){
  3446   3647     int rc;
  3447   3648     BtShared *pBt = p->pBt;
  3448   3649     MemPage *pPage1;
  3449   3650   
         3651  +  assert( writeOnly==1 || writeOnly==0 );
         3652  +  assert( tripCode==SQLITE_ABORT_ROLLBACK || tripCode==SQLITE_OK );
  3450   3653     sqlite3BtreeEnter(p);
  3451   3654     if( tripCode==SQLITE_OK ){
  3452   3655       rc = tripCode = saveAllCursors(pBt, 0, 0);
         3656  +    if( rc ) writeOnly = 0;
  3453   3657     }else{
  3454   3658       rc = SQLITE_OK;
  3455   3659     }
  3456   3660     if( tripCode ){
  3457         -    sqlite3BtreeTripAllCursors(p, tripCode);
         3661  +    int rc2 = sqlite3BtreeTripAllCursors(p, tripCode, writeOnly);
         3662  +    assert( rc==SQLITE_OK || (writeOnly==0 && rc2==SQLITE_OK) );
         3663  +    if( rc2!=SQLITE_OK ) rc = rc2;
  3458   3664     }
  3459   3665     btreeIntegrity(p);
  3460   3666   
  3461   3667     if( p->inTrans==TRANS_WRITE ){
  3462   3668       int rc2;
  3463   3669   
  3464   3670       assert( TRANS_WRITE==pBt->inTransaction );
................................................................................
  3485   3691   
  3486   3692     btreeEndTransaction(p);
  3487   3693     sqlite3BtreeLeave(p);
  3488   3694     return rc;
  3489   3695   }
  3490   3696   
  3491   3697   /*
  3492         -** Start a statement subtransaction. The subtransaction can can be rolled
         3698  +** Start a statement subtransaction. The subtransaction can be rolled
  3493   3699   ** back independently of the main transaction. You must start a transaction 
  3494   3700   ** before starting a subtransaction. The subtransaction is ended automatically 
  3495   3701   ** if the main transaction commits or rolls back.
  3496   3702   **
  3497   3703   ** Statement subtransactions are used around individual SQL statements
  3498   3704   ** that are contained within a BEGIN...COMMIT block.  If a constraint
  3499   3705   ** error occurs within the statement, the effect of that one statement
................................................................................
  3617   3823     assert( p->inTrans>TRANS_NONE );
  3618   3824     assert( wrFlag==0 || p->inTrans==TRANS_WRITE );
  3619   3825     assert( pBt->pPage1 && pBt->pPage1->aData );
  3620   3826   
  3621   3827     if( NEVER(wrFlag && (pBt->btsFlags & BTS_READ_ONLY)!=0) ){
  3622   3828       return SQLITE_READONLY;
  3623   3829     }
         3830  +  if( wrFlag ){
         3831  +    allocateTempSpace(pBt);
         3832  +    if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM;
         3833  +  }
  3624   3834     if( iTable==1 && btreePagecount(pBt)==0 ){
  3625   3835       assert( wrFlag==0 );
  3626   3836       iTable = 0;
  3627   3837     }
  3628   3838   
  3629   3839     /* Now that no other errors can occur, finish filling in the BtCursor
  3630   3840     ** variables and link the cursor into the BtShared list.  */
................................................................................
  3700   3910       if( pCur->pNext ){
  3701   3911         pCur->pNext->pPrev = pCur->pPrev;
  3702   3912       }
  3703   3913       for(i=0; i<=pCur->iPage; i++){
  3704   3914         releasePage(pCur->apPage[i]);
  3705   3915       }
  3706   3916       unlockBtreeIfUnused(pBt);
  3707         -    sqlite3DbFree(pBtree->db, pCur->aOverflow);
         3917  +    sqlite3_free(pCur->aOverflow);
  3708   3918       /* sqlite3_free(pCur); */
  3709   3919       sqlite3BtreeLeave(pBtree);
  3710   3920     }
  3711   3921     return SQLITE_OK;
  3712   3922   }
  3713   3923   
  3714   3924   /*
................................................................................
  3719   3929   ** BtCursor.info is a cache of the information in the current cell.
  3720   3930   ** Using this cache reduces the number of calls to btreeParseCell().
  3721   3931   **
  3722   3932   ** 2007-06-25:  There is a bug in some versions of MSVC that cause the
  3723   3933   ** compiler to crash when getCellInfo() is implemented as a macro.
  3724   3934   ** But there is a measureable speed advantage to using the macro on gcc
  3725   3935   ** (when less compiler optimizations like -Os or -O0 are used and the
  3726         -** compiler is not doing agressive inlining.)  So we use a real function
         3936  +** compiler is not doing aggressive inlining.)  So we use a real function
  3727   3937   ** for MSVC and a macro for everything else.  Ticket #2457.
  3728   3938   */
  3729   3939   #ifndef NDEBUG
  3730   3940     static void assertCellInfo(BtCursor *pCur){
  3731   3941       CellInfo info;
  3732   3942       int iPage = pCur->iPage;
  3733   3943       memset(&info, 0, sizeof(info));
................................................................................
  3781   3991   **
  3782   3992   ** The caller must position the cursor prior to invoking this routine.
  3783   3993   ** 
  3784   3994   ** This routine cannot fail.  It always returns SQLITE_OK.  
  3785   3995   */
  3786   3996   int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){
  3787   3997     assert( cursorHoldsMutex(pCur) );
  3788         -  assert( pCur->eState==CURSOR_INVALID || pCur->eState==CURSOR_VALID );
  3789         -  if( pCur->eState!=CURSOR_VALID ){
  3790         -    *pSize = 0;
  3791         -  }else{
  3792         -    getCellInfo(pCur);
  3793         -    *pSize = pCur->info.nKey;
  3794         -  }
         3998  +  assert( pCur->eState==CURSOR_VALID );
         3999  +  getCellInfo(pCur);
         4000  +  *pSize = pCur->info.nKey;
  3795   4001     return SQLITE_OK;
  3796   4002   }
  3797   4003   
  3798   4004   /*
  3799   4005   ** Set *pSize to the number of bytes of data in the entry the
  3800   4006   ** cursor currently points to.
  3801   4007   **
................................................................................
  3806   4012   ** Failure is not possible.  This function always returns SQLITE_OK.
  3807   4013   ** It might just as well be a procedure (returning void) but we continue
  3808   4014   ** to return an integer result code for historical reasons.
  3809   4015   */
  3810   4016   int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){
  3811   4017     assert( cursorHoldsMutex(pCur) );
  3812   4018     assert( pCur->eState==CURSOR_VALID );
         4019  +  assert( pCur->apPage[pCur->iPage]->intKeyLeaf==1 );
  3813   4020     getCellInfo(pCur);
  3814         -  *pSize = pCur->info.nData;
         4021  +  *pSize = pCur->info.nPayload;
  3815   4022     return SQLITE_OK;
  3816   4023   }
  3817   4024   
  3818   4025   /*
  3819   4026   ** Given the page number of an overflow page in the database (parameter
  3820   4027   ** ovfl), this function finds the page number of the next page in the 
  3821   4028   ** linked list of overflow pages. If possible, it uses the auto-vacuum
................................................................................
  3936   4143   ** Data is read to or from the buffer pBuf.
  3937   4144   **
  3938   4145   ** The content being read or written might appear on the main page
  3939   4146   ** or be scattered out on multiple overflow pages.
  3940   4147   **
  3941   4148   ** If the current cursor entry uses one or more overflow pages and the
  3942   4149   ** eOp argument is not 2, this function may allocate space for and lazily 
  3943         -** popluates the overflow page-list cache array (BtCursor.aOverflow). 
         4150  +** populates the overflow page-list cache array (BtCursor.aOverflow). 
  3944   4151   ** Subsequent calls use this cache to make seeking to the supplied offset 
  3945   4152   ** more efficient.
  3946   4153   **
  3947   4154   ** Once an overflow page-list cache has been allocated, it may be
  3948   4155   ** invalidated if some other cursor writes to the same table, or if
  3949   4156   ** the cursor is moved to a different row. Additionally, in auto-vacuum
  3950   4157   ** mode, the following events may invalidate an overflow page-list cache.
................................................................................
  3958   4165     u32 offset,          /* Begin reading this far into payload */
  3959   4166     u32 amt,             /* Read this many bytes */
  3960   4167     unsigned char *pBuf, /* Write the bytes into this buffer */ 
  3961   4168     int eOp              /* zero to read. non-zero to write. */
  3962   4169   ){
  3963   4170     unsigned char *aPayload;
  3964   4171     int rc = SQLITE_OK;
  3965         -  u32 nKey;
  3966   4172     int iIdx = 0;
  3967   4173     MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */
  3968   4174     BtShared *pBt = pCur->pBt;                  /* Btree this cursor belongs to */
  3969   4175   #ifdef SQLITE_DIRECT_OVERFLOW_READ
  3970         -  int bEnd;                                   /* True if reading to end of data */
         4176  +  unsigned char * const pBufStart = pBuf;
         4177  +  int bEnd;                                 /* True if reading to end of data */
  3971   4178   #endif
  3972   4179   
  3973   4180     assert( pPage );
  3974   4181     assert( pCur->eState==CURSOR_VALID );
  3975   4182     assert( pCur->aiIdx[pCur->iPage]<pPage->nCell );
  3976   4183     assert( cursorHoldsMutex(pCur) );
  3977         -  assert( eOp!=2 || offset==0 );      /* Always start from beginning for eOp==2 */
         4184  +  assert( eOp!=2 || offset==0 );    /* Always start from beginning for eOp==2 */
  3978   4185   
  3979   4186     getCellInfo(pCur);
  3980         -  aPayload = pCur->info.pCell + pCur->info.nHeader;
  3981         -  nKey = (pPage->intKey ? 0 : (int)pCur->info.nKey);
         4187  +  aPayload = pCur->info.pPayload;
  3982   4188   #ifdef SQLITE_DIRECT_OVERFLOW_READ
  3983         -  bEnd = (offset+amt==nKey+pCur->info.nData);
         4189  +  bEnd = offset+amt==pCur->info.nPayload;
  3984   4190   #endif
         4191  +  assert( offset+amt <= pCur->info.nPayload );
  3985   4192   
  3986         -  if( NEVER(offset+amt > nKey+pCur->info.nData) 
  3987         -   || &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize]
  3988         -  ){
         4193  +  if( &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] ){
  3989   4194       /* Trying to read or write past the end of the data is an error */
  3990   4195       return SQLITE_CORRUPT_BKPT;
  3991   4196     }
  3992   4197   
  3993   4198     /* Check if data must be read/written to/from the btree page itself. */
  3994   4199     if( offset<pCur->info.nLocal ){
  3995   4200       int a = amt;
................................................................................
  3999   4204       rc = copyPayload(&aPayload[offset], pBuf, a, (eOp & 0x01), pPage->pDbPage);
  4000   4205       offset = 0;
  4001   4206       pBuf += a;
  4002   4207       amt -= a;
  4003   4208     }else{
  4004   4209       offset -= pCur->info.nLocal;
  4005   4210     }
         4211  +
  4006   4212   
  4007   4213     if( rc==SQLITE_OK && amt>0 ){
  4008   4214       const u32 ovflSize = pBt->usableSize - 4;  /* Bytes content per ovfl page */
  4009   4215       Pgno nextPage;
  4010   4216   
  4011   4217       nextPage = get4byte(&aPayload[pCur->info.nLocal]);
  4012   4218   
................................................................................
  4017   4223       ** in the overflow chain. The page number of the first overflow page is
  4018   4224       ** stored in aOverflow[0], etc. A value of 0 in the aOverflow[] array
  4019   4225       ** means "not yet known" (the cache is lazily populated).
  4020   4226       */
  4021   4227       if( eOp!=2 && (pCur->curFlags & BTCF_ValidOvfl)==0 ){
  4022   4228         int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize;
  4023   4229         if( nOvfl>pCur->nOvflAlloc ){
  4024         -        Pgno *aNew = (Pgno*)sqlite3DbRealloc(
  4025         -            pCur->pBtree->db, pCur->aOverflow, nOvfl*2*sizeof(Pgno)
         4230  +        Pgno *aNew = (Pgno*)sqlite3Realloc(
         4231  +            pCur->aOverflow, nOvfl*2*sizeof(Pgno)
  4026   4232           );
  4027   4233           if( aNew==0 ){
  4028   4234             rc = SQLITE_NOMEM;
  4029   4235           }else{
  4030   4236             pCur->nOvflAlloc = nOvfl*2;
  4031   4237             pCur->aOverflow = aNew;
  4032   4238           }
................................................................................
  4037   4243         }
  4038   4244       }
  4039   4245   
  4040   4246       /* If the overflow page-list cache has been allocated and the
  4041   4247       ** entry for the first required overflow page is valid, skip
  4042   4248       ** directly to it.
  4043   4249       */
  4044         -    if( (pCur->curFlags & BTCF_ValidOvfl)!=0 && pCur->aOverflow[offset/ovflSize] ){
         4250  +    if( (pCur->curFlags & BTCF_ValidOvfl)!=0
         4251  +     && pCur->aOverflow[offset/ovflSize]
         4252  +    ){
  4045   4253         iIdx = (offset/ovflSize);
  4046   4254         nextPage = pCur->aOverflow[iIdx];
  4047   4255         offset = (offset%ovflSize);
  4048   4256       }
  4049   4257   
  4050   4258       for( ; rc==SQLITE_OK && amt>0 && nextPage; iIdx++){
  4051   4259   
................................................................................
  4063   4271           ** function.
  4064   4272           **
  4065   4273           ** Note that the aOverflow[] array must be allocated because eOp!=2
  4066   4274           ** here.  If eOp==2, then offset==0 and this branch is never taken.
  4067   4275           */
  4068   4276           assert( eOp!=2 );
  4069   4277           assert( pCur->curFlags & BTCF_ValidOvfl );
         4278  +        assert( pCur->pBtree->db==pBt->db );
  4070   4279           if( pCur->aOverflow[iIdx+1] ){
  4071   4280             nextPage = pCur->aOverflow[iIdx+1];
  4072   4281           }else{
  4073   4282             rc = getOverflowPage(pBt, nextPage, 0, &nextPage);
  4074   4283           }
  4075   4284           offset -= ovflSize;
  4076   4285         }else{
................................................................................
  4090   4299           **
  4091   4300           **   1) this is a read operation, and 
  4092   4301           **   2) data is required from the start of this overflow page, and
  4093   4302           **   3) the database is file-backed, and
  4094   4303           **   4) there is no open write-transaction, and
  4095   4304           **   5) the database is not a WAL database,
  4096   4305           **   6) all data from the page is being read.
         4306  +        **   7) at least 4 bytes have already been read into the output buffer 
  4097   4307           **
  4098   4308           ** then data can be read directly from the database file into the
  4099   4309           ** output buffer, bypassing the page-cache altogether. This speeds
  4100   4310           ** up loading large records that span many overflow pages.
  4101   4311           */
  4102   4312           if( (eOp&0x01)==0                                      /* (1) */
  4103   4313            && offset==0                                          /* (2) */
  4104   4314            && (bEnd || a==ovflSize)                              /* (6) */
  4105   4315            && pBt->inTransaction==TRANS_READ                     /* (4) */
  4106   4316            && (fd = sqlite3PagerFile(pBt->pPager))->pMethods     /* (3) */
  4107   4317            && pBt->pPage1->aData[19]==0x01                       /* (5) */
         4318  +         && &pBuf[-4]>=pBufStart                               /* (7) */
  4108   4319           ){
  4109   4320             u8 aSave[4];
  4110   4321             u8 *aWrite = &pBuf[-4];
         4322  +          assert( aWrite>=pBufStart );                         /* hence (7) */
  4111   4323             memcpy(aSave, aWrite, 4);
  4112   4324             rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1));
  4113   4325             nextPage = get4byte(aWrite);
  4114   4326             memcpy(aWrite, aSave, 4);
  4115   4327           }else
  4116   4328   #endif
  4117   4329   
................................................................................
  4138   4350       return SQLITE_CORRUPT_BKPT;
  4139   4351     }
  4140   4352     return rc;
  4141   4353   }
  4142   4354   
  4143   4355   /*
  4144   4356   ** Read part of the key associated with cursor pCur.  Exactly
  4145         -** "amt" bytes will be transfered into pBuf[].  The transfer
         4357  +** "amt" bytes will be transferred into pBuf[].  The transfer
  4146   4358   ** begins at "offset".
  4147   4359   **
  4148   4360   ** The caller must ensure that pCur is pointing to a valid row
  4149   4361   ** in the table.
  4150   4362   **
  4151   4363   ** Return SQLITE_OK on success or an error code if anything goes
  4152   4364   ** wrong.  An error is returned if "offset+amt" is larger than
................................................................................
  4215   4427     assert( pCur!=0 && pCur->iPage>=0 && pCur->apPage[pCur->iPage]);
  4216   4428     assert( pCur->eState==CURSOR_VALID );
  4217   4429     assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) );
  4218   4430     assert( cursorHoldsMutex(pCur) );
  4219   4431     assert( pCur->aiIdx[pCur->iPage]<pCur->apPage[pCur->iPage]->nCell );
  4220   4432     assert( pCur->info.nSize>0 );
  4221   4433     *pAmt = pCur->info.nLocal;
  4222         -  return (void*)(pCur->info.pCell + pCur->info.nHeader);
         4434  +  return (void*)pCur->info.pPayload;
  4223   4435   }
  4224   4436   
  4225   4437   
  4226   4438   /*
  4227   4439   ** For the entry that cursor pCur is point to, return as
  4228   4440   ** many bytes of the key or data as are available on the local
  4229   4441   ** b-tree page.  Write the number of available bytes into *pAmt.
................................................................................
  4458   4670   static int moveToRightmost(BtCursor *pCur){
  4459   4671     Pgno pgno;
  4460   4672     int rc = SQLITE_OK;
  4461   4673     MemPage *pPage = 0;
  4462   4674   
  4463   4675     assert( cursorHoldsMutex(pCur) );
  4464   4676     assert( pCur->eState==CURSOR_VALID );
  4465         -  while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){
         4677  +  while( !(pPage = pCur->apPage[pCur->iPage])->leaf ){
  4466   4678       pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]);
  4467   4679       pCur->aiIdx[pCur->iPage] = pPage->nCell;
  4468   4680       rc = moveToChild(pCur, pgno);
         4681  +    if( rc ) return rc;
  4469   4682     }
  4470         -  if( rc==SQLITE_OK ){
  4471         -    pCur->aiIdx[pCur->iPage] = pPage->nCell-1;
  4472         -    pCur->info.nSize = 0;
  4473         -    pCur->curFlags &= ~BTCF_ValidNKey;
  4474         -  }
  4475         -  return rc;
         4683  +  pCur->aiIdx[pCur->iPage] = pPage->nCell-1;
         4684  +  assert( pCur->info.nSize==0 );
         4685  +  assert( (pCur->curFlags & BTCF_ValidNKey)==0 );
         4686  +  return SQLITE_OK;
  4476   4687   }
  4477   4688   
  4478   4689   /* Move the cursor to the first entry in the table.  Return SQLITE_OK
  4479   4690   ** on success.  Set *pRes to 0 if the cursor actually points to something
  4480   4691   ** or set *pRes to 1 if the table is empty.
  4481   4692   */
  4482   4693   int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){
................................................................................
  4599   4810         *pRes = -1;
  4600   4811         return SQLITE_OK;
  4601   4812       }
  4602   4813     }
  4603   4814   
  4604   4815     if( pIdxKey ){
  4605   4816       xRecordCompare = sqlite3VdbeFindCompare(pIdxKey);
  4606         -    pIdxKey->isCorrupt = 0;
         4817  +    pIdxKey->errCode = 0;
  4607   4818       assert( pIdxKey->default_rc==1 
  4608   4819            || pIdxKey->default_rc==0 
  4609   4820            || pIdxKey->default_rc==-1
  4610   4821       );
  4611   4822     }else{
  4612   4823       xRecordCompare = 0; /* All keys are integers */
  4613   4824     }
................................................................................
  4644   4855       assert( biasRight==0 || biasRight==1 );
  4645   4856       idx = upr>>(1-biasRight); /* idx = biasRight ? upr : (lwr+upr)/2; */
  4646   4857       pCur->aiIdx[pCur->iPage] = (u16)idx;
  4647   4858       if( xRecordCompare==0 ){
  4648   4859         for(;;){
  4649   4860           i64 nCellKey;
  4650   4861           pCell = findCell(pPage, idx) + pPage->childPtrSize;
  4651         -        if( pPage->hasData ){
         4862  +        if( pPage->intKeyLeaf ){
  4652   4863             while( 0x80 <= *(pCell++) ){
  4653   4864               if( pCell>=pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT;
  4654   4865             }
  4655   4866           }
  4656   4867           getVarint(pCell, (u64*)&nCellKey);
  4657   4868           if( nCellKey<intKey ){
  4658   4869             lwr = idx+1;
................................................................................
  4692   4903           */
  4693   4904           nCell = pCell[0];
  4694   4905           if( nCell<=pPage->max1bytePayload ){
  4695   4906             /* This branch runs if the record-size field of the cell is a
  4696   4907             ** single byte varint and the record fits entirely on the main
  4697   4908             ** b-tree page.  */
  4698   4909             testcase( pCell+nCell+1==pPage->aDataEnd );
  4699         -          c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey, 0);
         4910  +          c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey);
  4700   4911           }else if( !(pCell[1] & 0x80) 
  4701   4912             && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal
  4702   4913           ){
  4703   4914             /* The record-size field is a 2 byte varint and the record 
  4704   4915             ** fits entirely on the main b-tree page.  */
  4705   4916             testcase( pCell+nCell+2==pPage->aDataEnd );
  4706         -          c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey, 0);
         4917  +          c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey);
  4707   4918           }else{
  4708   4919             /* The record flows over onto one or more overflow pages. In
  4709   4920             ** this case the whole cell needs to be parsed, a buffer allocated
  4710   4921             ** and accessPayload() used to retrieve the record into the
  4711   4922             ** buffer before VdbeRecordCompare() can be called. */
  4712   4923             void *pCellKey;
  4713   4924             u8 * const pCellBody = pCell - pPage->childPtrSize;
................................................................................
  4720   4931             }
  4721   4932             pCur->aiIdx[pCur->iPage] = (u16)idx;
  4722   4933             rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 2);
  4723   4934             if( rc ){
  4724   4935               sqlite3_free(pCellKey);
  4725   4936               goto moveto_finish;
  4726   4937             }
  4727         -          c = xRecordCompare(nCell, pCellKey, pIdxKey, 0);
         4938  +          c = xRecordCompare(nCell, pCellKey, pIdxKey);
  4728   4939             sqlite3_free(pCellKey);
  4729   4940           }
  4730         -        assert( pIdxKey->isCorrupt==0 || c==0 );
         4941  +        assert( 
         4942  +            (pIdxKey->errCode!=SQLITE_CORRUPT || c==0)
         4943  +         && (pIdxKey->errCode!=SQLITE_NOMEM || pCur->pBtree->db->mallocFailed)
         4944  +        );
  4731   4945           if( c<0 ){
  4732   4946             lwr = idx+1;
  4733   4947           }else if( c>0 ){
  4734   4948             upr = idx-1;
  4735   4949           }else{
  4736   4950             assert( c==0 );
  4737   4951             *pRes = 0;
  4738   4952             rc = SQLITE_OK;
  4739   4953             pCur->aiIdx[pCur->iPage] = (u16)idx;
  4740         -          if( pIdxKey->isCorrupt ) rc = SQLITE_CORRUPT;
         4954  +          if( pIdxKey->errCode ) rc = SQLITE_CORRUPT;
  4741   4955             goto moveto_finish;
  4742   4956           }
  4743   4957           if( lwr>upr ) break;
  4744   4958           assert( lwr+upr>=0 );
  4745   4959           idx = (lwr+upr)>>1;  /* idx = (lwr+upr)/2 */
  4746   4960         }
  4747   4961       }
................................................................................
  4787   5001   }
  4788   5002   
  4789   5003   /*
  4790   5004   ** Advance the cursor to the next entry in the database.  If
  4791   5005   ** successful then set *pRes=0.  If the cursor
  4792   5006   ** was already pointing to the last entry in the database before
  4793   5007   ** this routine was called, then set *pRes=1.
         5008  +**
         5009  +** The main entry point is sqlite3BtreeNext().  That routine is optimized
         5010  +** for the common case of merely incrementing the cell counter BtCursor.aiIdx
         5011  +** to the next cell on the current page.  The (slower) btreeNext() helper
         5012  +** routine is called when it is necessary to move to a different page or
         5013  +** to restore the cursor.
  4794   5014   **
  4795   5015   ** The calling function will set *pRes to 0 or 1.  The initial *pRes value
  4796   5016   ** will be 1 if the cursor being stepped corresponds to an SQL index and
  4797   5017   ** if this routine could have been skipped if that SQL index had been
  4798   5018   ** a unique index.  Otherwise the caller will have set *pRes to zero.
  4799   5019   ** Zero is the common case. The btree implementation is free to use the
  4800   5020   ** initial *pRes value as a hint to improve performance, but the current
  4801   5021   ** SQLite btree implementation does not. (Note that the comdb2 btree
  4802   5022   ** implementation does use this hint, however.)
  4803   5023   */
  4804         -int sqlite3BtreeNext(BtCursor *pCur, int *pRes){
         5024  +static SQLITE_NOINLINE int btreeNext(BtCursor *pCur, int *pRes){
  4805   5025     int rc;
  4806   5026     int idx;
  4807   5027     MemPage *pPage;
  4808   5028   
  4809   5029     assert( cursorHoldsMutex(pCur) );
  4810         -  assert( pRes!=0 );
  4811         -  assert( *pRes==0 || *pRes==1 );
  4812   5030     assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID );
         5031  +  assert( *pRes==0 );
  4813   5032     if( pCur->eState!=CURSOR_VALID ){
  4814         -    invalidateOverflowCache(pCur);
         5033  +    assert( (pCur->curFlags & BTCF_ValidOvfl)==0 );
  4815   5034       rc = restoreCursorPosition(pCur);
  4816   5035       if( rc!=SQLITE_OK ){
  4817         -      *pRes = 0;
  4818   5036         return rc;
  4819   5037       }
  4820   5038       if( CURSOR_INVALID==pCur->eState ){
  4821   5039         *pRes = 1;
  4822   5040         return SQLITE_OK;
  4823   5041       }
  4824   5042       if( pCur->skipNext ){
  4825   5043         assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_SKIPNEXT );
  4826   5044         pCur->eState = CURSOR_VALID;
  4827   5045         if( pCur->skipNext>0 ){
  4828   5046           pCur->skipNext = 0;
  4829         -        *pRes = 0;
  4830   5047           return SQLITE_OK;
  4831   5048         }
  4832   5049         pCur->skipNext = 0;
  4833   5050       }
  4834   5051     }
  4835   5052   
  4836   5053     pPage = pCur->apPage[pCur->iPage];
................................................................................
  4840   5057     /* If the database file is corrupt, it is possible for the value of idx 
  4841   5058     ** to be invalid here. This can only occur if a second cursor modifies
  4842   5059     ** the page while cursor pCur is holding a reference to it. Which can
  4843   5060     ** only happen if the database is corrupt in such a way as to link the
  4844   5061     ** page into more than one b-tree structure. */
  4845   5062     testcase( idx>pPage->nCell );
  4846   5063   
  4847         -  pCur->info.nSize = 0;
  4848         -  pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl);
  4849   5064     if( idx>=pPage->nCell ){
  4850   5065       if( !pPage->leaf ){
  4851   5066         rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8]));
  4852         -      if( rc ){
  4853         -        *pRes = 0;
  4854         -        return rc;
  4855         -      }
  4856         -      rc = moveToLeftmost(pCur);
  4857         -      *pRes = 0;
  4858         -      return rc;
         5067  +      if( rc ) return rc;
         5068  +      return moveToLeftmost(pCur);
  4859   5069       }
  4860   5070       do{
  4861   5071         if( pCur->iPage==0 ){
  4862   5072           *pRes = 1;
  4863   5073           pCur->eState = CURSOR_INVALID;
  4864   5074           return SQLITE_OK;
  4865   5075         }
  4866   5076         moveToParent(pCur);
  4867   5077         pPage = pCur->apPage[pCur->iPage];
  4868   5078       }while( pCur->aiIdx[pCur->iPage]>=pPage->nCell );
  4869         -    *pRes = 0;
  4870   5079       if( pPage->intKey ){
  4871         -      rc = sqlite3BtreeNext(pCur, pRes);
         5080  +      return sqlite3BtreeNext(pCur, pRes);
  4872   5081       }else{
  4873         -      rc = SQLITE_OK;
         5082  +      return SQLITE_OK;
  4874   5083       }
  4875         -    return rc;
  4876   5084     }
         5085  +  if( pPage->leaf ){
         5086  +    return SQLITE_OK;
         5087  +  }else{
         5088  +    return moveToLeftmost(pCur);
         5089  +  }
         5090  +}
         5091  +int sqlite3BtreeNext(BtCursor *pCur, int *pRes){
         5092  +  MemPage *pPage;
         5093  +  assert( cursorHoldsMutex(pCur) );
         5094  +  assert( pRes!=0 );
         5095  +  assert( *pRes==0 || *pRes==1 );
         5096  +  assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID );
         5097  +  pCur->info.nSize = 0;
         5098  +  pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl);
  4877   5099     *pRes = 0;
         5100  +  if( pCur->eState!=CURSOR_VALID ) return btreeNext(pCur, pRes);
         5101  +  pPage = pCur->apPage[pCur->iPage];
         5102  +  if( (++pCur->aiIdx[pCur->iPage])>=pPage->nCell ){
         5103  +    pCur->aiIdx[pCur->iPage]--;
         5104  +    return btreeNext(pCur, pRes);
         5105  +  }
  4878   5106     if( pPage->leaf ){
  4879   5107       return SQLITE_OK;
         5108  +  }else{
         5109  +    return moveToLeftmost(pCur);
  4880   5110     }
  4881         -  rc = moveToLeftmost(pCur);
  4882         -  return rc;
  4883   5111   }
  4884         -
  4885   5112   
  4886   5113   /*
  4887   5114   ** Step the cursor to the back to the previous entry in the database.  If
  4888   5115   ** successful then set *pRes=0.  If the cursor
  4889   5116   ** was already pointing to the first entry in the database before
  4890   5117   ** this routine was called, then set *pRes=1.
         5118  +**
         5119  +** The main entry point is sqlite3BtreePrevious().  That routine is optimized
         5120  +** for the common case of merely decrementing the cell counter BtCursor.aiIdx
         5121  +** to the previous cell on the current page.  The (slower) btreePrevious()
         5122  +** helper routine is called when it is necessary to move to a different page
         5123  +** or to restore the cursor.
  4891   5124   **
  4892   5125   ** The calling function will set *pRes to 0 or 1.  The initial *pRes value
  4893   5126   ** will be 1 if the cursor being stepped corresponds to an SQL index and
  4894   5127   ** if this routine could have been skipped if that SQL index had been
  4895   5128   ** a unique index.  Otherwise the caller will have set *pRes to zero.
  4896   5129   ** Zero is the common case. The btree implementation is free to use the
  4897   5130   ** initial *pRes value as a hint to improve performance, but the current
  4898   5131   ** SQLite btree implementation does not. (Note that the comdb2 btree
  4899   5132   ** implementation does use this hint, however.)
  4900   5133   */
  4901         -int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){
         5134  +static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur, int *pRes){
  4902   5135     int rc;
  4903   5136     MemPage *pPage;
  4904   5137   
  4905   5138     assert( cursorHoldsMutex(pCur) );
  4906   5139     assert( pRes!=0 );
  4907         -  assert( *pRes==0 || *pRes==1 );
         5140  +  assert( *pRes==0 );
  4908   5141     assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID );
  4909         -  pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl);
         5142  +  assert( (pCur->curFlags & (BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey))==0 );
         5143  +  assert( pCur->info.nSize==0 );
  4910   5144     if( pCur->eState!=CURSOR_VALID ){
  4911         -    if( ALWAYS(pCur->eState>=CURSOR_REQUIRESEEK) ){
  4912         -      rc = btreeRestoreCursorPosition(pCur);
  4913         -      if( rc!=SQLITE_OK ){
  4914         -        *pRes = 0;
  4915         -        return rc;
  4916         -      }
         5145  +    rc = restoreCursorPosition(pCur);
         5146  +    if( rc!=SQLITE_OK ){
         5147  +      return rc;
  4917   5148       }
  4918   5149       if( CURSOR_INVALID==pCur->eState ){
  4919   5150         *pRes = 1;
  4920   5151         return SQLITE_OK;
  4921   5152       }
  4922   5153       if( pCur->skipNext ){
  4923   5154         assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_SKIPNEXT );
  4924   5155         pCur->eState = CURSOR_VALID;
  4925   5156         if( pCur->skipNext<0 ){
  4926   5157           pCur->skipNext = 0;
  4927         -        *pRes = 0;
  4928   5158           return SQLITE_OK;
  4929   5159         }
  4930   5160         pCur->skipNext = 0;
  4931   5161       }
  4932   5162     }
  4933   5163   
  4934   5164     pPage = pCur->apPage[pCur->iPage];
  4935   5165     assert( pPage->isInit );
  4936   5166     if( !pPage->leaf ){
  4937   5167       int idx = pCur->aiIdx[pCur->iPage];
  4938   5168       rc = moveToChild(pCur, get4byte(findCell(pPage, idx)));
  4939         -    if( rc ){
  4940         -      *pRes = 0;
  4941         -      return rc;
  4942         -    }
         5169  +    if( rc ) return rc;
  4943   5170       rc = moveToRightmost(pCur);
  4944   5171     }else{
  4945   5172       while( pCur->aiIdx[pCur->iPage]==0 ){
  4946   5173         if( pCur->iPage==0 ){
  4947   5174           pCur->eState = CURSOR_INVALID;
  4948   5175           *pRes = 1;
  4949   5176           return SQLITE_OK;
  4950   5177         }
  4951   5178         moveToParent(pCur);
  4952   5179       }
  4953         -    pCur->info.nSize = 0;
  4954         -    pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl);
         5180  +    assert( pCur->info.nSize==0 );
         5181  +    assert( (pCur->curFlags & (BTCF_ValidNKey|BTCF_ValidOvfl))==0 );
  4955   5182   
  4956   5183       pCur->aiIdx[pCur->iPage]--;
  4957   5184       pPage = pCur->apPage[pCur->iPage];
  4958   5185       if( pPage->intKey && !pPage->leaf ){
  4959   5186         rc = sqlite3BtreePrevious(pCur, pRes);
  4960   5187       }else{
  4961   5188         rc = SQLITE_OK;
  4962   5189       }
  4963   5190     }
         5191  +  return rc;
         5192  +}
         5193  +int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){
         5194  +  assert( cursorHoldsMutex(pCur) );
         5195  +  assert( pRes!=0 );
         5196  +  assert( *pRes==0 || *pRes==1 );
         5197  +  assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID );
  4964   5198     *pRes = 0;
  4965         -  return rc;
         5199  +  pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey);
         5200  +  pCur->info.nSize = 0;
         5201  +  if( pCur->eState!=CURSOR_VALID
         5202  +   || pCur->aiIdx[pCur->iPage]==0
         5203  +   || pCur->apPage[pCur->iPage]->leaf==0
         5204  +  ){
         5205  +    return btreePrevious(pCur, pRes);
         5206  +  }
         5207  +  pCur->aiIdx[pCur->iPage]--;
         5208  +  return SQLITE_OK;
  4966   5209   }
  4967   5210   
  4968   5211   /*
  4969   5212   ** Allocate a new page from the database file.
  4970   5213   **
  4971   5214   ** The new page is marked as dirty.  (In other words, sqlite3PagerWrite()
  4972   5215   ** has already been called on the new page.)  The new page has also
................................................................................
  5003   5246     MemPage *pPrevTrunk = 0;
  5004   5247     Pgno mxPage;     /* Total size of the database file */
  5005   5248   
  5006   5249     assert( sqlite3_mutex_held(pBt->mutex) );
  5007   5250     assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) );
  5008   5251     pPage1 = pBt->pPage1;
  5009   5252     mxPage = btreePagecount(pBt);
         5253  +  /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36
         5254  +  ** stores stores the total number of pages on the freelist. */
  5010   5255     n = get4byte(&pPage1->aData[36]);
  5011   5256     testcase( n==mxPage-1 );
  5012   5257     if( n>=mxPage ){
  5013   5258       return SQLITE_CORRUPT_BKPT;
  5014   5259     }
  5015   5260     if( n>0 ){
  5016   5261       /* There are pages on the freelist.  Reuse one of those pages. */
................................................................................
  5049   5294       ** is not true. Otherwise, it runs once for each trunk-page on the
  5050   5295       ** free-list until the page 'nearby' is located (eMode==BTALLOC_EXACT)
  5051   5296       ** or until a page less than 'nearby' is located (eMode==BTALLOC_LT)
  5052   5297       */
  5053   5298       do {
  5054   5299         pPrevTrunk = pTrunk;
  5055   5300         if( pPrevTrunk ){
         5301  +        /* EVIDENCE-OF: R-01506-11053 The first integer on a freelist trunk page
         5302  +        ** is the page number of the next freelist trunk page in the list or
         5303  +        ** zero if this is the last freelist trunk page. */
  5056   5304           iTrunk = get4byte(&pPrevTrunk->aData[0]);
  5057   5305         }else{
         5306  +        /* EVIDENCE-OF: R-59841-13798 The 4-byte big-endian integer at offset 32
         5307  +        ** stores the page number of the first page of the freelist, or zero if
         5308  +        ** the freelist is empty. */
  5058   5309           iTrunk = get4byte(&pPage1->aData[32]);
  5059   5310         }
  5060   5311         testcase( iTrunk==mxPage );
  5061   5312         if( iTrunk>mxPage ){
  5062   5313           rc = SQLITE_CORRUPT_BKPT;
  5063   5314         }else{
  5064   5315           rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0);
................................................................................
  5065   5316         }
  5066   5317         if( rc ){
  5067   5318           pTrunk = 0;
  5068   5319           goto end_allocate_page;
  5069   5320         }
  5070   5321         assert( pTrunk!=0 );
  5071   5322         assert( pTrunk->aData!=0 );
  5072         -
  5073         -      k = get4byte(&pTrunk->aData[4]); /* # of leaves on this trunk page */
         5323  +      /* EVIDENCE-OF: R-13523-04394 The second integer on a freelist trunk page
         5324  +      ** is the number of leaf page pointers to follow. */
         5325  +      k = get4byte(&pTrunk->aData[4]);
  5074   5326         if( k==0 && !searchList ){
  5075   5327           /* The trunk has no leaves and the list is not being searched. 
  5076   5328           ** So extract the trunk page itself and use it as the newly 
  5077   5329           ** allocated page */
  5078   5330           assert( pPrevTrunk==0 );
  5079   5331           rc = sqlite3PagerWrite(pTrunk->pDbPage);
  5080   5332           if( rc ){
................................................................................
  5200   5452                    *pPgno, closest+1, k, pTrunk->pgno, n-1));
  5201   5453             rc = sqlite3PagerWrite(pTrunk->pDbPage);
  5202   5454             if( rc ) goto end_allocate_page;
  5203   5455             if( closest<k-1 ){
  5204   5456               memcpy(&aData[8+closest*4], &aData[4+k*4], 4);
  5205   5457             }
  5206   5458             put4byte(&aData[4], k-1);
  5207         -          noContent = !btreeGetHasContent(pBt, *pPgno) ? PAGER_GET_NOCONTENT : 0;
         5459  +          noContent = !btreeGetHasContent(pBt, *pPgno)? PAGER_GET_NOCONTENT : 0;
  5208   5460             rc = btreeGetPage(pBt, *pPgno, ppPage, noContent);
  5209   5461             if( rc==SQLITE_OK ){
  5210   5462               rc = sqlite3PagerWrite((*ppPage)->pDbPage);
  5211   5463               if( rc!=SQLITE_OK ){
  5212   5464                 releasePage(*ppPage);
  5213   5465               }
  5214   5466             }
................................................................................
  5233   5485       **
  5234   5486       ** Note that the pager will not actually attempt to load or journal 
  5235   5487       ** content for any page that really does lie past the end of the database
  5236   5488       ** file on disk. So the effects of disabling the no-content optimization
  5237   5489       ** here are confined to those pages that lie between the end of the
  5238   5490       ** database image and the end of the database file.
  5239   5491       */
  5240         -    int bNoContent = (0==IfNotOmitAV(pBt->bDoTruncate)) ? PAGER_GET_NOCONTENT : 0;
         5492  +    int bNoContent = (0==IfNotOmitAV(pBt->bDoTruncate))? PAGER_GET_NOCONTENT:0;
  5241   5493   
  5242   5494       rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
  5243   5495       if( rc ) return rc;
  5244   5496       pBt->nPage++;
  5245   5497       if( pBt->nPage==PENDING_BYTE_PAGE(pBt) ) pBt->nPage++;
  5246   5498   
  5247   5499   #ifndef SQLITE_OMIT_AUTOVACUUM
................................................................................
  5384   5636         ** 3.6.0, databases with freelist trunk pages holding more than
  5385   5637         ** usableSize/4 - 8 entries will be reported as corrupt.  In order
  5386   5638         ** to maintain backwards compatibility with older versions of SQLite,
  5387   5639         ** we will continue to restrict the number of entries to usableSize/4 - 8
  5388   5640         ** for now.  At some point in the future (once everyone has upgraded
  5389   5641         ** to 3.6.0 or later) we should consider fixing the conditional above
  5390   5642         ** to read "usableSize/4-2" instead of "usableSize/4-8".
         5643  +      **
         5644  +      ** EVIDENCE-OF: R-19920-11576 However, newer versions of SQLite still
         5645  +      ** avoid using the last six entries in the freelist trunk page array in
         5646  +      ** order that database files created by newer versions of SQLite can be
         5647  +      ** read by older versions of SQLite.
  5391   5648         */
  5392   5649         rc = sqlite3PagerWrite(pTrunk->pDbPage);
  5393   5650         if( rc==SQLITE_OK ){
  5394   5651           put4byte(&pTrunk->aData[4], nLeaf+1);
  5395   5652           put4byte(&pTrunk->aData[8+nLeaf*4], iPage);
  5396   5653           if( pPage && (pBt->btsFlags & BTS_SECURE_DELETE)==0 ){
  5397   5654             sqlite3PagerDontWrite(pPage->pDbPage);
................................................................................
  5432   5689   static void freePage(MemPage *pPage, int *pRC){
  5433   5690     if( (*pRC)==SQLITE_OK ){
  5434   5691       *pRC = freePage2(pPage->pBt, pPage, pPage->pgno);
  5435   5692     }
  5436   5693   }
  5437   5694   
  5438   5695   /*
  5439         -** Free any overflow pages associated with the given Cell.
         5696  +** Free any overflow pages associated with the given Cell.  Write the
         5697  +** local Cell size (the number of bytes on the original page, omitting
         5698  +** overflow) into *pnSize.
  5440   5699   */
  5441         -static int clearCell(MemPage *pPage, unsigned char *pCell){
         5700  +static int clearCell(
         5701  +  MemPage *pPage,          /* The page that contains the Cell */
         5702  +  unsigned char *pCell,    /* First byte of the Cell */
         5703  +  u16 *pnSize              /* Write the size of the Cell here */
         5704  +){
  5442   5705     BtShared *pBt = pPage->pBt;
  5443   5706     CellInfo info;
  5444   5707     Pgno ovflPgno;
  5445   5708     int rc;
  5446   5709     int nOvfl;
  5447   5710     u32 ovflPageSize;
  5448   5711   
  5449   5712     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  5450   5713     btreeParseCellPtr(pPage, pCell, &info);
         5714  +  *pnSize = info.nSize;
  5451   5715     if( info.iOverflow==0 ){
  5452   5716       return SQLITE_OK;  /* No overflow pages. Return without doing anything */
  5453   5717     }
  5454   5718     if( pCell+info.iOverflow+3 > pPage->aData+pPage->maskPage ){
  5455   5719       return SQLITE_CORRUPT_BKPT;  /* Cell extends past end of page */
  5456   5720     }
  5457   5721     ovflPgno = get4byte(&pCell[info.iOverflow]);
................................................................................
  5527   5791     MemPage *pOvfl = 0;
  5528   5792     MemPage *pToRelease = 0;
  5529   5793     unsigned char *pPrior;
  5530   5794     unsigned char *pPayload;
  5531   5795     BtShared *pBt = pPage->pBt;
  5532   5796     Pgno pgnoOvfl = 0;
  5533   5797     int nHeader;
  5534         -  CellInfo info;
  5535   5798   
  5536   5799     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  5537   5800   
  5538   5801     /* pPage is not necessarily writeable since pCell might be auxiliary
  5539   5802     ** buffer space that is separate from the pPage buffer area */
  5540   5803     assert( pCell<pPage->aData || pCell>=&pPage->aData[pBt->pageSize]
  5541   5804               || sqlite3PagerIswriteable(pPage->pDbPage) );
  5542   5805   
  5543   5806     /* Fill in the header. */
  5544         -  nHeader = 0;
  5545         -  if( !pPage->leaf ){
  5546         -    nHeader += 4;
  5547         -  }
  5548         -  if( pPage->hasData ){
  5549         -    nHeader += putVarint32(&pCell[nHeader], nData+nZero);
         5807  +  nHeader = pPage->childPtrSize;
         5808  +  nPayload = nData + nZero;
         5809  +  if( pPage->intKeyLeaf ){
         5810  +    nHeader += putVarint32(&pCell[nHeader], nPayload);
  5550   5811     }else{
  5551         -    nData = nZero = 0;
         5812  +    assert( nData==0 );
         5813  +    assert( nZero==0 );
  5552   5814     }
  5553   5815     nHeader += putVarint(&pCell[nHeader], *(u64*)&nKey);
  5554         -  btreeParseCellPtr(pPage, pCell, &info);
  5555         -  assert( info.nHeader==nHeader );
  5556         -  assert( info.nKey==nKey );
  5557         -  assert( info.nData==(u32)(nData+nZero) );
  5558   5816     
  5559         -  /* Fill in the payload */
  5560         -  nPayload = nData + nZero;
         5817  +  /* Fill in the payload size */
  5561   5818     if( pPage->intKey ){
  5562   5819       pSrc = pData;
  5563   5820       nSrc = nData;
  5564   5821       nData = 0;
  5565   5822     }else{ 
  5566   5823       if( NEVER(nKey>0x7fffffff || pKey==0) ){
  5567   5824         return SQLITE_CORRUPT_BKPT;
  5568   5825       }
  5569         -    nPayload += (int)nKey;
         5826  +    nPayload = (int)nKey;
  5570   5827       pSrc = pKey;
  5571   5828       nSrc = (int)nKey;
  5572   5829     }
  5573         -  *pnSize = info.nSize;
  5574         -  spaceLeft = info.nLocal;
         5830  +  if( nPayload<=pPage->maxLocal ){
         5831  +    n = nHeader + nPayload;
         5832  +    testcase( n==3 );
         5833  +    testcase( n==4 );
         5834  +    if( n<4 ) n = 4;
         5835  +    *pnSize = n;
         5836  +    spaceLeft = nPayload;
         5837  +    pPrior = pCell;
         5838  +  }else{
         5839  +    int mn = pPage->minLocal;
         5840  +    n = mn + (nPayload - mn) % (pPage->pBt->usableSize - 4);
         5841  +    testcase( n==pPage->maxLocal );
         5842  +    testcase( n==pPage->maxLocal+1 );
         5843  +    if( n > pPage->maxLocal ) n = mn;
         5844  +    spaceLeft = n;
         5845  +    *pnSize = n + nHeader + 4;
         5846  +    pPrior = &pCell[nHeader+n];
         5847  +  }
  5575   5848     pPayload = &pCell[nHeader];
  5576         -  pPrior = &pCell[info.iOverflow];
  5577   5849   
         5850  +  /* At this point variables should be set as follows:
         5851  +  **
         5852  +  **   nPayload           Total payload size in bytes
         5853  +  **   pPayload           Begin writing payload here
         5854  +  **   spaceLeft          Space available at pPayload.  If nPayload>spaceLeft,
         5855  +  **                      that means content must spill into overflow pages.
         5856  +  **   *pnSize            Size of the local cell (not counting overflow pages)
         5857  +  **   pPrior             Where to write the pgno of the first overflow page
         5858  +  **
         5859  +  ** Use a call to btreeParseCellPtr() to verify that the values above
         5860  +  ** were computed correctly.
         5861  +  */
         5862  +#if SQLITE_DEBUG
         5863  +  {
         5864  +    CellInfo info;
         5865  +    btreeParseCellPtr(pPage, pCell, &info);
         5866  +    assert( nHeader=(int)(info.pPayload - pCell) );
         5867  +    assert( info.nKey==nKey );
         5868  +    assert( *pnSize == info.nSize );
         5869  +    assert( spaceLeft == info.nLocal );
         5870  +    assert( pPrior == &pCell[info.iOverflow] );
         5871  +  }
         5872  +#endif
         5873  +
         5874  +  /* Write the payload into the local Cell and any extra into overflow pages */
  5578   5875     while( nPayload>0 ){
  5579   5876       if( spaceLeft==0 ){
  5580   5877   #ifndef SQLITE_OMIT_AUTOVACUUM
  5581   5878         Pgno pgnoPtrmap = pgnoOvfl; /* Overflow page pointer-map entry page */
  5582   5879         if( pBt->autoVacuum ){
  5583   5880           do{
  5584   5881             pgnoOvfl++;
................................................................................
  5695   5992     }
  5696   5993     rc = freeSpace(pPage, pc, sz);
  5697   5994     if( rc ){
  5698   5995       *pRC = rc;
  5699   5996       return;
  5700   5997     }
  5701   5998     pPage->nCell--;
  5702         -  memmove(ptr, ptr+2, 2*(pPage->nCell - idx));
  5703         -  put2byte(&data[hdr+3], pPage->nCell);
  5704         -  pPage->nFree += 2;
         5999  +  if( pPage->nCell==0 ){
         6000  +    memset(&data[hdr+1], 0, 4);
         6001  +    data[hdr+7] = 0;
         6002  +    put2byte(&data[hdr+5], pPage->pBt->usableSize);
         6003  +    pPage->nFree = pPage->pBt->usableSize - pPage->hdrOffset
         6004  +                       - pPage->childPtrSize - 8;
         6005  +  }else{
         6006  +    memmove(ptr, ptr+2, 2*(pPage->nCell - idx));
         6007  +    put2byte(&data[hdr+3], pPage->nCell);
         6008  +    pPage->nFree += 2;
         6009  +  }
  5705   6010   }
  5706   6011   
  5707   6012   /*
  5708   6013   ** Insert a new cell on pPage at cell index "i".  pCell points to the
  5709   6014   ** content of the cell.
  5710   6015   **
  5711   6016   ** If the cell content will fit on the page, then put it there.  If it
  5712   6017   ** will not fit, then make a copy of the cell content into pTemp if
  5713   6018   ** pTemp is not null.  Regardless of pTemp, allocate a new entry
  5714   6019   ** in pPage->apOvfl[] and make it point to the cell content (either
  5715   6020   ** in pTemp or the original pCell) and also record its index. 
  5716   6021   ** Allocating a new entry in pPage->aCell[] implies that 
  5717   6022   ** pPage->nOverflow is incremented.
  5718         -**
  5719         -** If nSkip is non-zero, then do not copy the first nSkip bytes of the
  5720         -** cell. The caller will overwrite them after this function returns. If
  5721         -** nSkip is non-zero, then pCell may not point to an invalid memory location 
  5722         -** (but pCell+nSkip is always valid).
  5723   6023   */
  5724   6024   static void insertCell(
  5725   6025     MemPage *pPage,   /* Page into which we are copying */
  5726   6026     int i,            /* New cell becomes the i-th cell of the page */
  5727   6027     u8 *pCell,        /* Content of the new cell */
  5728   6028     int sz,           /* Bytes of content in pCell */
  5729   6029     u8 *pTemp,        /* Temp storage space for pCell, if needed */
................................................................................
  5732   6032   ){
  5733   6033     int idx = 0;      /* Where to write new cell content in data[] */
  5734   6034     int j;            /* Loop counter */
  5735   6035     int end;          /* First byte past the last cell pointer in data[] */
  5736   6036     int ins;          /* Index in data[] where new cell pointer is inserted */
  5737   6037     int cellOffset;   /* Address of first cell pointer in data[] */
  5738   6038     u8 *data;         /* The content of the whole page */
  5739         -  int nSkip = (iChild ? 4 : 0);
  5740   6039   
  5741   6040     if( *pRC ) return;
  5742   6041   
  5743   6042     assert( i>=0 && i<=pPage->nCell+pPage->nOverflow );
  5744         -  assert( pPage->nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=10921 );
         6043  +  assert( MX_CELL(pPage->pBt)<=10921 );
         6044  +  assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB );
  5745   6045     assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) );
  5746   6046     assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) );
  5747   6047     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  5748   6048     /* The cell should normally be sized correctly.  However, when moving a
  5749   6049     ** malformed cell from a leaf page to an interior page, if the cell size
  5750   6050     ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size
  5751   6051     ** might be less than 8 (leaf-size + pointer) on the interior node.  Hence
  5752   6052     ** the term after the || in the following assert(). */
  5753   6053     assert( sz==cellSizePtr(pPage, pCell) || (sz==8 && iChild>0) );
  5754   6054     if( pPage->nOverflow || sz+2>pPage->nFree ){
  5755   6055       if( pTemp ){
  5756         -      memcpy(pTemp+nSkip, pCell+nSkip, sz-nSkip);
         6056  +      memcpy(pTemp, pCell, sz);
  5757   6057         pCell = pTemp;
  5758   6058       }
  5759   6059       if( iChild ){
  5760   6060         put4byte(pCell, iChild);
  5761   6061       }
  5762   6062       j = pPage->nOverflow++;
  5763   6063       assert( j<(int)(sizeof(pPage->apOvfl)/sizeof(pPage->apOvfl[0])) );
................................................................................
  5778   6078       if( rc ){ *pRC = rc; return; }
  5779   6079       /* The allocateSpace() routine guarantees the following two properties
  5780   6080       ** if it returns success */
  5781   6081       assert( idx >= end+2 );
  5782   6082       assert( idx+sz <= (int)pPage->pBt->usableSize );
  5783   6083       pPage->nCell++;
  5784   6084       pPage->nFree -= (u16)(2 + sz);
  5785         -    memcpy(&data[idx+nSkip], pCell+nSkip, sz-nSkip);
         6085  +    memcpy(&data[idx], pCell, sz);
  5786   6086       if( iChild ){
  5787   6087         put4byte(&data[idx], iChild);
  5788   6088       }
  5789   6089       memmove(&data[ins+2], &data[ins], end-ins);
  5790   6090       put2byte(&data[ins], idx);
  5791   6091       put2byte(&data[pPage->hdrOffset+3], pPage->nCell);
  5792   6092   #ifndef SQLITE_OMIT_AUTOVACUUM
................................................................................
  5797   6097         ptrmapPutOvflPtr(pPage, pCell, pRC);
  5798   6098       }
  5799   6099   #endif
  5800   6100     }
  5801   6101   }
  5802   6102   
  5803   6103   /*
  5804         -** Add a list of cells to a page.  The page should be initially empty.
  5805         -** The cells are guaranteed to fit on the page.
  5806         -*/
  5807         -static void assemblePage(
  5808         -  MemPage *pPage,   /* The page to be assemblied */
  5809         -  int nCell,        /* The number of cells to add to this page */
  5810         -  u8 **apCell,      /* Pointers to cell bodies */
  5811         -  u16 *aSize        /* Sizes of the cells */
         6104  +** Array apCell[] contains pointers to nCell b-tree page cells. The 
         6105  +** szCell[] array contains the size in bytes of each cell. This function
         6106  +** replaces the current contents of page pPg with the contents of the cell
         6107  +** array.
         6108  +**
         6109  +** Some of the cells in apCell[] may currently be stored in pPg. This
         6110  +** function works around problems caused by this by making a copy of any 
         6111  +** such cells before overwriting the page data.
         6112  +**
         6113  +** The MemPage.nFree field is invalidated by this function. It is the 
         6114  +** responsibility of the caller to set it correctly.
         6115  +*/
         6116  +static void rebuildPage(
         6117  +  MemPage *pPg,                   /* Edit this page */
         6118  +  int nCell,                      /* Final number of cells on page */
         6119  +  u8 **apCell,                    /* Array of cells */
         6120  +  u16 *szCell                     /* Array of cell sizes */
         6121  +){
         6122  +  const int hdr = pPg->hdrOffset;          /* Offset of header on pPg */
         6123  +  u8 * const aData = pPg->aData;           /* Pointer to data for pPg */
         6124  +  const int usableSize = pPg->pBt->usableSize;
         6125  +  u8 * const pEnd = &aData[usableSize];
         6126  +  int i;
         6127  +  u8 *pCellptr = pPg->aCellIdx;
         6128  +  u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager);
         6129  +  u8 *pData;
         6130  +
         6131  +  i = get2byte(&aData[hdr+5]);
         6132  +  memcpy(&pTmp[i], &aData[i], usableSize - i);
         6133  +
         6134  +  pData = pEnd;
         6135  +  for(i=0; i<nCell; i++){
         6136  +    u8 *pCell = apCell[i];
         6137  +    if( pCell>aData && pCell<pEnd ){
         6138  +      pCell = &pTmp[pCell - aData];
         6139  +    }
         6140  +    pData -= szCell[i];
         6141  +    memcpy(pData, pCell, szCell[i]);
         6142  +    put2byte(pCellptr, (pData - aData));
         6143  +    pCellptr += 2;
         6144  +    assert( szCell[i]==cellSizePtr(pPg, pCell) );
         6145  +  }
         6146  +
         6147  +  /* The pPg->nFree field is now set incorrectly. The caller will fix it. */
         6148  +  pPg->nCell = nCell;
         6149  +  pPg->nOverflow = 0;
         6150  +
         6151  +  put2byte(&aData[hdr+1], 0);
         6152  +  put2byte(&aData[hdr+3], pPg->nCell);
         6153  +  put2byte(&aData[hdr+5], pData - aData);
         6154  +  aData[hdr+7] = 0x00;
         6155  +}
         6156  +
         6157  +/*
         6158  +** Array apCell[] contains nCell pointers to b-tree cells. Array szCell
         6159  +** contains the size in bytes of each such cell. This function attempts to 
         6160  +** add the cells stored in the array to page pPg. If it cannot (because 
         6161  +** the page needs to be defragmented before the cells will fit), non-zero
         6162  +** is returned. Otherwise, if the cells are added successfully, zero is
         6163  +** returned.
         6164  +**
         6165  +** Argument pCellptr points to the first entry in the cell-pointer array
         6166  +** (part of page pPg) to populate. After cell apCell[0] is written to the
         6167  +** page body, a 16-bit offset is written to pCellptr. And so on, for each
         6168  +** cell in the array. It is the responsibility of the caller to ensure
         6169  +** that it is safe to overwrite this part of the cell-pointer array.
         6170  +**
         6171  +** When this function is called, *ppData points to the start of the 
         6172  +** content area on page pPg. If the size of the content area is extended,
         6173  +** *ppData is updated to point to the new start of the content area
         6174  +** before returning.
         6175  +**
         6176  +** Finally, argument pBegin points to the byte immediately following the
         6177  +** end of the space required by this page for the cell-pointer area (for
         6178  +** all cells - not just those inserted by the current call). If the content
         6179  +** area must be extended to before this point in order to accomodate all
         6180  +** cells in apCell[], then the cells do not fit and non-zero is returned.
         6181  +*/
         6182  +static int pageInsertArray(
         6183  +  MemPage *pPg,                   /* Page to add cells to */
         6184  +  u8 *pBegin,                     /* End of cell-pointer array */
         6185  +  u8 **ppData,                    /* IN/OUT: Page content -area pointer */
         6186  +  u8 *pCellptr,                   /* Pointer to cell-pointer area */
         6187  +  int nCell,                      /* Number of cells to add to pPg */
         6188  +  u8 **apCell,                    /* Array of cells */
         6189  +  u16 *szCell                     /* Array of cell sizes */
         6190  +){
         6191  +  int i;
         6192  +  u8 *aData = pPg->aData;
         6193  +  u8 *pData = *ppData;
         6194  +  const int bFreelist = aData[1] || aData[2];
         6195  +  assert( CORRUPT_DB || pPg->hdrOffset==0 );    /* Never called on page 1 */
         6196  +  for(i=0; i<nCell; i++){
         6197  +    int sz = szCell[i];
         6198  +    int rc;
         6199  +    u8 *pSlot;
         6200  +    if( bFreelist==0 || (pSlot = pageFindSlot(pPg, sz, &rc, 0))==0 ){
         6201  +      pData -= sz;
         6202  +      if( pData<pBegin ) return 1;
         6203  +      pSlot = pData;
         6204  +    }
         6205  +    memcpy(pSlot, apCell[i], sz);
         6206  +    put2byte(pCellptr, (pSlot - aData));
         6207  +    pCellptr += 2;
         6208  +  }
         6209  +  *ppData = pData;
         6210  +  return 0;
         6211  +}
         6212  +
         6213  +/*
         6214  +** Array apCell[] contains nCell pointers to b-tree cells. Array szCell 
         6215  +** contains the size in bytes of each such cell. This function adds the
         6216  +** space associated with each cell in the array that is currently stored 
         6217  +** within the body of pPg to the pPg free-list. The cell-pointers and other
         6218  +** fields of the page are not updated.
         6219  +**
         6220  +** This function returns the total number of cells added to the free-list.
         6221  +*/
         6222  +static int pageFreeArray(
         6223  +  MemPage *pPg,                   /* Page to edit */
         6224  +  int nCell,                      /* Cells to delete */
         6225  +  u8 **apCell,                    /* Array of cells */
         6226  +  u16 *szCell                     /* Array of cell sizes */
         6227  +){
         6228  +  u8 * const aData = pPg->aData;
         6229  +  u8 * const pEnd = &aData[pPg->pBt->usableSize];
         6230  +  u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize];
         6231  +  int nRet = 0;
         6232  +  int i;
         6233  +  u8 *pFree = 0;
         6234  +  int szFree = 0;
         6235  +
         6236  +  for(i=0; i<nCell; i++){
         6237  +    u8 *pCell = apCell[i];
         6238  +    if( pCell>=pStart && pCell<pEnd ){
         6239  +      int sz = szCell[i];
         6240  +      if( pFree!=(pCell + sz) ){
         6241  +        if( pFree ){
         6242  +          assert( pFree>aData && (pFree - aData)<65536 );
         6243  +          freeSpace(pPg, (u16)(pFree - aData), szFree);
         6244  +        }
         6245  +        pFree = pCell;
         6246  +        szFree = sz;
         6247  +        if( pFree+sz>pEnd ) return 0;
         6248  +      }else{
         6249  +        pFree = pCell;
         6250  +        szFree += sz;
         6251  +      }
         6252  +      nRet++;
         6253  +    }
         6254  +  }
         6255  +  if( pFree ){
         6256  +    assert( pFree>aData && (pFree - aData)<65536 );
         6257  +    freeSpace(pPg, (u16)(pFree - aData), szFree);
         6258  +  }
         6259  +  return nRet;
         6260  +}
         6261  +
         6262  +/*
         6263  +** apCell[] and szCell[] contains pointers to and sizes of all cells in the
         6264  +** pages being balanced.  The current page, pPg, has pPg->nCell cells starting
         6265  +** with apCell[iOld].  After balancing, this page should hold nNew cells
         6266  +** starting at apCell[iNew].
         6267  +**
         6268  +** This routine makes the necessary adjustments to pPg so that it contains
         6269  +** the correct cells after being balanced.
         6270  +**
         6271  +** The pPg->nFree field is invalid when this function returns. It is the
         6272  +** responsibility of the caller to set it correctly.
         6273  +*/
         6274  +static void editPage(
         6275  +  MemPage *pPg,                   /* Edit this page */
         6276  +  int iOld,                       /* Index of first cell currently on page */
         6277  +  int iNew,                       /* Index of new first cell on page */
         6278  +  int nNew,                       /* Final number of cells on page */
         6279  +  u8 **apCell,                    /* Array of cells */
         6280  +  u16 *szCell                     /* Array of cell sizes */
  5812   6281   ){
  5813         -  int i;            /* Loop counter */
  5814         -  u8 *pCellptr;     /* Address of next cell pointer */
  5815         -  int cellbody;     /* Address of next cell body */
  5816         -  u8 * const data = pPage->aData;             /* Pointer to data for pPage */
  5817         -  const int hdr = pPage->hdrOffset;           /* Offset of header on pPage */
  5818         -  const int nUsable = pPage->pBt->usableSize; /* Usable size of page */
  5819         -
  5820         -  assert( pPage->nOverflow==0 );
  5821         -  assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  5822         -  assert( nCell>=0 && nCell<=(int)MX_CELL(pPage->pBt)
  5823         -            && (int)MX_CELL(pPage->pBt)<=10921);
  5824         -  assert( sqlite3PagerIswriteable(pPage->pDbPage) );
  5825         -
  5826         -  /* Check that the page has just been zeroed by zeroPage() */
  5827         -  assert( pPage->nCell==0 );
  5828         -  assert( get2byteNotZero(&data[hdr+5])==nUsable );
  5829         -
  5830         -  pCellptr = &pPage->aCellIdx[nCell*2];
  5831         -  cellbody = nUsable;
  5832         -  for(i=nCell-1; i>=0; i--){
  5833         -    u16 sz = aSize[i];
  5834         -    pCellptr -= 2;
  5835         -    cellbody -= sz;
  5836         -    put2byte(pCellptr, cellbody);
  5837         -    memcpy(&data[cellbody], apCell[i], sz);
  5838         -  }
  5839         -  put2byte(&data[hdr+3], nCell);
  5840         -  put2byte(&data[hdr+5], cellbody);
  5841         -  pPage->nFree -= (nCell*2 + nUsable - cellbody);
  5842         -  pPage->nCell = (u16)nCell;
         6282  +  u8 * const aData = pPg->aData;
         6283  +  const int hdr = pPg->hdrOffset;
         6284  +  u8 *pBegin = &pPg->aCellIdx[nNew * 2];
         6285  +  int nCell = pPg->nCell;       /* Cells stored on pPg */
         6286  +  u8 *pData;
         6287  +  u8 *pCellptr;
         6288  +  int i;
         6289  +  int iOldEnd = iOld + pPg->nCell + pPg->nOverflow;
         6290  +  int iNewEnd = iNew + nNew;
         6291  +
         6292  +#ifdef SQLITE_DEBUG
         6293  +  u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager);
         6294  +  memcpy(pTmp, aData, pPg->pBt->usableSize);
         6295  +#endif
         6296  +
         6297  +  /* Remove cells from the start and end of the page */
         6298  +  if( iOld<iNew ){
         6299  +    int nShift = pageFreeArray(
         6300  +        pPg, iNew-iOld, &apCell[iOld], &szCell[iOld]
         6301  +    );
         6302  +    memmove(pPg->aCellIdx, &pPg->aCellIdx[nShift*2], nCell*2);
         6303  +    nCell -= nShift;
         6304  +  }
         6305  +  if( iNewEnd < iOldEnd ){
         6306  +    nCell -= pageFreeArray(
         6307  +        pPg, iOldEnd-iNewEnd, &apCell[iNewEnd], &szCell[iNewEnd]
         6308  +    );
         6309  +  }
         6310  +
         6311  +  pData = &aData[get2byteNotZero(&aData[hdr+5])];
         6312  +  if( pData<pBegin ) goto editpage_fail;
         6313  +
         6314  +  /* Add cells to the start of the page */
         6315  +  if( iNew<iOld ){
         6316  +    int nAdd = MIN(nNew,iOld-iNew);
         6317  +    assert( (iOld-iNew)<nNew || nCell==0 || CORRUPT_DB );
         6318  +    pCellptr = pPg->aCellIdx;
         6319  +    memmove(&pCellptr[nAdd*2], pCellptr, nCell*2);
         6320  +    if( pageInsertArray(
         6321  +          pPg, pBegin, &pData, pCellptr,
         6322  +          nAdd, &apCell[iNew], &szCell[iNew]
         6323  +    ) ) goto editpage_fail;
         6324  +    nCell += nAdd;
         6325  +  }
         6326  +
         6327  +  /* Add any overflow cells */
         6328  +  for(i=0; i<pPg->nOverflow; i++){
         6329  +    int iCell = (iOld + pPg->aiOvfl[i]) - iNew;
         6330  +    if( iCell>=0 && iCell<nNew ){
         6331  +      pCellptr = &pPg->aCellIdx[iCell * 2];
         6332  +      memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2);
         6333  +      nCell++;
         6334  +      if( pageInsertArray(
         6335  +            pPg, pBegin, &pData, pCellptr,
         6336  +            1, &apCell[iCell + iNew], &szCell[iCell + iNew]
         6337  +      ) ) goto editpage_fail;
         6338  +    }
         6339  +  }
         6340  +
         6341  +  /* Append cells to the end of the page */
         6342  +  pCellptr = &pPg->aCellIdx[nCell*2];
         6343  +  if( pageInsertArray(
         6344  +        pPg, pBegin, &pData, pCellptr,
         6345  +        nNew-nCell, &apCell[iNew+nCell], &szCell[iNew+nCell]
         6346  +  ) ) goto editpage_fail;
         6347  +
         6348  +  pPg->nCell = nNew;
         6349  +  pPg->nOverflow = 0;
         6350  +
         6351  +  put2byte(&aData[hdr+3], pPg->nCell);
         6352  +  put2byte(&aData[hdr+5], pData - aData);
         6353  +
         6354  +#ifdef SQLITE_DEBUG
         6355  +  for(i=0; i<nNew && !CORRUPT_DB; i++){
         6356  +    u8 *pCell = apCell[i+iNew];
         6357  +    int iOff = get2byte(&pPg->aCellIdx[i*2]);
         6358  +    if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){
         6359  +      pCell = &pTmp[pCell - aData];
         6360  +    }
         6361  +    assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) );
         6362  +  }
         6363  +#endif
         6364  +
         6365  +  return;
         6366  + editpage_fail:
         6367  +  /* Unable to edit this page. Rebuild it from scratch instead. */
         6368  +  rebuildPage(pPg, nNew, &apCell[iNew], &szCell[iNew]);
  5843   6369   }
  5844   6370   
  5845   6371   /*
  5846   6372   ** The following parameters determine how many adjacent pages get involved
  5847   6373   ** in a balancing operation.  NN is the number of neighbors on either side
  5848   6374   ** of the page that participate in the balancing operation.  NB is the
  5849   6375   ** total number of pages that participate, including the target page and
................................................................................
  5889   6415     Pgno pgnoNew;                        /* Page number of pNew */
  5890   6416   
  5891   6417     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  5892   6418     assert( sqlite3PagerIswriteable(pParent->pDbPage) );
  5893   6419     assert( pPage->nOverflow==1 );
  5894   6420   
  5895   6421     /* This error condition is now caught prior to reaching this function */
  5896         -  if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT;
         6422  +  if( NEVER(pPage->nCell==0) ) return SQLITE_CORRUPT_BKPT;
  5897   6423   
  5898   6424     /* Allocate a new page. This page will become the right-sibling of 
  5899   6425     ** pPage. Make the parent page writable, so that the new divider cell
  5900   6426     ** may be inserted. If both these operations are successful, proceed.
  5901   6427     */
  5902   6428     rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0);
  5903   6429   
................................................................................
  5907   6433       u8 *pCell = pPage->apOvfl[0];
  5908   6434       u16 szCell = cellSizePtr(pPage, pCell);
  5909   6435       u8 *pStop;
  5910   6436   
  5911   6437       assert( sqlite3PagerIswriteable(pNew->pDbPage) );
  5912   6438       assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) );
  5913   6439       zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF);
  5914         -    assemblePage(pNew, 1, &pCell, &szCell);
         6440  +    rebuildPage(pNew, 1, &pCell, &szCell);
         6441  +    pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell;
  5915   6442   
  5916   6443       /* If this is an auto-vacuum database, update the pointer map
  5917   6444       ** with entries for the new page, and any pointer from the 
  5918   6445       ** cell on the page to an overflow page. If either of these
  5919   6446       ** operations fails, the return code is set, but the contents
  5920   6447       ** of the parent page are still manipulated by thh code below.
  5921   6448       ** That is Ok, at this point the parent page is guaranteed to
................................................................................
  6126   6653     int usableSpace;             /* Bytes in pPage beyond the header */
  6127   6654     int pageFlags;               /* Value of pPage->aData[0] */
  6128   6655     int subtotal;                /* Subtotal of bytes in cells on one page */
  6129   6656     int iSpace1 = 0;             /* First unused byte of aSpace1[] */
  6130   6657     int iOvflSpace = 0;          /* First unused byte of aOvflSpace[] */
  6131   6658     int szScratch;               /* Size of scratch memory requested */
  6132   6659     MemPage *apOld[NB];          /* pPage and up to two siblings */
  6133         -  MemPage *apCopy[NB];         /* Private copies of apOld[] pages */
  6134   6660     MemPage *apNew[NB+2];        /* pPage and up to NB siblings after balancing */
  6135   6661     u8 *pRight;                  /* Location in parent of right-sibling pointer */
  6136   6662     u8 *apDiv[NB-1];             /* Divider cells in pParent */
  6137   6663     int cntNew[NB+2];            /* Index in aCell[] of cell after i-th page */
  6138         -  int szNew[NB+2];             /* Combined size of cells place on i-th page */
         6664  +  int cntOld[NB+2];            /* Old index in aCell[] after i-th page */
         6665  +  int szNew[NB+2];             /* Combined size of cells placed on i-th page */
  6139   6666     u8 **apCell = 0;             /* All cells begin balanced */
  6140   6667     u16 *szCell;                 /* Local size of all cells in apCell[] */
  6141   6668     u8 *aSpace1;                 /* Space for copies of dividers cells */
  6142   6669     Pgno pgno;                   /* Temp var to store a page number in */
         6670  +  u8 abDone[NB+2];             /* True after i'th new page is populated */
         6671  +  Pgno aPgno[NB+2];            /* Page numbers of new pages before shuffling */
         6672  +  Pgno aPgOrder[NB+2];         /* Copy of aPgno[] used for sorting pages */
         6673  +  u16 aPgFlags[NB+2];          /* flags field of new pages before shuffling */
  6143   6674   
         6675  +  memset(abDone, 0, sizeof(abDone));
  6144   6676     pBt = pParent->pBt;
  6145   6677     assert( sqlite3_mutex_held(pBt->mutex) );
  6146   6678     assert( sqlite3PagerIswriteable(pParent->pDbPage) );
  6147   6679   
  6148   6680   #if 0
  6149   6681     TRACE(("BALANCE: begin page %d child of %d\n", pPage->pgno, pParent->pgno));
  6150   6682   #endif
................................................................................
  6245   6777     /* Make nMaxCells a multiple of 4 in order to preserve 8-byte
  6246   6778     ** alignment */
  6247   6779     nMaxCells = (nMaxCells + 3)&~3;
  6248   6780   
  6249   6781     /*
  6250   6782     ** Allocate space for memory structures
  6251   6783     */
  6252         -  k = pBt->pageSize + ROUND8(sizeof(MemPage));
  6253   6784     szScratch =
  6254   6785          nMaxCells*sizeof(u8*)                       /* apCell */
  6255   6786        + nMaxCells*sizeof(u16)                       /* szCell */
  6256         -     + pBt->pageSize                               /* aSpace1 */
  6257         -     + k*nOld;                                     /* Page copies (apCopy) */
         6787  +     + pBt->pageSize;                              /* aSpace1 */
         6788  +
         6789  +  /* EVIDENCE-OF: R-28375-38319 SQLite will never request a scratch buffer
         6790  +  ** that is more than 6 times the database page size. */
         6791  +  assert( szScratch<=6*(int)pBt->pageSize );
  6258   6792     apCell = sqlite3ScratchMalloc( szScratch ); 
  6259   6793     if( apCell==0 ){
  6260   6794       rc = SQLITE_NOMEM;
  6261   6795       goto balance_cleanup;
  6262   6796     }
  6263   6797     szCell = (u16*)&apCell[nMaxCells];
  6264   6798     aSpace1 = (u8*)&szCell[nMaxCells];
  6265   6799     assert( EIGHT_BYTE_ALIGNMENT(aSpace1) );
  6266   6800   
  6267   6801     /*
  6268   6802     ** Load pointers to all cells on sibling pages and the divider cells
  6269   6803     ** into the local apCell[] array.  Make copies of the divider cells
  6270         -  ** into space obtained from aSpace1[] and remove the divider cells
  6271         -  ** from pParent.
         6804  +  ** into space obtained from aSpace1[]. The divider cells have already
         6805  +  ** been removed from pParent.
  6272   6806     **
  6273   6807     ** If the siblings are on leaf pages, then the child pointers of the
  6274   6808     ** divider cells are stripped from the cells before they are copied
  6275   6809     ** into aSpace1[].  In this way, all cells in apCell[] are without
  6276   6810     ** child pointers.  If siblings are not leaves, then all cell in
  6277   6811     ** apCell[] include child pointers.  Either way, all cells in apCell[]
  6278   6812     ** are alike.
  6279   6813     **
  6280   6814     ** leafCorrection:  4 if pPage is a leaf.  0 if pPage is not a leaf.
  6281   6815     **       leafData:  1 if pPage holds key+data and pParent holds only keys.
  6282   6816     */
  6283   6817     leafCorrection = apOld[0]->leaf*4;
  6284         -  leafData = apOld[0]->hasData;
         6818  +  leafData = apOld[0]->intKeyLeaf;
  6285   6819     for(i=0; i<nOld; i++){
  6286   6820       int limit;
  6287         -    
  6288         -    /* Before doing anything else, take a copy of the i'th original sibling
  6289         -    ** The rest of this function will use data from the copies rather
  6290         -    ** that the original pages since the original pages will be in the
  6291         -    ** process of being overwritten.  */
  6292         -    MemPage *pOld = apCopy[i] = (MemPage*)&aSpace1[pBt->pageSize + k*i];
  6293         -    memcpy(pOld, apOld[i], sizeof(MemPage));
  6294         -    pOld->aData = (void*)&pOld[1];
  6295         -    memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize);
         6821  +    MemPage *pOld = apOld[i];
  6296   6822   
  6297   6823       limit = pOld->nCell+pOld->nOverflow;
  6298   6824       if( pOld->nOverflow>0 ){
  6299   6825         for(j=0; j<limit; j++){
  6300   6826           assert( nCell<nMaxCells );
  6301   6827           apCell[nCell] = findOverflowCell(pOld, j);
  6302   6828           szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
................................................................................
  6309   6835         for(j=0; j<limit; j++){
  6310   6836           assert( nCell<nMaxCells );
  6311   6837           apCell[nCell] = findCellv2(aData, maskPage, cellOffset, j);
  6312   6838           szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
  6313   6839           nCell++;
  6314   6840         }
  6315   6841       }       
         6842  +    cntOld[i] = nCell;
  6316   6843       if( i<nOld-1 && !leafData){
  6317   6844         u16 sz = (u16)szNew[i];
  6318   6845         u8 *pTemp;
  6319   6846         assert( nCell<nMaxCells );
  6320   6847         szCell[nCell] = sz;
  6321   6848         pTemp = &aSpace1[iSpace1];
  6322   6849         iSpace1 += sz;
................................................................................
  6331   6858           assert( pOld->hdrOffset==0 );
  6332   6859           /* The right pointer of the child page pOld becomes the left
  6333   6860           ** pointer of the divider cell */
  6334   6861           memcpy(apCell[nCell], &pOld->aData[8], 4);
  6335   6862         }else{
  6336   6863           assert( leafCorrection==4 );
  6337   6864           if( szCell[nCell]<4 ){
  6338         -          /* Do not allow any cells smaller than 4 bytes. */
         6865  +          /* Do not allow any cells smaller than 4 bytes. If a smaller cell
         6866  +          ** does exist, pad it with 0x00 bytes. */
         6867  +          assert( szCell[nCell]==3 );
         6868  +          assert( apCell[nCell]==&aSpace1[iSpace1-3] );
         6869  +          aSpace1[iSpace1++] = 0x00;
  6339   6870             szCell[nCell] = 4;
  6340   6871           }
  6341   6872         }
  6342   6873         nCell++;
  6343   6874       }
  6344   6875     }
  6345   6876   
................................................................................
  6360   6891     ** 
  6361   6892     */
  6362   6893     usableSpace = pBt->usableSize - 12 + leafCorrection;
  6363   6894     for(subtotal=k=i=0; i<nCell; i++){
  6364   6895       assert( i<nMaxCells );
  6365   6896       subtotal += szCell[i] + 2;
  6366   6897       if( subtotal > usableSpace ){
  6367         -      szNew[k] = subtotal - szCell[i];
         6898  +      szNew[k] = subtotal - szCell[i] - 2;
  6368   6899         cntNew[k] = i;
  6369   6900         if( leafData ){ i--; }
  6370   6901         subtotal = 0;
  6371   6902         k++;
  6372   6903         if( k>NB+1 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; }
  6373   6904       }
  6374   6905     }
  6375   6906     szNew[k] = subtotal;
  6376   6907     cntNew[k] = nCell;
  6377   6908     k++;
  6378   6909   
  6379   6910     /*
  6380   6911     ** The packing computed by the previous block is biased toward the siblings
  6381         -  ** on the left side.  The left siblings are always nearly full, while the
  6382         -  ** right-most sibling might be nearly empty.  This block of code attempts
  6383         -  ** to adjust the packing of siblings to get a better balance.
         6912  +  ** on the left side (siblings with smaller keys). The left siblings are
         6913  +  ** always nearly full, while the right-most sibling might be nearly empty.
         6914  +  ** The next block of code attempts to adjust the packing of siblings to
         6915  +  ** get a better balance.
  6384   6916     **
  6385   6917     ** This adjustment is more than an optimization.  The packing above might
  6386   6918     ** be so out of balance as to be illegal.  For example, the right-most
  6387   6919     ** sibling might be completely empty.  This adjustment is not optional.
  6388   6920     */
  6389   6921     for(i=k-1; i>0; i--){
  6390   6922       int szRight = szNew[i];  /* Size of sibling on the right */
................................................................................
  6405   6937         r = cntNew[i-1] - 1;
  6406   6938         d = r + 1 - leafData;
  6407   6939       }
  6408   6940       szNew[i] = szRight;
  6409   6941       szNew[i-1] = szLeft;
  6410   6942     }
  6411   6943   
  6412         -  /* Either we found one or more cells (cntnew[0])>0) or pPage is
  6413         -  ** a virtual root page.  A virtual root page is when the real root
  6414         -  ** page is page 1 and we are the only child of that page.
  6415         -  **
  6416         -  ** UPDATE:  The assert() below is not necessarily true if the database
  6417         -  ** file is corrupt.  The corruption will be detected and reported later
  6418         -  ** in this procedure so there is no need to act upon it now.
         6944  +  /* Sanity check:  For a non-corrupt database file one of the follwing
         6945  +  ** must be true:
         6946  +  **    (1) We found one or more cells (cntNew[0])>0), or
         6947  +  **    (2) pPage is a virtual root page.  A virtual root page is when
         6948  +  **        the real root page is page 1 and we are the only child of
         6949  +  **        that page.
  6419   6950     */
  6420         -#if 0
  6421         -  assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) );
  6422         -#endif
  6423         -
  6424         -  TRACE(("BALANCE: old: %d %d %d  ",
  6425         -    apOld[0]->pgno, 
  6426         -    nOld>=2 ? apOld[1]->pgno : 0,
  6427         -    nOld>=3 ? apOld[2]->pgno : 0
         6951  +  assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) || CORRUPT_DB);
         6952  +  TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n",
         6953  +    apOld[0]->pgno, apOld[0]->nCell,
         6954  +    nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0,
         6955  +    nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0
  6428   6956     ));
  6429   6957   
  6430   6958     /*
  6431   6959     ** Allocate k new pages.  Reuse old pages where possible.
  6432   6960     */
  6433   6961     if( apOld[0]->pgno<=1 ){
  6434   6962       rc = SQLITE_CORRUPT_BKPT;
................................................................................
  6443   6971         rc = sqlite3PagerWrite(pNew->pDbPage);
  6444   6972         nNew++;
  6445   6973         if( rc ) goto balance_cleanup;
  6446   6974       }else{
  6447   6975         assert( i>0 );
  6448   6976         rc = allocateBtreePage(pBt, &pNew, &pgno, (bBulk ? 1 : pgno), 0);
  6449   6977         if( rc ) goto balance_cleanup;
         6978  +      zeroPage(pNew, pageFlags);
  6450   6979         apNew[i] = pNew;
  6451   6980         nNew++;
         6981  +      cntOld[i] = nCell;
  6452   6982   
  6453   6983         /* Set the pointer-map entry for the new sibling page. */
  6454   6984         if( ISAUTOVACUUM ){
  6455   6985           ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc);
  6456   6986           if( rc!=SQLITE_OK ){
  6457   6987             goto balance_cleanup;
  6458   6988           }
  6459   6989         }
  6460   6990       }
  6461   6991     }
  6462   6992   
  6463         -  /* Free any old pages that were not reused as new pages.
  6464         -  */
  6465         -  while( i<nOld ){
  6466         -    freePage(apOld[i], &rc);
  6467         -    if( rc ) goto balance_cleanup;
  6468         -    releasePage(apOld[i]);
  6469         -    apOld[i] = 0;
  6470         -    i++;
  6471         -  }
  6472         -
  6473   6993     /*
  6474         -  ** Put the new pages in accending order.  This helps to
  6475         -  ** keep entries in the disk file in order so that a scan
  6476         -  ** of the table is a linear scan through the file.  That
  6477         -  ** in turn helps the operating system to deliver pages
  6478         -  ** from the disk more rapidly.
  6479         -  **
  6480         -  ** An O(n^2) insertion sort algorithm is used, but since
  6481         -  ** n is never more than NB (a small constant), that should
  6482         -  ** not be a problem.
  6483         -  **
  6484         -  ** When NB==3, this one optimization makes the database
  6485         -  ** about 25% faster for large insertions and deletions.
  6486         -  */
  6487         -  for(i=0; i<k-1; i++){
  6488         -    int minV = apNew[i]->pgno;
  6489         -    int minI = i;
  6490         -    for(j=i+1; j<k; j++){
  6491         -      if( apNew[j]->pgno<(unsigned)minV ){
  6492         -        minI = j;
  6493         -        minV = apNew[j]->pgno;
  6494         -      }
  6495         -    }
  6496         -    if( minI>i ){
  6497         -      MemPage *pT;
  6498         -      pT = apNew[i];
  6499         -      apNew[i] = apNew[minI];
  6500         -      apNew[minI] = pT;
  6501         -    }
  6502         -  }
  6503         -  TRACE(("new: %d(%d) %d(%d) %d(%d) %d(%d) %d(%d)\n",
  6504         -    apNew[0]->pgno, szNew[0],
         6994  +  ** Reassign page numbers so that the new pages are in ascending order. 
         6995  +  ** This helps to keep entries in the disk file in order so that a scan
         6996  +  ** of the table is closer to a linear scan through the file. That in turn 
         6997  +  ** helps the operating system to deliver pages from the disk more rapidly.
         6998  +  **
         6999  +  ** An O(n^2) insertion sort algorithm is used, but since n is never more 
         7000  +  ** than (NB+2) (a small constant), that should not be a problem.
         7001  +  **
         7002  +  ** When NB==3, this one optimization makes the database about 25% faster 
         7003  +  ** for large insertions and deletions.
         7004  +  */
         7005  +  for(i=0; i<nNew; i++){
         7006  +    aPgOrder[i] = aPgno[i] = apNew[i]->pgno;
         7007  +    aPgFlags[i] = apNew[i]->pDbPage->flags;
         7008  +    for(j=0; j<i; j++){
         7009  +      if( aPgno[j]==aPgno[i] ){
         7010  +        /* This branch is taken if the set of sibling pages somehow contains
         7011  +        ** duplicate entries. This can happen if the database is corrupt. 
         7012  +        ** It would be simpler to detect this as part of the loop below, but
         7013  +        ** we do the detection here in order to avoid populating the pager
         7014  +        ** cache with two separate objects associated with the same
         7015  +        ** page number.  */
         7016  +        assert( CORRUPT_DB );
         7017  +        rc = SQLITE_CORRUPT_BKPT;
         7018  +        goto balance_cleanup;
         7019  +      }
         7020  +    }
         7021  +  }
         7022  +  for(i=0; i<nNew; i++){
         7023  +    int iBest = 0;                /* aPgno[] index of page number to use */
         7024  +    for(j=1; j<nNew; j++){
         7025  +      if( aPgOrder[j]<aPgOrder[iBest] ) iBest = j;
         7026  +    }
         7027  +    pgno = aPgOrder[iBest];
         7028  +    aPgOrder[iBest] = 0xffffffff;
         7029  +    if( iBest!=i ){
         7030  +      if( iBest>i ){
         7031  +        sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0);
         7032  +      }
         7033  +      sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]);
         7034  +      apNew[i]->pgno = pgno;
         7035  +    }
         7036  +  }
         7037  +
         7038  +  TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) "
         7039  +         "%d(%d nc=%d) %d(%d nc=%d)\n",
         7040  +    apNew[0]->pgno, szNew[0], cntNew[0],
  6505   7041       nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0,
         7042  +    nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0,
  6506   7043       nNew>=3 ? apNew[2]->pgno : 0, nNew>=3 ? szNew[2] : 0,
         7044  +    nNew>=3 ? cntNew[2] - cntNew[1] - !leafData : 0,
  6507   7045       nNew>=4 ? apNew[3]->pgno : 0, nNew>=4 ? szNew[3] : 0,
  6508         -    nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0));
         7046  +    nNew>=4 ? cntNew[3] - cntNew[2] - !leafData : 0,
         7047  +    nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0,
         7048  +    nNew>=5 ? cntNew[4] - cntNew[3] - !leafData : 0
         7049  +  ));
  6509   7050   
  6510   7051     assert( sqlite3PagerIswriteable(pParent->pDbPage) );
  6511   7052     put4byte(pRight, apNew[nNew-1]->pgno);
  6512   7053   
  6513         -  /*
  6514         -  ** Evenly distribute the data in apCell[] across the new pages.
  6515         -  ** Insert divider cells into pParent as necessary.
  6516         -  */
  6517         -  j = 0;
  6518         -  for(i=0; i<nNew; i++){
  6519         -    /* Assemble the new sibling page. */
  6520         -    MemPage *pNew = apNew[i];
  6521         -    assert( j<nMaxCells );
  6522         -    zeroPage(pNew, pageFlags);
  6523         -    assemblePage(pNew, cntNew[i]-j, &apCell[j], &szCell[j]);
  6524         -    assert( pNew->nCell>0 || (nNew==1 && cntNew[0]==0) );
  6525         -    assert( pNew->nOverflow==0 );
  6526         -
         7054  +  /* If the sibling pages are not leaves, ensure that the right-child pointer
         7055  +  ** of the right-most new sibling page is set to the value that was 
         7056  +  ** originally in the same field of the right-most old sibling page. */
         7057  +  if( (pageFlags & PTF_LEAF)==0 && nOld!=nNew ){
         7058  +    MemPage *pOld = (nNew>nOld ? apNew : apOld)[nOld-1];
         7059  +    memcpy(&apNew[nNew-1]->aData[8], &pOld->aData[8], 4);
         7060  +  }
         7061  +
         7062  +  /* Make any required updates to pointer map entries associated with 
         7063  +  ** cells stored on sibling pages following the balance operation. Pointer
         7064  +  ** map entries associated with divider cells are set by the insertCell()
         7065  +  ** routine. The associated pointer map entries are:
         7066  +  **
         7067  +  **   a) if the cell contains a reference to an overflow chain, the
         7068  +  **      entry associated with the first page in the overflow chain, and
         7069  +  **
         7070  +  **   b) if the sibling pages are not leaves, the child page associated
         7071  +  **      with the cell.
         7072  +  **
         7073  +  ** If the sibling pages are not leaves, then the pointer map entry 
         7074  +  ** associated with the right-child of each sibling may also need to be 
         7075  +  ** updated. This happens below, after the sibling pages have been 
         7076  +  ** populated, not here.
         7077  +  */
         7078  +  if( ISAUTOVACUUM ){
         7079  +    MemPage *pNew = apNew[0];
         7080  +    u8 *aOld = pNew->aData;
         7081  +    int cntOldNext = pNew->nCell + pNew->nOverflow;
         7082  +    int usableSize = pBt->usableSize;
         7083  +    int iNew = 0;
         7084  +    int iOld = 0;
         7085  +
         7086  +    for(i=0; i<nCell; i++){
         7087  +      u8 *pCell = apCell[i];
         7088  +      if( i==cntOldNext ){
         7089  +        MemPage *pOld = (++iOld)<nNew ? apNew[iOld] : apOld[iOld];
         7090  +        cntOldNext += pOld->nCell + pOld->nOverflow + !leafData;
         7091  +        aOld = pOld->aData;
         7092  +      }
         7093  +      if( i==cntNew[iNew] ){
         7094  +        pNew = apNew[++iNew];
         7095  +        if( !leafData ) continue;
         7096  +      }
         7097  +
         7098  +      /* Cell pCell is destined for new sibling page pNew. Originally, it
         7099  +      ** was either part of sibling page iOld (possibly an overflow cell), 
         7100  +      ** or else the divider cell to the left of sibling page iOld. So,
         7101  +      ** if sibling page iOld had the same page number as pNew, and if
         7102  +      ** pCell really was a part of sibling page iOld (not a divider or
         7103  +      ** overflow cell), we can skip updating the pointer map entries.  */
         7104  +      if( iOld>=nNew
         7105  +       || pNew->pgno!=aPgno[iOld]
         7106  +       || pCell<aOld
         7107  +       || pCell>=&aOld[usableSize]
         7108  +      ){
         7109  +        if( !leafCorrection ){
         7110  +          ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc);
         7111  +        }
         7112  +        if( szCell[i]>pNew->minLocal ){
         7113  +          ptrmapPutOvflPtr(pNew, pCell, &rc);
         7114  +        }
         7115  +      }
         7116  +    }
         7117  +  }
         7118  +
         7119  +  /* Insert new divider cells into pParent. */
         7120  +  for(i=0; i<nNew-1; i++){
         7121  +    u8 *pCell;
         7122  +    u8 *pTemp;
         7123  +    int sz;
         7124  +    MemPage *pNew = apNew[i];
  6527   7125       j = cntNew[i];
  6528   7126   
  6529         -    /* If the sibling page assembled above was not the right-most sibling,
  6530         -    ** insert a divider cell into the parent page.
  6531         -    */
  6532         -    assert( i<nNew-1 || j==nCell );
  6533         -    if( j<nCell ){
  6534         -      u8 *pCell;
  6535         -      u8 *pTemp;
  6536         -      int sz;
  6537         -
  6538         -      assert( j<nMaxCells );
  6539         -      pCell = apCell[j];
  6540         -      sz = szCell[j] + leafCorrection;
  6541         -      pTemp = &aOvflSpace[iOvflSpace];
  6542         -      if( !pNew->leaf ){
  6543         -        memcpy(&pNew->aData[8], pCell, 4);
  6544         -      }else if( leafData ){
  6545         -        /* If the tree is a leaf-data tree, and the siblings are leaves, 
  6546         -        ** then there is no divider cell in apCell[]. Instead, the divider 
  6547         -        ** cell consists of the integer key for the right-most cell of 
  6548         -        ** the sibling-page assembled above only.
  6549         -        */
  6550         -        CellInfo info;
  6551         -        j--;
  6552         -        btreeParseCellPtr(pNew, apCell[j], &info);
  6553         -        pCell = pTemp;
  6554         -        sz = 4 + putVarint(&pCell[4], info.nKey);
  6555         -        pTemp = 0;
         7127  +    assert( j<nMaxCells );
         7128  +    pCell = apCell[j];
         7129  +    sz = szCell[j] + leafCorrection;
         7130  +    pTemp = &aOvflSpace[iOvflSpace];
         7131  +    if( !pNew->leaf ){
         7132  +      memcpy(&pNew->aData[8], pCell, 4);
         7133  +    }else if( leafData ){
         7134  +      /* If the tree is a leaf-data tree, and the siblings are leaves, 
         7135  +      ** then there is no divider cell in apCell[]. Instead, the divider 
         7136  +      ** cell consists of the integer key for the right-most cell of 
         7137  +      ** the sibling-page assembled above only.
         7138  +      */
         7139  +      CellInfo info;
         7140  +      j--;
         7141  +      btreeParseCellPtr(pNew, apCell[j], &info);
         7142  +      pCell = pTemp;
         7143  +      sz = 4 + putVarint(&pCell[4], info.nKey);
         7144  +      pTemp = 0;
         7145  +    }else{
         7146  +      pCell -= 4;
         7147  +      /* Obscure case for non-leaf-data trees: If the cell at pCell was
         7148  +      ** previously stored on a leaf node, and its reported size was 4
         7149  +      ** bytes, then it may actually be smaller than this 
         7150  +      ** (see btreeParseCellPtr(), 4 bytes is the minimum size of
         7151  +      ** any cell). But it is important to pass the correct size to 
         7152  +      ** insertCell(), so reparse the cell now.
         7153  +      **
         7154  +      ** Note that this can never happen in an SQLite data file, as all
         7155  +      ** cells are at least 4 bytes. It only happens in b-trees used
         7156  +      ** to evaluate "IN (SELECT ...)" and similar clauses.
         7157  +      */
         7158  +      if( szCell[j]==4 ){
         7159  +        assert(leafCorrection==4);
         7160  +        sz = cellSizePtr(pParent, pCell);
         7161  +      }
         7162  +    }
         7163  +    iOvflSpace += sz;
         7164  +    assert( sz<=pBt->maxLocal+23 );
         7165  +    assert( iOvflSpace <= (int)pBt->pageSize );
         7166  +    insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno, &rc);
         7167  +    if( rc!=SQLITE_OK ) goto balance_cleanup;
         7168  +    assert( sqlite3PagerIswriteable(pParent->pDbPage) );
         7169  +  }
         7170  +
         7171  +  /* Now update the actual sibling pages. The order in which they are updated
         7172  +  ** is important, as this code needs to avoid disrupting any page from which
         7173  +  ** cells may still to be read. In practice, this means:
         7174  +  **
         7175  +  **  (1) If cells are moving left (from apNew[iPg] to apNew[iPg-1])
         7176  +  **      then it is not safe to update page apNew[iPg] until after
         7177  +  **      the left-hand sibling apNew[iPg-1] has been updated.
         7178  +  **
         7179  +  **  (2) If cells are moving right (from apNew[iPg] to apNew[iPg+1])
         7180  +  **      then it is not safe to update page apNew[iPg] until after
         7181  +  **      the right-hand sibling apNew[iPg+1] has been updated.
         7182  +  **
         7183  +  ** If neither of the above apply, the page is safe to update.
         7184  +  **
         7185  +  ** The iPg value in the following loop starts at nNew-1 goes down
         7186  +  ** to 0, then back up to nNew-1 again, thus making two passes over
         7187  +  ** the pages.  On the initial downward pass, only condition (1) above
         7188  +  ** needs to be tested because (2) will always be true from the previous
         7189  +  ** step.  On the upward pass, both conditions are always true, so the
         7190  +  ** upwards pass simply processes pages that were missed on the downward
         7191  +  ** pass.
         7192  +  */
         7193  +  for(i=1-nNew; i<nNew; i++){
         7194  +    int iPg = i<0 ? -i : i;
         7195  +    assert( iPg>=0 && iPg<nNew );
         7196  +    if( abDone[iPg] ) continue;         /* Skip pages already processed */
         7197  +    if( i>=0                            /* On the upwards pass, or... */
         7198  +     || cntOld[iPg-1]>=cntNew[iPg-1]    /* Condition (1) is true */
         7199  +    ){
         7200  +      int iNew;
         7201  +      int iOld;
         7202  +      int nNewCell;
         7203  +
         7204  +      /* Verify condition (1):  If cells are moving left, update iPg
         7205  +      ** only after iPg-1 has already been updated. */
         7206  +      assert( iPg==0 || cntOld[iPg-1]>=cntNew[iPg-1] || abDone[iPg-1] );
         7207  +
         7208  +      /* Verify condition (2):  If cells are moving right, update iPg
         7209  +      ** only after iPg+1 has already been updated. */
         7210  +      assert( cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1] );
         7211  +
         7212  +      if( iPg==0 ){
         7213  +        iNew = iOld = 0;
         7214  +        nNewCell = cntNew[0];
  6556   7215         }else{
  6557         -        pCell -= 4;
  6558         -        /* Obscure case for non-leaf-data trees: If the cell at pCell was
  6559         -        ** previously stored on a leaf node, and its reported size was 4
  6560         -        ** bytes, then it may actually be smaller than this 
  6561         -        ** (see btreeParseCellPtr(), 4 bytes is the minimum size of
  6562         -        ** any cell). But it is important to pass the correct size to 
  6563         -        ** insertCell(), so reparse the cell now.
  6564         -        **
  6565         -        ** Note that this can never happen in an SQLite data file, as all
  6566         -        ** cells are at least 4 bytes. It only happens in b-trees used
  6567         -        ** to evaluate "IN (SELECT ...)" and similar clauses.
  6568         -        */
  6569         -        if( szCell[j]==4 ){
  6570         -          assert(leafCorrection==4);
  6571         -          sz = cellSizePtr(pParent, pCell);
  6572         -        }
  6573         -      }
  6574         -      iOvflSpace += sz;
  6575         -      assert( sz<=pBt->maxLocal+23 );
  6576         -      assert( iOvflSpace <= (int)pBt->pageSize );
  6577         -      insertCell(pParent, nxDiv, pCell, sz, pTemp, pNew->pgno, &rc);
  6578         -      if( rc!=SQLITE_OK ) goto balance_cleanup;
  6579         -      assert( sqlite3PagerIswriteable(pParent->pDbPage) );
  6580         -
  6581         -      j++;
  6582         -      nxDiv++;
  6583         -    }
  6584         -  }
  6585         -  assert( j==nCell );
         7216  +        iOld = iPg<nOld ? (cntOld[iPg-1] + !leafData) : nCell;
         7217  +        iNew = cntNew[iPg-1] + !leafData;
         7218  +        nNewCell = cntNew[iPg] - iNew;
         7219  +      }
         7220  +
         7221  +      editPage(apNew[iPg], iOld, iNew, nNewCell, apCell, szCell);
         7222  +      abDone[iPg]++;
         7223  +      apNew[iPg]->nFree = usableSpace-szNew[iPg];
         7224  +      assert( apNew[iPg]->nOverflow==0 );
         7225  +      assert( apNew[iPg]->nCell==nNewCell );
         7226  +    }
         7227  +  }
         7228  +
         7229  +  /* All pages have been processed exactly once */
         7230  +  assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 );
         7231  +
  6586   7232     assert( nOld>0 );
  6587   7233     assert( nNew>0 );
  6588         -  if( (pageFlags & PTF_LEAF)==0 ){
  6589         -    u8 *zChild = &apCopy[nOld-1]->aData[8];
  6590         -    memcpy(&apNew[nNew-1]->aData[8], zChild, 4);
  6591         -  }
  6592   7234   
  6593   7235     if( isRoot && pParent->nCell==0 && pParent->hdrOffset<=apNew[0]->nFree ){
  6594   7236       /* The root page of the b-tree now contains no cells. The only sibling
  6595   7237       ** page is the right-child of the parent. Copy the contents of the
  6596   7238       ** child page into the parent, decreasing the overall height of the
  6597   7239       ** b-tree structure by one. This is described as the "balance-shallower"
  6598   7240       ** sub-algorithm in some documentation.
  6599   7241       **
  6600   7242       ** If this is an auto-vacuum database, the call to copyNodeContent() 
  6601   7243       ** sets all pointer-map entries corresponding to database image pages 
  6602   7244       ** for which the pointer is stored within the content being copied.
  6603   7245       **
  6604         -    ** The second assert below verifies that the child page is defragmented
  6605         -    ** (it must be, as it was just reconstructed using assemblePage()). This
  6606         -    ** is important if the parent page happens to be page 1 of the database
  6607         -    ** image.  */
         7246  +    ** It is critical that the child page be defragmented before being
         7247  +    ** copied into the parent, because if the parent is page 1 then it will
         7248  +    ** by smaller than the child due to the database header, and so all the
         7249  +    ** free space needs to be up front.
         7250  +    */
  6608   7251       assert( nNew==1 );
         7252  +    rc = defragmentPage(apNew[0]);
         7253  +    testcase( rc!=SQLITE_OK );
  6609   7254       assert( apNew[0]->nFree == 
  6610         -        (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) 
         7255  +        (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2)
         7256  +      || rc!=SQLITE_OK
  6611   7257       );
  6612   7258       copyNodeContent(apNew[0], pParent, &rc);
  6613   7259       freePage(apNew[0], &rc);
  6614         -  }else if( ISAUTOVACUUM ){
  6615         -    /* Fix the pointer-map entries for all the cells that were shifted around. 
  6616         -    ** There are several different types of pointer-map entries that need to
  6617         -    ** be dealt with by this routine. Some of these have been set already, but
  6618         -    ** many have not. The following is a summary:
  6619         -    **
  6620         -    **   1) The entries associated with new sibling pages that were not
  6621         -    **      siblings when this function was called. These have already
  6622         -    **      been set. We don't need to worry about old siblings that were
  6623         -    **      moved to the free-list - the freePage() code has taken care
  6624         -    **      of those.
  6625         -    **
  6626         -    **   2) The pointer-map entries associated with the first overflow
  6627         -    **      page in any overflow chains used by new divider cells. These 
  6628         -    **      have also already been taken care of by the insertCell() code.
  6629         -    **
  6630         -    **   3) If the sibling pages are not leaves, then the child pages of
  6631         -    **      cells stored on the sibling pages may need to be updated.
  6632         -    **
  6633         -    **   4) If the sibling pages are not internal intkey nodes, then any
  6634         -    **      overflow pages used by these cells may need to be updated
  6635         -    **      (internal intkey nodes never contain pointers to overflow pages).
  6636         -    **
  6637         -    **   5) If the sibling pages are not leaves, then the pointer-map
  6638         -    **      entries for the right-child pages of each sibling may need
  6639         -    **      to be updated.
  6640         -    **
  6641         -    ** Cases 1 and 2 are dealt with above by other code. The next
  6642         -    ** block deals with cases 3 and 4 and the one after that, case 5. Since
  6643         -    ** setting a pointer map entry is a relatively expensive operation, this
  6644         -    ** code only sets pointer map entries for child or overflow pages that have
  6645         -    ** actually moved between pages.  */
  6646         -    MemPage *pNew = apNew[0];
  6647         -    MemPage *pOld = apCopy[0];
  6648         -    int nOverflow = pOld->nOverflow;
  6649         -    int iNextOld = pOld->nCell + nOverflow;
  6650         -    int iOverflow = (nOverflow ? pOld->aiOvfl[0] : -1);
  6651         -    j = 0;                             /* Current 'old' sibling page */
  6652         -    k = 0;                             /* Current 'new' sibling page */
  6653         -    for(i=0; i<nCell; i++){
  6654         -      int isDivider = 0;
  6655         -      while( i==iNextOld ){
  6656         -        /* Cell i is the cell immediately following the last cell on old
  6657         -        ** sibling page j. If the siblings are not leaf pages of an
  6658         -        ** intkey b-tree, then cell i was a divider cell. */
  6659         -        assert( j+1 < ArraySize(apCopy) );
  6660         -        assert( j+1 < nOld );
  6661         -        pOld = apCopy[++j];
  6662         -        iNextOld = i + !leafData + pOld->nCell + pOld->nOverflow;
  6663         -        if( pOld->nOverflow ){
  6664         -          nOverflow = pOld->nOverflow;
  6665         -          iOverflow = i + !leafData + pOld->aiOvfl[0];
  6666         -        }
  6667         -        isDivider = !leafData;  
  6668         -      }
  6669         -
  6670         -      assert(nOverflow>0 || iOverflow<i );
  6671         -      assert(nOverflow<2 || pOld->aiOvfl[0]==pOld->aiOvfl[1]-1);
  6672         -      assert(nOverflow<3 || pOld->aiOvfl[1]==pOld->aiOvfl[2]-1);
  6673         -      if( i==iOverflow ){
  6674         -        isDivider = 1;
  6675         -        if( (--nOverflow)>0 ){
  6676         -          iOverflow++;
  6677         -        }
  6678         -      }
  6679         -
  6680         -      if( i==cntNew[k] ){
  6681         -        /* Cell i is the cell immediately following the last cell on new
  6682         -        ** sibling page k. If the siblings are not leaf pages of an
  6683         -        ** intkey b-tree, then cell i is a divider cell.  */
  6684         -        pNew = apNew[++k];
  6685         -        if( !leafData ) continue;
  6686         -      }
  6687         -      assert( j<nOld );
  6688         -      assert( k<nNew );
  6689         -
  6690         -      /* If the cell was originally divider cell (and is not now) or
  6691         -      ** an overflow cell, or if the cell was located on a different sibling
  6692         -      ** page before the balancing, then the pointer map entries associated
  6693         -      ** with any child or overflow pages need to be updated.  */
  6694         -      if( isDivider || pOld->pgno!=pNew->pgno ){
  6695         -        if( !leafCorrection ){
  6696         -          ptrmapPut(pBt, get4byte(apCell[i]), PTRMAP_BTREE, pNew->pgno, &rc);
  6697         -        }
  6698         -        if( szCell[i]>pNew->minLocal ){
  6699         -          ptrmapPutOvflPtr(pNew, apCell[i], &rc);
  6700         -        }
  6701         -      }
  6702         -    }
  6703         -
  6704         -    if( !leafCorrection ){
  6705         -      for(i=0; i<nNew; i++){
  6706         -        u32 key = get4byte(&apNew[i]->aData[8]);
  6707         -        ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc);
  6708         -      }
  6709         -    }
         7260  +  }else if( ISAUTOVACUUM && !leafCorrection ){
         7261  +    /* Fix the pointer map entries associated with the right-child of each
         7262  +    ** sibling page. All other pointer map entries have already been taken
         7263  +    ** care of.  */
         7264  +    for(i=0; i<nNew; i++){
         7265  +      u32 key = get4byte(&apNew[i]->aData[8]);
         7266  +      ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc);
         7267  +    }
         7268  +  }
         7269  +
         7270  +  assert( pParent->isInit );
         7271  +  TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n",
         7272  +          nOld, nNew, nCell));
         7273  +
         7274  +  /* Free any old pages that were not reused as new pages.
         7275  +  */
         7276  +  for(i=nNew; i<nOld; i++){
         7277  +    freePage(apOld[i], &rc);
         7278  +  }
  6710   7279   
  6711   7280   #if 0
         7281  +  if( ISAUTOVACUUM && rc==SQLITE_OK && apNew[0]->isInit ){
  6712   7282       /* The ptrmapCheckPages() contains assert() statements that verify that
  6713   7283       ** all pointer map pages are set correctly. This is helpful while 
  6714   7284       ** debugging. This is usually disabled because a corrupt database may
  6715   7285       ** cause an assert() statement to fail.  */
  6716   7286       ptrmapCheckPages(apNew, nNew);
  6717   7287       ptrmapCheckPages(&pParent, 1);
         7288  +  }
  6718   7289   #endif
  6719         -  }
  6720         -
  6721         -  assert( pParent->isInit );
  6722         -  TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n",
  6723         -          nOld, nNew, nCell));
  6724   7290   
  6725   7291     /*
  6726   7292     ** Cleanup before returning.
  6727   7293     */
  6728   7294   balance_cleanup:
  6729   7295     sqlite3ScratchFree(apCell);
  6730   7296     for(i=0; i<nOld; i++){
................................................................................
  6853   7419       }else{
  6854   7420         MemPage * const pParent = pCur->apPage[iPage-1];
  6855   7421         int const iIdx = pCur->aiIdx[iPage-1];
  6856   7422   
  6857   7423         rc = sqlite3PagerWrite(pParent->pDbPage);
  6858   7424         if( rc==SQLITE_OK ){
  6859   7425   #ifndef SQLITE_OMIT_QUICKBALANCE
  6860         -        if( pPage->hasData
         7426  +        if( pPage->intKeyLeaf
  6861   7427            && pPage->nOverflow==1
  6862   7428            && pPage->aiOvfl[0]==pPage->nCell
  6863   7429            && pParent->pgno!=1
  6864   7430            && pParent->nCell==iIdx
  6865   7431           ){
  6866   7432             /* Call balance_quick() to create a new sibling of pPage on which
  6867   7433             ** to store the overflow cell. balance_quick() inserts a new cell
  6868   7434             ** into pParent, which may cause pParent overflow. If this
  6869         -          ** happens, the next interation of the do-loop will balance pParent 
         7435  +          ** happens, the next iteration of the do-loop will balance pParent 
  6870   7436             ** use either balance_nonroot() or balance_deeper(). Until this
  6871   7437             ** happens, the overflow cell is stored in the aBalanceQuickSpace[]
  6872   7438             ** buffer. 
  6873   7439             **
  6874   7440             ** The purpose of the following assert() is to check that only a
  6875   7441             ** single call to balance_quick() is made for each call to this
  6876   7442             ** function. If this were not verified, a subtle bug involving reuse
................................................................................
  6939   7505   ** For an INTKEY table, only the nKey value of the key is used.  pKey is
  6940   7506   ** ignored.  For a ZERODATA table, the pData and nData are both ignored.
  6941   7507   **
  6942   7508   ** If the seekResult parameter is non-zero, then a successful call to
  6943   7509   ** MovetoUnpacked() to seek cursor pCur to (pKey, nKey) has already
  6944   7510   ** been performed. seekResult is the search result returned (a negative
  6945   7511   ** number if pCur points at an entry that is smaller than (pKey, nKey), or
  6946         -** a positive value if pCur points at an etry that is larger than 
         7512  +** a positive value if pCur points at an entry that is larger than 
  6947   7513   ** (pKey, nKey)). 
  6948   7514   **
  6949   7515   ** If the seekResult parameter is non-zero, then the caller guarantees that
  6950   7516   ** cursor pCur is pointing at the existing copy of a row that is to be
  6951   7517   ** overwritten.  If the seekResult parameter is 0, then cursor pCur may
  6952   7518   ** point to any entry or to no entry at all and so this function has to seek
  6953   7519   ** the cursor before the new key can be inserted.
................................................................................
  6972   7538   
  6973   7539     if( pCur->eState==CURSOR_FAULT ){
  6974   7540       assert( pCur->skipNext!=SQLITE_OK );
  6975   7541       return pCur->skipNext;
  6976   7542     }
  6977   7543   
  6978   7544     assert( cursorHoldsMutex(pCur) );
  6979         -  assert( (pCur->curFlags & BTCF_WriteFlag)!=0 && pBt->inTransaction==TRANS_WRITE
         7545  +  assert( (pCur->curFlags & BTCF_WriteFlag)!=0
         7546  +              && pBt->inTransaction==TRANS_WRITE
  6980   7547                 && (pBt->btsFlags & BTS_READ_ONLY)==0 );
  6981   7548     assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) );
  6982   7549   
  6983   7550     /* Assert that the caller has been consistent. If this cursor was opened
  6984   7551     ** expecting an index b-tree, then the caller should be inserting blob
  6985   7552     ** keys with no associated data. If the cursor was opened expecting an
  6986   7553     ** intkey table, the caller should be inserting integer keys with a
................................................................................
  7005   7572       /* If this is an insert into a table b-tree, invalidate any incrblob 
  7006   7573       ** cursors open on the row being replaced */
  7007   7574       invalidateIncrblobCursors(p, nKey, 0);
  7008   7575   
  7009   7576       /* If the cursor is currently on the last row and we are appending a
  7010   7577       ** new row onto the end, set the "loc" to avoid an unnecessary btreeMoveto()
  7011   7578       ** call */
  7012         -    if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 && pCur->info.nKey==nKey-1 ){
         7579  +    if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0
         7580  +      && pCur->info.nKey==nKey-1 ){
  7013   7581         loc = -1;
  7014   7582       }
  7015   7583     }
  7016   7584   
  7017   7585     if( !loc ){
  7018   7586       rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc);
  7019   7587       if( rc ) return rc;
................................................................................
  7024   7592     assert( pPage->intKey || nKey>=0 );
  7025   7593     assert( pPage->leaf || !pPage->intKey );
  7026   7594   
  7027   7595     TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n",
  7028   7596             pCur->pgnoRoot, nKey, nData, pPage->pgno,
  7029   7597             loc==0 ? "overwrite" : "new entry"));
  7030   7598     assert( pPage->isInit );
  7031         -  allocateTempSpace(pBt);
  7032   7599     newCell = pBt->pTmpSpace;
  7033         -  if( newCell==0 ) return SQLITE_NOMEM;
         7600  +  assert( newCell!=0 );
  7034   7601     rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew);
  7035   7602     if( rc ) goto end_insert;
  7036   7603     assert( szNew==cellSizePtr(pPage, newCell) );
  7037   7604     assert( szNew <= MX_CELL_SIZE(pBt) );
  7038   7605     idx = pCur->aiIdx[pCur->iPage];
  7039   7606     if( loc==0 ){
  7040   7607       u16 szOld;
................................................................................
  7043   7610       if( rc ){
  7044   7611         goto end_insert;
  7045   7612       }
  7046   7613       oldCell = findCell(pPage, idx);
  7047   7614       if( !pPage->leaf ){
  7048   7615         memcpy(newCell, oldCell, 4);
  7049   7616       }
  7050         -    szOld = cellSizePtr(pPage, oldCell);
  7051         -    rc = clearCell(pPage, oldCell);
         7617  +    rc = clearCell(pPage, oldCell, &szOld);
  7052   7618       dropCell(pPage, idx, szOld, &rc);
  7053   7619       if( rc ) goto end_insert;
  7054   7620     }else if( loc<0 && pPage->nCell>0 ){
  7055   7621       assert( pPage->leaf );
  7056   7622       idx = ++pCur->aiIdx[pCur->iPage];
  7057   7623     }else{
  7058   7624       assert( pPage->leaf );
................................................................................
  7096   7662   
  7097   7663   end_insert:
  7098   7664     return rc;
  7099   7665   }
  7100   7666   
  7101   7667   /*
  7102   7668   ** Delete the entry that the cursor is pointing to.  The cursor
  7103         -** is left pointing at a arbitrary location.
         7669  +** is left pointing at an arbitrary location.
  7104   7670   */
  7105   7671   int sqlite3BtreeDelete(BtCursor *pCur){
  7106   7672     Btree *p = pCur->pBtree;
  7107   7673     BtShared *pBt = p->pBt;              
  7108   7674     int rc;                              /* Return code */
  7109   7675     MemPage *pPage;                      /* Page to delete cell from */
  7110   7676     unsigned char *pCell;                /* Pointer to cell to delete */
  7111   7677     int iCellIdx;                        /* Index of cell to delete */
  7112   7678     int iCellDepth;                      /* Depth of node containing pCell */ 
         7679  +  u16 szCell;                          /* Size of the cell being deleted */
  7113   7680   
  7114   7681     assert( cursorHoldsMutex(pCur) );
  7115   7682     assert( pBt->inTransaction==TRANS_WRITE );
  7116   7683     assert( (pBt->btsFlags & BTS_READ_ONLY)==0 );
  7117   7684     assert( pCur->curFlags & BTCF_WriteFlag );
  7118   7685     assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) );
  7119   7686     assert( !hasReadConflicts(p, pCur->pgnoRoot) );
................................................................................
  7154   7721     ** invalidate any incrblob cursors open on the row being deleted.  */
  7155   7722     if( pCur->pKeyInfo==0 ){
  7156   7723       invalidateIncrblobCursors(p, pCur->info.nKey, 0);
  7157   7724     }
  7158   7725   
  7159   7726     rc = sqlite3PagerWrite(pPage->pDbPage);
  7160   7727     if( rc ) return rc;
  7161         -  rc = clearCell(pPage, pCell);
  7162         -  dropCell(pPage, iCellIdx, cellSizePtr(pPage, pCell), &rc);
         7728  +  rc = clearCell(pPage, pCell, &szCell);
         7729  +  dropCell(pPage, iCellIdx, szCell, &rc);
  7163   7730     if( rc ) return rc;
  7164   7731   
  7165   7732     /* If the cell deleted was not located on a leaf page, then the cursor
  7166   7733     ** is currently pointing to the largest entry in the sub-tree headed
  7167   7734     ** by the child-page of the cell that was just deleted from an internal
  7168   7735     ** node. The cell from the leaf node needs to be moved to the internal
  7169   7736     ** node to replace the deleted cell.  */
................................................................................
  7172   7739       int nCell;
  7173   7740       Pgno n = pCur->apPage[iCellDepth+1]->pgno;
  7174   7741       unsigned char *pTmp;
  7175   7742   
  7176   7743       pCell = findCell(pLeaf, pLeaf->nCell-1);
  7177   7744       nCell = cellSizePtr(pLeaf, pCell);
  7178   7745       assert( MX_CELL_SIZE(pBt) >= nCell );
  7179         -
  7180         -    allocateTempSpace(pBt);
  7181   7746       pTmp = pBt->pTmpSpace;
  7182         -
         7747  +    assert( pTmp!=0 );
  7183   7748       rc = sqlite3PagerWrite(pLeaf->pDbPage);
  7184   7749       insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc);
  7185   7750       dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc);
  7186   7751       if( rc ) return rc;
  7187   7752     }
  7188   7753   
  7189   7754     /* Balance the tree. If the entry deleted was located on a leaf page,
................................................................................
  7387   7952     int *pnChange            /* Add number of Cells freed to this counter */
  7388   7953   ){
  7389   7954     MemPage *pPage;
  7390   7955     int rc;
  7391   7956     unsigned char *pCell;
  7392   7957     int i;
  7393   7958     int hdr;
         7959  +  u16 szCell;
  7394   7960   
  7395   7961     assert( sqlite3_mutex_held(pBt->mutex) );
  7396   7962     if( pgno>btreePagecount(pBt) ){
  7397   7963       return SQLITE_CORRUPT_BKPT;
  7398   7964     }
  7399   7965   
  7400   7966     rc = getAndInitPage(pBt, pgno, &pPage, 0);
................................................................................
  7402   7968     hdr = pPage->hdrOffset;
  7403   7969     for(i=0; i<pPage->nCell; i++){
  7404   7970       pCell = findCell(pPage, i);
  7405   7971       if( !pPage->leaf ){
  7406   7972         rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange);
  7407   7973         if( rc ) goto cleardatabasepage_out;
  7408   7974       }
  7409         -    rc = clearCell(pPage, pCell);
         7975  +    rc = clearCell(pPage, pCell, &szCell);
  7410   7976       if( rc ) goto cleardatabasepage_out;
  7411   7977     }
  7412   7978     if( !pPage->leaf ){
  7413   7979       rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange);
  7414   7980       if( rc ) goto cleardatabasepage_out;
  7415   7981     }else if( pnChange ){
  7416   7982       assert( pPage->intKey );
................................................................................
  7608   8174   ** is the number of free pages currently in the database.  Meta[1]
  7609   8175   ** through meta[15] are available for use by higher layers.  Meta[0]
  7610   8176   ** is read-only, the others are read/write.
  7611   8177   ** 
  7612   8178   ** The schema layer numbers meta values differently.  At the schema
  7613   8179   ** layer (and the SetCookie and ReadCookie opcodes) the number of
  7614   8180   ** free pages is not visible.  So Cookie[0] is the same as Meta[1].
         8181  +**
         8182  +** This routine treats Meta[BTREE_DATA_VERSION] as a special case.  Instead
         8183  +** of reading the value out of the header, it instead loads the "DataVersion"
         8184  +** from the pager.  The BTREE_DATA_VERSION value is not actually stored in the
         8185  +** database file.  It is a number computed by the pager.  But its access
         8186  +** pattern is the same as header meta values, and so it is convenient to
         8187  +** read it from this routine.
  7615   8188   */
  7616   8189   void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){
  7617   8190     BtShared *pBt = p->pBt;
  7618   8191   
  7619   8192     sqlite3BtreeEnter(p);
  7620   8193     assert( p->inTrans>TRANS_NONE );
  7621   8194     assert( SQLITE_OK==querySharedCacheTableLock(p, MASTER_ROOT, READ_LOCK) );
  7622   8195     assert( pBt->pPage1 );
  7623   8196     assert( idx>=0 && idx<=15 );
  7624   8197   
  7625         -  *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]);
         8198  +  if( idx==BTREE_DATA_VERSION ){
         8199  +    *pMeta = sqlite3PagerDataVersion(pBt->pPager) + p->iDataVersion;
         8200  +  }else{
         8201  +    *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]);
         8202  +  }
  7626   8203   
  7627   8204     /* If auto-vacuum is disabled in this build and this is an auto-vacuum
  7628   8205     ** database, mark the database as read-only.  */
  7629   8206   #ifdef SQLITE_OMIT_AUTOVACUUM
  7630   8207     if( idx==BTREE_LARGEST_ROOT_PAGE && *pMeta>0 ){
  7631   8208       pBt->btsFlags |= BTS_READ_ONLY;
  7632   8209     }
................................................................................
  7709   8286       ** caller.
  7710   8287       */
  7711   8288       if( pPage->leaf ){
  7712   8289         do {
  7713   8290           if( pCur->iPage==0 ){
  7714   8291             /* All pages of the b-tree have been visited. Return successfully. */
  7715   8292             *pnEntry = nEntry;
  7716         -          return SQLITE_OK;
         8293  +          return moveToRoot(pCur);
  7717   8294           }
  7718   8295           moveToParent(pCur);
  7719   8296         }while ( pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell );
  7720   8297   
  7721   8298         pCur->aiIdx[pCur->iPage]++;
  7722   8299         pPage = pCur->apPage[pCur->iPage];
  7723   8300       }
................................................................................
  7748   8325   
  7749   8326   #ifndef SQLITE_OMIT_INTEGRITY_CHECK
  7750   8327   /*
  7751   8328   ** Append a message to the error message string.
  7752   8329   */
  7753   8330   static void checkAppendMsg(
  7754   8331     IntegrityCk *pCheck,
  7755         -  char *zMsg1,
  7756   8332     const char *zFormat,
  7757   8333     ...
  7758   8334   ){
  7759   8335     va_list ap;
         8336  +  char zBuf[200];
  7760   8337     if( !pCheck->mxErr ) return;
  7761   8338     pCheck->mxErr--;
  7762   8339     pCheck->nErr++;
  7763   8340     va_start(ap, zFormat);
  7764   8341     if( pCheck->errMsg.nChar ){
  7765   8342       sqlite3StrAccumAppend(&pCheck->errMsg, "\n", 1);
  7766   8343     }
  7767         -  if( zMsg1 ){
  7768         -    sqlite3StrAccumAppendAll(&pCheck->errMsg, zMsg1);
         8344  +  if( pCheck->zPfx ){
         8345  +    sqlite3_snprintf(sizeof(zBuf), zBuf, pCheck->zPfx, pCheck->v1, pCheck->v2);
         8346  +    sqlite3StrAccumAppendAll(&pCheck->errMsg, zBuf);
  7769   8347     }
  7770   8348     sqlite3VXPrintf(&pCheck->errMsg, 1, zFormat, ap);
  7771   8349     va_end(ap);
  7772   8350     if( pCheck->errMsg.accError==STRACCUM_NOMEM ){
  7773   8351       pCheck->mallocFailed = 1;
  7774   8352     }
  7775   8353   }
................................................................................
  7794   8372     pCheck->aPgRef[iPg/8] |= (1 << (iPg & 0x07));
  7795   8373   }
  7796   8374   
  7797   8375   
  7798   8376   /*
  7799   8377   ** Add 1 to the reference count for page iPage.  If this is the second
  7800   8378   ** reference to the page, add an error message to pCheck->zErrMsg.
  7801         -** Return 1 if there are 2 ore more references to the page and 0 if
         8379  +** Return 1 if there are 2 or more references to the page and 0 if
  7802   8380   ** if this is the first reference to the page.
  7803   8381   **
  7804   8382   ** Also check that the page number is in bounds.
  7805   8383   */
  7806         -static int checkRef(IntegrityCk *pCheck, Pgno iPage, char *zContext){
         8384  +static int checkRef(IntegrityCk *pCheck, Pgno iPage){
  7807   8385     if( iPage==0 ) return 1;
  7808   8386     if( iPage>pCheck->nPage ){
  7809         -    checkAppendMsg(pCheck, zContext, "invalid page number %d", iPage);
         8387  +    checkAppendMsg(pCheck, "invalid page number %d", iPage);
  7810   8388       return 1;
  7811   8389     }
  7812   8390     if( getPageReferenced(pCheck, iPage) ){
  7813         -    checkAppendMsg(pCheck, zContext, "2nd reference to page %d", iPage);
         8391  +    checkAppendMsg(pCheck, "2nd reference to page %d", iPage);
  7814   8392       return 1;
  7815   8393     }
  7816   8394     setPageReferenced(pCheck, iPage);
  7817   8395     return 0;
  7818   8396   }
  7819   8397   
  7820   8398   #ifndef SQLITE_OMIT_AUTOVACUUM
................................................................................
  7823   8401   ** page iParent, pointer type ptrType. If not, append an error message
  7824   8402   ** to pCheck.
  7825   8403   */
  7826   8404   static void checkPtrmap(
  7827   8405     IntegrityCk *pCheck,   /* Integrity check context */
  7828   8406     Pgno iChild,           /* Child page number */
  7829   8407     u8 eType,              /* Expected pointer map type */
  7830         -  Pgno iParent,          /* Expected pointer map parent page number */
  7831         -  char *zContext         /* Context description (used for error msg) */
         8408  +  Pgno iParent           /* Expected pointer map parent page number */
  7832   8409   ){
  7833   8410     int rc;
  7834   8411     u8 ePtrmapType;
  7835   8412     Pgno iPtrmapParent;
  7836   8413   
  7837   8414     rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent);
  7838   8415     if( rc!=SQLITE_OK ){
  7839   8416       if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) pCheck->mallocFailed = 1;
  7840         -    checkAppendMsg(pCheck, zContext, "Failed to read ptrmap key=%d", iChild);
         8417  +    checkAppendMsg(pCheck, "Failed to read ptrmap key=%d", iChild);
  7841   8418       return;
  7842   8419     }
  7843   8420   
  7844   8421     if( ePtrmapType!=eType || iPtrmapParent!=iParent ){
  7845         -    checkAppendMsg(pCheck, zContext, 
         8422  +    checkAppendMsg(pCheck,
  7846   8423         "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)", 
  7847   8424         iChild, eType, iParent, ePtrmapType, iPtrmapParent);
  7848   8425     }
  7849   8426   }
  7850   8427   #endif
  7851   8428   
  7852   8429   /*
................................................................................
  7853   8430   ** Check the integrity of the freelist or of an overflow page list.
  7854   8431   ** Verify that the number of pages on the list is N.
  7855   8432   */
  7856   8433   static void checkList(
  7857   8434     IntegrityCk *pCheck,  /* Integrity checking context */
  7858   8435     int isFreeList,       /* True for a freelist.  False for overflow page list */
  7859   8436     int iPage,            /* Page number for first page in the list */
  7860         -  int N,                /* Expected number of pages in the list */
  7861         -  char *zContext        /* Context for error messages */
         8437  +  int N                 /* Expected number of pages in the list */
  7862   8438   ){
  7863   8439     int i;
  7864   8440     int expected = N;
  7865   8441     int iFirst = iPage;
  7866   8442     while( N-- > 0 && pCheck->mxErr ){
  7867   8443       DbPage *pOvflPage;
  7868   8444       unsigned char *pOvflData;
  7869   8445       if( iPage<1 ){
  7870         -      checkAppendMsg(pCheck, zContext,
         8446  +      checkAppendMsg(pCheck,
  7871   8447            "%d of %d pages missing from overflow list starting at %d",
  7872   8448             N+1, expected, iFirst);
  7873   8449         break;
  7874   8450       }
  7875         -    if( checkRef(pCheck, iPage, zContext) ) break;
         8451  +    if( checkRef(pCheck, iPage) ) break;
  7876   8452       if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage) ){
  7877         -      checkAppendMsg(pCheck, zContext, "failed to get page %d", iPage);
         8453  +      checkAppendMsg(pCheck, "failed to get page %d", iPage);
  7878   8454         break;
  7879   8455       }
  7880   8456       pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage);
  7881   8457       if( isFreeList ){
  7882   8458         int n = get4byte(&pOvflData[4]);
  7883   8459   #ifndef SQLITE_OMIT_AUTOVACUUM
  7884   8460         if( pCheck->pBt->autoVacuum ){
  7885         -        checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0, zContext);
         8461  +        checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0);
  7886   8462         }
  7887   8463   #endif
  7888   8464         if( n>(int)pCheck->pBt->usableSize/4-2 ){
  7889         -        checkAppendMsg(pCheck, zContext,
         8465  +        checkAppendMsg(pCheck,
  7890   8466              "freelist leaf count too big on page %d", iPage);
  7891   8467           N--;
  7892   8468         }else{
  7893   8469           for(i=0; i<n; i++){
  7894   8470             Pgno iFreePage = get4byte(&pOvflData[8+i*4]);
  7895   8471   #ifndef SQLITE_OMIT_AUTOVACUUM
  7896   8472             if( pCheck->pBt->autoVacuum ){
  7897         -            checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0, zContext);
         8473  +            checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0);
  7898   8474             }
  7899   8475   #endif
  7900         -          checkRef(pCheck, iFreePage, zContext);
         8476  +          checkRef(pCheck, iFreePage);
  7901   8477           }
  7902   8478           N -= n;
  7903   8479         }
  7904   8480       }
  7905   8481   #ifndef SQLITE_OMIT_AUTOVACUUM
  7906   8482       else{
  7907   8483         /* If this database supports auto-vacuum and iPage is not the last
  7908   8484         ** page in this overflow list, check that the pointer-map entry for
  7909   8485         ** the following page matches iPage.
  7910   8486         */
  7911   8487         if( pCheck->pBt->autoVacuum && N>0 ){
  7912   8488           i = get4byte(pOvflData);
  7913         -        checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage, zContext);
         8489  +        checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage);
  7914   8490         }
  7915   8491       }
  7916   8492   #endif
  7917   8493       iPage = get4byte(pOvflData);
  7918   8494       sqlite3PagerUnref(pOvflPage);
  7919   8495     }
  7920   8496   }
................................................................................
  7938   8514   **      7.  Verify that the depth of all children is the same.
  7939   8515   **      8.  Make sure this page is at least 33% full or else it is
  7940   8516   **          the root of the tree.
  7941   8517   */
  7942   8518   static int checkTreePage(
  7943   8519     IntegrityCk *pCheck,  /* Context for the sanity check */
  7944   8520     int iPage,            /* Page number of the page to check */
  7945         -  char *zParentContext, /* Parent context */
  7946   8521     i64 *pnParentMinKey, 
  7947   8522     i64 *pnParentMaxKey
  7948   8523   ){
  7949   8524     MemPage *pPage;
  7950   8525     int i, rc, depth, d2, pgno, cnt;
  7951   8526     int hdr, cellStart;
  7952   8527     int nCell;
  7953   8528     u8 *data;
  7954   8529     BtShared *pBt;
  7955   8530     int usableSize;
  7956         -  char zContext[100];
  7957   8531     char *hit = 0;
  7958   8532     i64 nMinKey = 0;
  7959   8533     i64 nMaxKey = 0;
  7960         -
  7961         -  sqlite3_snprintf(sizeof(zContext), zContext, "Page %d: ", iPage);
         8534  +  const char *saved_zPfx = pCheck->zPfx;
         8535  +  int saved_v1 = pCheck->v1;
         8536  +  int saved_v2 = pCheck->v2;
  7962   8537   
  7963   8538     /* Check that the page exists
  7964   8539     */
  7965   8540     pBt = pCheck->pBt;
  7966   8541     usableSize = pBt->usableSize;
  7967   8542     if( iPage==0 ) return 0;
  7968         -  if( checkRef(pCheck, iPage, zParentContext) ) return 0;
         8543  +  if( checkRef(pCheck, iPage) ) return 0;
         8544  +  pCheck->zPfx = "Page %d: ";
         8545  +  pCheck->v1 = iPage;
  7969   8546     if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){
  7970         -    checkAppendMsg(pCheck, zContext,
         8547  +    checkAppendMsg(pCheck,
  7971   8548          "unable to get the page. error code=%d", rc);
  7972         -    return 0;
         8549  +    depth = -1;
         8550  +    goto end_of_check;
  7973   8551     }
  7974   8552   
  7975   8553     /* Clear MemPage.isInit to make sure the corruption detection code in
  7976   8554     ** btreeInitPage() is executed.  */
  7977   8555     pPage->isInit = 0;
  7978   8556     if( (rc = btreeInitPage(pPage))!=0 ){
  7979   8557       assert( rc==SQLITE_CORRUPT );  /* The only possible error from InitPage */
  7980         -    checkAppendMsg(pCheck, zContext, 
         8558  +    checkAppendMsg(pCheck,
  7981   8559                      "btreeInitPage() returns error code %d", rc);
  7982   8560       releasePage(pPage);
  7983         -    return 0;
         8561  +    depth = -1;
         8562  +    goto end_of_check;
  7984   8563     }
  7985   8564   
  7986   8565     /* Check out all the cells.
  7987   8566     */
  7988   8567     depth = 0;
  7989   8568     for(i=0; i<pPage->nCell && pCheck->mxErr; i++){
  7990   8569       u8 *pCell;
  7991   8570       u32 sz;
  7992   8571       CellInfo info;
  7993   8572   
  7994   8573       /* Check payload overflow pages
  7995   8574       */
  7996         -    sqlite3_snprintf(sizeof(zContext), zContext,
  7997         -             "On tree page %d cell %d: ", iPage, i);
         8575  +    pCheck->zPfx = "On tree page %d cell %d: ";
         8576  +    pCheck->v1 = iPage;
         8577  +    pCheck->v2 = i;
  7998   8578       pCell = findCell(pPage,i);
  7999   8579       btreeParseCellPtr(pPage, pCell, &info);
  8000         -    sz = info.nData;
  8001         -    if( !pPage->intKey ) sz += (int)info.nKey;
         8580  +    sz = info.nPayload;
  8002   8581       /* For intKey pages, check that the keys are in order.
  8003   8582       */
  8004         -    else if( i==0 ) nMinKey = nMaxKey = info.nKey;
  8005         -    else{
  8006         -      if( info.nKey <= nMaxKey ){
  8007         -        checkAppendMsg(pCheck, zContext, 
  8008         -            "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey);
         8583  +    if( pPage->intKey ){
         8584  +      if( i==0 ){
         8585  +        nMinKey = nMaxKey = info.nKey;
         8586  +      }else if( info.nKey <= nMaxKey ){
         8587  +        checkAppendMsg(pCheck,
         8588  +           "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey);
  8009   8589         }
  8010   8590         nMaxKey = info.nKey;
  8011   8591       }
  8012         -    assert( sz==info.nPayload );
  8013   8592       if( (sz>info.nLocal) 
  8014   8593        && (&pCell[info.iOverflow]<=&pPage->aData[pBt->usableSize])
  8015   8594       ){
  8016   8595         int nPage = (sz - info.nLocal + usableSize - 5)/(usableSize - 4);
  8017   8596         Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]);
  8018   8597   #ifndef SQLITE_OMIT_AUTOVACUUM
  8019   8598         if( pBt->autoVacuum ){
  8020         -        checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage, zContext);
         8599  +        checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage);
  8021   8600         }
  8022   8601   #endif
  8023         -      checkList(pCheck, 0, pgnoOvfl, nPage, zContext);
         8602  +      checkList(pCheck, 0, pgnoOvfl, nPage);
  8024   8603       }
  8025   8604   
  8026   8605       /* Check sanity of left child page.
  8027   8606       */
  8028   8607       if( !pPage->leaf ){
  8029   8608         pgno = get4byte(pCell);
  8030   8609   #ifndef SQLITE_OMIT_AUTOVACUUM
  8031   8610         if( pBt->autoVacuum ){
  8032         -        checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext);
         8611  +        checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage);
  8033   8612         }
  8034   8613   #endif
  8035         -      d2 = checkTreePage(pCheck, pgno, zContext, &nMinKey, i==0 ? NULL : &nMaxKey);
         8614  +      d2 = checkTreePage(pCheck, pgno, &nMinKey, i==0?NULL:&nMaxKey);
  8036   8615         if( i>0 && d2!=depth ){
  8037         -        checkAppendMsg(pCheck, zContext, "Child page depth differs");
         8616  +        checkAppendMsg(pCheck, "Child page depth differs");
  8038   8617         }
  8039   8618         depth = d2;
  8040   8619       }
  8041   8620     }
  8042   8621   
  8043   8622     if( !pPage->leaf ){
  8044   8623       pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]);
  8045         -    sqlite3_snprintf(sizeof(zContext), zContext, 
  8046         -                     "On page %d at right child: ", iPage);
         8624  +    pCheck->zPfx = "On page %d at right child: ";
         8625  +    pCheck->v1 = iPage;
  8047   8626   #ifndef SQLITE_OMIT_AUTOVACUUM
  8048   8627       if( pBt->autoVacuum ){
  8049         -      checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext);
         8628  +      checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage);
  8050   8629       }
  8051   8630   #endif
  8052         -    checkTreePage(pCheck, pgno, zContext, NULL, !pPage->nCell ? NULL : &nMaxKey);
         8631  +    checkTreePage(pCheck, pgno, NULL, !pPage->nCell?NULL:&nMaxKey);
  8053   8632     }
  8054   8633    
  8055   8634     /* For intKey leaf pages, check that the min/max keys are in order
  8056   8635     ** with any left/parent/right pages.
  8057   8636     */
         8637  +  pCheck->zPfx = "Page %d: ";
         8638  +  pCheck->v1 = iPage;
  8058   8639     if( pPage->leaf && pPage->intKey ){
  8059   8640       /* if we are a left child page */
  8060   8641       if( pnParentMinKey ){
  8061   8642         /* if we are the left most child page */
  8062   8643         if( !pnParentMaxKey ){
  8063   8644           if( nMaxKey > *pnParentMinKey ){
  8064         -          checkAppendMsg(pCheck, zContext, 
         8645  +          checkAppendMsg(pCheck,
  8065   8646                 "Rowid %lld out of order (max larger than parent min of %lld)",
  8066   8647                 nMaxKey, *pnParentMinKey);
  8067   8648           }
  8068   8649         }else{
  8069   8650           if( nMinKey <= *pnParentMinKey ){
  8070         -          checkAppendMsg(pCheck, zContext, 
         8651  +          checkAppendMsg(pCheck,
  8071   8652                 "Rowid %lld out of order (min less than parent min of %lld)",
  8072   8653                 nMinKey, *pnParentMinKey);
  8073   8654           }
  8074   8655           if( nMaxKey > *pnParentMaxKey ){
  8075         -          checkAppendMsg(pCheck, zContext, 
         8656  +          checkAppendMsg(pCheck,
  8076   8657                 "Rowid %lld out of order (max larger than parent max of %lld)",
  8077   8658                 nMaxKey, *pnParentMaxKey);
  8078   8659           }
  8079   8660           *pnParentMinKey = nMaxKey;
  8080   8661         }
  8081   8662       /* else if we're a right child page */
  8082   8663       } else if( pnParentMaxKey ){
  8083   8664         if( nMinKey <= *pnParentMaxKey ){
  8084         -        checkAppendMsg(pCheck, zContext, 
         8665  +        checkAppendMsg(pCheck,
  8085   8666               "Rowid %lld out of order (min less than parent max of %lld)",
  8086   8667               nMinKey, *pnParentMaxKey);
  8087   8668         }
  8088   8669       }
  8089   8670     }
  8090   8671   
  8091   8672     /* Check for complete coverage of the page
  8092   8673     */
  8093   8674     data = pPage->aData;
  8094   8675     hdr = pPage->hdrOffset;
  8095   8676     hit = sqlite3PageMalloc( pBt->pageSize );
         8677  +  pCheck->zPfx = 0;
  8096   8678     if( hit==0 ){
  8097   8679       pCheck->mallocFailed = 1;
  8098   8680     }else{
  8099   8681       int contentOffset = get2byteNotZero(&data[hdr+5]);
  8100   8682       assert( contentOffset<=usableSize );  /* Enforced by btreeInitPage() */
  8101   8683       memset(hit+contentOffset, 0, usableSize-contentOffset);
  8102   8684       memset(hit, 1, contentOffset);
         8685  +    /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the
         8686  +    ** number of cells on the page. */
  8103   8687       nCell = get2byte(&data[hdr+3]);
         8688  +    /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page
         8689  +    ** immediately follows the b-tree page header. */
  8104   8690       cellStart = hdr + 12 - 4*pPage->leaf;
         8691  +    /* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte
         8692  +    ** integer offsets to the cell contents. */
  8105   8693       for(i=0; i<nCell; i++){
  8106   8694         int pc = get2byte(&data[cellStart+i*2]);
  8107   8695         u32 size = 65536;
  8108   8696         int j;
  8109   8697         if( pc<=usableSize-4 ){
  8110   8698           size = cellSizePtr(pPage, &data[pc]);
  8111   8699         }
  8112   8700         if( (int)(pc+size-1)>=usableSize ){
  8113         -        checkAppendMsg(pCheck, 0, 
         8701  +        pCheck->zPfx = 0;
         8702  +        checkAppendMsg(pCheck,
  8114   8703               "Corruption detected in cell %d on page %d",i,iPage);
  8115   8704         }else{
  8116   8705           for(j=pc+size-1; j>=pc; j--) hit[j]++;
  8117   8706         }
  8118   8707       }
         8708  +    /* EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header
         8709  +    ** is the offset of the first freeblock, or zero if there are no
         8710  +    ** freeblocks on the page. */
  8119   8711       i = get2byte(&data[hdr+1]);
  8120   8712       while( i>0 ){
  8121   8713         int size, j;
  8122   8714         assert( i<=usableSize-4 );     /* Enforced by btreeInitPage() */
  8123   8715         size = get2byte(&data[i+2]);
  8124   8716         assert( i+size<=usableSize );  /* Enforced by btreeInitPage() */
  8125   8717         for(j=i+size-1; j>=i; j--) hit[j]++;
         8718  +      /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a
         8719  +      ** big-endian integer which is the offset in the b-tree page of the next
         8720  +      ** freeblock in the chain, or zero if the freeblock is the last on the
         8721  +      ** chain. */
  8126   8722         j = get2byte(&data[i]);
         8723  +      /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of
         8724  +      ** increasing offset. */
  8127   8725         assert( j==0 || j>i+size );  /* Enforced by btreeInitPage() */
  8128   8726         assert( j<=usableSize-4 );   /* Enforced by btreeInitPage() */
  8129   8727         i = j;
  8130   8728       }
  8131   8729       for(i=cnt=0; i<usableSize; i++){
  8132   8730         if( hit[i]==0 ){
  8133   8731           cnt++;
  8134   8732         }else if( hit[i]>1 ){
  8135         -        checkAppendMsg(pCheck, 0,
         8733  +        checkAppendMsg(pCheck,
  8136   8734             "Multiple uses for byte %d of page %d", i, iPage);
  8137   8735           break;
  8138   8736         }
  8139   8737       }
         8738  +    /* EVIDENCE-OF: R-43263-13491 The total number of bytes in all fragments
         8739  +    ** is stored in the fifth field of the b-tree page header.
         8740  +    ** EVIDENCE-OF: R-07161-27322 The one-byte integer at offset 7 gives the
         8741  +    ** number of fragmented free bytes within the cell content area.
         8742  +    */
  8140   8743       if( cnt!=data[hdr+7] ){
  8141         -      checkAppendMsg(pCheck, 0, 
         8744  +      checkAppendMsg(pCheck,
  8142   8745             "Fragmentation of %d bytes reported as %d on page %d",
  8143   8746             cnt, data[hdr+7], iPage);
  8144   8747       }
  8145   8748     }
  8146   8749     sqlite3PageFree(hit);
  8147   8750     releasePage(pPage);
         8751  +
         8752  +end_of_check:
         8753  +  pCheck->zPfx = saved_zPfx;
         8754  +  pCheck->v1 = saved_v1;
         8755  +  pCheck->v2 = saved_v2;
  8148   8756     return depth+1;
  8149   8757   }
  8150   8758   #endif /* SQLITE_OMIT_INTEGRITY_CHECK */
  8151   8759   
  8152   8760   #ifndef SQLITE_OMIT_INTEGRITY_CHECK
  8153   8761   /*
  8154   8762   ** This routine does a complete check of the given BTree file.  aRoot[] is
................................................................................
  8181   8789     nRef = sqlite3PagerRefcount(pBt->pPager);
  8182   8790     sCheck.pBt = pBt;
  8183   8791     sCheck.pPager = pBt->pPager;
  8184   8792     sCheck.nPage = btreePagecount(sCheck.pBt);
  8185   8793     sCheck.mxErr = mxErr;
  8186   8794     sCheck.nErr = 0;
  8187   8795     sCheck.mallocFailed = 0;
         8796  +  sCheck.zPfx = 0;
         8797  +  sCheck.v1 = 0;
         8798  +  sCheck.v2 = 0;
  8188   8799     *pnErr = 0;
  8189   8800     if( sCheck.nPage==0 ){
  8190   8801       sqlite3BtreeLeave(p);
  8191   8802       return 0;
  8192   8803     }
  8193   8804   
  8194   8805     sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1);
................................................................................
  8200   8811     i = PENDING_BYTE_PAGE(pBt);
  8201   8812     if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i);
  8202   8813     sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), SQLITE_MAX_LENGTH);
  8203   8814     sCheck.errMsg.useMalloc = 2;
  8204   8815   
  8205   8816     /* Check the integrity of the freelist
  8206   8817     */
         8818  +  sCheck.zPfx = "Main freelist: ";
  8207   8819     checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]),
  8208         -            get4byte(&pBt->pPage1->aData[36]), "Main freelist: ");
         8820  +            get4byte(&pBt->pPage1->aData[36]));
         8821  +  sCheck.zPfx = 0;
  8209   8822   
  8210   8823     /* Check all the tables.
  8211   8824     */
  8212   8825     for(i=0; (int)i<nRoot && sCheck.mxErr; i++){
  8213   8826       if( aRoot[i]==0 ) continue;
  8214   8827   #ifndef SQLITE_OMIT_AUTOVACUUM
  8215   8828       if( pBt->autoVacuum && aRoot[i]>1 ){
  8216         -      checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0, 0);
         8829  +      checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0);
  8217   8830       }
  8218   8831   #endif
  8219         -    checkTreePage(&sCheck, aRoot[i], "List of tree roots: ", NULL, NULL);
         8832  +    sCheck.zPfx = "List of tree roots: ";
         8833  +    checkTreePage(&sCheck, aRoot[i], NULL, NULL);
         8834  +    sCheck.zPfx = 0;
  8220   8835     }
  8221   8836   
  8222   8837     /* Make sure every page in the file is referenced
  8223   8838     */
  8224   8839     for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){
  8225   8840   #ifdef SQLITE_OMIT_AUTOVACUUM
  8226   8841       if( getPageReferenced(&sCheck, i)==0 ){
  8227         -      checkAppendMsg(&sCheck, 0, "Page %d is never used", i);
         8842  +      checkAppendMsg(&sCheck, "Page %d is never used", i);
  8228   8843       }
  8229   8844   #else
  8230   8845       /* If the database supports auto-vacuum, make sure no tables contain
  8231   8846       ** references to pointer-map pages.
  8232   8847       */
  8233   8848       if( getPageReferenced(&sCheck, i)==0 && 
  8234   8849          (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){
  8235         -      checkAppendMsg(&sCheck, 0, "Page %d is never used", i);
         8850  +      checkAppendMsg(&sCheck, "Page %d is never used", i);
  8236   8851       }
  8237   8852       if( getPageReferenced(&sCheck, i)!=0 && 
  8238   8853          (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){
  8239         -      checkAppendMsg(&sCheck, 0, "Pointer map page %d is referenced", i);
         8854  +      checkAppendMsg(&sCheck, "Pointer map page %d is referenced", i);
  8240   8855       }
  8241   8856   #endif
  8242   8857     }
  8243   8858   
  8244   8859     /* Make sure this analysis did not leave any unref() pages.
  8245   8860     ** This is an internal consistency check; an integrity check
  8246   8861     ** of the integrity check.
  8247   8862     */
  8248   8863     if( NEVER(nRef != sqlite3PagerRefcount(pBt->pPager)) ){
  8249         -    checkAppendMsg(&sCheck, 0, 
         8864  +    checkAppendMsg(&sCheck,
  8250   8865         "Outstanding page count goes from %d to %d during this analysis",
  8251   8866         nRef, sqlite3PagerRefcount(pBt->pPager)
  8252   8867       );
  8253   8868     }
  8254   8869   
  8255   8870     /* Clean  up and report errors.
  8256   8871     */
................................................................................
  8438   9053       return SQLITE_ABORT;
  8439   9054     }
  8440   9055   
  8441   9056     /* Save the positions of all other cursors open on this table. This is
  8442   9057     ** required in case any of them are holding references to an xFetch
  8443   9058     ** version of the b-tree page modified by the accessPayload call below.
  8444   9059     **
  8445         -  ** Note that pCsr must be open on a BTREE_INTKEY table and saveCursorPosition()
         9060  +  ** Note that pCsr must be open on a INTKEY table and saveCursorPosition()
  8446   9061     ** and hence saveAllCursors() cannot fail on a BTREE_INTKEY table, hence
  8447   9062     ** saveAllCursors can only return SQLITE_OK.
  8448   9063     */
  8449   9064     VVA_ONLY(rc =) saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr);
  8450   9065     assert( rc==SQLITE_OK );
  8451   9066   
  8452   9067     /* Check some assumptions: 
................................................................................
  8523   9138   
  8524   9139   /*
  8525   9140   ** Return true if the given Btree is read-only.
  8526   9141   */
  8527   9142   int sqlite3BtreeIsReadonly(Btree *p){
  8528   9143     return (p->pBt->btsFlags & BTS_READ_ONLY)!=0;
  8529   9144   }
         9145  +
         9146  +/*
         9147  +** Return the size of the header added to each page by this module.
         9148  +*/
         9149  +int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); }

Changes to src/btree.h.

    15     15   */
    16     16   #ifndef _BTREE_H_
    17     17   #define _BTREE_H_
    18     18   
    19     19   /* TODO: This definition is just included so other modules compile. It
    20     20   ** needs to be revisited.
    21     21   */
    22         -#define SQLITE_N_BTREE_META 10
           22  +#define SQLITE_N_BTREE_META 16
    23     23   
    24     24   /*
    25     25   ** If defined as non-zero, auto-vacuum is enabled by default. Otherwise
    26     26   ** it must be turned on for each database using "PRAGMA auto_vacuum = 1".
    27     27   */
    28     28   #ifndef SQLITE_DEFAULT_AUTOVACUUM
    29     29     #define SQLITE_DEFAULT_AUTOVACUUM 0
................................................................................
    79     79   #endif
    80     80   int sqlite3BtreeSetAutoVacuum(Btree *, int);
    81     81   int sqlite3BtreeGetAutoVacuum(Btree *);
    82     82   int sqlite3BtreeBeginTrans(Btree*,int);
    83     83   int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster);
    84     84   int sqlite3BtreeCommitPhaseTwo(Btree*, int);
    85     85   int sqlite3BtreeCommit(Btree*);
    86         -int sqlite3BtreeRollback(Btree*,int);
           86  +int sqlite3BtreeRollback(Btree*,int,int);
    87     87   int sqlite3BtreeBeginStmt(Btree*,int);
    88     88   int sqlite3BtreeCreateTable(Btree*, int*, int flags);
    89     89   int sqlite3BtreeIsInTrans(Btree*);
    90     90   int sqlite3BtreeIsInReadTrans(Btree*);
    91     91   int sqlite3BtreeIsInBackup(Btree*);
    92     92   void *sqlite3BtreeSchema(Btree *, int, void(*)(void *));
    93     93   int sqlite3BtreeSchemaLocked(Btree *pBtree);
................................................................................
   112    112   */
   113    113   #define BTREE_INTKEY     1    /* Table has only 64-bit signed integer keys */
   114    114   #define BTREE_BLOBKEY    2    /* Table has keys only - no data */
   115    115   
   116    116   int sqlite3BtreeDropTable(Btree*, int, int*);
   117    117   int sqlite3BtreeClearTable(Btree*, int, int*);
   118    118   int sqlite3BtreeClearTableOfCursor(BtCursor*);
   119         -void sqlite3BtreeTripAllCursors(Btree*, int);
          119  +int sqlite3BtreeTripAllCursors(Btree*, int, int);
   120    120   
   121    121   void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue);
   122    122   int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value);
   123    123   
   124    124   int sqlite3BtreeNewDb(Btree *p);
   125    125   
   126    126   /*
................................................................................
   130    130   ** SQLite database header may be found using the following formula:
   131    131   **
   132    132   **   offset = 36 + (idx * 4)
   133    133   **
   134    134   ** For example, the free-page-count field is located at byte offset 36 of
   135    135   ** the database file header. The incr-vacuum-flag field is located at
   136    136   ** byte offset 64 (== 36+4*7).
          137  +**
          138  +** The BTREE_DATA_VERSION value is not really a value stored in the header.
          139  +** It is a read-only number computed by the pager.  But we merge it with
          140  +** the header value access routines since its access pattern is the same.
          141  +** Call it a "virtual meta value".
   137    142   */
   138    143   #define BTREE_FREE_PAGE_COUNT     0
   139    144   #define BTREE_SCHEMA_VERSION      1
   140    145   #define BTREE_FILE_FORMAT         2
   141    146   #define BTREE_DEFAULT_CACHE_SIZE  3
   142    147   #define BTREE_LARGEST_ROOT_PAGE   4
   143    148   #define BTREE_TEXT_ENCODING       5
   144    149   #define BTREE_USER_VERSION        6
   145    150   #define BTREE_INCR_VACUUM         7
   146    151   #define BTREE_APPLICATION_ID      8
          152  +#define BTREE_DATA_VERSION        15  /* A virtual meta-value */
   147    153   
   148    154   /*
   149    155   ** Values that may be OR'd together to form the second argument of an
   150    156   ** sqlite3BtreeCursorHints() call.
   151    157   */
   152    158   #define BTREE_BULKLOAD 0x00000001
   153    159   
................................................................................
   165    171   int sqlite3BtreeMovetoUnpacked(
   166    172     BtCursor*,
   167    173     UnpackedRecord *pUnKey,
   168    174     i64 intKey,
   169    175     int bias,
   170    176     int *pRes
   171    177   );
   172         -int sqlite3BtreeCursorHasMoved(BtCursor*, int*);
          178  +int sqlite3BtreeCursorHasMoved(BtCursor*);
          179  +int sqlite3BtreeCursorRestore(BtCursor*, int*);
   173    180   int sqlite3BtreeDelete(BtCursor*);
   174    181   int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey,
   175    182                                     const void *pData, int nData,
   176    183                                     int nZero, int bias, int seekResult);
   177    184   int sqlite3BtreeFirst(BtCursor*, int *pRes);
   178    185   int sqlite3BtreeLast(BtCursor*, int *pRes);
   179    186   int sqlite3BtreeNext(BtCursor*, int *pRes);
................................................................................
   191    198   
   192    199   int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*);
   193    200   void sqlite3BtreeIncrblobCursor(BtCursor *);
   194    201   void sqlite3BtreeClearCursor(BtCursor *);
   195    202   int sqlite3BtreeSetVersion(Btree *pBt, int iVersion);
   196    203   void sqlite3BtreeCursorHints(BtCursor *, unsigned int mask);
   197    204   int sqlite3BtreeIsReadonly(Btree *pBt);
          205  +int sqlite3HeaderSizeBtree(void);
   198    206   
   199    207   #ifndef NDEBUG
   200    208   int sqlite3BtreeCursorIsValid(BtCursor*);
   201    209   #endif
   202    210   
   203    211   #ifndef SQLITE_OMIT_BTREECOUNT
   204    212   int sqlite3BtreeCount(BtCursor *, i64 *);

Changes to src/btreeInt.h.

     5      5   ** a legal notice, here is a blessing:
     6      6   **
     7      7   **    May you do good and not evil.
     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   *************************************************************************
    12         -** This file implements a external (disk-based) database using BTrees.
           12  +** This file implements an external (disk-based) database using BTrees.
    13     13   ** For a detailed discussion of BTrees, refer to
    14     14   **
    15     15   **     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
    16     16   **     "Sorting And Searching", pages 473-480. Addison-Wesley
    17     17   **     Publishing Company, Reading, Massachusetts.
    18     18   **
    19     19   ** The basic idea is that each page of the file contains N database
................................................................................
   131    131   **      3       2      number of cells on this page
   132    132   **      5       2      first byte of the cell content area
   133    133   **      7       1      number of fragmented free bytes
   134    134   **      8       4      Right child (the Ptr(N) value).  Omitted on leaves.
   135    135   **
   136    136   ** The flags define the format of this btree page.  The leaf flag means that
   137    137   ** this page has no children.  The zerodata flag means that this page carries
   138         -** only keys and no data.  The intkey flag means that the key is a integer
          138  +** only keys and no data.  The intkey flag means that the key is an integer
   139    139   ** which is stored in the key size entry of the cell header rather than in
   140    140   ** the payload area.
   141    141   **
   142    142   ** The cell pointer array begins on the first byte after the page header.
   143    143   ** The cell pointer array contains zero or more 2-byte numbers which are
   144    144   ** offsets from the beginning of the page to the cell content in the cell
   145    145   ** content area.  The cell pointers occur in sorted order.  The system strives
................................................................................
   269    269   **
   270    270   ** Access to all fields of this structure is controlled by the mutex
   271    271   ** stored in MemPage.pBt->mutex.
   272    272   */
   273    273   struct MemPage {
   274    274     u8 isInit;           /* True if previously initialized. MUST BE FIRST! */
   275    275     u8 nOverflow;        /* Number of overflow cell bodies in aCell[] */
   276         -  u8 intKey;           /* True if intkey flag is set */
   277         -  u8 leaf;             /* True if leaf flag is set */
   278         -  u8 hasData;          /* True if this page stores data */
          276  +  u8 intKey;           /* True if table b-trees.  False for index b-trees */
          277  +  u8 intKeyLeaf;       /* True if the leaf of an intKey table */
          278  +  u8 noPayload;        /* True if internal intKey page (thus w/o data) */
          279  +  u8 leaf;             /* True if a leaf page */
   279    280     u8 hdrOffset;        /* 100 for page 1.  0 otherwise */
   280    281     u8 childPtrSize;     /* 0 if leaf==1.  4 if leaf==0 */
   281    282     u8 max1bytePayload;  /* min(maxLocal,127) */
   282    283     u16 maxLocal;        /* Copy of BtShared.maxLocal or BtShared.maxLeaf */
   283    284     u16 minLocal;        /* Copy of BtShared.minLocal or BtShared.minLeaf */
   284    285     u16 cellOffset;      /* Index in aData of first cell pointer */
   285    286     u16 nFree;           /* Number of free bytes on the page */
................................................................................
   346    347     sqlite3 *db;       /* The database connection holding this btree */
   347    348     BtShared *pBt;     /* Sharable content of this btree */
   348    349     u8 inTrans;        /* TRANS_NONE, TRANS_READ or TRANS_WRITE */
   349    350     u8 sharable;       /* True if we can share pBt with another db */
   350    351     u8 locked;         /* True if db currently has pBt locked */
   351    352     int wantToLock;    /* Number of nested calls to sqlite3BtreeEnter() */
   352    353     int nBackup;       /* Number of backup operations reading this btree */
          354  +  u32 iDataVersion;  /* Combines with pBt->pPager->iDataVersion */
   353    355     Btree *pNext;      /* List of other sharable Btrees from the same db */
   354    356     Btree *pPrev;      /* Back pointer of the same list */
   355    357   #ifndef SQLITE_OMIT_SHARED_CACHE
   356    358     BtLock lock;       /* Object used to lock page 1 */
   357    359   #endif
   358    360   };
   359    361   
................................................................................
   431    433     Bitvec *pHasContent;  /* Set of pages moved to free-list this transaction */
   432    434   #ifndef SQLITE_OMIT_SHARED_CACHE
   433    435     int nRef;             /* Number of references to this structure */
   434    436     BtShared *pNext;      /* Next on a list of sharable BtShared structs */
   435    437     BtLock *pLock;        /* List of locks held on this shared-btree struct */
   436    438     Btree *pWriter;       /* Btree with currently open write transaction */
   437    439   #endif
   438         -  u8 *pTmpSpace;        /* BtShared.pageSize bytes of space for tmp use */
          440  +  u8 *pTmpSpace;        /* Temp space sufficient to hold a single cell */
   439    441   };
   440    442   
   441    443   /*
   442    444   ** Allowed values for BtShared.btsFlags
   443    445   */
   444    446   #define BTS_READ_ONLY        0x0001   /* Underlying file is readonly */
   445    447   #define BTS_PAGESIZE_FIXED   0x0002   /* Page size can no longer be changed */
................................................................................
   452    454   /*
   453    455   ** An instance of the following structure is used to hold information
   454    456   ** about a cell.  The parseCellPtr() function fills in this structure
   455    457   ** based on information extract from the raw disk page.
   456    458   */
   457    459   typedef struct CellInfo CellInfo;
   458    460   struct CellInfo {
   459         -  i64 nKey;      /* The key for INTKEY tables, or number of bytes in key */
   460         -  u8 *pCell;     /* Pointer to the start of cell content */
   461         -  u32 nData;     /* Number of bytes of data */
   462         -  u32 nPayload;  /* Total amount of payload */
   463         -  u16 nHeader;   /* Size of the cell content header in bytes */
   464         -  u16 nLocal;    /* Amount of payload held locally */
          461  +  i64 nKey;      /* The key for INTKEY tables, or nPayload otherwise */
          462  +  u8 *pPayload;  /* Pointer to the start of payload */
          463  +  u32 nPayload;  /* Bytes of payload */
          464  +  u16 nLocal;    /* Amount of payload held locally, not on overflow */
   465    465     u16 iOverflow; /* Offset to overflow page number.  Zero if no overflow */
   466    466     u16 nSize;     /* Size of the cell content on the main b-tree page */
   467    467   };
   468    468   
   469    469   /*
   470    470   ** Maximum depth of an SQLite B-Tree structure. Any B-Tree deeper than
   471    471   ** this will be declared corrupt. This value is calculated based on a
................................................................................
   486    486   **
   487    487   ** A single database file can be shared by two more database connections,
   488    488   ** but cursors cannot be shared.  Each cursor is associated with a
   489    489   ** particular database connection identified BtCursor.pBtree.db.
   490    490   **
   491    491   ** Fields in this structure are accessed under the BtShared.mutex
   492    492   ** found at self->pBt->mutex. 
          493  +**
          494  +** skipNext meaning:
          495  +**    eState==SKIPNEXT && skipNext>0:  Next sqlite3BtreeNext() is no-op.
          496  +**    eState==SKIPNEXT && skipNext<0:  Next sqlite3BtreePrevious() is no-op.
          497  +**    eState==FAULT:                   Cursor fault with skipNext as error code.
   493    498   */
   494    499   struct BtCursor {
   495    500     Btree *pBtree;            /* The Btree to which this cursor belongs */
   496    501     BtShared *pBt;            /* The BtShared this cursor points to */
   497    502     BtCursor *pNext, *pPrev;  /* Forms a linked list of all cursors */
   498    503     struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */
   499    504     Pgno *aOverflow;          /* Cache of overflow page locations */
   500    505     CellInfo info;            /* A parse of the cell we are pointing at */
   501    506     i64 nKey;                 /* Size of pKey, or last integer key */
   502    507     void *pKey;               /* Saved key that was cursor last known position */
   503    508     Pgno pgnoRoot;            /* The root page of this tree */
   504    509     int nOvflAlloc;           /* Allocated size of aOverflow[] array */
   505         -  int skipNext;    /* Prev() is noop if negative. Next() is noop if positive */
          510  +  int skipNext;    /* Prev() is noop if negative. Next() is noop if positive.
          511  +                   ** Error code if eState==CURSOR_FAULT */
   506    512     u8 curFlags;              /* zero or more BTCF_* flags defined below */
   507    513     u8 eState;                /* One of the CURSOR_XXX constants (see below) */
   508    514     u8 hints;                             /* As configured by CursorSetHints() */
   509    515     i16 iPage;                            /* Index of current page in apPage */
   510    516     u16 aiIdx[BTCURSOR_MAX_DEPTH];        /* Current index in apPage[i] */
   511    517     MemPage *apPage[BTCURSOR_MAX_DEPTH];  /* Pages from root to current page */
   512    518   };
................................................................................
   540    546   **   The table that this cursor was opened on still exists, but has been 
   541    547   **   modified since the cursor was last used. The cursor position is saved
   542    548   **   in variables BtCursor.pKey and BtCursor.nKey. When a cursor is in 
   543    549   **   this state, restoreCursorPosition() can be called to attempt to
   544    550   **   seek the cursor to the saved position.
   545    551   **
   546    552   ** CURSOR_FAULT:
   547         -**   A unrecoverable error (an I/O error or a malloc failure) has occurred
          553  +**   An unrecoverable error (an I/O error or a malloc failure) has occurred
   548    554   **   on a different connection that shares the BtShared cache with this
   549    555   **   cursor.  The error has left the cache in an inconsistent state.
   550    556   **   Do nothing else with this cursor.  Any attempt to use the cursor
   551         -**   should return the error code stored in BtCursor.skip
          557  +**   should return the error code stored in BtCursor.skipNext
   552    558   */
   553    559   #define CURSOR_INVALID           0
   554    560   #define CURSOR_VALID             1
   555    561   #define CURSOR_SKIPNEXT          2
   556    562   #define CURSOR_REQUIRESEEK       3
   557    563   #define CURSOR_FAULT             4
   558    564   
................................................................................
   654    660     BtShared *pBt;    /* The tree being checked out */
   655    661     Pager *pPager;    /* The associated pager.  Also accessible by pBt->pPager */
   656    662     u8 *aPgRef;       /* 1 bit per page in the db (see above) */
   657    663     Pgno nPage;       /* Number of pages in the database */
   658    664     int mxErr;        /* Stop accumulating errors when this reaches zero */
   659    665     int nErr;         /* Number of messages written to zErrMsg so far */
   660    666     int mallocFailed; /* A memory allocation error has occurred */
          667  +  const char *zPfx; /* Error message prefix */
          668  +  int v1, v2;       /* Values for up to two %d fields in zPfx */
   661    669     StrAccum errMsg;  /* Accumulate the error message text here */
   662    670   };
   663    671   
   664    672   /*
   665    673   ** Routines to read or write a two- and four-byte big-endian integer values.
   666    674   */
   667    675   #define get2byte(x)   ((x)[0]<<8 | (x)[1])
   668    676   #define put2byte(p,v) ((p)[0] = (u8)((v)>>8), (p)[1] = (u8)(v))
   669    677   #define get4byte sqlite3Get4byte
   670    678   #define put4byte sqlite3Put4byte

Changes to src/build.c.

   109    109                         p->zName, P4_STATIC);
   110    110     }
   111    111   }
   112    112   #else
   113    113     #define codeTableLocks(x)
   114    114   #endif
   115    115   
          116  +/*
          117  +** Return TRUE if the given yDbMask object is empty - if it contains no
          118  +** 1 bits.  This routine is used by the DbMaskAllZero() and DbMaskNotZero()
          119  +** macros when SQLITE_MAX_ATTACHED is greater than 30.
          120  +*/
          121  +#if SQLITE_MAX_ATTACHED>30
          122  +int sqlite3DbMaskAllZero(yDbMask m){
          123  +  int i;
          124  +  for(i=0; i<sizeof(yDbMask); i++) if( m[i] ) return 0;
          125  +  return 1;
          126  +}
          127  +#endif
          128  +
   116    129   /*
   117    130   ** This routine is called after a single SQL statement has been
   118    131   ** parsed and a VDBE program to execute that statement has been
   119    132   ** prepared.  This routine puts the finishing touches on the
   120    133   ** VDBE program and resets the pParse structure for the next
   121    134   ** parse.
   122    135   **
................................................................................
   138    151     */
   139    152     v = sqlite3GetVdbe(pParse);
   140    153     assert( !pParse->isMultiWrite 
   141    154          || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort));
   142    155     if( v ){
   143    156       while( sqlite3VdbeDeletePriorOpcode(v, OP_Close) ){}
   144    157       sqlite3VdbeAddOp0(v, OP_Halt);
          158  +
          159  +#if SQLITE_USER_AUTHENTICATION
          160  +    if( pParse->nTableLock>0 && db->init.busy==0 ){
          161  +      sqlite3UserAuthInit(db);
          162  +      if( db->auth.authLevel<UAUTH_User ){
          163  +        pParse->rc = SQLITE_AUTH_USER;
          164  +        sqlite3ErrorMsg(pParse, "user not authenticated");
          165  +        return;
          166  +      }
          167  +    }
          168  +#endif
   145    169   
   146    170       /* The cookie mask contains one bit for each database file open.
   147    171       ** (Bit 0 is for main, bit 1 is for temp, and so forth.)  Bits are
   148    172       ** set for each database that is used.  Generate code to start a
   149    173       ** transaction on each used database and to verify the schema cookie
   150    174       ** on each used database.
   151    175       */
   152         -    if( db->mallocFailed==0 && (pParse->cookieMask || pParse->pConstExpr) ){
   153         -      yDbMask mask;
          176  +    if( db->mallocFailed==0 
          177  +     && (DbMaskNonZero(pParse->cookieMask) || pParse->pConstExpr)
          178  +    ){
   154    179         int iDb, i;
   155    180         assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init );
   156    181         sqlite3VdbeJumpHere(v, 0);
   157         -      for(iDb=0, mask=1; iDb<db->nDb; mask<<=1, iDb++){
   158         -        if( (mask & pParse->cookieMask)==0 ) continue;
          182  +      for(iDb=0; iDb<db->nDb; iDb++){
          183  +        if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue;
   159    184           sqlite3VdbeUsesBtree(v, iDb);
   160    185           sqlite3VdbeAddOp4Int(v,
   161    186             OP_Transaction,                    /* Opcode */
   162    187             iDb,                               /* P1 */
   163         -          (mask & pParse->writeMask)!=0,     /* P2 */
          188  +          DbMaskTest(pParse->writeMask,iDb), /* P2 */
   164    189             pParse->cookieValue[iDb],          /* P3 */
   165    190             db->aDb[iDb].pSchema->iGeneration  /* P4 */
   166    191           );
   167    192           if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1);
   168    193         }
   169    194   #ifndef SQLITE_OMIT_VIRTUALTABLE
   170    195         for(i=0; i<pParse->nVtabLock; i++){
................................................................................
   212    237     }else{
   213    238       pParse->rc = SQLITE_ERROR;
   214    239     }
   215    240     pParse->nTab = 0;
   216    241     pParse->nMem = 0;
   217    242     pParse->nSet = 0;
   218    243     pParse->nVar = 0;
   219         -  pParse->cookieMask = 0;
          244  +  DbMaskZero(pParse->cookieMask);
   220    245   }
   221    246   
   222    247   /*
   223    248   ** Run the parser and code generator recursively in order to generate
   224    249   ** code for the SQL statement given onto the end of the pParse context
   225    250   ** currently under construction.  When the parser is run recursively
   226    251   ** this way, the final OP_Halt is not appended and other initialization
................................................................................
   253    278     sqlite3RunParser(pParse, zSql, &zErrMsg);
   254    279     sqlite3DbFree(db, zErrMsg);
   255    280     sqlite3DbFree(db, zSql);
   256    281     memcpy(&pParse->nVar, saveBuf, SAVE_SZ);
   257    282     pParse->nested--;
   258    283   }
   259    284   
          285  +#if SQLITE_USER_AUTHENTICATION
          286  +/*
          287  +** Return TRUE if zTable is the name of the system table that stores the
          288  +** list of users and their access credentials.
          289  +*/
          290  +int sqlite3UserAuthTable(const char *zTable){
          291  +  return sqlite3_stricmp(zTable, "sqlite_user")==0;
          292  +}
          293  +#endif
          294  +
   260    295   /*
   261    296   ** Locate the in-memory structure that describes a particular database
   262    297   ** table given the name of that table and (optionally) the name of the
   263    298   ** database containing the table.  Return NULL if not found.
   264    299   **
   265    300   ** If zDatabase is 0, all databases are searched for the table and the
   266    301   ** first matching table is returned.  (No checking for duplicate table
................................................................................
   268    303   ** auxiliary databases added using the ATTACH command.
   269    304   **
   270    305   ** See also sqlite3LocateTable().
   271    306   */
   272    307   Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){
   273    308     Table *p = 0;
   274    309     int i;
   275         -  int nName;
   276         -  assert( zName!=0 );
   277         -  nName = sqlite3Strlen30(zName);
          310  +
          311  +#ifdef SQLITE_ENABLE_API_ARMOR
          312  +  if( !sqlite3SafetyCheckOk(db) || zName==0 ) return 0;
          313  +#endif
          314  +
   278    315     /* All mutexes are required for schema access.  Make sure we hold them. */
   279    316     assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) );
          317  +#if SQLITE_USER_AUTHENTICATION
          318  +  /* Only the admin user is allowed to know that the sqlite_user table
          319  +  ** exists */
          320  +  if( db->auth.authLevel<UAUTH_Admin && sqlite3UserAuthTable(zName)!=0 ){
          321  +    return 0;
          322  +  }
          323  +#endif
   280    324     for(i=OMIT_TEMPDB; i<db->nDb; i++){
   281    325       int j = (i<2) ? i^1 : i;   /* Search TEMP before MAIN */
   282    326       if( zDatabase!=0 && sqlite3StrICmp(zDatabase, db->aDb[j].zName) ) continue;
   283    327       assert( sqlite3SchemaMutexHeld(db, j, 0) );
   284         -    p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName, nName);
          328  +    p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName);
   285    329       if( p ) break;
   286    330     }
   287    331     return p;
   288    332   }
   289    333   
   290    334   /*
   291    335   ** Locate the in-memory structure that describes a particular database
................................................................................
   317    361       if( zDbase ){
   318    362         sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName);
   319    363       }else{
   320    364         sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName);
   321    365       }
   322    366       pParse->checkSchema = 1;
   323    367     }
          368  +#if SQLITE_USER_AUTHENICATION
          369  +  else if( pParse->db->auth.authLevel<UAUTH_User ){
          370  +    sqlite3ErrorMsg(pParse, "user not authenticated");
          371  +    p = 0;
          372  +  }
          373  +#endif
   324    374     return p;
   325    375   }
   326    376   
   327    377   /*
   328    378   ** Locate the table identified by *p.
   329    379   **
   330    380   ** This is a wrapper around sqlite3LocateTable(). The difference between
................................................................................
   360    410   ** for duplicate index names is done.)  The search order is
   361    411   ** TEMP first, then MAIN, then any auxiliary databases added
   362    412   ** using the ATTACH command.
   363    413   */
   364    414   Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){
   365    415     Index *p = 0;
   366    416     int i;
   367         -  int nName = sqlite3Strlen30(zName);
   368    417     /* All mutexes are required for schema access.  Make sure we hold them. */
   369    418     assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) );
   370    419     for(i=OMIT_TEMPDB; i<db->nDb; i++){
   371    420       int j = (i<2) ? i^1 : i;  /* Search TEMP before MAIN */
   372    421       Schema *pSchema = db->aDb[j].pSchema;
   373    422       assert( pSchema );
   374    423       if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zName) ) continue;
   375    424       assert( sqlite3SchemaMutexHeld(db, j, 0) );
   376         -    p = sqlite3HashFind(&pSchema->idxHash, zName, nName);
          425  +    p = sqlite3HashFind(&pSchema->idxHash, zName);
   377    426       if( p ) break;
   378    427     }
   379    428     return p;
   380    429   }
   381    430   
   382    431   /*
   383    432   ** Reclaim the memory used by an index
   384    433   */
   385    434   static void freeIndex(sqlite3 *db, Index *p){
   386    435   #ifndef SQLITE_OMIT_ANALYZE
   387    436     sqlite3DeleteIndexSamples(db, p);
   388    437   #endif
   389         -  if( db==0 || db->pnBytesFreed==0 ) sqlite3KeyInfoUnref(p->pKeyInfo);
   390    438     sqlite3ExprDelete(db, p->pPartIdxWhere);
   391    439     sqlite3DbFree(db, p->zColAff);
   392    440     if( p->isResized ) sqlite3DbFree(db, p->azColl);
          441  +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4
          442  +  sqlite3_free(p->aiRowEst);
          443  +#endif
   393    444     sqlite3DbFree(db, p);
   394    445   }
   395    446   
   396    447   /*
   397    448   ** For the index called zIdxName which is found in the database iDb,
   398    449   ** unlike that index from its Table then remove the index from
   399    450   ** the index hash table and free all memory structures associated
   400    451   ** with the index.
   401    452   */
   402    453   void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){
   403    454     Index *pIndex;
   404         -  int len;
   405    455     Hash *pHash;
   406    456   
   407    457     assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
   408    458     pHash = &db->aDb[iDb].pSchema->idxHash;
   409         -  len = sqlite3Strlen30(zIdxName);
   410         -  pIndex = sqlite3HashInsert(pHash, zIdxName, len, 0);
          459  +  pIndex = sqlite3HashInsert(pHash, zIdxName, 0);
   411    460     if( ALWAYS(pIndex) ){
   412    461       if( pIndex->pTable->pIndex==pIndex ){
   413    462         pIndex->pTable->pIndex = pIndex->pNext;
   414    463       }else{
   415    464         Index *p;
   416    465         /* Justification of ALWAYS();  The index must be on the list of
   417    466         ** indices. */
................................................................................
   563    612     /* Delete all indices associated with this table. */
   564    613     for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){
   565    614       pNext = pIndex->pNext;
   566    615       assert( pIndex->pSchema==pTable->pSchema );
   567    616       if( !db || db->pnBytesFreed==0 ){
   568    617         char *zName = pIndex->zName; 
   569    618         TESTONLY ( Index *pOld = ) sqlite3HashInsert(
   570         -         &pIndex->pSchema->idxHash, zName, sqlite3Strlen30(zName), 0
          619  +         &pIndex->pSchema->idxHash, zName, 0
   571    620         );
   572    621         assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) );
   573    622         assert( pOld==pIndex || pOld==0 );
   574    623       }
   575    624       freeIndex(db, pIndex);
   576    625     }
   577    626   
................................................................................
   606    655   
   607    656     assert( db!=0 );
   608    657     assert( iDb>=0 && iDb<db->nDb );
   609    658     assert( zTabName );
   610    659     assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
   611    660     testcase( zTabName[0]==0 );  /* Zero-length table names are allowed */
   612    661     pDb = &db->aDb[iDb];
   613         -  p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName,
   614         -                        sqlite3Strlen30(zTabName),0);
          662  +  p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, 0);
   615    663     sqlite3DeleteTable(db, p);
   616    664     db->flags |= SQLITE_InternChanges;
   617    665   }
   618    666   
   619    667   /*
   620    668   ** Given a token, return a string that consists of the text of that
   621    669   ** token.  Space to hold the returned string
................................................................................
  1131   1179       }
  1132   1180     }
  1133   1181   
  1134   1182     /* If pszEst is not NULL, store an estimate of the field size.  The
  1135   1183     ** estimate is scaled so that the size of an integer is 1.  */
  1136   1184     if( pszEst ){
  1137   1185       *pszEst = 1;   /* default size is approx 4 bytes */
  1138         -    if( aff<=SQLITE_AFF_NONE ){
         1186  +    if( aff<SQLITE_AFF_NUMERIC ){
  1139   1187         if( zChar ){
  1140   1188           while( zChar[0] ){
  1141   1189             if( sqlite3Isdigit(zChar[0]) ){
  1142   1190               int v = 0;
  1143   1191               sqlite3GetInt32(zChar, &v);
  1144   1192               v = v/4 + 1;
  1145   1193               if( v>255 ) v = 255;
................................................................................
  1190   1238   void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){
  1191   1239     Table *p;
  1192   1240     Column *pCol;
  1193   1241     sqlite3 *db = pParse->db;
  1194   1242     p = pParse->pNewTable;
  1195   1243     if( p!=0 ){
  1196   1244       pCol = &(p->aCol[p->nCol-1]);
  1197         -    if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr) ){
         1245  +    if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr, db->init.busy) ){
  1198   1246         sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant",
  1199   1247             pCol->zName);
  1200   1248       }else{
  1201   1249         /* A copy of pExpr is used instead of the original, as pExpr contains
  1202   1250         ** tokens that point to volatile memory. The 'span' of the expression
  1203   1251         ** is required by pragma table_info.
  1204   1252         */
................................................................................
  1502   1550     }
  1503   1551     sqlite3_snprintf(n, zStmt, "CREATE TABLE ");
  1504   1552     k = sqlite3Strlen30(zStmt);
  1505   1553     identPut(zStmt, &k, p->zName);
  1506   1554     zStmt[k++] = '(';
  1507   1555     for(pCol=p->aCol, i=0; i<p->nCol; i++, pCol++){
  1508   1556       static const char * const azType[] = {
  1509         -        /* SQLITE_AFF_TEXT    */ " TEXT",
  1510   1557           /* SQLITE_AFF_NONE    */ "",
         1558  +        /* SQLITE_AFF_TEXT    */ " TEXT",
  1511   1559           /* SQLITE_AFF_NUMERIC */ " NUM",
  1512   1560           /* SQLITE_AFF_INTEGER */ " INT",
  1513   1561           /* SQLITE_AFF_REAL    */ " REAL"
  1514   1562       };
  1515   1563       int len;
  1516   1564       const char *zType;
  1517   1565   
  1518   1566       sqlite3_snprintf(n-k, &zStmt[k], zSep);
  1519   1567       k += sqlite3Strlen30(&zStmt[k]);
  1520   1568       zSep = zSep2;
  1521   1569       identPut(zStmt, &k, pCol->zName);
  1522         -    assert( pCol->affinity-SQLITE_AFF_TEXT >= 0 );
  1523         -    assert( pCol->affinity-SQLITE_AFF_TEXT < ArraySize(azType) );
  1524         -    testcase( pCol->affinity==SQLITE_AFF_TEXT );
         1570  +    assert( pCol->affinity-SQLITE_AFF_NONE >= 0 );
         1571  +    assert( pCol->affinity-SQLITE_AFF_NONE < ArraySize(azType) );
  1525   1572       testcase( pCol->affinity==SQLITE_AFF_NONE );
         1573  +    testcase( pCol->affinity==SQLITE_AFF_TEXT );
  1526   1574       testcase( pCol->affinity==SQLITE_AFF_NUMERIC );
  1527   1575       testcase( pCol->affinity==SQLITE_AFF_INTEGER );
  1528   1576       testcase( pCol->affinity==SQLITE_AFF_REAL );
  1529   1577       
  1530         -    zType = azType[pCol->affinity - SQLITE_AFF_TEXT];
         1578  +    zType = azType[pCol->affinity - SQLITE_AFF_NONE];
  1531   1579       len = sqlite3Strlen30(zType);
  1532   1580       assert( pCol->affinity==SQLITE_AFF_NONE 
  1533   1581               || pCol->affinity==sqlite3AffinityType(zType, 0) );
  1534   1582       memcpy(&zStmt[k], zType, len);
  1535   1583       k += len;
  1536   1584       assert( k<=n );
  1537   1585     }
................................................................................
  1607   1655   ** are appropriate for a WITHOUT ROWID table instead of a rowid table.
  1608   1656   ** Changes include:
  1609   1657   **
  1610   1658   **     (1)  Convert the OP_CreateTable into an OP_CreateIndex.  There is
  1611   1659   **          no rowid btree for a WITHOUT ROWID.  Instead, the canonical
  1612   1660   **          data storage is a covering index btree.
  1613   1661   **     (2)  Bypass the creation of the sqlite_master table entry
  1614         -**          for the PRIMARY KEY as the the primary key index is now
         1662  +**          for the PRIMARY KEY as the primary key index is now
  1615   1663   **          identified by the sqlite_master table entry of the table itself.
  1616   1664   **     (3)  Set the Index.tnum of the PRIMARY KEY Index object in the
  1617   1665   **          schema to the rootpage from the main table.
  1618   1666   **     (4)  Set all columns of the PRIMARY KEY schema object to be NOT NULL.
  1619   1667   **     (5)  Add all table columns to the PRIMARY KEY Index object
  1620   1668   **          so that the PRIMARY KEY is a covering index.  The surplus
  1621   1669   **          columns are part of KeyInfo.nXField and are not used for
................................................................................
  1628   1676     Index *pPk;
  1629   1677     int nPk;
  1630   1678     int i, j;
  1631   1679     sqlite3 *db = pParse->db;
  1632   1680     Vdbe *v = pParse->pVdbe;
  1633   1681   
  1634   1682     /* Convert the OP_CreateTable opcode that would normally create the
  1635         -  ** root-page for the table into a OP_CreateIndex opcode.  The index
         1683  +  ** root-page for the table into an OP_CreateIndex opcode.  The index
  1636   1684     ** created will become the PRIMARY KEY index.
  1637   1685     */
  1638   1686     if( pParse->addrCrTab ){
  1639   1687       assert( v );
  1640   1688       sqlite3VdbeGetOp(v, pParse->addrCrTab)->opcode = OP_CreateIndex;
  1641   1689     }
  1642   1690   
................................................................................
  1661   1709       assert( pParse->pNewTable==pTab );
  1662   1710       pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0);
  1663   1711       if( pPk==0 ) return;
  1664   1712       pPk->idxType = SQLITE_IDXTYPE_PRIMARYKEY;
  1665   1713       pTab->iPKey = -1;
  1666   1714     }else{
  1667   1715       pPk = sqlite3PrimaryKeyIndex(pTab);
         1716  +    /*
         1717  +    ** Remove all redundant columns from the PRIMARY KEY.  For example, change
         1718  +    ** "PRIMARY KEY(a,b,a,b,c,b,c,d)" into just "PRIMARY KEY(a,b,c,d)".  Later
         1719  +    ** code assumes the PRIMARY KEY contains no repeated columns.
         1720  +    */
         1721  +    for(i=j=1; i<pPk->nKeyCol; i++){
         1722  +      if( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ){
         1723  +        pPk->nColumn--;
         1724  +      }else{
         1725  +        pPk->aiColumn[j++] = pPk->aiColumn[i];
         1726  +      }
         1727  +    }
         1728  +    pPk->nKeyCol = j;
  1668   1729     }
  1669   1730     pPk->isCovering = 1;
  1670   1731     assert( pPk!=0 );
  1671   1732     nPk = pPk->nKeyCol;
  1672   1733   
  1673   1734     /* Make sure every column of the PRIMARY KEY is NOT NULL */
  1674   1735     for(i=0; i<nPk; i++){
................................................................................
  1929   1990   
  1930   1991     /* Add the table to the in-memory representation of the database.
  1931   1992     */
  1932   1993     if( db->init.busy ){
  1933   1994       Table *pOld;
  1934   1995       Schema *pSchema = p->pSchema;
  1935   1996       assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
  1936         -    pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName,
  1937         -                             sqlite3Strlen30(p->zName),p);
         1997  +    pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, p);
  1938   1998       if( pOld ){
  1939   1999         assert( p==pOld );  /* Malloc must have failed inside HashInsert() */
  1940   2000         db->mallocFailed = 1;
  1941   2001         return;
  1942   2002       }
  1943   2003       pParse->pNewTable = 0;
  1944   2004       db->flags |= SQLITE_InternChanges;
................................................................................
  2041   2101   */
  2042   2102   int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){
  2043   2103     Table *pSelTab;   /* A fake table from which we get the result set */
  2044   2104     Select *pSel;     /* Copy of the SELECT that implements the view */
  2045   2105     int nErr = 0;     /* Number of errors encountered */
  2046   2106     int n;            /* Temporarily holds the number of cursors assigned */
  2047   2107     sqlite3 *db = pParse->db;  /* Database connection for malloc errors */
  2048         -  int (*xAuth)(void*,int,const char*,const char*,const char*,const char*);
         2108  +  sqlite3_xauth xAuth;       /* Saved xAuth pointer */
  2049   2109   
  2050   2110     assert( pTable );
  2051   2111   
  2052   2112   #ifndef SQLITE_OMIT_VIRTUALTABLE
  2053   2113     if( sqlite3VtabCallConnect(pParse, pTable) ){
  2054   2114       return SQLITE_ERROR;
  2055   2115     }
................................................................................
  2112   2172         assert( pTable->aCol==0 );
  2113   2173         pTable->nCol = pSelTab->nCol;
  2114   2174         pTable->aCol = pSelTab->aCol;
  2115   2175         pSelTab->nCol = 0;
  2116   2176         pSelTab->aCol = 0;
  2117   2177         sqlite3DeleteTable(db, pSelTab);
  2118   2178         assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) );
  2119         -      pTable->pSchema->flags |= DB_UnresetViews;
         2179  +      pTable->pSchema->schemaFlags |= DB_UnresetViews;
  2120   2180       }else{
  2121   2181         pTable->nCol = 0;
  2122   2182         nErr++;
  2123   2183       }
  2124   2184       sqlite3SelectDelete(db, pSel);
  2125   2185     } else {
  2126   2186       nErr++;
................................................................................
  2580   2640     }
  2581   2641     pFKey->isDeferred = 0;
  2582   2642     pFKey->aAction[0] = (u8)(flags & 0xff);            /* ON DELETE action */
  2583   2643     pFKey->aAction[1] = (u8)((flags >> 8 ) & 0xff);    /* ON UPDATE action */
  2584   2644   
  2585   2645     assert( sqlite3SchemaMutexHeld(db, 0, p->pSchema) );
  2586   2646     pNextTo = (FKey *)sqlite3HashInsert(&p->pSchema->fkeyHash, 
  2587         -      pFKey->zTo, sqlite3Strlen30(pFKey->zTo), (void *)pFKey
         2647  +      pFKey->zTo, (void *)pFKey
  2588   2648     );
  2589   2649     if( pNextTo==pFKey ){
  2590   2650       db->mallocFailed = 1;
  2591   2651       goto fk_end;
  2592   2652     }
  2593   2653     if( pNextTo ){
  2594   2654       assert( pNextTo->pPrevTo==0 );
................................................................................
  2643   2703     int iSorter;                   /* Cursor opened by OpenSorter (if in use) */
  2644   2704     int addr1;                     /* Address of top of loop */
  2645   2705     int addr2;                     /* Address to jump to for next iteration */
  2646   2706     int tnum;                      /* Root page of index */
  2647   2707     int iPartIdxLabel;             /* Jump to this label to skip a row */
  2648   2708     Vdbe *v;                       /* Generate code into this virtual machine */
  2649   2709     KeyInfo *pKey;                 /* KeyInfo for index */
  2650         -  int regRecord;                 /* Register holding assemblied index record */
         2710  +  int regRecord;                 /* Register holding assembled index record */
  2651   2711     sqlite3 *db = pParse->db;      /* The database connection */
  2652   2712     int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema);
  2653   2713   
  2654   2714   #ifndef SQLITE_OMIT_AUTHORIZATION
  2655   2715     if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0,
  2656   2716         db->aDb[iDb].zName ) ){
  2657   2717       return;
................................................................................
  2668   2728     }else{
  2669   2729       tnum = pIndex->tnum;
  2670   2730     }
  2671   2731     pKey = sqlite3KeyInfoOfIndex(pParse, pIndex);
  2672   2732   
  2673   2733     /* Open the sorter cursor if we are to use one. */
  2674   2734     iSorter = pParse->nTab++;
  2675         -  sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*)
         2735  +  sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, pIndex->nKeyCol, (char*)
  2676   2736                       sqlite3KeyInfoRef(pKey), P4_KEYINFO);
  2677   2737   
  2678   2738     /* Open the table. Loop through all rows of the table, inserting index
  2679   2739     ** records into the sorter. */
  2680   2740     sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead);
  2681   2741     addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); VdbeCoverage(v);
  2682   2742     regRecord = sqlite3GetTempReg(pParse);
................................................................................
  2689   2749     if( memRootPage<0 ) sqlite3VdbeAddOp2(v, OP_Clear, tnum, iDb);
  2690   2750     sqlite3VdbeAddOp4(v, OP_OpenWrite, iIdx, tnum, iDb, 
  2691   2751                       (char *)pKey, P4_KEYINFO);
  2692   2752     sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0));
  2693   2753   
  2694   2754     addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); VdbeCoverage(v);
  2695   2755     assert( pKey!=0 || db->mallocFailed || pParse->nErr );
  2696         -  if( pIndex->onError!=OE_None && pKey!=0 ){
         2756  +  if( IsUniqueIndex(pIndex) && pKey!=0 ){
  2697   2757       int j2 = sqlite3VdbeCurrentAddr(v) + 3;
  2698   2758       sqlite3VdbeAddOp2(v, OP_Goto, 0, j2);
  2699   2759       addr2 = sqlite3VdbeCurrentAddr(v);
  2700   2760       sqlite3VdbeAddOp4Int(v, OP_SorterCompare, iSorter, j2, regRecord,
  2701         -                         pKey->nField - pIndex->nKeyCol); VdbeCoverage(v);
         2761  +                         pIndex->nKeyCol); VdbeCoverage(v);
  2702   2762       sqlite3UniqueConstraint(pParse, OE_Abort, pIndex);
  2703   2763     }else{
  2704   2764       addr2 = sqlite3VdbeCurrentAddr(v);
  2705   2765     }
  2706         -  sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord);
         2766  +  sqlite3VdbeAddOp3(v, OP_SorterData, iSorter, regRecord, iIdx);
  2707   2767     sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1);
  2708   2768     sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
  2709   2769     sqlite3ReleaseTempReg(pParse, regRecord);
  2710   2770     sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); VdbeCoverage(v);
  2711   2771     sqlite3VdbeJumpHere(v, addr1);
  2712   2772   
  2713   2773     sqlite3VdbeAddOp1(v, OP_Close, iTab);
................................................................................
  2856   2916       iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
  2857   2917     }
  2858   2918     pDb = &db->aDb[iDb];
  2859   2919   
  2860   2920     assert( pTab!=0 );
  2861   2921     assert( pParse->nErr==0 );
  2862   2922     if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 
         2923  +       && db->init.busy==0
         2924  +#if SQLITE_USER_AUTHENTICATION
         2925  +       && sqlite3UserAuthTable(pTab->zName)==0
         2926  +#endif
  2863   2927          && sqlite3StrNICmp(&pTab->zName[7],"altertab_",9)!=0 ){
  2864   2928       sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName);
  2865   2929       goto exit_create_index;
  2866   2930     }
  2867   2931   #ifndef SQLITE_OMIT_VIEW
  2868   2932     if( pTab->pSelect ){
  2869   2933       sqlite3ErrorMsg(pParse, "views may not be indexed");
................................................................................
  3017   3081       }
  3018   3082       if( j>=pTab->nCol ){
  3019   3083         sqlite3ErrorMsg(pParse, "table %s has no column named %s",
  3020   3084           pTab->zName, zColName);
  3021   3085         pParse->checkSchema = 1;
  3022   3086         goto exit_create_index;
  3023   3087       }
  3024         -    assert( pTab->nCol<=0x7fff && j<=0x7fff );
         3088  +    assert( j<=0x7fff );
  3025   3089       pIndex->aiColumn[i] = (i16)j;
  3026   3090       if( pListItem->pExpr ){
  3027   3091         int nColl;
  3028   3092         assert( pListItem->pExpr->op==TK_COLLATE );
  3029   3093         zColl = pListItem->pExpr->u.zToken;
  3030   3094         nColl = sqlite3Strlen30(zColl) + 1;
  3031   3095         assert( nExtra>=nColl );
................................................................................
  3086   3150       ** If there are different collating sequences or if the columns of
  3087   3151       ** the constraint occur in different orders, then the constraints are
  3088   3152       ** considered distinct and both result in separate indices.
  3089   3153       */
  3090   3154       Index *pIdx;
  3091   3155       for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
  3092   3156         int k;
  3093         -      assert( pIdx->onError!=OE_None );
         3157  +      assert( IsUniqueIndex(pIdx) );
  3094   3158         assert( pIdx->idxType!=SQLITE_IDXTYPE_APPDEF );
  3095         -      assert( pIndex->onError!=OE_None );
         3159  +      assert( IsUniqueIndex(pIndex) );
  3096   3160   
  3097   3161         if( pIdx->nKeyCol!=pIndex->nKeyCol ) continue;
  3098   3162         for(k=0; k<pIdx->nKeyCol; k++){
  3099   3163           const char *z1;
  3100   3164           const char *z2;
  3101   3165           if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break;
  3102   3166           z1 = pIdx->azColl[k];
................................................................................
  3128   3192     /* Link the new Index structure to its table and to the other
  3129   3193     ** in-memory database structures. 
  3130   3194     */
  3131   3195     if( db->init.busy ){
  3132   3196       Index *p;
  3133   3197       assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) );
  3134   3198       p = sqlite3HashInsert(&pIndex->pSchema->idxHash, 
  3135         -                          pIndex->zName, sqlite3Strlen30(pIndex->zName),
  3136         -                          pIndex);
         3199  +                          pIndex->zName, pIndex);
  3137   3200       if( p ){
  3138   3201         assert( p==pIndex );  /* Malloc must have failed */
  3139   3202         db->mallocFailed = 1;
  3140   3203         goto exit_create_index;
  3141   3204       }
  3142   3205       db->flags |= SQLITE_InternChanges;
  3143   3206       if( pTblName!=0 ){
................................................................................
  3244   3307     return pRet;
  3245   3308   }
  3246   3309   
  3247   3310   /*
  3248   3311   ** Fill the Index.aiRowEst[] array with default information - information
  3249   3312   ** to be used when we have not run the ANALYZE command.
  3250   3313   **
  3251         -** aiRowEst[0] is suppose to contain the number of elements in the index.
         3314  +** aiRowEst[0] is supposed to contain the number of elements in the index.
  3252   3315   ** Since we do not know, guess 1 million.  aiRowEst[1] is an estimate of the
  3253   3316   ** number of rows in the table that match any particular value of the
  3254   3317   ** first column of the index.  aiRowEst[2] is an estimate of the number
  3255   3318   ** of rows that match any particular combination of the first 2 columns
  3256   3319   ** of the index.  And so forth.  It must always be the case that
  3257   3320   *
  3258   3321   **           aiRowEst[N]<=aiRowEst[N-1]
................................................................................
  3279   3342     ** 6 and each subsequent value (if any) is 5.  */
  3280   3343     memcpy(&a[1], aVal, nCopy*sizeof(LogEst));
  3281   3344     for(i=nCopy+1; i<=pIdx->nKeyCol; i++){
  3282   3345       a[i] = 23;                    assert( 23==sqlite3LogEst(5) );
  3283   3346     }
  3284   3347   
  3285   3348     assert( 0==sqlite3LogEst(1) );
  3286         -  if( pIdx->onError!=OE_None ) a[pIdx->nKeyCol] = 0;
         3349  +  if( IsUniqueIndex(pIdx) ) a[pIdx->nKeyCol] = 0;
  3287   3350   }
  3288   3351   
  3289   3352   /*
  3290   3353   ** This routine will drop an existing named index.  This routine
  3291   3354   ** implements the DROP INDEX statement.
  3292   3355   */
  3293   3356   void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){
................................................................................
  3623   3686   /*
  3624   3687   ** This routine is called by the parser to add a new term to the
  3625   3688   ** end of a growing FROM clause.  The "p" parameter is the part of
  3626   3689   ** the FROM clause that has already been constructed.  "p" is NULL
  3627   3690   ** if this is the first term of the FROM clause.  pTable and pDatabase
  3628   3691   ** are the name of the table and database named in the FROM clause term.
  3629   3692   ** pDatabase is NULL if the database name qualifier is missing - the
  3630         -** usual case.  If the term has a alias, then pAlias points to the
         3693  +** usual case.  If the term has an alias, then pAlias points to the
  3631   3694   ** alias token.  If the term is a subquery, then pSubquery is the
  3632   3695   ** SELECT statement that the subquery encodes.  The pTable and
  3633   3696   ** pDatabase parameters are NULL for subqueries.  The pOn and pUsing
  3634   3697   ** parameters are the content of the ON and USING clauses.
  3635   3698   **
  3636   3699   ** Return a new SrcList which encodes is the FROM with the new
  3637   3700   ** term added.
................................................................................
  3839   3902   ** for database iDb.  The code to actually verify the schema cookie
  3840   3903   ** will occur at the end of the top-level VDBE and will be generated
  3841   3904   ** later, by sqlite3FinishCoding().
  3842   3905   */
  3843   3906   void sqlite3CodeVerifySchema(Parse *pParse, int iDb){
  3844   3907     Parse *pToplevel = sqlite3ParseToplevel(pParse);
  3845   3908     sqlite3 *db = pToplevel->db;
  3846         -  yDbMask mask;
  3847   3909   
  3848   3910     assert( iDb>=0 && iDb<db->nDb );
  3849   3911     assert( db->aDb[iDb].pBt!=0 || iDb==1 );
  3850   3912     assert( iDb<SQLITE_MAX_ATTACHED+2 );
  3851   3913     assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
  3852         -  mask = ((yDbMask)1)<<iDb;
  3853         -  if( (pToplevel->cookieMask & mask)==0 ){
  3854         -    pToplevel->cookieMask |= mask;
         3914  +  if( DbMaskTest(pToplevel->cookieMask, iDb)==0 ){
         3915  +    DbMaskSet(pToplevel->cookieMask, iDb);
  3855   3916       pToplevel->cookieValue[iDb] = db->aDb[iDb].pSchema->schema_cookie;
  3856   3917       if( !OMIT_TEMPDB && iDb==1 ){
  3857   3918         sqlite3OpenTempDatabase(pToplevel);
  3858   3919       }
  3859   3920     }
  3860   3921   }
  3861   3922   
................................................................................
  3886   3947   ** rollback the whole transaction.  For operations where all constraints
  3887   3948   ** can be checked before any changes are made to the database, it is never
  3888   3949   ** necessary to undo a write and the checkpoint should not be set.
  3889   3950   */
  3890   3951   void sqlite3BeginWriteOperation(Parse *pParse, int setStatement, int iDb){
  3891   3952     Parse *pToplevel = sqlite3ParseToplevel(pParse);
  3892   3953     sqlite3CodeVerifySchema(pParse, iDb);
  3893         -  pToplevel->writeMask |= ((yDbMask)1)<<iDb;
         3954  +  DbMaskSet(pToplevel->writeMask, iDb);
  3894   3955     pToplevel->isMultiWrite |= setStatement;
  3895   3956   }
  3896   3957   
  3897   3958   /*
  3898   3959   ** Indicate that the statement currently under construction might write
  3899   3960   ** more than one entry (example: deleting one row then inserting another,
  3900   3961   ** inserting multiple rows in a table, or inserting a row and index entries.)
................................................................................
  4137   4198   ** So there might be multiple references to the returned pointer.  The
  4138   4199   ** caller should not try to modify the KeyInfo object.
  4139   4200   **
  4140   4201   ** The caller should invoke sqlite3KeyInfoUnref() on the returned object
  4141   4202   ** when it has finished using it.
  4142   4203   */
  4143   4204   KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){
         4205  +  int i;
         4206  +  int nCol = pIdx->nColumn;
         4207  +  int nKey = pIdx->nKeyCol;
         4208  +  KeyInfo *pKey;
  4144   4209     if( pParse->nErr ) return 0;
  4145         -#ifndef SQLITE_OMIT_SHARED_CACHE
  4146         -  if( pIdx->pKeyInfo && pIdx->pKeyInfo->db!=pParse->db ){
  4147         -    sqlite3KeyInfoUnref(pIdx->pKeyInfo);
  4148         -    pIdx->pKeyInfo = 0;
         4210  +  if( pIdx->uniqNotNull ){
         4211  +    pKey = sqlite3KeyInfoAlloc(pParse->db, nKey, nCol-nKey);
         4212  +  }else{
         4213  +    pKey = sqlite3KeyInfoAlloc(pParse->db, nCol, 0);
  4149   4214     }
  4150         -#endif
  4151         -  if( pIdx->pKeyInfo==0 ){
  4152         -    int i;
  4153         -    int nCol = pIdx->nColumn;
  4154         -    int nKey = pIdx->nKeyCol;
  4155         -    KeyInfo *pKey;
  4156         -    if( pIdx->uniqNotNull ){
  4157         -      pKey = sqlite3KeyInfoAlloc(pParse->db, nKey, nCol-nKey);
  4158         -    }else{
  4159         -      pKey = sqlite3KeyInfoAlloc(pParse->db, nCol, 0);
         4215  +  if( pKey ){
         4216  +    assert( sqlite3KeyInfoIsWriteable(pKey) );
         4217  +    for(i=0; i<nCol; i++){
         4218  +      char *zColl = pIdx->azColl[i];
         4219  +      assert( zColl!=0 );
         4220  +      pKey->aColl[i] = strcmp(zColl,"BINARY")==0 ? 0 :
         4221  +                        sqlite3LocateCollSeq(pParse, zColl);
         4222  +      pKey->aSortOrder[i] = pIdx->aSortOrder[i];
  4160   4223       }
  4161         -    if( pKey ){
  4162         -      assert( sqlite3KeyInfoIsWriteable(pKey) );
  4163         -      for(i=0; i<nCol; i++){
  4164         -        char *zColl = pIdx->azColl[i];
  4165         -        assert( zColl!=0 );
  4166         -        pKey->aColl[i] = strcmp(zColl,"BINARY")==0 ? 0 :
  4167         -                          sqlite3LocateCollSeq(pParse, zColl);
  4168         -        pKey->aSortOrder[i] = pIdx->aSortOrder[i];
  4169         -      }
  4170         -      if( pParse->nErr ){
  4171         -        sqlite3KeyInfoUnref(pKey);
  4172         -      }else{
  4173         -        pIdx->pKeyInfo = pKey;
  4174         -      }
         4224  +    if( pParse->nErr ){
         4225  +      sqlite3KeyInfoUnref(pKey);
         4226  +      pKey = 0;
  4175   4227       }
  4176   4228     }
  4177         -  return sqlite3KeyInfoRef(pIdx->pKeyInfo);
         4229  +  return pKey;
  4178   4230   }
  4179   4231   
  4180   4232   #ifndef SQLITE_OMIT_CTE
  4181   4233   /* 
  4182   4234   ** This routine is invoked once per CTE by the parser while parsing a 
  4183   4235   ** WITH clause. 
  4184   4236   */

Changes to src/callback.c.

   138    138   /*
   139    139   ** Locate and return an entry from the db.aCollSeq hash table. If the entry
   140    140   ** specified by zName and nName is not found and parameter 'create' is
   141    141   ** true, then create a new entry. Otherwise return NULL.
   142    142   **
   143    143   ** Each pointer stored in the sqlite3.aCollSeq hash table contains an
   144    144   ** array of three CollSeq structures. The first is the collation sequence
   145         -** prefferred for UTF-8, the second UTF-16le, and the third UTF-16be.
          145  +** preferred for UTF-8, the second UTF-16le, and the third UTF-16be.
   146    146   **
   147    147   ** Stored immediately after the three collation sequences is a copy of
   148    148   ** the collation sequence name. A pointer to this string is stored in
   149    149   ** each collation sequence structure.
   150    150   */
   151    151   static CollSeq *findCollSeqEntry(
   152    152     sqlite3 *db,          /* Database connection */
   153    153     const char *zName,    /* Name of the collating sequence */
   154    154     int create            /* Create a new entry if true */
   155    155   ){
   156    156     CollSeq *pColl;
   157         -  int nName = sqlite3Strlen30(zName);
   158         -  pColl = sqlite3HashFind(&db->aCollSeq, zName, nName);
          157  +  pColl = sqlite3HashFind(&db->aCollSeq, zName);
   159    158   
   160    159     if( 0==pColl && create ){
   161         -    pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName + 1 );
          160  +    int nName = sqlite3Strlen30(zName);
          161  +    pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName + 1);
   162    162       if( pColl ){
   163    163         CollSeq *pDel = 0;
   164    164         pColl[0].zName = (char*)&pColl[3];
   165    165         pColl[0].enc = SQLITE_UTF8;
   166    166         pColl[1].zName = (char*)&pColl[3];
   167    167         pColl[1].enc = SQLITE_UTF16LE;
   168    168         pColl[2].zName = (char*)&pColl[3];
   169    169         pColl[2].enc = SQLITE_UTF16BE;
   170    170         memcpy(pColl[0].zName, zName, nName);
   171    171         pColl[0].zName[nName] = 0;
   172         -      pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, nName, pColl);
          172  +      pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, pColl);
   173    173   
   174    174         /* If a malloc() failure occurred in sqlite3HashInsert(), it will 
   175    175         ** return the pColl pointer to be deleted (because it wasn't added
   176    176         ** to the hash table).
   177    177         */
   178    178         assert( pDel==0 || pDel==pColl );
   179    179         if( pDel!=0 ){
................................................................................
   443    443     for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){
   444    444       Table *pTab = sqliteHashData(pElem);
   445    445       sqlite3DeleteTable(0, pTab);
   446    446     }
   447    447     sqlite3HashClear(&temp1);
   448    448     sqlite3HashClear(&pSchema->fkeyHash);
   449    449     pSchema->pSeqTab = 0;
   450         -  if( pSchema->flags & DB_SchemaLoaded ){
          450  +  if( pSchema->schemaFlags & DB_SchemaLoaded ){
   451    451       pSchema->iGeneration++;
   452         -    pSchema->flags &= ~DB_SchemaLoaded;
          452  +    pSchema->schemaFlags &= ~DB_SchemaLoaded;
   453    453     }
   454    454   }
   455    455   
   456    456   /*
   457    457   ** Find and return the schema associated with a BTree.  Create
   458    458   ** a new one if necessary.
   459    459   */

Changes to src/complete.c.

    66     66   **   (2) NORMAL    We are in the middle of statement which ends with a single
    67     67   **                 semicolon.
    68     68   **
    69     69   **   (3) EXPLAIN   The keyword EXPLAIN has been seen at the beginning of 
    70     70   **                 a statement.
    71     71   **
    72     72   **   (4) CREATE    The keyword CREATE has been seen at the beginning of a
    73         -**                 statement, possibly preceeded by EXPLAIN and/or followed by
           73  +**                 statement, possibly preceded by EXPLAIN and/or followed by
    74     74   **                 TEMP or TEMPORARY
    75     75   **
    76     76   **   (5) TRIGGER   We are in the middle of a trigger definition that must be
    77     77   **                 ended by a semicolon, the keyword END, and another semicolon.
    78     78   **
    79     79   **   (6) SEMI      We've seen the first semicolon in the ";END;" that occurs at
    80     80   **                 the end of a trigger definition.
    81     81   **
    82     82   **   (7) END       We've seen the ";END" of the ";END;" that occurs at the end
    83         -**                 of a trigger difinition.
           83  +**                 of a trigger definition.
    84     84   **
    85     85   ** Transitions between states above are determined by tokens extracted
    86     86   ** from the input.  The following tokens are significant:
    87     87   **
    88     88   **   (0) tkSEMI      A semicolon.
    89     89   **   (1) tkWS        Whitespace.
    90     90   **   (2) tkOTHER     Any other SQL token.
................................................................................
   119    119        /* 4  CREATE: */ {    1,  4,     2,       2,      2,    4,       5,   2, },
   120    120        /* 5 TRIGGER: */ {    6,  5,     5,       5,      5,    5,       5,   5, },
   121    121        /* 6    SEMI: */ {    6,  6,     5,       5,      5,    5,       5,   7, },
   122    122        /* 7     END: */ {    1,  7,     5,       5,      5,    5,       5,   5, },
   123    123     };
   124    124   #else
   125    125     /* If triggers are not supported by this compile then the statement machine
   126         -  ** used to detect the end of a statement is much simplier
          126  +  ** used to detect the end of a statement is much simpler
   127    127     */
   128    128     static const u8 trans[3][3] = {
   129    129                        /* Token:           */
   130    130        /* State:       **  SEMI  WS  OTHER */
   131    131        /* 0 INVALID: */ {    1,  0,     2, },
   132    132        /* 1   START: */ {    1,  1,     2, },
   133    133        /* 2  NORMAL: */ {    1,  2,     2, },
   134    134     };
   135    135   #endif /* SQLITE_OMIT_TRIGGER */
          136  +
          137  +#ifdef SQLITE_ENABLE_API_ARMOR
          138  +  if( zSql==0 ){
          139  +    (void)SQLITE_MISUSE_BKPT;
          140  +    return 0;
          141  +  }
          142  +#endif
   136    143   
   137    144     while( *zSql ){
   138    145       switch( *zSql ){
   139    146         case ';': {  /* A semicolon */
   140    147           token = tkSEMI;
   141    148           break;
   142    149         }

Changes to src/ctime.c.

    58     58     "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE),
    59     59   #endif
    60     60   #ifdef SQLITE_DISABLE_DIRSYNC
    61     61     "DISABLE_DIRSYNC",
    62     62   #endif
    63     63   #ifdef SQLITE_DISABLE_LFS
    64     64     "DISABLE_LFS",
           65  +#endif
           66  +#ifdef SQLITE_ENABLE_API_ARMOR
           67  +  "ENABLE_API_ARMOR",
    65     68   #endif
    66     69   #ifdef SQLITE_ENABLE_ATOMIC_WRITE
    67     70     "ENABLE_ATOMIC_WRITE",
    68     71   #endif
    69     72   #ifdef SQLITE_ENABLE_CEROD
    70     73     "ENABLE_CEROD",
    71     74   #endif
................................................................................
   363    366     "TEST",
   364    367   #endif
   365    368   #if defined(SQLITE_THREADSAFE)
   366    369     "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE),
   367    370   #endif
   368    371   #ifdef SQLITE_USE_ALLOCA
   369    372     "USE_ALLOCA",
          373  +#endif
          374  +#ifdef SQLITE_USER_AUTHENTICATION
          375  +  "USER_AUTHENTICATION",
   370    376   #endif
   371    377   #ifdef SQLITE_WIN32_MALLOC
   372    378     "WIN32_MALLOC",
   373    379   #endif
   374    380   #ifdef SQLITE_ZERO_MALLOC
   375    381     "ZERO_MALLOC"
   376    382   #endif
................................................................................
   381    387   ** was used and false if not.
   382    388   **
   383    389   ** The name can optionally begin with "SQLITE_" but the "SQLITE_" prefix
   384    390   ** is not required for a match.
   385    391   */
   386    392   int sqlite3_compileoption_used(const char *zOptName){
   387    393     int i, n;
          394  +
          395  +#ifdef SQLITE_ENABLE_API_ARMOR
          396  +  if( zOptName==0 ){
          397  +    (void)SQLITE_MISUSE_BKPT;
          398  +    return 0;
          399  +  }
          400  +#endif
   388    401     if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7;
   389    402     n = sqlite3Strlen30(zOptName);
   390    403   
   391    404     /* Since ArraySize(azCompileOpt) is normally in single digits, a
   392    405     ** linear search is adequate.  No need for a binary search. */
   393    406     for(i=0; i<ArraySize(azCompileOpt); i++){
   394    407       if( sqlite3StrNICmp(zOptName, azCompileOpt[i], n)==0
   395         -     && sqlite3CtypeMap[(unsigned char)azCompileOpt[i][n]]==0
          408  +     && sqlite3IsIdChar((unsigned char)azCompileOpt[i][n])==0
   396    409       ){
   397    410         return 1;
   398    411       }
   399    412     }
   400    413     return 0;
   401    414   }
   402    415   

Changes to src/date.c.

    12     12   ** This file contains the C functions that implement date and time
    13     13   ** functions for SQLite.  
    14     14   **
    15     15   ** There is only one exported symbol in this file - the function
    16     16   ** sqlite3RegisterDateTimeFunctions() found at the bottom of the file.
    17     17   ** All other code has file scope.
    18     18   **
    19         -** SQLite processes all times and dates as Julian Day numbers.  The
           19  +** SQLite processes all times and dates as julian day numbers.  The
    20     20   ** dates and times are stored as the number of days since noon
    21     21   ** in Greenwich on November 24, 4714 B.C. according to the Gregorian
    22     22   ** calendar system. 
    23     23   **
    24     24   ** 1970-01-01 00:00:00 is JD 2440587.5
    25     25   ** 2000-01-01 00:00:00 is JD 2451544.5
    26     26   **
    27         -** This implemention requires years to be expressed as a 4-digit number
           27  +** This implementation requires years to be expressed as a 4-digit number
    28     28   ** which means that only dates between 0000-01-01 and 9999-12-31 can
    29     29   ** be represented, even though julian day numbers allow a much wider
    30     30   ** range of dates.
    31     31   **
    32     32   ** The Gregorian calendar system is used for all dates and times,
    33     33   ** even those that predate the Gregorian calendar.  Historians usually
    34         -** use the Julian calendar for dates prior to 1582-10-15 and for some
           34  +** use the julian calendar for dates prior to 1582-10-15 and for some
    35     35   ** dates afterwards, depending on locale.  Beware of this difference.
    36     36   **
    37     37   ** The conversion algorithms are implemented based on descriptions
    38     38   ** in the following text:
    39     39   **
    40     40   **      Jean Meeus
    41     41   **      Astronomical Algorithms, 2nd Edition, 1998
................................................................................
   300    300       return 0;
   301    301     }else{
   302    302       return 1;
   303    303     }
   304    304   }
   305    305   
   306    306   /*
   307         -** Attempt to parse the given string into a Julian Day Number.  Return
          307  +** Attempt to parse the given string into a julian day number.  Return
   308    308   ** the number of errors.
   309    309   **
   310    310   ** The following are acceptable forms for the input string:
   311    311   **
   312    312   **      YYYY-MM-DD HH:MM:SS.FFF  +/-HH:MM
   313    313   **      DDDD.DD 
   314    314   **      now
................................................................................
   871    871   **
   872    872   ** Return a string described by FORMAT.  Conversions as follows:
   873    873   **
   874    874   **   %d  day of month
   875    875   **   %f  ** fractional seconds  SS.SSS
   876    876   **   %H  hour 00-24
   877    877   **   %j  day of year 000-366
   878         -**   %J  ** Julian day number
          878  +**   %J  ** julian day number
   879    879   **   %m  month 01-12
   880    880   **   %M  minute 00-59
   881    881   **   %s  seconds since 1970-01-01
   882    882   **   %S  seconds 00-59
   883    883   **   %w  day of week 0-6  sunday==0
   884    884   **   %W  week of year 00-53
   885    885   **   %Y  year 0000-9999

Changes to src/delete.c.

    86     86   ** pWhere argument is an optional WHERE clause that restricts the
    87     87   ** set of rows in the view that are to be added to the ephemeral table.
    88     88   */
    89     89   void sqlite3MaterializeView(
    90     90     Parse *pParse,       /* Parsing context */
    91     91     Table *pView,        /* View definition */
    92     92     Expr *pWhere,        /* Optional WHERE clause to be added */
    93         -  int iCur             /* Cursor number for ephemerial table */
           93  +  int iCur             /* Cursor number for ephemeral table */
    94     94   ){
    95     95     SelectDest dest;
    96     96     Select *pSel;
    97     97     SrcList *pFrom;
    98     98     sqlite3 *db = pParse->db;
    99     99     int iDb = sqlite3SchemaToIndex(db, pView->pSchema);
   100    100     pWhere = sqlite3ExprDup(db, pWhere, 0);
................................................................................
   244    244     int iKey;              /* Memory cell holding key of row to be deleted */
   245    245     i16 nKey;              /* Number of memory cells in the row key */
   246    246     int iEphCur = 0;       /* Ephemeral table holding all primary key values */
   247    247     int iRowSet = 0;       /* Register for rowset of rows to delete */
   248    248     int addrBypass = 0;    /* Address of jump over the delete logic */
   249    249     int addrLoop = 0;      /* Top of the delete loop */
   250    250     int addrDelete = 0;    /* Jump directly to the delete logic */
   251         -  int addrEphOpen = 0;   /* Instruction to open the Ephermeral table */
          251  +  int addrEphOpen = 0;   /* Instruction to open the Ephemeral table */
   252    252    
   253    253   #ifndef SQLITE_OMIT_TRIGGER
   254    254     int isView;                  /* True if attempting to delete from a view */
   255    255     Trigger *pTrigger;           /* List of table triggers, if required */
   256    256   #endif
   257    257   
   258    258     memset(&sContext, 0, sizeof(sContext));
................................................................................
   324    324     if( v==0 ){
   325    325       goto delete_from_cleanup;
   326    326     }
   327    327     if( pParse->nested==0 ) sqlite3VdbeCountChanges(v);
   328    328     sqlite3BeginWriteOperation(pParse, 1, iDb);
   329    329   
   330    330     /* If we are trying to delete from a view, realize that view into
   331         -  ** a ephemeral table.
          331  +  ** an ephemeral table.
   332    332     */
   333    333   #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER)
   334    334     if( isView ){
   335    335       sqlite3MaterializeView(pParse, pTab, pWhere, iTabCur);
   336    336       iDataCur = iIdxCur = iTabCur;
   337    337     }
   338    338   #endif
................................................................................
   378    378       if( HasRowid(pTab) ){
   379    379         /* For a rowid table, initialize the RowSet to an empty set */
   380    380         pPk = 0;
   381    381         nPk = 1;
   382    382         iRowSet = ++pParse->nMem;
   383    383         sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet);
   384    384       }else{
   385         -      /* For a WITHOUT ROWID table, create an ephermeral table used to
          385  +      /* For a WITHOUT ROWID table, create an ephemeral table used to
   386    386         ** hold all primary keys for rows to be deleted. */
   387    387         pPk = sqlite3PrimaryKeyIndex(pTab);
   388    388         assert( pPk!=0 );
   389    389         nPk = pPk->nKeyCol;
   390    390         iPk = pParse->nMem+1;
   391    391         pParse->nMem += nPk;
   392    392         iEphCur = pParse->nTab++;
................................................................................
   462    462     
   463    463       /* Unless this is a view, open cursors for the table we are 
   464    464       ** deleting from and all its indices. If this is a view, then the
   465    465       ** only effect this statement has is to fire the INSTEAD OF 
   466    466       ** triggers.
   467    467       */
   468    468       if( !isView ){
          469  +      testcase( IsVirtual(pTab) );
   469    470         sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iTabCur, aToOpen,
   470    471                                    &iDataCur, &iIdxCur);
   471         -      assert( pPk || iDataCur==iTabCur );
   472         -      assert( pPk || iIdxCur==iDataCur+1 );
          472  +      assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur );
          473  +      assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 );
   473    474       }
   474    475     
   475    476       /* Set up a loop over the rowids/primary-keys that were found in the
   476    477       ** where-clause loop above.
   477    478       */
   478    479       if( okOnePass ){
   479    480         /* Just one row.  Hence the top-of-loop is a no-op */
   480         -      assert( nKey==nPk ); /* OP_Found will use an unpacked key */
          481  +      assert( nKey==nPk );  /* OP_Found will use an unpacked key */
          482  +      assert( !IsVirtual(pTab) );
   481    483         if( aToOpen[iDataCur-iTabCur] ){
   482         -        assert( pPk!=0 );
          484  +        assert( pPk!=0 || pTab->pSelect!=0 );
   483    485           sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey);
   484    486           VdbeCoverage(v);
   485    487         }
   486    488       }else if( pPk ){
   487    489         addrLoop = sqlite3VdbeAddOp1(v, OP_Rewind, iEphCur); VdbeCoverage(v);
   488    490         sqlite3VdbeAddOp2(v, OP_RowKey, iEphCur, iKey);
   489    491         assert( nKey==0 );  /* OP_Found will use a composite key */
................................................................................
   551    553     sqlite3AuthContextPop(&sContext);
   552    554     sqlite3SrcListDelete(db, pTabList);
   553    555     sqlite3ExprDelete(db, pWhere);
   554    556     sqlite3DbFree(db, aToOpen);
   555    557     return;
   556    558   }
   557    559   /* Make sure "isView" and other macros defined above are undefined. Otherwise
   558         -** thely may interfere with compilation of other functions in this file
          560  +** they may interfere with compilation of other functions in this file
   559    561   ** (or in another file, if this file becomes part of the amalgamation).  */
   560    562   #ifdef isView
   561    563    #undef isView
   562    564   #endif
   563    565   #ifdef pTrigger
   564    566    #undef pTrigger
   565    567   #endif

Changes to src/expr.c.

    18     18   ** Return the 'affinity' of the expression pExpr if any.
    19     19   **
    20     20   ** If pExpr is a column, a reference to a column via an 'AS' alias,
    21     21   ** or a sub-select with a column as the return value, then the 
    22     22   ** affinity of that column is returned. Otherwise, 0x00 is returned,
    23     23   ** indicating no affinity for the expression.
    24     24   **
    25         -** i.e. the WHERE clause expresssions in the following statements all
           25  +** i.e. the WHERE clause expressions in the following statements all
    26     26   ** have an affinity:
    27     27   **
    28     28   ** CREATE TABLE t1(a);
    29     29   ** SELECT * FROM t1 WHERE a;
    30     30   ** SELECT a AS b FROM t1 WHERE b;
    31     31   ** SELECT * FROM t1 WHERE (select a from t1);
    32     32   */
    33     33   char sqlite3ExprAffinity(Expr *pExpr){
    34     34     int op;
    35     35     pExpr = sqlite3ExprSkipCollate(pExpr);
    36         -  if( pExpr->flags & EP_Generic ) return SQLITE_AFF_NONE;
           36  +  if( pExpr->flags & EP_Generic ) return 0;
    37     37     op = pExpr->op;
    38     38     if( op==TK_SELECT ){
    39     39       assert( pExpr->flags&EP_xIsSelect );
    40     40       return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr);
    41     41     }
    42     42   #ifndef SQLITE_OMIT_CAST
    43     43     if( op==TK_CAST ){
................................................................................
   497    497         pRoot->flags |= EP_Collate & pLeft->flags;
   498    498       }
   499    499       exprSetHeight(pRoot);
   500    500     }
   501    501   }
   502    502   
   503    503   /*
   504         -** Allocate a Expr node which joins as many as two subtrees.
          504  +** Allocate an Expr node which joins as many as two subtrees.
   505    505   **
   506    506   ** One or both of the subtrees can be NULL.  Return a pointer to the new
   507    507   ** Expr node.  Or, if an OOM error occurs, set pParse->db->mallocFailed,
   508    508   ** free the subtrees and return NULL.
   509    509   */
   510    510   Expr *sqlite3PExpr(
   511    511     Parse *pParse,          /* Parsing context */
................................................................................
   607    607   **
   608    608   ** Wildcards of the form "?nnn" are assigned the number "nnn".  We make
   609    609   ** sure "nnn" is not too be to avoid a denial of service attack when
   610    610   ** the SQL statement comes from an external source.
   611    611   **
   612    612   ** Wildcards of the form ":aaa", "@aaa", or "$aaa" are assigned the same number
   613    613   ** as the previous instance of the same wildcard.  Or if this is the first
   614         -** instance of the wildcard, the next sequenial variable number is
          614  +** instance of the wildcard, the next sequential variable number is
   615    615   ** assigned.
   616    616   */
   617    617   void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){
   618    618     sqlite3 *db = pParse->db;
   619    619     const char *z;
   620    620   
   621    621     if( pExpr==0 ) return;
................................................................................
   742    742   ** return value with EP_Reduced|EP_TokenOnly.
   743    743   **
   744    744   ** Note that with flags==EXPRDUP_REDUCE, this routines works on full-size
   745    745   ** (unreduced) Expr objects as they or originally constructed by the parser.
   746    746   ** During expression analysis, extra information is computed and moved into
   747    747   ** later parts of teh Expr object and that extra information might get chopped
   748    748   ** off if the expression is reduced.  Note also that it does not work to
   749         -** make a EXPRDUP_REDUCE copy of a reduced expression.  It is only legal
          749  +** make an EXPRDUP_REDUCE copy of a reduced expression.  It is only legal
   750    750   ** to reduce a pristine expression tree from the parser.  The implementation
   751    751   ** of dupedExprStructSize() contain multiple assert() statements that attempt
   752    752   ** to enforce this constraint.
   753    753   */
   754    754   static int dupedExprStructSize(Expr *p, int flags){
   755    755     int nSize;
   756    756     assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */
................................................................................
   811    811   }
   812    812   
   813    813   /*
   814    814   ** This function is similar to sqlite3ExprDup(), except that if pzBuffer 
   815    815   ** is not NULL then *pzBuffer is assumed to point to a buffer large enough 
   816    816   ** to store the copy of expression p, the copies of p->u.zToken
   817    817   ** (if applicable), and the copies of the p->pLeft and p->pRight expressions,
   818         -** if any. Before returning, *pzBuffer is set to the first byte passed the
          818  +** if any. Before returning, *pzBuffer is set to the first byte past the
   819    819   ** portion of the buffer copied into by this function.
   820    820   */
   821    821   static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){
   822    822     Expr *pNew = 0;                      /* Value to return */
   823    823     if( p ){
   824    824       const int isReduced = (flags&EXPRDUP_REDUCE);
   825    825       u8 *zAlloc;
................................................................................
  1065   1065     pNew->iLimit = 0;
  1066   1066     pNew->iOffset = 0;
  1067   1067     pNew->selFlags = p->selFlags & ~SF_UsesEphemeral;
  1068   1068     pNew->addrOpenEphm[0] = -1;
  1069   1069     pNew->addrOpenEphm[1] = -1;
  1070   1070     pNew->nSelectRow = p->nSelectRow;
  1071   1071     pNew->pWith = withDup(db, p->pWith);
         1072  +  sqlite3SelectSetName(pNew, p->zSelName);
  1072   1073     return pNew;
  1073   1074   }
  1074   1075   #else
  1075   1076   Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){
  1076   1077     assert( p==0 );
  1077   1078     return 0;
  1078   1079   }
................................................................................
  1205   1206       sqlite3DbFree(db, pItem->zSpan);
  1206   1207     }
  1207   1208     sqlite3DbFree(db, pList->a);
  1208   1209     sqlite3DbFree(db, pList);
  1209   1210   }
  1210   1211   
  1211   1212   /*
  1212         -** These routines are Walker callbacks.  Walker.u.pi is a pointer
  1213         -** to an integer.  These routines are checking an expression to see
  1214         -** if it is a constant.  Set *Walker.u.pi to 0 if the expression is
  1215         -** not constant.
         1213  +** These routines are Walker callbacks used to check expressions to
         1214  +** see if they are "constant" for some definition of constant.  The
         1215  +** Walker.eCode value determines the type of "constant" we are looking
         1216  +** for.
  1216   1217   **
  1217   1218   ** These callback routines are used to implement the following:
  1218   1219   **
  1219         -**     sqlite3ExprIsConstant()
  1220         -**     sqlite3ExprIsConstantNotJoin()
  1221         -**     sqlite3ExprIsConstantOrFunction()
         1220  +**     sqlite3ExprIsConstant()                  pWalker->eCode==1
         1221  +**     sqlite3ExprIsConstantNotJoin()           pWalker->eCode==2
         1222  +**     sqlite3ExprRefOneTableOnly()             pWalker->eCode==3
         1223  +**     sqlite3ExprIsConstantOrFunction()        pWalker->eCode==4 or 5
  1222   1224   **
         1225  +** In all cases, the callbacks set Walker.eCode=0 and abort if the expression
         1226  +** is found to not be a constant.
         1227  +**
         1228  +** The sqlite3ExprIsConstantOrFunction() is used for evaluating expressions
         1229  +** in a CREATE TABLE statement.  The Walker.eCode value is 5 when parsing
         1230  +** an existing schema and 4 when processing a new statement.  A bound
         1231  +** parameter raises an error for new statements, but is silently converted
         1232  +** to NULL for existing schemas.  This allows sqlite_master tables that 
         1233  +** contain a bound parameter because they were generated by older versions
         1234  +** of SQLite to be parsed by newer versions of SQLite without raising a
         1235  +** malformed schema error.
  1223   1236   */
  1224   1237   static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){
  1225   1238   
  1226         -  /* If pWalker->u.i is 3 then any term of the expression that comes from
  1227         -  ** the ON or USING clauses of a join disqualifies the expression
         1239  +  /* If pWalker->eCode is 2 then any term of the expression that comes from
         1240  +  ** the ON or USING clauses of a left join disqualifies the expression
  1228   1241     ** from being considered constant. */
  1229         -  if( pWalker->u.i==3 && ExprHasProperty(pExpr, EP_FromJoin) ){
  1230         -    pWalker->u.i = 0;
         1242  +  if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_FromJoin) ){
         1243  +    pWalker->eCode = 0;
  1231   1244       return WRC_Abort;
  1232   1245     }
  1233   1246   
  1234   1247     switch( pExpr->op ){
  1235   1248       /* Consider functions to be constant if all their arguments are constant
  1236         -    ** and either pWalker->u.i==2 or the function as the SQLITE_FUNC_CONST
  1237         -    ** flag. */
         1249  +    ** and either pWalker->eCode==4 or 5 or the function has the
         1250  +    ** SQLITE_FUNC_CONST flag. */
  1238   1251       case TK_FUNCTION:
  1239         -      if( pWalker->u.i==2 || ExprHasProperty(pExpr,EP_Constant) ){
         1252  +      if( pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_Constant) ){
  1240   1253           return WRC_Continue;
         1254  +      }else{
         1255  +        pWalker->eCode = 0;
         1256  +        return WRC_Abort;
  1241   1257         }
  1242         -      /* Fall through */
  1243   1258       case TK_ID:
  1244   1259       case TK_COLUMN:
  1245   1260       case TK_AGG_FUNCTION:
  1246   1261       case TK_AGG_COLUMN:
  1247   1262         testcase( pExpr->op==TK_ID );
  1248   1263         testcase( pExpr->op==TK_COLUMN );
  1249   1264         testcase( pExpr->op==TK_AGG_FUNCTION );
  1250   1265         testcase( pExpr->op==TK_AGG_COLUMN );
  1251         -      pWalker->u.i = 0;
  1252         -      return WRC_Abort;
         1266  +      if( pWalker->eCode==3 && pExpr->iTable==pWalker->u.iCur ){
         1267  +        return WRC_Continue;
         1268  +      }else{
         1269  +        pWalker->eCode = 0;
         1270  +        return WRC_Abort;
         1271  +      }
         1272  +    case TK_VARIABLE:
         1273  +      if( pWalker->eCode==5 ){
         1274  +        /* Silently convert bound parameters that appear inside of CREATE
         1275  +        ** statements into a NULL when parsing the CREATE statement text out
         1276  +        ** of the sqlite_master table */
         1277  +        pExpr->op = TK_NULL;
         1278  +      }else if( pWalker->eCode==4 ){
         1279  +        /* A bound parameter in a CREATE statement that originates from
         1280  +        ** sqlite3_prepare() causes an error */
         1281  +        pWalker->eCode = 0;
         1282  +        return WRC_Abort;
         1283  +      }
         1284  +      /* Fall through */
  1253   1285       default:
  1254   1286         testcase( pExpr->op==TK_SELECT ); /* selectNodeIsConstant will disallow */
  1255   1287         testcase( pExpr->op==TK_EXISTS ); /* selectNodeIsConstant will disallow */
  1256   1288         return WRC_Continue;
  1257   1289     }
  1258   1290   }
  1259   1291   static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){
  1260   1292     UNUSED_PARAMETER(NotUsed);
  1261         -  pWalker->u.i = 0;
         1293  +  pWalker->eCode = 0;
  1262   1294     return WRC_Abort;
  1263   1295   }
  1264         -static int exprIsConst(Expr *p, int initFlag){
         1296  +static int exprIsConst(Expr *p, int initFlag, int iCur){
  1265   1297     Walker w;
  1266   1298     memset(&w, 0, sizeof(w));
  1267         -  w.u.i = initFlag;
         1299  +  w.eCode = initFlag;
  1268   1300     w.xExprCallback = exprNodeIsConstant;
  1269   1301     w.xSelectCallback = selectNodeIsConstant;
         1302  +  w.u.iCur = iCur;
  1270   1303     sqlite3WalkExpr(&w, p);
  1271         -  return w.u.i;
         1304  +  return w.eCode;
  1272   1305   }
  1273   1306   
  1274   1307   /*
  1275         -** Walk an expression tree.  Return 1 if the expression is constant
         1308  +** Walk an expression tree.  Return non-zero if the expression is constant
  1276   1309   ** and 0 if it involves variables or function calls.
  1277   1310   **
  1278   1311   ** For the purposes of this function, a double-quoted string (ex: "abc")
  1279   1312   ** is considered a variable but a single-quoted string (ex: 'abc') is
  1280   1313   ** a constant.
  1281   1314   */
  1282   1315   int sqlite3ExprIsConstant(Expr *p){
  1283         -  return exprIsConst(p, 1);
         1316  +  return exprIsConst(p, 1, 0);
  1284   1317   }
  1285   1318   
  1286   1319   /*
  1287         -** Walk an expression tree.  Return 1 if the expression is constant
         1320  +** Walk an expression tree.  Return non-zero if the expression is constant
  1288   1321   ** that does no originate from the ON or USING clauses of a join.
  1289   1322   ** Return 0 if it involves variables or function calls or terms from
  1290   1323   ** an ON or USING clause.
  1291   1324   */
  1292   1325   int sqlite3ExprIsConstantNotJoin(Expr *p){
  1293         -  return exprIsConst(p, 3);
         1326  +  return exprIsConst(p, 2, 0);
  1294   1327   }
  1295   1328   
  1296   1329   /*
  1297         -** Walk an expression tree.  Return 1 if the expression is constant
         1330  +** Walk an expression tree.  Return non-zero if the expression constant
         1331  +** for any single row of the table with cursor iCur.  In other words, the
         1332  +** expression must not refer to any non-deterministic function nor any
         1333  +** table other than iCur.
         1334  +*/
         1335  +int sqlite3ExprIsTableConstant(Expr *p, int iCur){
         1336  +  return exprIsConst(p, 3, iCur);
         1337  +}
         1338  +
         1339  +/*
         1340  +** Walk an expression tree.  Return non-zero if the expression is constant
  1298   1341   ** or a function call with constant arguments.  Return and 0 if there
  1299   1342   ** are any variables.
  1300   1343   **
  1301   1344   ** For the purposes of this function, a double-quoted string (ex: "abc")
  1302   1345   ** is considered a variable but a single-quoted string (ex: 'abc') is
  1303   1346   ** a constant.
  1304   1347   */
  1305         -int sqlite3ExprIsConstantOrFunction(Expr *p){
  1306         -  return exprIsConst(p, 2);
         1348  +int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){
         1349  +  assert( isInit==0 || isInit==1 );
         1350  +  return exprIsConst(p, 4+isInit, 0);
  1307   1351   }
  1308   1352   
  1309   1353   /*
  1310   1354   ** If the expression p codes a constant integer that is small enough
  1311   1355   ** to fit in a 32-bit integer, return 1 and put the value of the integer
  1312   1356   ** in *pValue.  If the expression is not an integer or if it is too big
  1313   1357   ** to fit in a signed 32-bit integer, return 0 and leave *pValue unchanged.
................................................................................
  1364   1408     if( op==TK_REGISTER ) op = p->op2;
  1365   1409     switch( op ){
  1366   1410       case TK_INTEGER:
  1367   1411       case TK_STRING:
  1368   1412       case TK_FLOAT:
  1369   1413       case TK_BLOB:
  1370   1414         return 0;
         1415  +    case TK_COLUMN:
         1416  +      assert( p->pTab!=0 );
         1417  +      return ExprHasProperty(p, EP_CanBeNull) ||
         1418  +             (p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0);
  1371   1419       default:
  1372   1420         return 1;
  1373   1421     }
  1374   1422   }
  1375   1423   
  1376   1424   /*
  1377   1425   ** Return TRUE if the given expression is a constant which would be
................................................................................
  1471   1519   ** address of the new instruction.
  1472   1520   */
  1473   1521   int sqlite3CodeOnce(Parse *pParse){
  1474   1522     Vdbe *v = sqlite3GetVdbe(pParse);      /* Virtual machine being coded */
  1475   1523     return sqlite3VdbeAddOp1(v, OP_Once, pParse->nOnce++);
  1476   1524   }
  1477   1525   
         1526  +/*
         1527  +** Generate code that checks the left-most column of index table iCur to see if
         1528  +** it contains any NULL entries.  Cause the register at regHasNull to be set
         1529  +** to a non-NULL value if iCur contains no NULLs.  Cause register regHasNull
         1530  +** to be set to NULL if iCur contains one or more NULL values.
         1531  +*/
         1532  +static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){
         1533  +  int j1;
         1534  +  sqlite3VdbeAddOp2(v, OP_Integer, 0, regHasNull);
         1535  +  j1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
         1536  +  sqlite3VdbeAddOp3(v, OP_Column, iCur, 0, regHasNull);
         1537  +  sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG);
         1538  +  VdbeComment((v, "first_entry_in(%d)", iCur));
         1539  +  sqlite3VdbeJumpHere(v, j1);
         1540  +}
         1541  +
         1542  +
         1543  +#ifndef SQLITE_OMIT_SUBQUERY
         1544  +/*
         1545  +** The argument is an IN operator with a list (not a subquery) on the 
         1546  +** right-hand side.  Return TRUE if that list is constant.
         1547  +*/
         1548  +static int sqlite3InRhsIsConstant(Expr *pIn){
         1549  +  Expr *pLHS;
         1550  +  int res;
         1551  +  assert( !ExprHasProperty(pIn, EP_xIsSelect) );
         1552  +  pLHS = pIn->pLeft;
         1553  +  pIn->pLeft = 0;
         1554  +  res = sqlite3ExprIsConstant(pIn);
         1555  +  pIn->pLeft = pLHS;
         1556  +  return res;
         1557  +}
         1558  +#endif
         1559  +
  1478   1560   /*
  1479   1561   ** This function is used by the implementation of the IN (...) operator.
  1480   1562   ** The pX parameter is the expression on the RHS of the IN operator, which
  1481   1563   ** might be either a list of expressions or a subquery.
  1482   1564   **
  1483   1565   ** The job of this routine is to find or create a b-tree object that can
  1484   1566   ** be used either to test for membership in the RHS set or to iterate through
  1485   1567   ** all members of the RHS set, skipping duplicates.
  1486   1568   **
  1487         -** A cursor is opened on the b-tree object that the RHS of the IN operator
         1569  +** A cursor is opened on the b-tree object that is the RHS of the IN operator
  1488   1570   ** and pX->iTable is set to the index of that cursor.
  1489   1571   **
  1490   1572   ** The returned value of this function indicates the b-tree type, as follows:
  1491   1573   **
  1492   1574   **   IN_INDEX_ROWID      - The cursor was opened on a database table.
  1493   1575   **   IN_INDEX_INDEX_ASC  - The cursor was opened on an ascending index.
  1494   1576   **   IN_INDEX_INDEX_DESC - The cursor was opened on a descending index.
  1495   1577   **   IN_INDEX_EPH        - The cursor was opened on a specially created and
  1496   1578   **                         populated epheremal table.
         1579  +**   IN_INDEX_NOOP       - No cursor was allocated.  The IN operator must be
         1580  +**                         implemented as a sequence of comparisons.
  1497   1581   **
  1498   1582   ** An existing b-tree might be used if the RHS expression pX is a simple
  1499   1583   ** subquery such as:
  1500   1584   **
  1501   1585   **     SELECT <column> FROM <table>
  1502   1586   **
  1503   1587   ** If the RHS of the IN operator is a list or a more complex subquery, then
  1504   1588   ** an ephemeral table might need to be generated from the RHS and then
  1505         -** pX->iTable made to point to the ephermeral table instead of an
  1506         -** existing table.  
         1589  +** pX->iTable made to point to the ephemeral table instead of an
         1590  +** existing table.
  1507   1591   **
  1508         -** If the prNotFound parameter is 0, then the b-tree will be used to iterate
  1509         -** through the set members, skipping any duplicates. In this case an
  1510         -** epheremal table must be used unless the selected <column> is guaranteed
         1592  +** The inFlags parameter must contain exactly one of the bits
         1593  +** IN_INDEX_MEMBERSHIP or IN_INDEX_LOOP.  If inFlags contains
         1594  +** IN_INDEX_MEMBERSHIP, then the generated table will be used for a
         1595  +** fast membership test.  When the IN_INDEX_LOOP bit is set, the
         1596  +** IN index will be used to loop over all values of the RHS of the
         1597  +** IN operator.
         1598  +**
         1599  +** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate
         1600  +** through the set members) then the b-tree must not contain duplicates.
         1601  +** An epheremal table must be used unless the selected <column> is guaranteed
  1511   1602   ** to be unique - either because it is an INTEGER PRIMARY KEY or it
  1512   1603   ** has a UNIQUE constraint or UNIQUE index.
  1513   1604   **
  1514         -** If the prNotFound parameter is not 0, then the b-tree will be used 
  1515         -** for fast set membership tests. In this case an epheremal table must 
         1605  +** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used 
         1606  +** for fast set membership tests) then an epheremal table must 
  1516   1607   ** be used unless <column> is an INTEGER PRIMARY KEY or an index can 
  1517   1608   ** be found with <column> as its left-most column.
         1609  +**
         1610  +** If the IN_INDEX_NOOP_OK and IN_INDEX_MEMBERSHIP are both set and
         1611  +** if the RHS of the IN operator is a list (not a subquery) then this
         1612  +** routine might decide that creating an ephemeral b-tree for membership
         1613  +** testing is too expensive and return IN_INDEX_NOOP.  In that case, the
         1614  +** calling routine should implement the IN operator using a sequence
         1615  +** of Eq or Ne comparison operations.
  1518   1616   **
  1519   1617   ** When the b-tree is being used for membership tests, the calling function
  1520         -** needs to know whether or not the structure contains an SQL NULL 
  1521         -** value in order to correctly evaluate expressions like "X IN (Y, Z)".
  1522         -** If there is any chance that the (...) might contain a NULL value at
         1618  +** might need to know whether or not the RHS side of the IN operator
         1619  +** contains a NULL.  If prRhsHasNull is not a NULL pointer and 
         1620  +** if there is any chance that the (...) might contain a NULL value at
  1523   1621   ** runtime, then a register is allocated and the register number written
  1524         -** to *prNotFound. If there is no chance that the (...) contains a
  1525         -** NULL value, then *prNotFound is left unchanged.
         1622  +** to *prRhsHasNull. If there is no chance that the (...) contains a
         1623  +** NULL value, then *prRhsHasNull is left unchanged.
  1526   1624   **
  1527         -** If a register is allocated and its location stored in *prNotFound, then
  1528         -** its initial value is NULL.  If the (...) does not remain constant
  1529         -** for the duration of the query (i.e. the SELECT within the (...)
  1530         -** is a correlated subquery) then the value of the allocated register is
  1531         -** reset to NULL each time the subquery is rerun. This allows the
  1532         -** caller to use vdbe code equivalent to the following:
  1533         -**
  1534         -**   if( register==NULL ){
  1535         -**     has_null = <test if data structure contains null>
  1536         -**     register = 1
  1537         -**   }
  1538         -**
  1539         -** in order to avoid running the <test if data structure contains null>
  1540         -** test more often than is necessary.
         1625  +** If a register is allocated and its location stored in *prRhsHasNull, then
         1626  +** the value in that register will be NULL if the b-tree contains one or more
         1627  +** NULL values, and it will be some non-NULL value if the b-tree contains no
         1628  +** NULL values.
  1541   1629   */
  1542   1630   #ifndef SQLITE_OMIT_SUBQUERY
  1543         -int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){
         1631  +int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){
  1544   1632     Select *p;                            /* SELECT to the right of IN operator */
  1545   1633     int eType = 0;                        /* Type of RHS table. IN_INDEX_* */
  1546   1634     int iTab = pParse->nTab++;            /* Cursor of the RHS table */
  1547         -  int mustBeUnique = (prNotFound==0);   /* True if RHS must be unique */
         1635  +  int mustBeUnique;                     /* True if RHS must be unique */
  1548   1636     Vdbe *v = sqlite3GetVdbe(pParse);     /* Virtual machine being coded */
  1549   1637   
  1550   1638     assert( pX->op==TK_IN );
         1639  +  mustBeUnique = (inFlags & IN_INDEX_LOOP)!=0;
  1551   1640   
  1552   1641     /* Check to see if an existing table or index can be used to
  1553   1642     ** satisfy the query.  This is preferable to generating a new 
  1554   1643     ** ephemeral table.
  1555   1644     */
  1556   1645     p = (ExprHasProperty(pX, EP_xIsSelect) ? pX->x.pSelect : 0);
  1557   1646     if( ALWAYS(pParse->nErr==0) && isCandidateForInOpt(p) ){
................................................................................
  1600   1689         ** it is not, it is not possible to use any index.
  1601   1690         */
  1602   1691         int affinity_ok = sqlite3IndexAffinityOk(pX, pTab->aCol[iCol].affinity);
  1603   1692   
  1604   1693         for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){
  1605   1694           if( (pIdx->aiColumn[0]==iCol)
  1606   1695            && sqlite3FindCollSeq(db, ENC(db), pIdx->azColl[0], 0)==pReq
  1607         -         && (!mustBeUnique || (pIdx->nKeyCol==1 && pIdx->onError!=OE_None))
         1696  +         && (!mustBeUnique || (pIdx->nKeyCol==1 && IsUniqueIndex(pIdx)))
  1608   1697           ){
  1609   1698             int iAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v);
  1610   1699             sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb);
  1611   1700             sqlite3VdbeSetP4KeyInfo(pParse, pIdx);
  1612   1701             VdbeComment((v, "%s", pIdx->zName));
  1613   1702             assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 );
  1614   1703             eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0];
  1615   1704   
  1616         -          if( prNotFound && !pTab->aCol[iCol].notNull ){
  1617         -            *prNotFound = ++pParse->nMem;
  1618         -            sqlite3VdbeAddOp2(v, OP_Null, 0, *prNotFound);
         1705  +          if( prRhsHasNull && !pTab->aCol[iCol].notNull ){
         1706  +            *prRhsHasNull = ++pParse->nMem;
         1707  +            sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull);
  1619   1708             }
  1620   1709             sqlite3VdbeJumpHere(v, iAddr);
  1621   1710           }
  1622   1711         }
  1623   1712       }
  1624   1713     }
         1714  +
         1715  +  /* If no preexisting index is available for the IN clause
         1716  +  ** and IN_INDEX_NOOP is an allowed reply
         1717  +  ** and the RHS of the IN operator is a list, not a subquery
         1718  +  ** and the RHS is not contant or has two or fewer terms,
         1719  +  ** then it is not worth creating an ephemeral table to evaluate
         1720  +  ** the IN operator so return IN_INDEX_NOOP.
         1721  +  */
         1722  +  if( eType==0
         1723  +   && (inFlags & IN_INDEX_NOOP_OK)
         1724  +   && !ExprHasProperty(pX, EP_xIsSelect)
         1725  +   && (!sqlite3InRhsIsConstant(pX) || pX->x.pList->nExpr<=2)
         1726  +  ){
         1727  +    eType = IN_INDEX_NOOP;
         1728  +  }
         1729  +     
  1625   1730   
  1626   1731     if( eType==0 ){
  1627         -    /* Could not found an existing table or index to use as the RHS b-tree.
         1732  +    /* Could not find an existing table or index to use as the RHS b-tree.
  1628   1733       ** We will have to generate an ephemeral table to do the job.
  1629   1734       */
  1630   1735       u32 savedNQueryLoop = pParse->nQueryLoop;
  1631   1736       int rMayHaveNull = 0;
  1632   1737       eType = IN_INDEX_EPH;
  1633         -    if( prNotFound ){
  1634         -      *prNotFound = rMayHaveNull = ++pParse->nMem;
  1635         -      sqlite3VdbeAddOp2(v, OP_Null, 0, *prNotFound);
  1636         -    }else{
         1738  +    if( inFlags & IN_INDEX_LOOP ){
  1637   1739         pParse->nQueryLoop = 0;
  1638   1740         if( pX->pLeft->iColumn<0 && !ExprHasProperty(pX, EP_xIsSelect) ){
  1639   1741           eType = IN_INDEX_ROWID;
  1640   1742         }
         1743  +    }else if( prRhsHasNull ){
         1744  +      *prRhsHasNull = rMayHaveNull = ++pParse->nMem;
  1641   1745       }
  1642   1746       sqlite3CodeSubselect(pParse, pX, rMayHaveNull, eType==IN_INDEX_ROWID);
  1643   1747       pParse->nQueryLoop = savedNQueryLoop;
  1644   1748     }else{
  1645   1749       pX->iTable = iTab;
  1646   1750     }
  1647   1751     return eType;
................................................................................
  1664   1768   ** to be of the form "<rowid> IN (?, ?, ?)", where <rowid> is a reference
  1665   1769   ** to some integer key column of a table B-Tree. In this case, use an
  1666   1770   ** intkey B-Tree to store the set of IN(...) values instead of the usual
  1667   1771   ** (slower) variable length keys B-Tree.
  1668   1772   **
  1669   1773   ** If rMayHaveNull is non-zero, that means that the operation is an IN
  1670   1774   ** (not a SELECT or EXISTS) and that the RHS might contains NULLs.
  1671         -** Furthermore, the IN is in a WHERE clause and that we really want
  1672         -** to iterate over the RHS of the IN operator in order to quickly locate
  1673         -** all corresponding LHS elements.  All this routine does is initialize
  1674         -** the register given by rMayHaveNull to NULL.  Calling routines will take
  1675         -** care of changing this register value to non-NULL if the RHS is NULL-free.
  1676         -**
  1677         -** If rMayHaveNull is zero, that means that the subquery is being used
  1678         -** for membership testing only.  There is no need to initialize any
  1679         -** registers to indicate the presence or absence of NULLs on the RHS.
         1775  +** All this routine does is initialize the register given by rMayHaveNull
         1776  +** to NULL.  Calling routines will take care of changing this register
         1777  +** value to non-NULL if the RHS is NULL-free.
  1680   1778   **
  1681   1779   ** For a SELECT or EXISTS operator, return the register that holds the
  1682   1780   ** result.  For IN operators or if an error occurs, the return value is 0.
  1683   1781   */
  1684   1782   #ifndef SQLITE_OMIT_SUBQUERY
  1685   1783   int sqlite3CodeSubselect(
  1686   1784     Parse *pParse,          /* Parsing context */
  1687   1785     Expr *pExpr,            /* The IN, SELECT, or EXISTS operator */
  1688         -  int rMayHaveNull,       /* Register that records whether NULLs exist in RHS */
         1786  +  int rHasNullFlag,       /* Register that records whether NULLs exist in RHS */
  1689   1787     int isRowid             /* If true, LHS of IN operator is a rowid */
  1690   1788   ){
  1691         -  int testAddr = -1;                      /* One-time test address */
         1789  +  int jmpIfDynamic = -1;                      /* One-time test address */
  1692   1790     int rReg = 0;                           /* Register storing resulting */
  1693   1791     Vdbe *v = sqlite3GetVdbe(pParse);
  1694   1792     if( NEVER(v==0) ) return 0;
  1695   1793     sqlite3ExprCachePush(pParse);
  1696   1794   
  1697   1795     /* This code must be run in its entirety every time it is encountered
  1698   1796     ** if any of the following is true:
................................................................................
  1701   1799     **    *  The right-hand side is an expression list containing variables
  1702   1800     **    *  We are inside a trigger
  1703   1801     **
  1704   1802     ** If all of the above are false, then we can run this code just once
  1705   1803     ** save the results, and reuse the same result on subsequent invocations.
  1706   1804     */
  1707   1805     if( !ExprHasProperty(pExpr, EP_VarSelect) ){
  1708         -    testAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v);
         1806  +    jmpIfDynamic = sqlite3CodeOnce(pParse); VdbeCoverage(v);
  1709   1807     }
  1710   1808   
  1711   1809   #ifndef SQLITE_OMIT_EXPLAIN
  1712   1810     if( pParse->explain==2 ){
  1713   1811       char *zMsg = sqlite3MPrintf(
  1714         -        pParse->db, "EXECUTE %s%s SUBQUERY %d", testAddr>=0?"":"CORRELATED ",
         1812  +        pParse->db, "EXECUTE %s%s SUBQUERY %d", jmpIfDynamic>=0?"":"CORRELATED ",
  1715   1813           pExpr->op==TK_IN?"LIST":"SCALAR", pParse->iNextSelectId
  1716   1814       );
  1717   1815       sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC);
  1718   1816     }
  1719   1817   #endif
  1720   1818   
  1721   1819     switch( pExpr->op ){
  1722   1820       case TK_IN: {
  1723   1821         char affinity;              /* Affinity of the LHS of the IN */
  1724   1822         int addr;                   /* Address of OP_OpenEphemeral instruction */
  1725   1823         Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */
  1726   1824         KeyInfo *pKeyInfo = 0;      /* Key information */
  1727   1825   
  1728         -      if( rMayHaveNull ){
  1729         -        sqlite3VdbeAddOp2(v, OP_Null, 0, rMayHaveNull);
  1730         -      }
  1731         -
  1732   1826         affinity = sqlite3ExprAffinity(pLeft);
  1733   1827   
  1734   1828         /* Whether this is an 'x IN(SELECT...)' or an 'x IN(<exprlist>)'
  1735   1829         ** expression it is handled the same way.  An ephemeral table is 
  1736   1830         ** filled with single-field index keys representing the results
  1737   1831         ** from the SELECT or the <exprlist>.
  1738   1832         **
................................................................................
  1750   1844   
  1751   1845         if( ExprHasProperty(pExpr, EP_xIsSelect) ){
  1752   1846           /* Case 1:     expr IN (SELECT ...)
  1753   1847           **
  1754   1848           ** Generate code to write the results of the select into the temporary
  1755   1849           ** table allocated and opened above.
  1756   1850           */
         1851  +        Select *pSelect = pExpr->x.pSelect;
  1757   1852           SelectDest dest;
  1758   1853           ExprList *pEList;
  1759   1854   
  1760   1855           assert( !isRowid );
  1761   1856           sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable);
  1762   1857           dest.affSdst = (u8)affinity;
  1763   1858           assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable );
  1764         -        pExpr->x.pSelect->iLimit = 0;
         1859  +        pSelect->iLimit = 0;
         1860  +        testcase( pSelect->selFlags & SF_Distinct );
  1765   1861           testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */
  1766         -        if( sqlite3Select(pParse, pExpr->x.pSelect, &dest) ){
         1862  +        if( sqlite3Select(pParse, pSelect, &dest) ){
  1767   1863             sqlite3KeyInfoUnref(pKeyInfo);
  1768   1864             return 0;
  1769   1865           }
  1770         -        pEList = pExpr->x.pSelect->pEList;
         1866  +        pEList = pSelect->pEList;
  1771   1867           assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */
  1772   1868           assert( pEList!=0 );
  1773   1869           assert( pEList->nExpr>0 );
  1774   1870           assert( sqlite3KeyInfoIsWriteable(pKeyInfo) );
  1775   1871           pKeyInfo->aColl[0] = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft,
  1776   1872                                                            pEList->a[0].pExpr);
  1777   1873         }else if( ALWAYS(pExpr->x.pList!=0) ){
................................................................................
  1794   1890             assert( sqlite3KeyInfoIsWriteable(pKeyInfo) );
  1795   1891             pKeyInfo->aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft);
  1796   1892           }
  1797   1893   
  1798   1894           /* Loop through each expression in <exprlist>. */
  1799   1895           r1 = sqlite3GetTempReg(pParse);
  1800   1896           r2 = sqlite3GetTempReg(pParse);
  1801         -        sqlite3VdbeAddOp2(v, OP_Null, 0, r2);
         1897  +        if( isRowid ) sqlite3VdbeAddOp2(v, OP_Null, 0, r2);
  1802   1898           for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){
  1803   1899             Expr *pE2 = pItem->pExpr;
  1804   1900             int iValToIns;
  1805   1901   
  1806   1902             /* If the expression is not constant then we will need to
  1807   1903             ** disable the test that was generated above that makes sure
  1808   1904             ** this code only executes once.  Because for a non-constant
  1809   1905             ** expression we need to rerun this code each time.
  1810   1906             */
  1811         -          if( testAddr>=0 && !sqlite3ExprIsConstant(pE2) ){
  1812         -            sqlite3VdbeChangeToNoop(v, testAddr);
  1813         -            testAddr = -1;
         1907  +          if( jmpIfDynamic>=0 && !sqlite3ExprIsConstant(pE2) ){
         1908  +            sqlite3VdbeChangeToNoop(v, jmpIfDynamic);
         1909  +            jmpIfDynamic = -1;
  1814   1910             }
  1815   1911   
  1816   1912             /* Evaluate the expression and insert it into the temp table */
  1817   1913             if( isRowid && sqlite3ExprIsInteger(pE2, &iValToIns) ){
  1818   1914               sqlite3VdbeAddOp3(v, OP_InsertInt, pExpr->iTable, r2, iValToIns);
  1819   1915             }else{
  1820   1916               r3 = sqlite3ExprCodeTarget(pParse, pE2, r1);
................................................................................
  1856   1952         assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT );
  1857   1953   
  1858   1954         assert( ExprHasProperty(pExpr, EP_xIsSelect) );
  1859   1955         pSel = pExpr->x.pSelect;
  1860   1956         sqlite3SelectDestInit(&dest, 0, ++pParse->nMem);
  1861   1957         if( pExpr->op==TK_SELECT ){
  1862   1958           dest.eDest = SRT_Mem;
         1959  +        dest.iSdst = dest.iSDParm;
  1863   1960           sqlite3VdbeAddOp2(v, OP_Null, 0, dest.iSDParm);
  1864   1961           VdbeComment((v, "Init subquery result"));
  1865   1962         }else{
  1866   1963           dest.eDest = SRT_Exists;
  1867   1964           sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iSDParm);
  1868   1965           VdbeComment((v, "Init EXISTS result"));
  1869   1966         }
................................................................................
  1876   1973         }
  1877   1974         rReg = dest.iSDParm;
  1878   1975         ExprSetVVAProperty(pExpr, EP_NoReduce);
  1879   1976         break;
  1880   1977       }
  1881   1978     }
  1882   1979   
  1883         -  if( testAddr>=0 ){
  1884         -    sqlite3VdbeJumpHere(v, testAddr);
         1980  +  if( rHasNullFlag ){
         1981  +    sqlite3SetHasNullFlag(v, pExpr->iTable, rHasNullFlag);
         1982  +  }
         1983  +
         1984  +  if( jmpIfDynamic>=0 ){
         1985  +    sqlite3VdbeJumpHere(v, jmpIfDynamic);
  1885   1986     }
  1886   1987     sqlite3ExprCachePop(pParse);
  1887   1988   
  1888   1989     return rReg;
  1889   1990   }
  1890   1991   #endif /* SQLITE_OMIT_SUBQUERY */
  1891   1992   
................................................................................
  1898   1999   **
  1899   2000   ** The left-hand side (LHS) is a scalar expression.  The right-hand side (RHS)
  1900   2001   ** is an array of zero or more values.  The expression is true if the LHS is
  1901   2002   ** contained within the RHS.  The value of the expression is unknown (NULL)
  1902   2003   ** if the LHS is NULL or if the LHS is not contained within the RHS and the
  1903   2004   ** RHS contains one or more NULL values.
  1904   2005   **
  1905         -** This routine generates code will jump to destIfFalse if the LHS is not 
         2006  +** This routine generates code that jumps to destIfFalse if the LHS is not 
  1906   2007   ** contained within the RHS.  If due to NULLs we cannot determine if the LHS
  1907   2008   ** is contained in the RHS then jump to destIfNull.  If the LHS is contained
  1908   2009   ** within the RHS then fall through.
  1909   2010   */
  1910   2011   static void sqlite3ExprCodeIN(
  1911   2012     Parse *pParse,        /* Parsing and code generating context */
  1912   2013     Expr *pExpr,          /* The IN expression */
................................................................................
  1921   2022   
  1922   2023     /* Compute the RHS.   After this step, the table with cursor
  1923   2024     ** pExpr->iTable will contains the values that make up the RHS.
  1924   2025     */
  1925   2026     v = pParse->pVdbe;
  1926   2027     assert( v!=0 );       /* OOM detected prior to this routine */
  1927   2028     VdbeNoopComment((v, "begin IN expr"));
  1928         -  eType = sqlite3FindInIndex(pParse, pExpr, &rRhsHasNull);
         2029  +  eType = sqlite3FindInIndex(pParse, pExpr,
         2030  +                             IN_INDEX_MEMBERSHIP | IN_INDEX_NOOP_OK,
         2031  +                             destIfFalse==destIfNull ? 0 : &rRhsHasNull);
  1929   2032   
  1930   2033     /* Figure out the affinity to use to create a key from the results
  1931   2034     ** of the expression. affinityStr stores a static string suitable for
  1932   2035     ** P4 of OP_MakeRecord.
  1933   2036     */
  1934   2037     affinity = comparisonAffinity(pExpr);
  1935   2038   
  1936   2039     /* Code the LHS, the <expr> from "<expr> IN (...)".
  1937   2040     */
  1938   2041     sqlite3ExprCachePush(pParse);
  1939   2042     r1 = sqlite3GetTempReg(pParse);
  1940   2043     sqlite3ExprCode(pParse, pExpr->pLeft, r1);
  1941   2044   
  1942         -  /* If the LHS is NULL, then the result is either false or NULL depending
  1943         -  ** on whether the RHS is empty or not, respectively.
         2045  +  /* If sqlite3FindInIndex() did not find or create an index that is
         2046  +  ** suitable for evaluating the IN operator, then evaluate using a
         2047  +  ** sequence of comparisons.
  1944   2048     */
  1945         -  if( destIfNull==destIfFalse ){
  1946         -    /* Shortcut for the common case where the false and NULL outcomes are
  1947         -    ** the same. */
  1948         -    sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v);
  1949         -  }else{
  1950         -    int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v);
  1951         -    sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse);
  1952         -    VdbeCoverage(v);
  1953         -    sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull);
  1954         -    sqlite3VdbeJumpHere(v, addr1);
  1955         -  }
  1956         -
  1957         -  if( eType==IN_INDEX_ROWID ){
  1958         -    /* In this case, the RHS is the ROWID of table b-tree
  1959         -    */
  1960         -    sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); VdbeCoverage(v);
  1961         -    sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1);
  1962         -    VdbeCoverage(v);
  1963         -  }else{
  1964         -    /* In this case, the RHS is an index b-tree.
         2049  +  if( eType==IN_INDEX_NOOP ){
         2050  +    ExprList *pList = pExpr->x.pList;
         2051  +    CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft);
         2052  +    int labelOk = sqlite3VdbeMakeLabel(v);
         2053  +    int r2, regToFree;
         2054  +    int regCkNull = 0;
         2055  +    int ii;
         2056  +    assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
         2057  +    if( destIfNull!=destIfFalse ){
         2058  +      regCkNull = sqlite3GetTempReg(pParse);
         2059  +      sqlite3VdbeAddOp3(v, OP_BitAnd, r1, r1, regCkNull);
         2060  +    }
         2061  +    for(ii=0; ii<pList->nExpr; ii++){
         2062  +      r2 = sqlite3ExprCodeTemp(pParse, pList->a[ii].pExpr, &regToFree);
         2063  +      if( regCkNull && sqlite3ExprCanBeNull(pList->a[ii].pExpr) ){
         2064  +        sqlite3VdbeAddOp3(v, OP_BitAnd, regCkNull, r2, regCkNull);
         2065  +      }
         2066  +      if( ii<pList->nExpr-1 || destIfNull!=destIfFalse ){
         2067  +        sqlite3VdbeAddOp4(v, OP_Eq, r1, labelOk, r2,
         2068  +                          (void*)pColl, P4_COLLSEQ);
         2069  +        VdbeCoverageIf(v, ii<pList->nExpr-1);
         2070  +        VdbeCoverageIf(v, ii==pList->nExpr-1);
         2071  +        sqlite3VdbeChangeP5(v, affinity);
         2072  +      }else{
         2073  +        assert( destIfNull==destIfFalse );
         2074  +        sqlite3VdbeAddOp4(v, OP_Ne, r1, destIfFalse, r2,
         2075  +                          (void*)pColl, P4_COLLSEQ); VdbeCoverage(v);
         2076  +        sqlite3VdbeChangeP5(v, affinity | SQLITE_JUMPIFNULL);
         2077  +      }
         2078  +      sqlite3ReleaseTempReg(pParse, regToFree);
         2079  +    }
         2080  +    if( regCkNull ){
         2081  +      sqlite3VdbeAddOp2(v, OP_IsNull, regCkNull, destIfNull); VdbeCoverage(v);
         2082  +      sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse);
         2083  +    }
         2084  +    sqlite3VdbeResolveLabel(v, labelOk);
         2085  +    sqlite3ReleaseTempReg(pParse, regCkNull);
         2086  +  }else{
         2087  +  
         2088  +    /* If the LHS is NULL, then the result is either false or NULL depending
         2089  +    ** on whether the RHS is empty or not, respectively.
  1965   2090       */
  1966         -    sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1);
  1967         -
  1968         -    /* If the set membership test fails, then the result of the 
  1969         -    ** "x IN (...)" expression must be either 0 or NULL. If the set
  1970         -    ** contains no NULL values, then the result is 0. If the set 
  1971         -    ** contains one or more NULL values, then the result of the
  1972         -    ** expression is also NULL.
  1973         -    */
  1974         -    if( rRhsHasNull==0 || destIfFalse==destIfNull ){
  1975         -      /* This branch runs if it is known at compile time that the RHS
  1976         -      ** cannot contain NULL values. This happens as the result
  1977         -      ** of a "NOT NULL" constraint in the database schema.
  1978         -      **
  1979         -      ** Also run this branch if NULL is equivalent to FALSE
  1980         -      ** for this particular IN operator.
         2091  +    if( sqlite3ExprCanBeNull(pExpr->pLeft) ){
         2092  +      if( destIfNull==destIfFalse ){
         2093  +        /* Shortcut for the common case where the false and NULL outcomes are
         2094  +        ** the same. */
         2095  +        sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v);
         2096  +      }else{
         2097  +        int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v);
         2098  +        sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse);
         2099  +        VdbeCoverage(v);
         2100  +        sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull);
         2101  +        sqlite3VdbeJumpHere(v, addr1);
         2102  +      }
         2103  +    }
         2104  +  
         2105  +    if( eType==IN_INDEX_ROWID ){
         2106  +      /* In this case, the RHS is the ROWID of table b-tree
  1981   2107         */
  1982         -      sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1);
         2108  +      sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); VdbeCoverage(v);
         2109  +      sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1);
  1983   2110         VdbeCoverage(v);
  1984   2111       }else{
  1985         -      /* In this branch, the RHS of the IN might contain a NULL and
  1986         -      ** the presence of a NULL on the RHS makes a difference in the
  1987         -      ** outcome.
         2112  +      /* In this case, the RHS is an index b-tree.
  1988   2113         */
  1989         -      int j1, j2;
  1990         -
  1991         -      /* First check to see if the LHS is contained in the RHS.  If so,
  1992         -      ** then the presence of NULLs in the RHS does not matter, so jump
  1993         -      ** over all of the code that follows.
         2114  +      sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1);
         2115  +  
         2116  +      /* If the set membership test fails, then the result of the 
         2117  +      ** "x IN (...)" expression must be either 0 or NULL. If the set
         2118  +      ** contains no NULL values, then the result is 0. If the set 
         2119  +      ** contains one or more NULL values, then the result of the
         2120  +      ** expression is also NULL.
  1994   2121         */
  1995         -      j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1);
  1996         -      VdbeCoverage(v);
  1997         -
  1998         -      /* Here we begin generating code that runs if the LHS is not
  1999         -      ** contained within the RHS.  Generate additional code that