Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -165,11 +165,11 @@ # Object files for the SQLite library (non-amalgamation). # LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ backup.lo bitvec.lo btmutex.lo btree.lo build.lo \ - callback.lo complete.lo ctime.lo date.lo delete.lo \ + callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \ expr.lo fault.lo fkey.lo \ fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \ fts3_tokenize_vtab.lo \ fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ @@ -211,10 +211,11 @@ $(TOP)/src/build.c \ $(TOP)/src/callback.c \ $(TOP)/src/complete.c \ $(TOP)/src/ctime.c \ $(TOP)/src/date.c \ + $(TOP)/src/dbstat.c \ $(TOP)/src/delete.c \ $(TOP)/src/expr.c \ $(TOP)/src/fault.c \ $(TOP)/src/fkey.c \ $(TOP)/src/func.c \ @@ -387,11 +388,10 @@ $(TOP)/src/test_rtree.c \ $(TOP)/src/test_schema.c \ $(TOP)/src/test_server.c \ $(TOP)/src/test_superlock.c \ $(TOP)/src/test_syscall.c \ - $(TOP)/src/test_stat.c \ $(TOP)/src/test_tclvar.c \ $(TOP)/src/test_thread.c \ $(TOP)/src/test_vfs.c \ $(TOP)/src/test_wsd.c \ $(TOP)/ext/fts3/fts3_term.c \ @@ -421,10 +421,11 @@ $(TOP)/src/bitvec.c \ $(TOP)/src/btree.c \ $(TOP)/src/build.c \ $(TOP)/src/ctime.c \ $(TOP)/src/date.c \ + $(TOP)/src/dbstat.c \ $(TOP)/src/expr.c \ $(TOP)/src/func.c \ $(TOP)/src/insert.c \ $(TOP)/src/wal.c \ $(TOP)/src/main.c \ @@ -506,10 +507,19 @@ $(TOP)/ext/rtree/rtree.h EXTHDR += \ $(TOP)/ext/icu/sqliteicu.h EXTHDR += \ $(TOP)/ext/rtree/sqlite3rtree.h + +# executables needed for testing +# +TESTPROGS = \ + testfixture$(TEXE) \ + sqlite3$(TEXE) \ + sqlite3_analyzer$(TEXE) \ + sqldiff$(TEXE) + # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la) @@ -534,23 +544,23 @@ sqlite3$(TEXE): $(TOP)/src/shell.c libsqlite3.la sqlite3.h $(LTLINK) $(READLINE_FLAGS) \ -o $@ $(TOP)/src/shell.c libsqlite3.la \ $(LIBREADLINE) $(TLIBS) -rpath "$(libdir)" -sqldiff$(EXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h +sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) -fuzzershell$(EXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h +fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS) -mptester$(EXE): sqlite3.c $(TOP)/mptest/mptest.c +mptester$(TEXE): sqlite3.c $(TOP)/mptest/mptest.c $(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \ $(TLIBS) -rpath "$(libdir)" -MPTEST1=./mptester$(EXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20 -MPTEST2=./mptester$(EXE) mptest.db $(TOP)/mptest/multiwrite01.test --repeat 20 -mptest: mptester$(EXE) +MPTEST1=./mptester$(TEXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20 +MPTEST2=./mptester$(TEXE) mptest.db $(TOP)/mptest/multiwrite01.test --repeat 20 +mptest: mptester$(TEXE) rm -f mptest.db $(MPTEST1) --journalmode DELETE $(MPTEST2) --journalmode WAL $(MPTEST1) --journalmode WAL $(MPTEST2) --journalmode PERSIST @@ -650,10 +660,13 @@ $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/ctime.c date.lo: $(TOP)/src/date.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/date.c +dbstat.lo: $(TOP)/src/dbstat.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbstat.c + delete.lo: $(TOP)/src/delete.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/delete.c expr.lo: $(TOP)/src/expr.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/expr.c @@ -949,48 +962,52 @@ testfixture$(TEXE): $(TESTFIXTURE_SRC) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \ -o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS) # A very detailed test running most or all test cases -fulltest: testfixture$(TEXE) sqlite3$(TEXE) fuzztest +fulltest: $(TESTPROGS) fuzztest ./testfixture$(TEXE) $(TOP)/test/all.test # Really really long testing -soaktest: testfixture$(TEXE) sqlite3$(TEXE) +soaktest: $(TESTPROGS) fuzzoomtest ./testfixture$(TEXE) $(TOP)/test/all.test -soak=1 -# Do extra testing but not aeverything. -fulltestonly: testfixture$(TEXE) sqlite3$(TEXE) +# Do extra testing but not everything. +fulltestonly: $(TESTPROGS) ./testfixture$(TEXE) $(TOP)/test/full.test # Fuzz testing fuzztest: fuzzershell$(TEXE) - ./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt + ./fuzzershell$(TEXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt + +fuzzoomtest: fuzzershell$(TEXE) + ./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt --oom # This is the common case. Run many tests but not those that take # a really long time. # -test: testfixture$(TEXE) sqlite3$(TEXE) fuzztest +test: $(TESTPROGS) fuzztest ./testfixture$(TEXE) $(TOP)/test/veryquick.test # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # -valgrindtest: testfixture$(TEXE) sqlite3$(TEXE) fuzzershell$(TEXE) +valgrindtest: $(TESTPROGS) fuzzershell$(TEXE) valgrind -v ./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind # A very fast test that checks basic sanity. The name comes from # the 60s-era electronics testing: "Turn it on and see if smoke # comes out." # -smoketest: testfixture$(TEXE) fuzzershell$(TEXE) +smoketest: $(TESTPROGS) fuzzershell$(TEXE) ./testfixture$(TEXE) $(TOP)/test/main.test -sqlite3_analyzer.c: sqlite3.c $(TOP)/src/test_stat.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl +sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl echo "#define TCLSH 2" > $@ - cat sqlite3.c $(TOP)/src/test_stat.c $(TOP)/src/tclsqlite.c >> $@ + echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@ + cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@ echo "static const char *tclsh_main_loop(void){" >> $@ echo "static const char *zMainloop = " >> $@ $(NAWK) -f $(TOP)/tool/tostr.awk $(TOP)/tool/spaceanal.tcl >> $@ echo "; return zMainloop; }" >> $@ @@ -1096,10 +1113,12 @@ rm -f sqlite3rc.h rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c rm -f sqlite-*-output.vsix rm -f mptester mptester.exe + rm -f fuzzershell fuzzershell.exe + rm -f sqldiff sqldiff.exe distclean: clean rm -f config.log config.status libtool Makefile sqlite3.pc # Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -33,10 +33,17 @@ # link library. # !IFNDEF DYNAMIC_SHELL DYNAMIC_SHELL = 0 !ENDIF + +# Set this non-0 to enable extra code that attempts to detect misuse of the +# SQLite API. +# +!IFNDEF API_ARMOR +API_ARMOR = 0 +!ENDIF # If necessary, create a list of harmless compiler warnings to disable when # compiling the various tools. For the SQLite source code itself, warnings, # if any, will be disabled from within it. # @@ -145,10 +152,17 @@ # Set this to non-0 to use the Win32 native heap subsystem. # !IFNDEF WIN32HEAP WIN32HEAP = 0 !ENDIF + +# Set this to non-0 to enable OSTRACE() macros, which can be useful when +# debugging. +# +!IFNDEF OSTRACE +OSTRACE = 0 +!ENDIF # Set this to one of the following values to enable various debugging # features. Each level includes the debugging options from the previous # levels. Currently, the recognized values for DEBUG are: # @@ -482,28 +496,28 @@ TCC = $(TCC) -DNDEBUG BCC = $(BCC) -DNDEBUG RCC = $(RCC) -DNDEBUG !ENDIF -!IF $(DEBUG)>0 -TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR -RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR +!IF $(DEBUG)>0 || $(API_ARMOR)!=0 +TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1 +RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !ENDIF !IF $(DEBUG)>2 -TCC = $(TCC) -DSQLITE_DEBUG -RCC = $(RCC) -DSQLITE_DEBUG +TCC = $(TCC) -DSQLITE_DEBUG=1 +RCC = $(RCC) -DSQLITE_DEBUG=1 !ENDIF -!IF $(DEBUG)>4 -TCC = $(TCC) -DSQLITE_DEBUG_OS_TRACE=1 -RCC = $(RCC) -DSQLITE_DEBUG_OS_TRACE=1 +!IF $(DEBUG)>4 || $(OSTRACE)!=0 +TCC = $(TCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1 +RCC = $(RCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1 !ENDIF !IF $(DEBUG)>5 -TCC = $(TCC) -DSQLITE_ENABLE_IOTRACE -RCC = $(RCC) -DSQLITE_ENABLE_IOTRACE +TCC = $(TCC) -DSQLITE_ENABLE_IOTRACE=1 +RCC = $(RCC) -DSQLITE_ENABLE_IOTRACE=1 !ENDIF # Prevent warnings about "insecure" MSVC runtime library functions # being used. # @@ -806,11 +820,11 @@ # Object files for the SQLite library (non-amalgamation). # LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ backup.lo bitvec.lo btmutex.lo btree.lo build.lo \ - callback.lo complete.lo ctime.lo date.lo delete.lo \ + callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \ expr.lo fault.lo fkey.lo \ fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \ fts3_tokenize_vtab.lo fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ func.lo global.lo hash.lo \ @@ -862,10 +876,11 @@ $(TOP)\src\build.c \ $(TOP)\src\callback.c \ $(TOP)\src\complete.c \ $(TOP)\src\ctime.c \ $(TOP)\src\date.c \ + $(TOP)\src\dbstat.c \ $(TOP)\src\delete.c \ $(TOP)\src\expr.c \ $(TOP)\src\fault.c \ $(TOP)\src\fkey.c \ $(TOP)\src\func.c \ @@ -1039,11 +1054,10 @@ $(TOP)\src\test_rtree.c \ $(TOP)\src\test_schema.c \ $(TOP)\src\test_server.c \ $(TOP)\src\test_superlock.c \ $(TOP)\src\test_syscall.c \ - $(TOP)\src\test_stat.c \ $(TOP)\src\test_tclvar.c \ $(TOP)\src\test_thread.c \ $(TOP)\src\test_vfs.c \ $(TOP)\src\test_wsd.c \ $(TOP)\ext\fts3\fts3_term.c \ @@ -1074,10 +1088,11 @@ $(TOP)\src\bitvec.c \ $(TOP)\src\btree.c \ $(TOP)\src\build.c \ $(TOP)\src\ctime.c \ $(TOP)\src\date.c \ + $(TOP)\src\dbstat.c \ $(TOP)\src\expr.c \ $(TOP)\src\func.c \ $(TOP)\src\insert.c \ $(TOP)\src\wal.c \ $(TOP)\src\main.c \ @@ -1161,10 +1176,19 @@ $(TOP)\ext\rtree\rtree.h EXTHDR = $(EXTHDR) \ $(TOP)\ext\icu\sqliteicu.h EXTHDR = $(EXTHDR) \ $(TOP)\ext\rtree\sqlite3rtree.h + +# executables needed for testing +# +TESTPROGS = \ + testfixture.exe \ + sqlite3.exe \ + sqlite3_analyzer.exe \ + sqldiff.exe + # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib @@ -1317,10 +1341,13 @@ $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\ctime.c date.lo: $(TOP)\src\date.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\date.c +dbstat.lo: $(TOP)\src\date.c $(HDR) + $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\dbstat.c + delete.lo: $(TOP)\src\delete.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\delete.c expr.lo: $(TOP)\src\expr.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\expr.c @@ -1622,40 +1649,45 @@ /link $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) extensiontest: testfixture.exe testloadext.dll .\testfixture.exe $(TOP)\test\loadext.test -fulltest: testfixture.exe sqlite3.exe fuzztest +fulltest: $(TESTPROGS) fuzztest .\testfixture.exe $(TOP)\test\all.test -soaktest: testfixture.exe sqlite3.exe +soaktest: $(TESTPROGS) fuzzoomtest .\testfixture.exe $(TOP)\test\all.test -soak=1 -fulltestonly: testfixture.exe sqlite3.exe fuzztest +fulltestonly: $(TESTPROGS) fuzztest .\testfixture.exe $(TOP)\test\full.test queryplantest: testfixture.exe sqlite3.exe .\testfixture.exe $(TOP)\test\permutations.test queryplanner fuzztest: fuzzershell.exe - .\fuzzershell.exe -f $(TOP)\test\fuzzdata1.txt + .\fuzzershell.exe $(TOP)\test\fuzzdata1.txt $(TOP)\test\fuzzdata2.txt + +fuzzoomtest: fuzzershell.exe + .\fuzzershell.exe -f $(TOP)\test\fuzzdata1.txt --oom -test: testfixture.exe sqlite3.exe fuzztest +test: $(TESTPROGS) fuzztest .\testfixture.exe $(TOP)\test\veryquick.test -smoketest: testfixture.exe +smoketest: $(TESTPROGS) fuzzershell.exe .\testfixture.exe $(TOP)\test\main.test -sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\test_stat.c $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl - copy $(SQLITE3C) + $(TOP)\src\test_stat.c + $(TOP)\src\tclsqlite.c $@ +sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl + echo #define TCLSH 2 > $@ + echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@ + copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@ echo static const char *tclsh_main_loop(void){ >> $@ echo static const char *zMainloop = >> $@ $(NAWK) -f $(TOP)\tool\tostr.awk $(TOP)\tool\spaceanal.tcl >> $@ echo ; return zMainloop; } >> $@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) - $(LTLINK) $(NO_WARN) -DBUILD_sqlite -DTCLSH=2 -I$(TCLINCDIR) sqlite3_analyzer.c \ + $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \ /link $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) testloadext.lo: $(TOP)\src\test_loadext.c $(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c @@ -1721,10 +1753,11 @@ del /Q sqlite3.c sqlite3-*.c 2>NUL del /Q sqlite3rc.h 2>NUL del /Q shell.c sqlite3ext.h 2>NUL del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL del /Q sqlite-*-output.vsix 2>NUL + del /Q fuzzershell.exe sqldiff.exe 2>NUL # Dynamic link library section. # dll: sqlite3.dll Index: VERSION ================================================================== --- VERSION +++ VERSION @@ -1,1 +1,1 @@ -3.8.10 +3.8.10.1 Index: autoconf/Makefile.am ================================================================== --- autoconf/Makefile.am +++ autoconf/Makefile.am @@ -4,13 +4,13 @@ lib_LTLIBRARIES = libsqlite3.la libsqlite3_la_SOURCES = sqlite3.c libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8 bin_PROGRAMS = sqlite3 -sqlite3_SOURCES = shell.c sqlite3.h -sqlite3_LDADD = $(top_builddir)/libsqlite3.la @READLINE_LIBS@ -sqlite3_DEPENDENCIES = $(top_builddir)/libsqlite3.la +sqlite3_SOURCES = shell.c sqlite3.c sqlite3.h +sqlite3_LDADD = @READLINE_LIBS@ +sqlite3_CFLAGS = $(AM_CFLAGS) include_HEADERS = sqlite3.h sqlite3ext.h EXTRA_DIST = sqlite3.1 tea pkgconfigdir = ${libdir}/pkgconfig Index: configure ================================================================== --- configure +++ configure @@ -1,8 +1,8 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.8.10. +# Generated by GNU Autoconf 2.62 for sqlite 3.8.10.1. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. # This configure script is free software; the Free Software Foundation # gives unlimited permission to copy, distribute and modify it. @@ -741,12 +741,12 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.10' -PACKAGE_STRING='sqlite 3.8.10' +PACKAGE_VERSION='3.8.10.1' +PACKAGE_STRING='sqlite 3.8.10.1' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. ac_includes_default="\ #include @@ -1479,11 +1479,11 @@ # if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.10 to adapt to many kinds of systems. +\`configure' configures sqlite 3.8.10.1 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... To assign environment variables (e.g., CC, CFLAGS...), specify them as VAR=VALUE. See below for descriptions of some of the useful variables. @@ -1544,11 +1544,11 @@ _ACEOF fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.10:";; + short | recursive ) echo "Configuration of sqlite 3.8.10.1:";; esac cat <<\_ACEOF Optional Features: --disable-option-checking ignore unrecognized --enable/--with options @@ -1658,11 +1658,11 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.10 +sqlite configure 3.8.10.1 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation @@ -1672,11 +1672,11 @@ fi cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.10, which was +It was created by sqlite $as_me 3.8.10.1, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ _ACEOF @@ -13951,11 +13951,11 @@ # Save the log message, to keep $[0] and so on meaningful, and to # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.10, which was +This file was extended by sqlite $as_me 3.8.10.1, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS CONFIG_LINKS = $CONFIG_LINKS @@ -14004,11 +14004,11 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.8.10 +sqlite config.status 3.8.10.1 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" Copyright (C) 2008 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -505,10 +505,21 @@ p->pTokenizer->pModule->xDestroy(p->pTokenizer); sqlite3_free(p); return SQLITE_OK; } + +/* +** Write an error message into *pzErr +*/ +void sqlite3Fts3ErrMsg(char **pzErr, const char *zFormat, ...){ + va_list ap; + sqlite3_free(*pzErr); + va_start(ap, zFormat); + *pzErr = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} /* ** Construct one or more SQL statements from the format string given ** and then evaluate those statements. The success code is written ** into *pRc. @@ -1037,11 +1048,11 @@ if( !zSql ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + sqlite3Fts3ErrMsg(pzErr, "%s", sqlite3_errmsg(db)); } } sqlite3_free(zSql); if( rc==SQLITE_OK ){ @@ -1205,17 +1216,17 @@ if( nKey==pOp->nOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ break; } } if( iOpt==SizeofArray(aFts4Opt) ){ - *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); + sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); rc = SQLITE_ERROR; }else{ switch( iOpt ){ case 0: /* MATCHINFO */ if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ - *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); rc = SQLITE_ERROR; } bNoDocsize = 1; break; @@ -1239,11 +1250,11 @@ case 4: /* ORDER */ if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) ){ - *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal); + sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); rc = SQLITE_ERROR; } bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); break; @@ -1325,11 +1336,11 @@ assert( pTokenizer ); rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex); if( rc==SQLITE_ERROR ){ assert( zPrefix ); - *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix); + sqlite3Fts3ErrMsg(pzErr, "error parsing prefix parameter: %s", zPrefix); } if( rc!=SQLITE_OK ) goto fts3_init_out; /* Allocate and populate the Fts3Table structure. */ nByte = sizeof(Fts3Table) + /* Fts3Table */ @@ -1407,19 +1418,19 @@ } } } for(i=0; izReadExprlist = fts3ReadExprList(p, zUncompress, &rc); p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); if( rc!=SQLITE_OK ) goto fts3_init_out; @@ -1662,11 +1673,11 @@ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); sqlite3_finalize(pCsr->pStmt); sqlite3Fts3ExprFree(pCsr->pExpr); sqlite3Fts3FreeDeferredTokens(pCsr); sqlite3_free(pCsr->aDoclist); - sqlite3_free(pCsr->aMatchinfo); + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); sqlite3_free(pCsr); return SQLITE_OK; } @@ -3163,11 +3174,11 @@ assert( iIdx==nVal ); /* In case the cursor has been used before, clear it now. */ sqlite3_finalize(pCsr->pStmt); sqlite3_free(pCsr->aDoclist); - sqlite3_free(pCsr->aMatchinfo); + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); /* Set the lower and upper bounds on docids to return */ pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); Index: ext/fts3/fts3Int.h ================================================================== --- ext/fts3/fts3Int.h +++ ext/fts3/fts3Int.h @@ -195,10 +195,12 @@ typedef struct Fts3SegFilter Fts3SegFilter; typedef struct Fts3DeferredToken Fts3DeferredToken; typedef struct Fts3SegReader Fts3SegReader; typedef struct Fts3MultiSegReader Fts3MultiSegReader; +typedef struct MatchinfoBuffer MatchinfoBuffer; + /* ** A connection to a fulltext index is an instance of the following ** structure. The xCreate and xConnect methods create an instance ** of this structure and xDestroy and xDisconnect free that instance. ** All other methods receive a pointer to the structure as one of their @@ -304,13 +306,11 @@ int nRowAvg; /* Average size of database rows, in pages */ sqlite3_int64 nDoc; /* Documents in table */ i64 iMinDocid; /* Minimum docid to return */ i64 iMaxDocid; /* Maximum docid to return */ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ - u32 *aMatchinfo; /* Information about most recent match */ - int nMatchinfo; /* Number of elements in aMatchinfo[] */ - char *zMatchinfo; /* Matchinfo specification */ + MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */ }; #define FTS3_EVAL_FILTER 0 #define FTS3_EVAL_NEXT 1 #define FTS3_EVAL_MATCHINFO 2 @@ -426,11 +426,13 @@ sqlite3_int64 iDocid; /* Current docid */ u8 bEof; /* True this expression is at EOF already */ u8 bStart; /* True if iDocid is valid */ u8 bDeferred; /* True if this expression is entirely deferred */ - u32 *aMI; + /* The following are used by the fts3_snippet.c module. */ + int iPhrase; /* Index of this phrase in matchinfo() results */ + u32 *aMI; /* See above */ }; /* ** Candidate values for Fts3Query.eType. Note that the order of the first ** four values is in order of precedence when parsing expressions. For @@ -537,10 +539,11 @@ #define fts3GetVarint32(p, piVal) ( \ (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ ) /* fts3.c */ +void sqlite3Fts3ErrMsg(char**,const char*,...); int sqlite3Fts3PutVarint(char *, sqlite3_int64); int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); @@ -561,10 +564,11 @@ void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, const char *, const char *, int, int ); void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); +void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p); /* fts3_expr.c */ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, char **, int, int, int, const char *, int, Fts3Expr **, char ** ); Index: ext/fts3/fts3_aux.c ================================================================== --- ext/fts3/fts3_aux.c +++ ext/fts3/fts3_aux.c @@ -114,11 +114,11 @@ *ppVtab = (sqlite3_vtab *)p; return SQLITE_OK; bad_args: - *pzErr = sqlite3_mprintf("invalid arguments to fts4aux constructor"); + sqlite3Fts3ErrMsg(pzErr, "invalid arguments to fts4aux constructor"); return SQLITE_ERROR; } /* ** This function does the work for both the xDisconnect and xDestroy methods. Index: ext/fts3/fts3_expr.c ================================================================== --- ext/fts3/fts3_expr.c +++ ext/fts3/fts3_expr.c @@ -1020,17 +1020,17 @@ if( rc!=SQLITE_OK ){ sqlite3Fts3ExprFree(*ppExpr); *ppExpr = 0; if( rc==SQLITE_TOOBIG ){ - *pzErr = sqlite3_mprintf( + sqlite3Fts3ErrMsg(pzErr, "FTS expression tree is too large (maximum depth %d)", SQLITE_FTS3_MAX_EXPR_DEPTH ); rc = SQLITE_ERROR; }else if( rc==SQLITE_ERROR ){ - *pzErr = sqlite3_mprintf("malformed MATCH expression: [%s]", z); + sqlite3Fts3ErrMsg(pzErr, "malformed MATCH expression: [%s]", z); } } return rc; } Index: ext/fts3/fts3_snippet.c ================================================================== --- ext/fts3/fts3_snippet.c +++ ext/fts3/fts3_snippet.c @@ -25,10 +25,12 @@ #define FTS3_MATCHINFO_NDOC 'n' /* 1 value */ #define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */ #define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */ #define FTS3_MATCHINFO_LCS 's' /* nCol values */ #define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */ /* ** The default value for the second argument to matchinfo(). */ #define FTS3_MATCHINFO_DEFAULT "pcx" @@ -86,13 +88,26 @@ struct MatchInfo { Fts3Cursor *pCursor; /* FTS3 Cursor */ int nCol; /* Number of columns in table */ int nPhrase; /* Number of matchable phrases in query */ sqlite3_int64 nDoc; /* Number of docs in database */ + char flag; u32 *aMatchinfo; /* Pre-allocated buffer */ }; +/* +** An instance of this structure is used to manage a pair of buffers, each +** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below +** for details. +*/ +struct MatchinfoBuffer { + u8 aRef[3]; + int nElem; + int bGlobal; /* Set if global data is loaded */ + char *zMatchinfo; + u32 aMatchinfo[0]; +}; /* ** The snippet() and offsets() functions both return text values. An instance ** of the following structure is used to accumulate those values while the @@ -103,10 +118,101 @@ char *z; /* Pointer to buffer containing string */ int n; /* Length of z in bytes (excl. nul-term) */ int nAlloc; /* Allocated size of buffer z in bytes */ }; + +/************************************************************************* +** Start of MatchinfoBuffer code. +*/ + +/* +** Allocate a two-slot MatchinfoBuffer object. +*/ +static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){ + MatchinfoBuffer *pRet; + int nByte = sizeof(u32) * (2*nElem + 2) + sizeof(MatchinfoBuffer); + int nStr = strlen(zMatchinfo); + + pRet = sqlite3_malloc(nByte + nStr+1); + if( pRet ){ + memset(pRet, 0, nByte); + pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; + pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1); + pRet->nElem = nElem; + pRet->zMatchinfo = ((char*)pRet) + nByte; + memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1); + pRet->aRef[0] = 1; + } + + return pRet; +} + +static void fts3MIBufferFree(void *p){ + MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); + + assert( (u32*)p==&pBuf->aMatchinfo[1] + || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] + ); + if( (u32*)p==&pBuf->aMatchinfo[1] ){ + pBuf->aRef[1] = 0; + }else{ + pBuf->aRef[2] = 0; + } + + if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){ + sqlite3_free(pBuf); + } +} + +static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ + void (*xRet)(void*) = 0; + u32 *aOut = 0; + + if( p->aRef[1]==0 ){ + p->aRef[1] = 1; + aOut = &p->aMatchinfo[1]; + xRet = fts3MIBufferFree; + } + else if( p->aRef[2]==0 ){ + p->aRef[2] = 1; + aOut = &p->aMatchinfo[p->nElem+2]; + xRet = fts3MIBufferFree; + }else{ + aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32)); + if( aOut ){ + xRet = sqlite3_free; + if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); + } + } + + *paOut = aOut; + return xRet; +} + +static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ + p->bGlobal = 1; + memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); +} + +/* +** Free a MatchinfoBuffer object allocated using fts3MIBufferNew() +*/ +void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){ + if( p ){ + assert( p->aRef[0]==1 ); + p->aRef[0] = 0; + if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){ + sqlite3_free(p); + } + } +} + +/* +** End of MatchinfoBuffer code. +*************************************************************************/ + /* ** This function is used to help iterate through a position-list. A position ** list is a list of unique integers, sorted from smallest to largest. Each ** element of the list is represented by an FTS3 varint that takes the value @@ -140,11 +246,11 @@ int *piPhrase, /* Pointer to phrase counter */ int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ void *pCtx /* Second argument to pass to callback */ ){ int rc; /* Return code */ - int eType = pExpr->eType; /* Type of expression node pExpr */ + int eType = pExpr->eType; /* Type of expression node pExpr */ if( eType!=FTSQUERY_PHRASE ){ assert( pExpr->pLeft && pExpr->pRight ); rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx); if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ @@ -173,10 +279,11 @@ void *pCtx /* Second argument to pass to callback */ ){ int iPhrase = 0; /* Variable used as the phrase counter */ return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); } + /* ** This is an fts3ExprIterate() callback used while loading the doclists ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also ** fts3ExprLoadDoclists(). @@ -218,12 +325,11 @@ return rc; } static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ (*(int *)ctx)++; - UNUSED_PARAMETER(pExpr); - UNUSED_PARAMETER(iPhrase); + pExpr->iPhrase = iPhrase; return SQLITE_OK; } static int fts3ExprPhraseCount(Fts3Expr *pExpr){ int nPhrase = 0; (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); @@ -440,11 +546,11 @@ sIter.pCsr = pCsr; sIter.iCol = iCol; sIter.nSnippet = nSnippet; sIter.nPhrase = nList; sIter.iCurrent = -1; - rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void *)&sIter); + rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter); if( rc==SQLITE_OK ){ /* Set the *pmSeen output variable. */ for(i=0; ipCursor->base.pVtab; + int iStart; + Fts3Phrase *pPhrase = pExpr->pPhrase; + char *pIter = pPhrase->doclist.pList; + int iCol = 0; + + assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS ); + if( p->flag==FTS3_MATCHINFO_LHITS ){ + iStart = pExpr->iPhrase * p->nCol; + }else{ + iStart = pExpr->iPhrase * ((p->nCol + 31) / 32); + } + + while( 1 ){ + int nHit = fts3ColumnlistCount(&pIter); + if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){ + if( p->flag==FTS3_MATCHINFO_LHITS ){ + p->aMatchinfo[iStart + iCol] = (u32)nHit; + }else if( nHit ){ + p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F)); + } + } + assert( *pIter==0x00 || *pIter==0x01 ); + if( *pIter!=0x01 ) break; + pIter++; + pIter += fts3GetVarint32(pIter, &iCol); + } +} + +/* +** Gather the results for matchinfo directives 'y' and 'b'. +*/ +static void fts3ExprLHitGather( + Fts3Expr *pExpr, + MatchInfo *p +){ + assert( (pExpr->pLeft==0)==(pExpr->pRight==0) ); + if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){ + if( pExpr->pLeft ){ + fts3ExprLHitGather(pExpr->pLeft, p); + fts3ExprLHitGather(pExpr->pRight, p); + }else{ + fts3ExprLHits(pExpr, p); + } + } +} /* ** fts3ExprIterate() callback used to collect the "global" matchinfo stats ** for a single query. ** @@ -819,14 +979,16 @@ || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4) || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4) || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) || (cArg==FTS3_MATCHINFO_LCS) || (cArg==FTS3_MATCHINFO_HITS) + || (cArg==FTS3_MATCHINFO_LHITS) + || (cArg==FTS3_MATCHINFO_LHITS_BM) ){ return SQLITE_OK; } - *pzErr = sqlite3_mprintf("unrecognized matchinfo request: %c", cArg); + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg); return SQLITE_ERROR; } static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ int nVal; /* Number of integers output by cArg */ @@ -841,10 +1003,18 @@ case FTS3_MATCHINFO_AVGLENGTH: case FTS3_MATCHINFO_LENGTH: case FTS3_MATCHINFO_LCS: nVal = pInfo->nCol; break; + + case FTS3_MATCHINFO_LHITS: + nVal = pInfo->nCol * pInfo->nPhrase; + break; + + case FTS3_MATCHINFO_LHITS_BM: + nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32); + break; default: assert( cArg==FTS3_MATCHINFO_HITS ); nVal = pInfo->nCol * pInfo->nPhrase * 3; break; @@ -1036,11 +1206,11 @@ int i; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; sqlite3_stmt *pSelect = 0; for(i=0; rc==SQLITE_OK && zArg[i]; i++){ - + pInfo->flag = zArg[i]; switch( zArg[i] ){ case FTS3_MATCHINFO_NPHRASE: if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase; break; @@ -1095,10 +1265,18 @@ rc = fts3ExprLoadDoclists(pCsr, 0, 0); if( rc==SQLITE_OK ){ rc = fts3MatchinfoLcs(pCsr, pInfo); } break; + + case FTS3_MATCHINFO_LHITS_BM: + case FTS3_MATCHINFO_LHITS: { + int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); + memset(pInfo->aMatchinfo, 0, nZero); + fts3ExprLHitGather(pCsr->pExpr, pInfo); + break; + } default: { Fts3Expr *pExpr; assert( zArg[i]==FTS3_MATCHINFO_HITS ); pExpr = pCsr->pExpr; @@ -1127,73 +1305,90 @@ /* ** Populate pCsr->aMatchinfo[] with data for the current row. The ** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). */ -static int fts3GetMatchinfo( +static void fts3GetMatchinfo( + sqlite3_context *pCtx, /* Return results here */ Fts3Cursor *pCsr, /* FTS3 Cursor object */ const char *zArg /* Second argument to matchinfo() function */ ){ MatchInfo sInfo; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int rc = SQLITE_OK; int bGlobal = 0; /* Collect 'global' stats as well as local */ + u32 *aOut = 0; + void (*xDestroyOut)(void*) = 0; + memset(&sInfo, 0, sizeof(MatchInfo)); sInfo.pCursor = pCsr; sInfo.nCol = pTab->nColumn; /* If there is cached matchinfo() data, but the format string for the ** cache does not match the format string for this request, discard ** the cached data. */ - if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){ - assert( pCsr->aMatchinfo ); - sqlite3_free(pCsr->aMatchinfo); - pCsr->zMatchinfo = 0; - pCsr->aMatchinfo = 0; + if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){ + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + pCsr->pMIBuffer = 0; } - /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the + /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the ** matchinfo function has been called for this query. In this case ** allocate the array used to accumulate the matchinfo data and ** initialize those elements that are constant for every row. */ - if( pCsr->aMatchinfo==0 ){ + if( pCsr->pMIBuffer==0 ){ int nMatchinfo = 0; /* Number of u32 elements in match-info */ - int nArg; /* Bytes in zArg */ int i; /* Used to iterate through zArg */ /* Determine the number of phrases in the query */ pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr); sInfo.nPhrase = pCsr->nPhrase; /* Determine the number of integers in the buffer returned by this call. */ for(i=0; zArg[i]; i++){ + char *zErr = 0; + if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + return; + } nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]); } /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */ - nArg = (int)strlen(zArg); - pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1); - if( !pCsr->aMatchinfo ) return SQLITE_NOMEM; - - pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo]; - pCsr->nMatchinfo = nMatchinfo; - memcpy(pCsr->zMatchinfo, zArg, nArg+1); - memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo); + pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg); + if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM; + pCsr->isMatchinfoNeeded = 1; bGlobal = 1; } - sInfo.aMatchinfo = pCsr->aMatchinfo; - sInfo.nPhrase = pCsr->nPhrase; - if( pCsr->isMatchinfoNeeded ){ + if( rc==SQLITE_OK ){ + xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut); + if( xDestroyOut==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + sInfo.aMatchinfo = aOut; + sInfo.nPhrase = pCsr->nPhrase; rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); - pCsr->isMatchinfoNeeded = 0; + if( bGlobal ){ + fts3MIBufferSetGlobal(pCsr->pMIBuffer); + } } - return rc; + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + if( xDestroyOut ) xDestroyOut(aOut); + }else{ + int n = pCsr->pMIBuffer->nElem * sizeof(u32); + sqlite3_result_blob(pCtx, aOut, n, xDestroyOut); + } } /* ** Implementation of snippet() function. */ @@ -1395,11 +1590,11 @@ ** no way that this operation can fail, so the return code from ** fts3ExprIterate() can be discarded. */ sCtx.iCol = iCol; sCtx.iTerm = 0; - (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx); + (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx); /* Retreive the text stored in column iCol. If an SQL NULL is stored ** in column iCol, jump immediately to the next iteration of the loop. ** If an OOM occurs while retrieving the data (this can happen if SQLite ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM @@ -1487,41 +1682,24 @@ sqlite3_context *pContext, /* Function call context */ Fts3Cursor *pCsr, /* FTS3 table cursor */ const char *zArg /* Second arg to matchinfo() function */ ){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc; - int i; const char *zFormat; if( zArg ){ - for(i=0; zArg[i]; i++){ - char *zErr = 0; - if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ - sqlite3_result_error(pContext, zErr, -1); - sqlite3_free(zErr); - return; - } - } zFormat = zArg; }else{ zFormat = FTS3_MATCHINFO_DEFAULT; } if( !pCsr->pExpr ){ sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); return; - } - - /* Retrieve matchinfo() data. */ - rc = fts3GetMatchinfo(pCsr, zFormat); - sqlite3Fts3SegmentsClose(pTab); - - if( rc!=SQLITE_OK ){ - sqlite3_result_error_code(pContext, rc); }else{ - int n = pCsr->nMatchinfo * sizeof(u32); - sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT); + /* Retrieve matchinfo() data. */ + fts3GetMatchinfo(pContext, pCsr, zFormat); + sqlite3Fts3SegmentsClose(pTab); } } #endif Index: ext/fts3/fts3_term.c ================================================================== --- ext/fts3/fts3_term.c +++ ext/fts3/fts3_term.c @@ -79,11 +79,11 @@ argc--; } /* The user should specify a single argument - the name of an fts3 table. */ if( argc!=4 ){ - *pzErr = sqlite3_mprintf( + sqlite3Fts3ErrMsg(pzErr, "wrong number of arguments to fts4term constructor" ); return SQLITE_ERROR; } Index: ext/fts3/fts3_tokenize_vtab.c ================================================================== --- ext/fts3/fts3_tokenize_vtab.c +++ ext/fts3/fts3_tokenize_vtab.c @@ -83,11 +83,11 @@ sqlite3_tokenizer_module *p; int nName = (int)strlen(zName); p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); if( !p ){ - *pzErr = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", zName); return SQLITE_ERROR; } *pp = p; return SQLITE_OK; Index: ext/fts3/fts3_tokenizer.c ================================================================== --- ext/fts3/fts3_tokenizer.c +++ ext/fts3/fts3_tokenizer.c @@ -170,11 +170,11 @@ z[n] = '\0'; sqlite3Fts3Dequote(z); m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1); if( !m ){ - *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z); + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", z); rc = SQLITE_ERROR; }else{ char const **aArg = 0; int iArg = 0; z = &z[n+1]; @@ -193,11 +193,11 @@ z = &z[n+1]; } rc = m->xCreate(iArg, aArg, ppTok); assert( rc!=SQLITE_OK || *ppTok ); if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("unknown tokenizer"); + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer"); }else{ (*ppTok)->pModule = m; } sqlite3_free((void *)aArg); } Index: ext/rtree/rtree.c ================================================================== --- ext/rtree/rtree.c +++ ext/rtree/rtree.c @@ -2818,15 +2818,23 @@ ** conflict-handling mode specified by the user. */ if( nData>1 ){ int ii; - /* Populate the cell.aCoord[] array. The first coordinate is azData[3]. */ - assert( nData==(pRtree->nDim*2 + 3) ); + /* Populate the cell.aCoord[] array. The first coordinate is azData[3]. + ** + ** NB: nData can only be less than nDim*2+3 if the rtree is mis-declared + ** with "column" that are interpreted as table constraints. + ** Example: CREATE VIRTUAL TABLE bad USING rtree(x,y,CHECK(y>5)); + ** This problem was discovered after years of use, so we silently ignore + ** these kinds of misdeclared tables to avoid breaking any legacy. + */ + assert( nData<=(pRtree->nDim*2 + 3) ); + #ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + for(ii=0; iicell.aCoord[ii+1].f ){ rc = SQLITE_CONSTRAINT; goto constraint; @@ -2833,11 +2841,11 @@ } } }else #endif { - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + for(ii=0; iicell.aCoord[ii+1].i ){ rc = SQLITE_CONSTRAINT; goto constraint; Index: ext/rtree/rtree9.test ================================================================== --- ext/rtree/rtree9.test +++ ext/rtree/rtree9.test @@ -85,11 +85,10 @@ #------------------------------------------------------------------------- # Test the example 2d "circle" geometry callback. # register_circle_geom db -breakpoint do_execsql_test rtree9-5.1 { CREATE VIRTUAL TABLE rt2 USING rtree(id, xmin, xmax, ymin, ymax); INSERT INTO rt2 VALUES(1, 1, 2, 1, 2); INSERT INTO rt2 VALUES(2, 1, 2, -2, -1); Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -52,11 +52,11 @@ # Object files for the SQLite library. # LIBOBJ+= vdbe.o parse.o \ alter.o analyze.o attach.o auth.o \ backup.o bitvec.o btmutex.o btree.o build.o \ - callback.o complete.o ctime.o date.o delete.o expr.o fault.o fkey.o \ + callback.o complete.o ctime.o date.o dbstat.o delete.o expr.o fault.o fkey.o \ fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o \ fts3_tokenize_vtab.o \ fts3_unicode.o fts3_unicode2.o \ fts3_write.o func.o global.o hash.o \ @@ -103,10 +103,11 @@ $(TOP)/src/build.c \ $(TOP)/src/callback.c \ $(TOP)/src/complete.c \ $(TOP)/src/ctime.c \ $(TOP)/src/date.c \ + $(TOP)/src/dbstat.c \ $(TOP)/src/delete.c \ $(TOP)/src/expr.c \ $(TOP)/src/fault.c \ $(TOP)/src/fkey.c \ $(TOP)/src/func.c \ @@ -298,11 +299,10 @@ $(TOP)/src/test_pcache.c \ $(TOP)/src/test_quota.c \ $(TOP)/src/test_rtree.c \ $(TOP)/src/test_schema.c \ $(TOP)/src/test_server.c \ - $(TOP)/src/test_stat.c \ $(TOP)/src/test_sqllog.c \ $(TOP)/src/test_superlock.c \ $(TOP)/src/test_syscall.c \ $(TOP)/src/test_tclvar.c \ $(TOP)/src/test_thread.c \ @@ -335,10 +335,11 @@ $(TOP)/src/attach.c \ $(TOP)/src/backup.c \ $(TOP)/src/btree.c \ $(TOP)/src/build.c \ $(TOP)/src/date.c \ + $(TOP)/src/dbstat.c \ $(TOP)/src/expr.c \ $(TOP)/src/func.c \ $(TOP)/src/insert.c \ $(TOP)/src/wal.c \ $(TOP)/src/main.c \ @@ -422,10 +423,18 @@ $(TOP)/ext/fts5/fts5Int.h \ fts5parse.h \ $(TOP)/ext/fts5/fts5.h EXTHDR += \ $(TOP)/ext/userauth/sqlite3userauth.h + +# executables needed for testing +# +TESTPROGS = \ + testfixture$(EXE) \ + sqlite3$(EXE) \ + sqlite3_analyzer$(EXE) \ + sqldiff$(EXE) # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: sqlite3.h libsqlite3.a sqlite3$(EXE) @@ -682,13 +691,14 @@ # tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a $(TCCX) $(TCL_FLAGS) -DTCLSH=1 -o tclsqlite3 \ $(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB) -sqlite3_analyzer.c: sqlite3.c $(TOP)/src/test_stat.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl +sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl echo "#define TCLSH 2" > $@ - cat sqlite3.c $(TOP)/src/test_stat.c $(TOP)/src/tclsqlite.c >> $@ + echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@ + cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@ echo "static const char *tclsh_main_loop(void){" >> $@ echo "static const char *zMainloop = " >> $@ $(NAWK) -f $(TOP)/tool/tostr.awk $(TOP)/tool/spaceanal.tcl >> $@ echo "; return zMainloop; }" >> $@ @@ -714,34 +724,44 @@ $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ -DSQLITE_ENABLE_FTS3=1 \ $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c \ -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) -fulltest: testfixture$(EXE) sqlite3$(EXE) fuzztest +fulltest: $(TESTPROGS) fuzztest ./testfixture$(EXE) $(TOP)/test/all.test -soaktest: testfixture$(EXE) sqlite3$(EXE) +soaktest: $(TESTPROGS) fuzzoomtest ./testfixture$(EXE) $(TOP)/test/all.test -soak=1 -fulltestonly: testfixture$(EXE) sqlite3$(EXE) fuzztest +fulltestonly: $(TESTPROGS) fuzztest ./testfixture$(EXE) $(TOP)/test/full.test queryplantest: testfixture$(EXE) sqlite3$(EXE) ./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner fuzztest: fuzzershell$(EXE) - ./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt + ./fuzzershell$(EXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt + +fuzzoomtest: fuzzershell$(EXE) + ./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt --oom -test: testfixture$(EXE) sqlite3$(EXE) fuzztest +test: $(TESTPROGS) fuzztest ./testfixture$(EXE) $(TOP)/test/veryquick.test # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # -valgrindtest: testfixture$(EXE) sqlite3$(EXE) fuzzershell$(EXE) +valgrindtest: $(TESTPROGS) fuzzershell$(EXE) valgrind -v ./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt OMIT_MISUSE=1 valgrind -v ./testfixture$(EXE) $(TOP)/test/permutations.test valgrind + +# A very fast test that checks basic sanity. The name comes from +# the 60s-era electronics testing: "Turn it on and see if smoke +# comes out." +# +smoketest: $(TESTPROGS) fuzzershell$(EXE) + ./testfixture$(EXE) $(TOP)/test/main.test # The next two rules are used to support the "threadtest" target. Building # threadtest runs a few thread-safety tests that are implemented in C. This # target is invoked by the releasetest.tcl script. # @@ -848,5 +868,7 @@ rm -f sqlite3rc.h rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c rm -f sqlite-*-output.vsix rm -f mptester mptester.exe + rm -f fuzzershell fuzzershell.exe + rm -f sqldiff sqldiff.exe Index: src/alter.c ================================================================== --- src/alter.c +++ src/alter.c @@ -124,10 +124,11 @@ do { z += n; n = sqlite3GetToken(z, &token); }while( token==TK_SPACE ); + if( token==TK_ILLEGAL ) break; zParent = sqlite3DbStrNDup(db, (const char *)z, n); if( zParent==0 ) break; sqlite3Dequote(zParent); if( 0==sqlite3StrICmp((const char *)zOld, zParent) ){ char *zOut = sqlite3MPrintf(db, "%s%.*s\"%w\"", Index: src/bitvec.c ================================================================== --- src/bitvec.c +++ src/bitvec.c @@ -339,11 +339,11 @@ /* Allocate the Bitvec to be tested and a linear array of ** bits to act as the reference */ pBitvec = sqlite3BitvecCreate( sz ); pV = sqlite3MallocZero( (sz+7)/8 + 1 ); - pTmpSpace = sqlite3_malloc(BITVEC_SZ); + pTmpSpace = sqlite3_malloc64(BITVEC_SZ); if( pBitvec==0 || pV==0 || pTmpSpace==0 ) goto bitvec_end; /* NULL pBitvec tests */ sqlite3BitvecSet(0, 1); sqlite3BitvecClear(0, 1, pTmpSpace); Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -2427,11 +2427,11 @@ } assert( nReserve>=0 && nReserve<=255 ); if( pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE && ((pageSize-1)&pageSize)==0 ){ assert( (pageSize & 7)==0 ); - assert( !pBt->pPage1 && !pBt->pCursor ); + assert( !pBt->pCursor ); pBt->pageSize = (u32)pageSize; freeTempSpace(pBt); } rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, nReserve); pBt->usableSize = pBt->pageSize - (u16)nReserve; @@ -8535,16 +8535,16 @@ ** root element. The daughter nodes of aHeap[N] are aHeap[N*2] ** and aHeap[N*2+1]. ** ** The heap property is this: Every node is less than or equal to both ** of its daughter nodes. A consequence of the heap property is that the -** root node aHeap[1] is always the minimum value current in the heap. +** root node aHeap[1] is always the minimum value currently in the heap. ** ** The btreeHeapInsert() routine inserts an unsigned 32-bit number onto ** the heap, preserving the heap property. The btreeHeapPull() routine ** removes the root element from the heap (the minimum value in the heap) -** and then move other nodes around as necessary to preserve the heap +** and then moves other nodes around as necessary to preserve the heap ** property. ** ** This heap is used for cell overlap and coverage testing. Each u32 ** entry represents the span of a cell or freeblock on a btree page. ** The upper 16 bits are the index of the first byte of a range and the @@ -8897,12 +8897,11 @@ sqlite3BtreeLeave(p); return 0; } i = PENDING_BYTE_PAGE(pBt); if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); - sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); - sCheck.errMsg.useMalloc = 2; + sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); /* Check the integrity of the freelist */ sCheck.zPfx = "Main freelist: "; checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -4022,12 +4022,11 @@ char *zErr; int j; StrAccum errMsg; Table *pTab = pIdx->pTable; - sqlite3StrAccumInit(&errMsg, 0, 0, 200); - errMsg.db = pParse->db; + sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, 200); for(j=0; jnKeyCol; j++){ char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); sqlite3StrAccumAppendAll(&errMsg, pTab->zName); sqlite3StrAccumAppend(&errMsg, ".", 1); Index: src/ctime.c ================================================================== --- src/ctime.c +++ src/ctime.c @@ -72,10 +72,13 @@ #if SQLITE_ENABLE_CEROD "ENABLE_CEROD", #endif #if SQLITE_ENABLE_COLUMN_METADATA "ENABLE_COLUMN_METADATA", +#endif +#if SQLITE_ENABLE_DBSTAT_VTAB + "ENABLE_DBSTAT_VTAB", #endif #if SQLITE_ENABLE_EXPENSIVE_ASSERT "ENABLE_EXPENSIVE_ASSERT", #endif #if SQLITE_ENABLE_FTS1 ADDED src/dbstat.c Index: src/dbstat.c ================================================================== --- /dev/null +++ src/dbstat.c @@ -0,0 +1,651 @@ +/* +** 2010 July 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an implementation of the "dbstat" virtual table. +** +** The dbstat virtual table is used to extract low-level formatting +** information from an SQLite database in order to implement the +** "sqlite3_analyzer" utility. See the ../tool/spaceanal.tcl script +** for an example implementation. +*/ + +#include "sqliteInt.h" /* Requires access to internal data structures */ +#if (defined(SQLITE_ENABLE_DBSTAT_VTAB) || defined(SQLITE_TEST)) \ + && !defined(SQLITE_OMIT_VIRTUALTABLE) + +/* +** Page paths: +** +** The value of the 'path' column describes the path taken from the +** root-node of the b-tree structure to each page. The value of the +** root-node path is '/'. +** +** The value of the path for the left-most child page of the root of +** a b-tree is '/000/'. (Btrees store content ordered from left to right +** so the pages to the left have smaller keys than the pages to the right.) +** The next to left-most child of the root page is +** '/001', and so on, each sibling page identified by a 3-digit hex +** value. The children of the 451st left-most sibling have paths such +** as '/1c2/000/, '/1c2/001/' etc. +** +** Overflow pages are specified by appending a '+' character and a +** six-digit hexadecimal value to the path to the cell they are linked +** from. For example, the three overflow pages in a chain linked from +** the left-most cell of the 450th child of the root page are identified +** by the paths: +** +** '/1c2/000+000000' // First page in overflow chain +** '/1c2/000+000001' // Second page in overflow chain +** '/1c2/000+000002' // Third page in overflow chain +** +** If the paths are sorted using the BINARY collation sequence, then +** the overflow pages associated with a cell will appear earlier in the +** sort-order than its child page: +** +** '/1c2/000/' // Left-most child of 451st child of root +*/ +#define VTAB_SCHEMA \ + "CREATE TABLE xx( " \ + " name STRING, /* Name of table or index */" \ + " path INTEGER, /* Path to page from root */" \ + " pageno INTEGER, /* Page number */" \ + " pagetype STRING, /* 'internal', 'leaf' or 'overflow' */" \ + " ncell INTEGER, /* Cells on page (0 for overflow) */" \ + " payload INTEGER, /* Bytes of payload on this page */" \ + " unused INTEGER, /* Bytes of unused space on this page */" \ + " mx_payload INTEGER, /* Largest payload size of all cells */" \ + " pgoffset INTEGER, /* Offset of page in file */" \ + " pgsize INTEGER /* Size of the page */" \ + ");" + + +typedef struct StatTable StatTable; +typedef struct StatCursor StatCursor; +typedef struct StatPage StatPage; +typedef struct StatCell StatCell; + +struct StatCell { + int nLocal; /* Bytes of local payload */ + u32 iChildPg; /* Child node (or 0 if this is a leaf) */ + int nOvfl; /* Entries in aOvfl[] */ + u32 *aOvfl; /* Array of overflow page numbers */ + int nLastOvfl; /* Bytes of payload on final overflow page */ + int iOvfl; /* Iterates through aOvfl[] */ +}; + +struct StatPage { + u32 iPgno; + DbPage *pPg; + int iCell; + + char *zPath; /* Path to this page */ + + /* Variables populated by statDecodePage(): */ + u8 flags; /* Copy of flags byte */ + int nCell; /* Number of cells on page */ + int nUnused; /* Number of unused bytes on page */ + StatCell *aCell; /* Array of parsed cells */ + u32 iRightChildPg; /* Right-child page number (or 0) */ + int nMxPayload; /* Largest payload of any cell on this page */ +}; + +struct StatCursor { + sqlite3_vtab_cursor base; + sqlite3_stmt *pStmt; /* Iterates through set of root pages */ + int isEof; /* After pStmt has returned SQLITE_DONE */ + + StatPage aPage[32]; + int iPage; /* Current entry in aPage[] */ + + /* Values to return. */ + char *zName; /* Value of 'name' column */ + char *zPath; /* Value of 'path' column */ + u32 iPageno; /* Value of 'pageno' column */ + char *zPagetype; /* Value of 'pagetype' column */ + int nCell; /* Value of 'ncell' column */ + int nPayload; /* Value of 'payload' column */ + int nUnused; /* Value of 'unused' column */ + int nMxPayload; /* Value of 'mx_payload' column */ + i64 iOffset; /* Value of 'pgOffset' column */ + int szPage; /* Value of 'pgSize' column */ +}; + +struct StatTable { + sqlite3_vtab base; + sqlite3 *db; + int iDb; /* Index of database to analyze */ +}; + +#ifndef get2byte +# define get2byte(x) ((x)[0]<<8 | (x)[1]) +#endif + +/* +** Connect to or create a statvfs virtual table. +*/ +static int statConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + StatTable *pTab = 0; + int rc = SQLITE_OK; + int iDb; + + if( argc>=4 ){ + iDb = sqlite3FindDbName(db, argv[3]); + if( iDb<0 ){ + *pzErr = sqlite3_mprintf("no such database: %s", argv[3]); + return SQLITE_ERROR; + } + }else{ + iDb = 0; + } + rc = sqlite3_declare_vtab(db, VTAB_SCHEMA); + if( rc==SQLITE_OK ){ + pTab = (StatTable *)sqlite3_malloc64(sizeof(StatTable)); + if( pTab==0 ) rc = SQLITE_NOMEM; + } + + assert( rc==SQLITE_OK || pTab==0 ); + if( rc==SQLITE_OK ){ + memset(pTab, 0, sizeof(StatTable)); + pTab->db = db; + pTab->iDb = iDb; + } + + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Disconnect from or destroy a statvfs virtual table. +*/ +static int statDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** There is no "best-index". This virtual table always does a linear +** scan of the binary VFS log file. +*/ +static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + + /* Records are always returned in ascending order of (name, path). + ** If this will satisfy the client, set the orderByConsumed flag so that + ** SQLite does not do an external sort. + */ + if( ( pIdxInfo->nOrderBy==1 + && pIdxInfo->aOrderBy[0].iColumn==0 + && pIdxInfo->aOrderBy[0].desc==0 + ) || + ( pIdxInfo->nOrderBy==2 + && pIdxInfo->aOrderBy[0].iColumn==0 + && pIdxInfo->aOrderBy[0].desc==0 + && pIdxInfo->aOrderBy[1].iColumn==1 + && pIdxInfo->aOrderBy[1].desc==0 + ) + ){ + pIdxInfo->orderByConsumed = 1; + } + + pIdxInfo->estimatedCost = 10.0; + return SQLITE_OK; +} + +/* +** Open a new statvfs cursor. +*/ +static int statOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + StatTable *pTab = (StatTable *)pVTab; + StatCursor *pCsr; + int rc; + + pCsr = (StatCursor *)sqlite3_malloc64(sizeof(StatCursor)); + if( pCsr==0 ){ + rc = SQLITE_NOMEM; + }else{ + char *zSql; + memset(pCsr, 0, sizeof(StatCursor)); + pCsr->base.pVtab = pVTab; + + zSql = sqlite3_mprintf( + "SELECT 'sqlite_master' AS name, 1 AS rootpage, 'table' AS type" + " UNION ALL " + "SELECT name, rootpage, type" + " FROM \"%w\".sqlite_master WHERE rootpage!=0" + " ORDER BY name", pTab->db->aDb[pTab->iDb].zName); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + if( rc!=SQLITE_OK ){ + sqlite3_free(pCsr); + pCsr = 0; + } + } + + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + return rc; +} + +static void statClearPage(StatPage *p){ + int i; + if( p->aCell ){ + for(i=0; inCell; i++){ + sqlite3_free(p->aCell[i].aOvfl); + } + sqlite3_free(p->aCell); + } + sqlite3PagerUnref(p->pPg); + sqlite3_free(p->zPath); + memset(p, 0, sizeof(StatPage)); +} + +static void statResetCsr(StatCursor *pCsr){ + int i; + sqlite3_reset(pCsr->pStmt); + for(i=0; iaPage); i++){ + statClearPage(&pCsr->aPage[i]); + } + pCsr->iPage = 0; + sqlite3_free(pCsr->zPath); + pCsr->zPath = 0; +} + +/* +** Close a statvfs cursor. +*/ +static int statClose(sqlite3_vtab_cursor *pCursor){ + StatCursor *pCsr = (StatCursor *)pCursor; + statResetCsr(pCsr); + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +static void getLocalPayload( + int nUsable, /* Usable bytes per page */ + u8 flags, /* Page flags */ + int nTotal, /* Total record (payload) size */ + int *pnLocal /* OUT: Bytes stored locally */ +){ + int nLocal; + int nMinLocal; + int nMaxLocal; + + if( flags==0x0D ){ /* Table leaf node */ + nMinLocal = (nUsable - 12) * 32 / 255 - 23; + nMaxLocal = nUsable - 35; + }else{ /* Index interior and leaf nodes */ + nMinLocal = (nUsable - 12) * 32 / 255 - 23; + nMaxLocal = (nUsable - 12) * 64 / 255 - 23; + } + + nLocal = nMinLocal + (nTotal - nMinLocal) % (nUsable - 4); + if( nLocal>nMaxLocal ) nLocal = nMinLocal; + *pnLocal = nLocal; +} + +static int statDecodePage(Btree *pBt, StatPage *p){ + int nUnused; + int iOff; + int nHdr; + int isLeaf; + int szPage; + + u8 *aData = sqlite3PagerGetData(p->pPg); + u8 *aHdr = &aData[p->iPgno==1 ? 100 : 0]; + + p->flags = aHdr[0]; + p->nCell = get2byte(&aHdr[3]); + p->nMxPayload = 0; + + isLeaf = (p->flags==0x0A || p->flags==0x0D); + nHdr = 12 - isLeaf*4 + (p->iPgno==1)*100; + + nUnused = get2byte(&aHdr[5]) - nHdr - 2*p->nCell; + nUnused += (int)aHdr[7]; + iOff = get2byte(&aHdr[1]); + while( iOff ){ + nUnused += get2byte(&aData[iOff+2]); + iOff = get2byte(&aData[iOff]); + } + p->nUnused = nUnused; + p->iRightChildPg = isLeaf ? 0 : sqlite3Get4byte(&aHdr[8]); + szPage = sqlite3BtreeGetPageSize(pBt); + + if( p->nCell ){ + int i; /* Used to iterate through cells */ + int nUsable; /* Usable bytes per page */ + + sqlite3BtreeEnter(pBt); + nUsable = szPage - sqlite3BtreeGetReserveNoMutex(pBt); + sqlite3BtreeLeave(pBt); + p->aCell = sqlite3_malloc64((p->nCell+1) * sizeof(StatCell)); + if( p->aCell==0 ) return SQLITE_NOMEM; + memset(p->aCell, 0, (p->nCell+1) * sizeof(StatCell)); + + for(i=0; inCell; i++){ + StatCell *pCell = &p->aCell[i]; + + iOff = get2byte(&aData[nHdr+i*2]); + if( !isLeaf ){ + pCell->iChildPg = sqlite3Get4byte(&aData[iOff]); + iOff += 4; + } + if( p->flags==0x05 ){ + /* A table interior node. nPayload==0. */ + }else{ + u32 nPayload; /* Bytes of payload total (local+overflow) */ + int nLocal; /* Bytes of payload stored locally */ + iOff += getVarint32(&aData[iOff], nPayload); + if( p->flags==0x0D ){ + u64 dummy; + iOff += sqlite3GetVarint(&aData[iOff], &dummy); + } + if( nPayload>(u32)p->nMxPayload ) p->nMxPayload = nPayload; + getLocalPayload(nUsable, p->flags, nPayload, &nLocal); + pCell->nLocal = nLocal; + assert( nLocal>=0 ); + assert( nPayload>=(u32)nLocal ); + assert( nLocal<=(nUsable-35) ); + if( nPayload>(u32)nLocal ){ + int j; + int nOvfl = ((nPayload - nLocal) + nUsable-4 - 1) / (nUsable - 4); + pCell->nLastOvfl = (nPayload-nLocal) - (nOvfl-1) * (nUsable-4); + pCell->nOvfl = nOvfl; + pCell->aOvfl = sqlite3_malloc64(sizeof(u32)*nOvfl); + if( pCell->aOvfl==0 ) return SQLITE_NOMEM; + pCell->aOvfl[0] = sqlite3Get4byte(&aData[iOff+nLocal]); + for(j=1; jaOvfl[j-1]; + DbPage *pPg = 0; + rc = sqlite3PagerGet(sqlite3BtreePager(pBt), iPrev, &pPg); + if( rc!=SQLITE_OK ){ + assert( pPg==0 ); + return rc; + } + pCell->aOvfl[j] = sqlite3Get4byte(sqlite3PagerGetData(pPg)); + sqlite3PagerUnref(pPg); + } + } + } + } + } + + return SQLITE_OK; +} + +/* +** Populate the pCsr->iOffset and pCsr->szPage member variables. Based on +** the current value of pCsr->iPageno. +*/ +static void statSizeAndOffset(StatCursor *pCsr){ + StatTable *pTab = (StatTable *)((sqlite3_vtab_cursor *)pCsr)->pVtab; + Btree *pBt = pTab->db->aDb[pTab->iDb].pBt; + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3_file *fd; + sqlite3_int64 x[2]; + + /* The default page size and offset */ + pCsr->szPage = sqlite3BtreeGetPageSize(pBt); + pCsr->iOffset = (i64)pCsr->szPage * (pCsr->iPageno - 1); + + /* If connected to a ZIPVFS backend, override the page size and + ** offset with actual values obtained from ZIPVFS. + */ + fd = sqlite3PagerFile(pPager); + x[0] = pCsr->iPageno; + if( fd->pMethods!=0 && sqlite3OsFileControl(fd, 230440, &x)==SQLITE_OK ){ + pCsr->iOffset = x[0]; + pCsr->szPage = (int)x[1]; + } +} + +/* +** Move a statvfs cursor to the next entry in the file. +*/ +static int statNext(sqlite3_vtab_cursor *pCursor){ + int rc; + int nPayload; + char *z; + StatCursor *pCsr = (StatCursor *)pCursor; + StatTable *pTab = (StatTable *)pCursor->pVtab; + Btree *pBt = pTab->db->aDb[pTab->iDb].pBt; + Pager *pPager = sqlite3BtreePager(pBt); + + sqlite3_free(pCsr->zPath); + pCsr->zPath = 0; + +statNextRestart: + if( pCsr->aPage[0].pPg==0 ){ + rc = sqlite3_step(pCsr->pStmt); + if( rc==SQLITE_ROW ){ + int nPage; + u32 iRoot = (u32)sqlite3_column_int64(pCsr->pStmt, 1); + sqlite3PagerPagecount(pPager, &nPage); + if( nPage==0 ){ + pCsr->isEof = 1; + return sqlite3_reset(pCsr->pStmt); + } + rc = sqlite3PagerGet(pPager, iRoot, &pCsr->aPage[0].pPg); + pCsr->aPage[0].iPgno = iRoot; + pCsr->aPage[0].iCell = 0; + pCsr->aPage[0].zPath = z = sqlite3_mprintf("/"); + pCsr->iPage = 0; + if( z==0 ) rc = SQLITE_NOMEM; + }else{ + pCsr->isEof = 1; + return sqlite3_reset(pCsr->pStmt); + } + }else{ + + /* Page p itself has already been visited. */ + StatPage *p = &pCsr->aPage[pCsr->iPage]; + + while( p->iCellnCell ){ + StatCell *pCell = &p->aCell[p->iCell]; + if( pCell->iOvflnOvfl ){ + int nUsable; + sqlite3BtreeEnter(pBt); + nUsable = sqlite3BtreeGetPageSize(pBt) - + sqlite3BtreeGetReserveNoMutex(pBt); + sqlite3BtreeLeave(pBt); + pCsr->zName = (char *)sqlite3_column_text(pCsr->pStmt, 0); + pCsr->iPageno = pCell->aOvfl[pCell->iOvfl]; + pCsr->zPagetype = "overflow"; + pCsr->nCell = 0; + pCsr->nMxPayload = 0; + pCsr->zPath = z = sqlite3_mprintf( + "%s%.3x+%.6x", p->zPath, p->iCell, pCell->iOvfl + ); + if( pCell->iOvflnOvfl-1 ){ + pCsr->nUnused = 0; + pCsr->nPayload = nUsable - 4; + }else{ + pCsr->nPayload = pCell->nLastOvfl; + pCsr->nUnused = nUsable - 4 - pCsr->nPayload; + } + pCell->iOvfl++; + statSizeAndOffset(pCsr); + return z==0 ? SQLITE_NOMEM : SQLITE_OK; + } + if( p->iRightChildPg ) break; + p->iCell++; + } + + if( !p->iRightChildPg || p->iCell>p->nCell ){ + statClearPage(p); + if( pCsr->iPage==0 ) return statNext(pCursor); + pCsr->iPage--; + goto statNextRestart; /* Tail recursion */ + } + pCsr->iPage++; + assert( p==&pCsr->aPage[pCsr->iPage-1] ); + + if( p->iCell==p->nCell ){ + p[1].iPgno = p->iRightChildPg; + }else{ + p[1].iPgno = p->aCell[p->iCell].iChildPg; + } + rc = sqlite3PagerGet(pPager, p[1].iPgno, &p[1].pPg); + p[1].iCell = 0; + p[1].zPath = z = sqlite3_mprintf("%s%.3x/", p->zPath, p->iCell); + p->iCell++; + if( z==0 ) rc = SQLITE_NOMEM; + } + + + /* Populate the StatCursor fields with the values to be returned + ** by the xColumn() and xRowid() methods. + */ + if( rc==SQLITE_OK ){ + int i; + StatPage *p = &pCsr->aPage[pCsr->iPage]; + pCsr->zName = (char *)sqlite3_column_text(pCsr->pStmt, 0); + pCsr->iPageno = p->iPgno; + + rc = statDecodePage(pBt, p); + if( rc==SQLITE_OK ){ + statSizeAndOffset(pCsr); + + switch( p->flags ){ + case 0x05: /* table internal */ + case 0x02: /* index internal */ + pCsr->zPagetype = "internal"; + break; + case 0x0D: /* table leaf */ + case 0x0A: /* index leaf */ + pCsr->zPagetype = "leaf"; + break; + default: + pCsr->zPagetype = "corrupted"; + break; + } + pCsr->nCell = p->nCell; + pCsr->nUnused = p->nUnused; + pCsr->nMxPayload = p->nMxPayload; + pCsr->zPath = z = sqlite3_mprintf("%s", p->zPath); + if( z==0 ) rc = SQLITE_NOMEM; + nPayload = 0; + for(i=0; inCell; i++){ + nPayload += p->aCell[i].nLocal; + } + pCsr->nPayload = nPayload; + } + } + + return rc; +} + +static int statEof(sqlite3_vtab_cursor *pCursor){ + StatCursor *pCsr = (StatCursor *)pCursor; + return pCsr->isEof; +} + +static int statFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + StatCursor *pCsr = (StatCursor *)pCursor; + + statResetCsr(pCsr); + return statNext(pCursor); +} + +static int statColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int i +){ + StatCursor *pCsr = (StatCursor *)pCursor; + switch( i ){ + case 0: /* name */ + sqlite3_result_text(ctx, pCsr->zName, -1, SQLITE_TRANSIENT); + break; + case 1: /* path */ + sqlite3_result_text(ctx, pCsr->zPath, -1, SQLITE_TRANSIENT); + break; + case 2: /* pageno */ + sqlite3_result_int64(ctx, pCsr->iPageno); + break; + case 3: /* pagetype */ + sqlite3_result_text(ctx, pCsr->zPagetype, -1, SQLITE_STATIC); + break; + case 4: /* ncell */ + sqlite3_result_int(ctx, pCsr->nCell); + break; + case 5: /* payload */ + sqlite3_result_int(ctx, pCsr->nPayload); + break; + case 6: /* unused */ + sqlite3_result_int(ctx, pCsr->nUnused); + break; + case 7: /* mx_payload */ + sqlite3_result_int(ctx, pCsr->nMxPayload); + break; + case 8: /* pgoffset */ + sqlite3_result_int64(ctx, pCsr->iOffset); + break; + default: /* pgsize */ + assert( i==9 ); + sqlite3_result_int(ctx, pCsr->szPage); + break; + } + return SQLITE_OK; +} + +static int statRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + StatCursor *pCsr = (StatCursor *)pCursor; + *pRowid = pCsr->iPageno; + return SQLITE_OK; +} + +/* +** Invoke this routine to register the "dbstat" virtual table module +*/ +int sqlite3DbstatRegister(sqlite3 *db){ + static sqlite3_module dbstat_module = { + 0, /* iVersion */ + statConnect, /* xCreate */ + statConnect, /* xConnect */ + statBestIndex, /* xBestIndex */ + statDisconnect, /* xDisconnect */ + statDisconnect, /* xDestroy */ + statOpen, /* xOpen - open a cursor */ + statClose, /* xClose - close a cursor */ + statFilter, /* xFilter - configure scan constraints */ + statNext, /* xNext - advance a cursor */ + statEof, /* xEof - check for end of scan */ + statColumn, /* xColumn - read data */ + statRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + }; + return sqlite3_create_module(db, "dbstat", &dbstat_module, 0); +} +#elif defined(SQLITE_ENABLE_DBSTAT_VTAB) +int sqlite3DbstatRegister(sqlite3 *db){ return SQLITE_OK; } +#endif /* SQLITE_ENABLE_DBSTAT_VTAB */ Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -1250,11 +1250,11 @@ int i; u32 m = 0; if( pList ){ for(i=0; inExpr; i++){ Expr *pExpr = pList->a[i].pExpr; - if( ALWAYS(pExpr) ) m |= pList->a[i].pExpr->flags; + if( ALWAYS(pExpr) ) m |= pExpr->flags; } } return m; } Index: src/func.c ================================================================== --- src/func.c +++ src/func.c @@ -230,17 +230,17 @@ ){ PrintfArguments x; StrAccum str; const char *zFormat; int n; + sqlite3 *db = sqlite3_context_db_handle(context); if( argc>=1 && (zFormat = (const char*)sqlite3_value_text(argv[0]))!=0 ){ x.nArg = argc-1; x.nUsed = 0; x.apArg = argv+1; - sqlite3StrAccumInit(&str, 0, 0, SQLITE_MAX_LENGTH); - str.db = sqlite3_context_db_handle(context); + sqlite3StrAccumInit(&str, db, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); sqlite3XPrintf(&str, SQLITE_PRINTF_SQLFUNC, zFormat, &x); n = str.nChar; sqlite3_result_text(context, sqlite3StrAccumFinish(&str), n, SQLITE_DYNAMIC); } @@ -386,11 +386,11 @@ sqlite3_result_double(context, r); } #endif /* -** Allocate nByte bytes of space using sqlite3_malloc(). If the +** Allocate nByte bytes of space using sqlite3Malloc(). If the ** allocation fails, call sqlite3_result_error_nomem() to notify ** the database handle that malloc() has failed and return NULL. ** If nByte is larger than the maximum string or blob length, then ** raise an SQLITE_TOOBIG exception and return NULL. */ @@ -1055,11 +1055,11 @@ int argc, sqlite3_value **argv ){ unsigned char *z, *zOut; int i; - zOut = z = sqlite3_malloc( argc*4+1 ); + zOut = z = sqlite3_malloc64( argc*4+1 ); if( z==0 ){ sqlite3_result_error_nomem(context); return; } for(i=0; iuseMalloc==0; - pAccum->useMalloc = 2; + int firstTerm = pAccum->mxAlloc==0; pAccum->mxAlloc = db->aLimit[SQLITE_LIMIT_LENGTH]; if( !firstTerm ){ if( argc==2 ){ zSep = (char*)sqlite3_value_text(argv[1]); nSep = sqlite3_value_bytes(argv[1]); Index: src/loadext.c ================================================================== --- src/loadext.c +++ src/loadext.c @@ -428,11 +428,11 @@ int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); char *zErrmsg = 0; const char *zEntry; char *zAltEntry = 0; void **aHandle; - int nMsg = 300 + sqlite3Strlen30(zFile); + u64 nMsg = 300 + sqlite3Strlen30(zFile); int ii; /* Shared library endings to try if zFile cannot be loaded as written */ static const char *azEndings[] = { #if SQLITE_OS_WIN @@ -471,11 +471,11 @@ sqlite3_free(zAltFile); } #endif if( handle==0 ){ if( pzErrMsg ){ - *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); + *pzErrMsg = zErrmsg = sqlite3_malloc64(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, "unable to open shared library [%s]", zFile); sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); } @@ -497,11 +497,11 @@ ** C:/lib/mathfuncs.dll ==> sqlite3_mathfuncs_init */ if( xInit==0 && zProc==0 ){ int iFile, iEntry, c; int ncFile = sqlite3Strlen30(zFile); - zAltEntry = sqlite3_malloc(ncFile+30); + zAltEntry = sqlite3_malloc64(ncFile+30); if( zAltEntry==0 ){ sqlite3OsDlClose(pVfs, handle); return SQLITE_NOMEM; } memcpy(zAltEntry, "sqlite3_", 8); @@ -519,11 +519,11 @@ sqlite3OsDlSym(pVfs, handle, zEntry); } if( xInit==0 ){ if( pzErrMsg ){ nMsg += sqlite3Strlen30(zEntry); - *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); + *pzErrMsg = zErrmsg = sqlite3_malloc64(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, "no entry point [%s] in shared library [%s]", zEntry, zFile); sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); } @@ -618,11 +618,11 @@ ** This list is shared across threads. The SQLITE_MUTEX_STATIC_MASTER ** mutex must be held while accessing this list. */ typedef struct sqlite3AutoExtList sqlite3AutoExtList; static SQLITE_WSD struct sqlite3AutoExtList { - int nExt; /* Number of entries in aExt[] */ + u32 nExt; /* Number of entries in aExt[] */ void (**aExt)(void); /* Pointers to the extension init functions */ } sqlite3Autoext = { 0, 0 }; /* The "wsdAutoext" macro will resolve to the autoextension ** state vector. If writable static data is unsupported on the target, @@ -651,23 +651,23 @@ if( rc ){ return rc; }else #endif { - int i; + u32 i; #if SQLITE_THREADSAFE sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); #endif wsdAutoextInit; sqlite3_mutex_enter(mutex); for(i=0; i=0; i--){ + for(i=(int)wsdAutoext.nExt-1; i>=0; i--){ if( wsdAutoext.aExt[i]==xInit ){ wsdAutoext.nExt--; wsdAutoext.aExt[i] = wsdAutoext.aExt[wsdAutoext.nExt]; n++; break; @@ -733,11 +733,11 @@ ** Load all automatic extensions. ** ** If anything goes wrong, set an error in the database connection. */ void sqlite3AutoLoadExtensions(sqlite3 *db){ - int i; + u32 i; int go = 1; int rc; int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); wsdAutoextInit; Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -56,10 +56,22 @@ ** zero if and only if SQLite was compiled with mutexing code omitted due to ** the SQLITE_THREADSAFE compile-time option being set to 0. */ int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } +/* +** When compiling the test fixture or with debugging enabled (on Win32), +** this variable being set to non-zero will cause OSTRACE macros to emit +** extra diagnostic information. +*/ +#ifdef SQLITE_HAVE_OS_TRACE +# ifndef SQLITE_DEBUG_OS_TRACE +# define SQLITE_DEBUG_OS_TRACE 0 +# endif + int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; +#endif + #if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) /* ** If the following function pointer is not NULL and if ** SQLITE_ENABLE_IOTRACE is enabled, then messages describing ** I/O active are written using this function. These messages @@ -1195,11 +1207,11 @@ /* ** Return a static string containing the name corresponding to the error code ** specified in the argument. */ -#if (defined(SQLITE_DEBUG) && SQLITE_OS_WIN) || defined(SQLITE_TEST) +#if defined(SQLITE_NEED_ERR_NAME) const char *sqlite3ErrName(int rc){ const char *zName = 0; int i, origRc = rc; for(i=0; i<2 && zName==0; i++, rc &= 0xff){ switch( rc ){ @@ -2420,18 +2432,18 @@ ){ char *zOpt; int eState; /* Parser state when parsing URI */ int iIn; /* Input character index */ int iOut = 0; /* Output character index */ - int nByte = nUri+2; /* Bytes of space to allocate */ + u64 nByte = nUri+2; /* Bytes of space to allocate */ /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen ** method that there may be extra parameters following the file-name. */ flags |= SQLITE_OPEN_URI; for(iIn=0; iInmallocFailed && rc==SQLITE_OK){ rc = sqlite3RtreeInit(db); } #endif + +#ifdef SQLITE_ENABLE_DBSTAT_VTAB + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3DbstatRegister(db); + } +#endif /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking ** mode. Doing nothing at all also makes NORMAL the default. */ Index: src/malloc.c ================================================================== --- src/malloc.c +++ src/malloc.c @@ -224,28 +224,24 @@ /* ** Return the amount of memory currently checked out. */ sqlite3_int64 sqlite3_memory_used(void){ - int n, mx; - sqlite3_int64 res; - sqlite3_status(SQLITE_STATUS_MEMORY_USED, &n, &mx, 0); - res = (sqlite3_int64)n; /* Work around bug in Borland C. Ticket #3216 */ + sqlite3_int64 res, mx; + sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &res, &mx, 0); return res; } /* ** Return the maximum amount of memory that has ever been ** checked out since either the beginning of this process ** or since the most recent reset. */ sqlite3_int64 sqlite3_memory_highwater(int resetFlag){ - int n, mx; - sqlite3_int64 res; - sqlite3_status(SQLITE_STATUS_MEMORY_USED, &n, &mx, resetFlag); - res = (sqlite3_int64)mx; /* Work around bug in Borland C. Ticket #3216 */ - return res; + sqlite3_int64 res, mx; + sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &res, &mx, resetFlag); + return mx; } /* ** Trigger the alarm */ Index: src/os_common.h ================================================================== --- src/os_common.h +++ src/os_common.h @@ -27,20 +27,10 @@ */ #ifdef MEMORY_DEBUG # error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." #endif -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) -# ifndef SQLITE_DEBUG_OS_TRACE -# define SQLITE_DEBUG_OS_TRACE 0 -# endif - int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; -# define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X -#else -# define OSTRACE(X) -#endif - /* ** Macros for performance tracing. Normally turned off. Only works ** on i486 hardware. */ #ifdef SQLITE_PERFORMANCE_TRACE Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -639,11 +639,11 @@ return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } #endif -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) +#ifdef SQLITE_HAVE_OS_TRACE /* ** Helper function for printing out trace information from debugging ** binaries. This returns the string representation of the supplied ** integer lock-type. */ @@ -902,11 +902,11 @@ struct vxworksFileId *pCandidate; /* For looping over existing file IDs */ int n; /* Length of zAbsoluteName string */ assert( zAbsoluteName[0]=='/' ); n = (int)strlen(zAbsoluteName); - pNew = sqlite3_malloc( sizeof(*pNew) + (n+1) ); + pNew = sqlite3_malloc64( sizeof(*pNew) + (n+1) ); if( pNew==0 ) return 0; pNew->zCanonicalName = (char*)&pNew[1]; memcpy(pNew->zCanonicalName, zAbsoluteName, n+1); n = vxworksSimplifyName(pNew->zCanonicalName, n); @@ -1306,11 +1306,11 @@ pInode = inodeList; while( pInode && memcmp(&fileId, &pInode->fileId, sizeof(fileId)) ){ pInode = pInode->pNext; } if( pInode==0 ){ - pInode = sqlite3_malloc( sizeof(*pInode) ); + pInode = sqlite3_malloc64( sizeof(*pInode) ); if( pInode==0 ){ return SQLITE_NOMEM; } memset(pInode, 0, sizeof(*pInode)); memcpy(&pInode->fileId, &fileId, sizeof(fileId)); @@ -3827,11 +3827,11 @@ case SQLITE_FCNTL_VFSNAME: { *(char**)pArg = sqlite3_mprintf("%s", pFile->pVfs->zName); return SQLITE_OK; } case SQLITE_FCNTL_TEMPFILENAME: { - char *zTFile = sqlite3_malloc( pFile->pVfs->mxPathname ); + char *zTFile = sqlite3_malloc64( pFile->pVfs->mxPathname ); if( zTFile ){ unixGetTempname(pFile->pVfs->mxPathname, zTFile); *(char**)pArg = zTFile; } return SQLITE_OK; @@ -4268,11 +4268,11 @@ unixInodeInfo *pInode; /* The inode of fd */ char *zShmFilename; /* Name of the file used for SHM */ int nShmFilename; /* Size of the SHM filename in bytes */ /* Allocate space for the new unixShm object. */ - p = sqlite3_malloc( sizeof(*p) ); + p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ) return SQLITE_NOMEM; memset(p, 0, sizeof(*p)); assert( pDbFd->pShm==0 ); /* Check to see if a unixShmNode object already exists. Reuse an existing @@ -4299,11 +4299,11 @@ #ifdef SQLITE_SHM_DIRECTORY nShmFilename = sizeof(SQLITE_SHM_DIRECTORY) + 31; #else nShmFilename = 6 + (int)strlen(zBasePath); #endif - pShmNode = sqlite3_malloc( sizeof(*pShmNode) + nShmFilename ); + pShmNode = sqlite3_malloc64( sizeof(*pShmNode) + nShmFilename ); if( pShmNode==0 ){ rc = SQLITE_NOMEM; goto shm_open_err; } memset(pShmNode, 0, sizeof(*pShmNode)+nShmFilename); @@ -4509,11 +4509,11 @@ if( pMem==MAP_FAILED ){ rc = unixLogError(SQLITE_IOERR_SHMMAP, "mmap", pShmNode->zFilename); goto shmpage_out; } }else{ - pMem = sqlite3_malloc(szRegion); + pMem = sqlite3_malloc64(szRegion); if( pMem==0 ){ rc = SQLITE_NOMEM; goto shmpage_out; } memset(pMem, 0, szRegion); @@ -5346,11 +5346,11 @@ else if( pLockingStyle == &afpIoMethods ){ /* AFP locking uses the file path so it needs to be included in ** the afpLockingContext. */ afpLockingContext *pCtx; - pNew->lockingContext = pCtx = sqlite3_malloc( sizeof(*pCtx) ); + pNew->lockingContext = pCtx = sqlite3_malloc64( sizeof(*pCtx) ); if( pCtx==0 ){ rc = SQLITE_NOMEM; }else{ /* NB: zFilename exists and remains valid until the file is closed ** according to requirement F11141. So we do not need to make a @@ -5376,11 +5376,11 @@ */ char *zLockFile; int nFilename; assert( zFilename!=0 ); nFilename = (int)strlen(zFilename) + 6; - zLockFile = (char *)sqlite3_malloc(nFilename); + zLockFile = (char *)sqlite3_malloc64(nFilename); if( zLockFile==0 ){ rc = SQLITE_NOMEM; }else{ sqlite3_snprintf(nFilename, zLockFile, "%s" DOTLOCK_SUFFIX, zFilename); } @@ -5753,11 +5753,11 @@ UnixUnusedFd *pUnused; pUnused = findReusableFd(zName, flags); if( pUnused ){ fd = pUnused->fd; }else{ - pUnused = sqlite3_malloc(sizeof(*pUnused)); + pUnused = sqlite3_malloc64(sizeof(*pUnused)); if( !pUnused ){ return SQLITE_NOMEM; } } p->pUnused = pUnused; @@ -6133,11 +6133,11 @@ ** that we always use the same random number sequence. This makes the ** tests repeatable. */ memset(zBuf, 0, nBuf); randomnessPid = osGetpid(0); -#if !defined(SQLITE_TEST) +#if !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) { int fd, got; fd = robust_open("/dev/urandom", O_RDONLY, 0); if( fd<0 ){ time_t t; @@ -6545,11 +6545,11 @@ */ pUnused = findReusableFd(path, openFlags); if( pUnused ){ fd = pUnused->fd; }else{ - pUnused = sqlite3_malloc(sizeof(*pUnused)); + pUnused = sqlite3_malloc64(sizeof(*pUnused)); if( !pUnused ){ return SQLITE_NOMEM; } } if( fd<0 ){ @@ -6578,11 +6578,11 @@ default: return SQLITE_CANTOPEN_BKPT; } } - pNew = (unixFile *)sqlite3_malloc(sizeof(*pNew)); + pNew = (unixFile *)sqlite3_malloc64(sizeof(*pNew)); if( pNew==NULL ){ rc = SQLITE_NOMEM; goto end_create_proxy; } memset(pNew, 0, sizeof(unixFile)); @@ -7040,11 +7040,11 @@ return rc; } /* ** Given the name of a database file, compute the name of its conch file. -** Store the conch filename in memory obtained from sqlite3_malloc(). +** Store the conch filename in memory obtained from sqlite3_malloc64(). ** Make *pConchPath point to the new name. Return SQLITE_OK on success ** or SQLITE_NOMEM if unable to obtain memory. ** ** The caller is responsible for ensuring that the allocated memory ** space is eventually freed. @@ -7056,11 +7056,11 @@ int len = (int)strlen(dbPath); /* Length of database filename - dbPath */ char *conchPath; /* buffer in which to construct conch name */ /* Allocate space for the conch filename and initialize the name to ** the name of the original database file. */ - *pConchPath = conchPath = (char *)sqlite3_malloc(len + 8); + *pConchPath = conchPath = (char *)sqlite3_malloc64(len + 8); if( conchPath==0 ){ return SQLITE_NOMEM; } memcpy(conchPath, dbPath, len+1); @@ -7172,11 +7172,11 @@ } OSTRACE(("TRANSPROXY %d for %s pid=%d\n", pFile->h, (lockPath ? lockPath : ":auto:"), osGetpid(0))); - pCtx = sqlite3_malloc( sizeof(*pCtx) ); + pCtx = sqlite3_malloc64( sizeof(*pCtx) ); if( pCtx==0 ){ return SQLITE_NOMEM; } memset(pCtx, 0, sizeof(*pCtx)); Index: src/os_win.c ================================================================== --- src/os_win.c +++ src/os_win.c @@ -2755,11 +2755,11 @@ ** Used only when SQLITE_NO_SYNC is not defined. */ BOOL rc; #endif #if !defined(NDEBUG) || !defined(SQLITE_NO_SYNC) || \ - (defined(SQLITE_TEST) && defined(SQLITE_DEBUG)) + defined(SQLITE_HAVE_OS_TRACE) /* ** Used when SQLITE_NO_SYNC is not defined and by the assert() and/or ** OSTRACE() macros. */ winFile *pFile = (winFile*)id; @@ -3432,11 +3432,11 @@ DWORD lastErrno; /* The Windows errno from the last I/O error */ int nRef; /* Number of winShm objects pointing to this */ winShm *pFirst; /* All winShm objects pointing to this */ winShmNode *pNext; /* Next in list of all winShmNode objects */ -#ifdef SQLITE_DEBUG +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) u8 nextShmId; /* Next available winShm.id value */ #endif }; /* @@ -3463,11 +3463,11 @@ winShmNode *pShmNode; /* The underlying winShmNode object */ winShm *pNext; /* Next winShm with the same winShmNode */ u8 hasMutex; /* True if holding the winShmNode mutex */ u16 sharedMask; /* Mask of shared locks held */ u16 exclMask; /* Mask of exclusive locks held */ -#ifdef SQLITE_DEBUG +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) u8 id; /* Id of this connection with its winShmNode */ #endif }; /* @@ -3654,11 +3654,11 @@ if( rc ) goto shm_open_err; } /* Make the new connection a child of the winShmNode */ p->pShmNode = pShmNode; -#ifdef SQLITE_DEBUG +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) p->id = pShmNode->nextShmId++; #endif pShmNode->nRef++; pDbFd->pShm = p; winShmLeaveMutex(); @@ -3923,11 +3923,11 @@ goto shmpage_out; } } /* Map the requested memory region into this processes address space. */ - apNew = (struct ShmRegion *)sqlite3_realloc( + apNew = (struct ShmRegion *)sqlite3_realloc64( pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0]) ); if( !apNew ){ rc = SQLITE_IOERR_NOMEM; goto shmpage_out; @@ -5370,11 +5370,11 @@ ** Write up to nBuf bytes of randomness into zBuf. */ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ int n = 0; UNUSED_PARAMETER(pVfs); -#if defined(SQLITE_TEST) +#if defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) n = nBuf; memset(zBuf, 0, nBuf); #else if( sizeof(SYSTEMTIME)<=nBuf-n ){ SYSTEMTIME x; @@ -5404,27 +5404,27 @@ LARGE_INTEGER i; osQueryPerformanceCounter(&i); memcpy(&zBuf[n], &i, sizeof(i)); n += sizeof(i); } -#endif #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID if( sizeof(UUID)<=nBuf-n ){ UUID id; memset(&id, 0, sizeof(UUID)); osUuidCreate(&id); - memcpy(zBuf, &id, sizeof(UUID)); + memcpy(&zBuf[n], &id, sizeof(UUID)); n += sizeof(UUID); } if( sizeof(UUID)<=nBuf-n ){ UUID id; memset(&id, 0, sizeof(UUID)); osUuidCreateSequential(&id); - memcpy(zBuf, &id, sizeof(UUID)); + memcpy(&zBuf[n], &id, sizeof(UUID)); n += sizeof(UUID); } #endif +#endif /* defined(SQLITE_TEST) || defined(SQLITE_ZERO_PRNG_SEED) */ return n; } /* Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -7008,10 +7008,12 @@ }else if( state==PAGER_OPEN ){ pager_unlock(pPager); } assert( state==pPager->eState ); } + }else if( eMode==PAGER_JOURNALMODE_OFF ){ + sqlite3OsClose(pPager->jfd); } } /* Return the new journal mode */ return (int)pPager->journalMode; Index: src/printf.c ================================================================== --- src/printf.c +++ src/printf.c @@ -136,10 +136,11 @@ /* ** Set the StrAccum object to an error mode. */ static void setStrAccumError(StrAccum *p, u8 eError){ + assert( eError==STRACCUM_NOMEM || eError==STRACCUM_TOOBIG ); p->accError = eError; p->nAlloc = 0; } /* @@ -250,11 +251,10 @@ case '0': flag_zeropad = 1; break; default: done = 1; break; } }while( !done && (c=(*++fmt))!=0 ); /* Get the field width */ - width = 0; if( c=='*' ){ if( bArgList ){ width = (int)getIntArg(pArgList); }else{ width = va_arg(ap,int); @@ -274,11 +274,10 @@ width = wx & 0x7fffffff; } /* Get the precision */ if( c=='.' ){ - precision = 0; c = *++fmt; if( c=='*' ){ if( bArgList ){ precision = (int)getIntArg(pArgList); }else{ @@ -753,11 +752,11 @@ if( p->accError ){ testcase(p->accError==STRACCUM_TOOBIG); testcase(p->accError==STRACCUM_NOMEM); return 0; } - if( !p->useMalloc ){ + if( p->mxAlloc==0 ){ N = p->nAlloc - p->nChar - 1; setStrAccumError(p, STRACCUM_TOOBIG); return N; }else{ char *zOld = (p->zText==p->zBase ? 0 : p->zText); @@ -773,14 +772,14 @@ setStrAccumError(p, STRACCUM_TOOBIG); return 0; }else{ p->nAlloc = (int)szNew; } - if( p->useMalloc==1 ){ + if( p->db ){ zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc); }else{ - zNew = sqlite3_realloc(zOld, p->nAlloc); + zNew = sqlite3_realloc64(zOld, p->nAlloc); } if( zNew ){ assert( p->zText!=0 || p->nChar==0 ); if( zOld==0 && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); p->zText = zNew; @@ -853,16 +852,12 @@ ** pointer if any kind of error was encountered. */ char *sqlite3StrAccumFinish(StrAccum *p){ if( p->zText ){ p->zText[p->nChar] = 0; - if( p->useMalloc && p->zText==p->zBase ){ - if( p->useMalloc==1 ){ - p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); - }else{ - p->zText = sqlite3_malloc(p->nChar+1); - } + if( p->mxAlloc>0 && p->zText==p->zBase ){ + p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); if( p->zText ){ memcpy(p->zText, p->zBase, p->nChar+1); }else{ setStrAccumError(p, STRACCUM_NOMEM); } @@ -874,29 +869,35 @@ /* ** Reset an StrAccum string. Reclaim all malloced memory. */ void sqlite3StrAccumReset(StrAccum *p){ if( p->zText!=p->zBase ){ - if( p->useMalloc==1 ){ - sqlite3DbFree(p->db, p->zText); - }else{ - sqlite3_free(p->zText); - } + sqlite3DbFree(p->db, p->zText); } p->zText = 0; } /* -** Initialize a string accumulator +** Initialize a string accumulator. +** +** p: The accumulator to be initialized. +** db: Pointer to a database connection. May be NULL. Lookaside +** memory is used if not NULL. db->mallocFailed is set appropriately +** when not NULL. +** zBase: An initial buffer. May be NULL in which case the initial buffer +** is malloced. +** n: Size of zBase in bytes. If total space requirements never exceed +** n then no memory allocations ever occur. +** mx: Maximum number of bytes to accumulate. If mx==0 then no memory +** allocations will ever occur. */ -void sqlite3StrAccumInit(StrAccum *p, char *zBase, int n, int mx){ +void sqlite3StrAccumInit(StrAccum *p, sqlite3 *db, char *zBase, int n, int mx){ p->zText = p->zBase = zBase; - p->db = 0; + p->db = db; p->nChar = 0; p->nAlloc = n; p->mxAlloc = mx; - p->useMalloc = 1; p->accError = 0; } /* ** Print into memory obtained from sqliteMalloc(). Use the internal @@ -905,13 +906,12 @@ char *sqlite3VMPrintf(sqlite3 *db, const char *zFormat, va_list ap){ char *z; char zBase[SQLITE_PRINT_BUF_SIZE]; StrAccum acc; assert( db!=0 ); - sqlite3StrAccumInit(&acc, zBase, sizeof(zBase), + sqlite3StrAccumInit(&acc, db, zBase, sizeof(zBase), db->aLimit[SQLITE_LIMIT_LENGTH]); - acc.db = db; sqlite3VXPrintf(&acc, SQLITE_PRINTF_INTERNAL, zFormat, ap); z = sqlite3StrAccumFinish(&acc); if( acc.accError==STRACCUM_NOMEM ){ db->mallocFailed = 1; } @@ -965,12 +965,11 @@ } #endif #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif - sqlite3StrAccumInit(&acc, zBase, sizeof(zBase), SQLITE_MAX_LENGTH); - acc.useMalloc = 2; + sqlite3StrAccumInit(&acc, 0, zBase, sizeof(zBase), SQLITE_MAX_LENGTH); sqlite3VXPrintf(&acc, 0, zFormat, ap); z = sqlite3StrAccumFinish(&acc); return z; } @@ -1011,12 +1010,11 @@ (void)SQLITE_MISUSE_BKPT; if( zBuf ) zBuf[0] = 0; return zBuf; } #endif - sqlite3StrAccumInit(&acc, zBuf, n, 0); - acc.useMalloc = 0; + sqlite3StrAccumInit(&acc, 0, zBuf, n, 0); sqlite3VXPrintf(&acc, 0, zFormat, ap); return sqlite3StrAccumFinish(&acc); } char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ char *z; @@ -1038,12 +1036,11 @@ */ static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ StrAccum acc; /* String accumulator */ char zMsg[SQLITE_PRINT_BUF_SIZE*3]; /* Complete log message */ - sqlite3StrAccumInit(&acc, zMsg, sizeof(zMsg), 0); - acc.useMalloc = 0; + sqlite3StrAccumInit(&acc, 0, zMsg, sizeof(zMsg), 0); sqlite3VXPrintf(&acc, 0, zFormat, ap); sqlite3GlobalConfig.xLog(sqlite3GlobalConfig.pLogArg, iErrCode, sqlite3StrAccumFinish(&acc)); } @@ -1057,22 +1054,21 @@ renderLogMsg(iErrCode, zFormat, ap); va_end(ap); } } -#if defined(SQLITE_DEBUG) +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) /* ** A version of printf() that understands %lld. Used for debugging. ** The printf() built into some versions of windows does not understand %lld ** and segfaults if you give it a long long int. */ void sqlite3DebugPrintf(const char *zFormat, ...){ va_list ap; StrAccum acc; char zBuf[500]; - sqlite3StrAccumInit(&acc, zBuf, sizeof(zBuf), 0); - acc.useMalloc = 0; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); va_start(ap,zFormat); sqlite3VXPrintf(&acc, 0, zFormat, ap); va_end(ap); sqlite3StrAccumFinish(&acc); fprintf(stdout,"%s", zBuf); @@ -1095,11 +1091,11 @@ */ /* Add a new subitem to the tree. The moreToFollow flag indicates that this ** is not the last item in the tree. */ TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ if( p==0 ){ - p = sqlite3_malloc( sizeof(*p) ); + p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ) return 0; memset(p, 0, sizeof(*p)); }else{ p->iLevel++; } @@ -1118,12 +1114,11 @@ void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ va_list ap; int i; StrAccum acc; char zBuf[500]; - sqlite3StrAccumInit(&acc, zBuf, sizeof(zBuf), 0); - acc.useMalloc = 0; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); if( p ){ for(i=0; iiLevel && ibLine)-1; i++){ sqlite3StrAccumAppend(&acc, p->bLine[i] ? "| " : " ", 4); } sqlite3StrAccumAppend(&acc, p->bLine[i] ? "|-- " : "'-- ", 4); Index: src/resolve.c ================================================================== --- src/resolve.c +++ src/resolve.c @@ -97,11 +97,10 @@ sqlite3 *db; /* The database connection */ assert( iCol>=0 && iColnExpr ); pOrig = pEList->a[iCol].pExpr; assert( pOrig!=0 ); - assert( (pOrig->flags & EP_Resolved)!=0 || zType[0]==0 ); db = pParse->db; pDup = sqlite3ExprDup(db, pOrig, 0); if( pDup==0 ) return; if( pOrig->op!=TK_COLUMN && zType[0]!='G' ){ incrAggFunctionDepth(pDup, nSubquery); Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -2601,11 +2601,11 @@ ** then there should be a single item on the stack. Write this ** item into the set table with bogus data. */ case SRT_Set: { int r1; - assert( pIn->nSdst==1 ); + assert( pIn->nSdst==1 || pParse->nErr>0 ); pDest->affSdst = sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affSdst); r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, 1, r1, &pDest->affSdst,1); sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, 1); @@ -5537,12 +5537,11 @@ sqlite3TreeViewLine(pView, "FROM"); for(i=0; ipSrc->nSrc; i++){ struct SrcList_item *pItem = &p->pSrc->a[i]; StrAccum x; char zLine[100]; - sqlite3StrAccumInit(&x, zLine, sizeof(zLine), 0); - x.useMalloc = 0; + sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); sqlite3XPrintf(&x, 0, "{%d,*}", pItem->iCursor); if( pItem->zDatabase ){ sqlite3XPrintf(&x, 0, " %s.%s", pItem->zDatabase, pItem->zName); }else if( pItem->zName ){ sqlite3XPrintf(&x, 0, " %s", pItem->zName); Index: src/shell.c ================================================================== --- src/shell.c +++ src/shell.c @@ -988,11 +988,20 @@ break; } case MODE_Insert: { p->cnt++; if( azArg==0 ) break; - fprintf(p->out,"INSERT INTO %s VALUES(",p->zDestTable); + fprintf(p->out,"INSERT INTO %s",p->zDestTable); + if( p->showHeader ){ + fprintf(p->out,"("); + for(i=0; i0 ? ",": ""; + fprintf(p->out, "%s%s", zSep, azCol[i]); + } + fprintf(p->out,")"); + } + fprintf(p->out," VALUES("); for(i=0; i0 ? ",": ""; if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ fprintf(p->out,"%sNULL",zSep); }else if( aiType && aiType[i]==SQLITE_TEXT ){ @@ -1189,11 +1198,11 @@ */ static char *save_err_msg( sqlite3 *db /* Database to query */ ){ int nErrMsg = 1+strlen30(sqlite3_errmsg(db)); - char *zErrMsg = sqlite3_malloc(nErrMsg); + char *zErrMsg = sqlite3_malloc64(nErrMsg); if( zErrMsg ){ memcpy(zErrMsg, sqlite3_errmsg(db), nErrMsg); } return zErrMsg; } @@ -1426,12 +1435,12 @@ int p2op = (p2 + (iOp-iAddr)); /* Grow the p->aiIndent array as required */ if( iOp>=nAlloc ){ nAlloc += 100; - p->aiIndent = (int*)sqlite3_realloc(p->aiIndent, nAlloc*sizeof(int)); - abYield = (int*)sqlite3_realloc(abYield, nAlloc*sizeof(int)); + p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int)); + abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); } abYield[iOp] = str_in_array(zOp, azYield); p->aiIndent[iOp] = 0; p->nIndent = iOp+1; @@ -1544,11 +1553,11 @@ if( SQLITE_ROW == rc ){ /* if we have a callback... */ if( xCallback ){ /* allocate space for col name ptr, value ptr, and type */ int nCol = sqlite3_column_count(pStmt); - void *pData = sqlite3_malloc(3*nCol*sizeof(const char*) + 1); + void *pData = sqlite3_malloc64(3*nCol*sizeof(const char*) + 1); if( !pData ){ rc = SQLITE_NOMEM; }else{ char **azCols = (char **)pData; /* Names of result columns */ char **azVals = &azCols[nCol]; /* Results */ @@ -1770,10 +1779,11 @@ ** Text of a help message */ static char zHelp[] = ".backup ?DB? FILE Backup DB (default \"main\") to FILE\n" ".bail on|off Stop after hitting an error. Default OFF\n" + ".binary on|off Turn binary output on or off. Default OFF\n" ".clone NEWDB Clone data into NEWDB from the existing database\n" ".databases List names and files of attached databases\n" ".dbinfo ?DB? Show status information about the database\n" ".dump ?TABLE? ... Dump the database in an SQL text format\n" " If TABLE specified, only dump tables matching\n" @@ -1791,10 +1801,11 @@ " If TABLE specified, only show indexes for tables\n" " matching LIKE pattern TABLE.\n" #ifdef SQLITE_ENABLE_IOTRACE ".iotrace FILE Enable I/O diagnostic logging to FILE\n" #endif + ".limit ?LIMIT? ?VAL? Display or change the value of an SQLITE_LIMIT\n" #ifndef SQLITE_OMIT_LOAD_EXTENSION ".load FILE ?ENTRY? Load an extension library\n" #endif ".log FILE|off Turn logging on or off. FILE can be stderr/stdout\n" ".mode MODE ?TABLE? Set output mode where MODE is one of:\n" @@ -1860,11 +1871,11 @@ in = fopen(zName, "rb"); if( in==0 ) return; fseek(in, 0, SEEK_END); nIn = ftell(in); rewind(in); - pBuf = sqlite3_malloc( nIn ); + pBuf = sqlite3_malloc64( nIn ); if( pBuf && 1==fread(pBuf, nIn, 1, in) ){ sqlite3_result_blob(context, pBuf, nIn, sqlite3_free); }else{ sqlite3_free(pBuf); } @@ -1931,30 +1942,48 @@ } /* ** Do C-language style dequoting. ** +** \a -> alarm +** \b -> backspace ** \t -> tab ** \n -> newline +** \v -> vertical tab +** \f -> form feed ** \r -> carriage return +** \s -> space ** \" -> " -** \NNN -> ascii character NNN in octal +** \' -> ' ** \\ -> backslash +** \NNN -> ascii character NNN in octal */ static void resolve_backslashes(char *z){ int i, j; char c; while( *z && *z!='\\' ) z++; for(i=j=0; (c = z[i])!=0; i++, j++){ if( c=='\\' && z[i+1]!=0 ){ c = z[++i]; - if( c=='n' ){ - c = '\n'; + if( c=='a' ){ + c = '\a'; + }else if( c=='b' ){ + c = '\b'; }else if( c=='t' ){ c = '\t'; + }else if( c=='n' ){ + c = '\n'; + }else if( c=='v' ){ + c = '\v'; + }else if( c=='f' ){ + c = '\f'; }else if( c=='r' ){ c = '\r'; + }else if( c=='"' ){ + c = '"'; + }else if( c=='\'' ){ + c = '\''; }else if( c=='\\' ){ c = '\\'; }else if( c>='0' && c<='7' ){ c -= '0'; if( z[i+1]>='0' && z[i+1]<='7' ){ @@ -2120,11 +2149,11 @@ /* Append a single byte to z[] */ static void import_append_char(ImportCtx *p, int c){ if( p->n+1>=p->nAlloc ){ p->nAlloc += p->nAlloc + 100; - p->z = sqlite3_realloc(p->z, p->nAlloc); + p->z = sqlite3_realloc64(p->z, p->nAlloc); if( p->z==0 ){ fprintf(stderr, "out of memory\n"); exit(1); } } @@ -2134,11 +2163,11 @@ /* Read a single field of CSV text. Compatible with rfc4180 and extended ** with the option of having a separator other than ",". ** ** + Input comes from p->in. ** + Store results in p->z of length p->n. Space to hold p->z comes -** from sqlite3_malloc(). +** from sqlite3_malloc64(). ** + Use p->cSep as the column separator. The default is ",". ** + Use p->rSep as the row separator. The default is "\n". ** + Keep track of the line number in p->nLine. ** + Store the character that terminates the field in p->cTerm. Store ** EOF on end-of-file. @@ -2208,11 +2237,11 @@ /* Read a single field of ASCII delimited text. ** ** + Input comes from p->in. ** + Store results in p->z of length p->n. Space to hold p->z comes -** from sqlite3_malloc(). +** from sqlite3_malloc64(). ** + Use p->cSep as the column separator. The default is "\x1F". ** + Use p->rSep as the row separator. The default is "\x1E". ** + Keep track of the row number in p->nLine. ** + Store the character that terminates the field in p->cTerm. Store ** EOF on end-of-file. @@ -2268,11 +2297,11 @@ sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery); goto end_data_xfer; } n = sqlite3_column_count(pQuery); - zInsert = sqlite3_malloc(200 + nTable + n*3); + zInsert = sqlite3_malloc64(200 + nTable + n*3); if( zInsert==0 ){ fprintf(stderr, "out of memory\n"); goto end_data_xfer; } sqlite3_snprintf(200+nTable,zInsert, @@ -2683,10 +2712,23 @@ }else{ fprintf(stderr, "Usage: .bail on|off\n"); rc = 1; } }else + + if( c=='b' && n>=3 && strncmp(azArg[0], "binary", n)==0 ){ + if( nArg==2 ){ + if( booleanValue(azArg[1]) ){ + setBinaryMode(p->out); + }else{ + setTextMode(p->out); + } + }else{ + fprintf(stderr, "Usage: .binary on|off\n"); + rc = 1; + } + }else /* The undocumented ".breakpoint" command causes a call to the no-op ** routine named test_breakpoint(). */ if( c=='b' && n>=3 && strncmp(azArg[0], "breakpoint", n)==0 ){ @@ -3023,11 +3065,11 @@ } nCol = sqlite3_column_count(pStmt); sqlite3_finalize(pStmt); pStmt = 0; if( nCol==0 ) return 0; /* no columns, no error */ - zSql = sqlite3_malloc( nByte*2 + 20 + nCol*2 ); + zSql = sqlite3_malloc64( nByte*2 + 20 + nCol*2 ); if( zSql==0 ){ fprintf(stderr, "Error: out of memory\n"); xCloser(sCtx.in); return 1; } @@ -3163,10 +3205,68 @@ sqlite3IoTrace = iotracePrintf; } } }else #endif + if( c=='l' && n>=5 && strncmp(azArg[0], "limits", n)==0 ){ + static const struct { + const char *zLimitName; /* Name of a limit */ + int limitCode; /* Integer code for that limit */ + } aLimit[] = { + { "length", SQLITE_LIMIT_LENGTH }, + { "sql_length", SQLITE_LIMIT_SQL_LENGTH }, + { "column", SQLITE_LIMIT_COLUMN }, + { "expr_depth", SQLITE_LIMIT_EXPR_DEPTH }, + { "compound_select", SQLITE_LIMIT_COMPOUND_SELECT }, + { "vdbe_op", SQLITE_LIMIT_VDBE_OP }, + { "function_arg", SQLITE_LIMIT_FUNCTION_ARG }, + { "attached", SQLITE_LIMIT_ATTACHED }, + { "like_pattern_length", SQLITE_LIMIT_LIKE_PATTERN_LENGTH }, + { "variable_number", SQLITE_LIMIT_VARIABLE_NUMBER }, + { "trigger_depth", SQLITE_LIMIT_TRIGGER_DEPTH }, + { "worker_threads", SQLITE_LIMIT_WORKER_THREADS }, + }; + int i, n2; + open_db(p, 0); + if( nArg==1 ){ + for(i=0; idb, aLimit[i].limitCode, -1)); + } + }else if( nArg>3 ){ + fprintf(stderr, "Usage: .limit NAME ?NEW-VALUE?\n"); + rc = 1; + goto meta_command_exit; + }else{ + int iLimit = -1; + n2 = strlen30(azArg[1]); + for(i=0; idb, aLimit[iLimit].limitCode, + (int)integerValue(azArg[2])); + } + printf("%20s %d\n", aLimit[iLimit].zLimitName, + sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1)); + } + }else #ifndef SQLITE_OMIT_LOAD_EXTENSION if( c=='l' && strncmp(azArg[0], "load", n)==0 ){ const char *zFile, *zProc; char *zErrMsg = 0; @@ -3648,11 +3748,11 @@ } while( sqlite3_step(pStmt)==SQLITE_ROW ){ if( nRow>=nAlloc ){ char **azNew; int n2 = nAlloc*2 + 10; - azNew = sqlite3_realloc(azResult, sizeof(azResult[0])*n2); + azNew = sqlite3_realloc64(azResult, sizeof(azResult[0])*n2); if( azNew==0 ){ fprintf(stderr, "Error: out of memory\n"); break; } nAlloc = n2; Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -3891,12 +3891,10 @@ /* ** CAPI3REF: Result Values From A Query ** KEYWORDS: {column access functions} ** METHOD: sqlite3_stmt ** -** These routines form the "result set" interface. -** ** ^These routines return information about a single column of the current ** result row of a query. ^In every case the first argument is a pointer ** to the [prepared statement] that is being evaluated (the [sqlite3_stmt*] ** that was returned from [sqlite3_prepare_v2()] or one of its variants) ** and the second argument is the index of the column for which information @@ -3952,17 +3950,18 @@ ** ** ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(), ** even empty strings, are always zero-terminated. ^The return ** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** -** ^The object returned by [sqlite3_column_value()] is an -** [unprotected sqlite3_value] object. An unprotected sqlite3_value object -** may only be used with [sqlite3_bind_value()] and [sqlite3_result_value()]. +** Warning: ^The object returned by [sqlite3_column_value()] is an +** [unprotected sqlite3_value] object. In a multithreaded environment, +** an unprotected sqlite3_value object may only be used safely with +** [sqlite3_bind_value()] and [sqlite3_result_value()]. ** If the [unprotected sqlite3_value] object returned by ** [sqlite3_column_value()] is used in any other way, including calls ** to routines like [sqlite3_value_int()], [sqlite3_value_text()], -** or [sqlite3_value_bytes()], then the behavior is undefined. +** or [sqlite3_value_bytes()], the behavior is not threadsafe. ** ** These routines attempt to convert the value where appropriate. ^For ** example, if the internal representation is FLOAT and a text result ** is requested, [sqlite3_snprintf()] is used internally to perform the ** conversion automatically. ^(The following table details the conversions @@ -3989,16 +3988,10 @@ ** BLOB FLOAT [CAST] to REAL ** BLOB TEXT Add a zero terminator if needed ** ** )^ ** -** The table above makes reference to standard C library functions atoi() -** and atof(). SQLite does not really use these functions. It has its -** own equivalent internal routines. The atoi() and atof() names are -** used in the table for brevity and because they are familiar to most -** C programmers. -** ** Note that when type conversions occur, pointers returned by prior ** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or ** sqlite3_column_text16() may be invalidated. ** Type conversions and pointer invalidations might occur ** in the following cases: @@ -4019,11 +4012,11 @@ ** not invalidate a prior pointer, though of course the content of the buffer ** that the prior pointer references will have been modified. Other kinds ** of conversion are done in place when it is possible, but sometimes they ** are not possible and in those cases prior pointers are invalidated. ** -** The safest and easiest to remember policy is to invoke these routines +** The safest policy is to invoke these routines ** in one of the following ways: ** **
    **
  • sqlite3_column_text() followed by sqlite3_column_bytes()
    • **
  • sqlite3_column_blob() followed by sqlite3_column_bytes()
    • @@ -4039,11 +4032,11 @@ ** with calls to sqlite3_column_bytes(). ** ** ^The pointers returned are valid until a type conversion occurs as ** described above, or until [sqlite3_step()] or [sqlite3_reset()] or ** [sqlite3_finalize()] is called. ^The memory space used to hold strings -** and BLOBs is freed automatically. Do not pass the pointers returned +** and BLOBs is freed automatically. Do not pass the pointers returned ** from [sqlite3_column_blob()], [sqlite3_column_text()], etc. into ** [sqlite3_free()]. ** ** ^(If a memory allocation error occurs during the evaluation of any ** of these routines, a default value is returned. The default value Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -360,10 +360,36 @@ #else # define ALWAYS(X) (X) # define NEVER(X) (X) #endif +/* +** Declarations used for tracing the operating system interfaces. +*/ +#if defined(SQLITE_FORCE_OS_TRACE) || defined(SQLITE_TEST) || \ + (defined(SQLITE_DEBUG) && SQLITE_OS_WIN) + extern int sqlite3OSTrace; +# define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X +# define SQLITE_HAVE_OS_TRACE +#else +# define OSTRACE(X) +# undef SQLITE_HAVE_OS_TRACE +#endif + +/* +** Is the sqlite3ErrName() function needed in the build? Currently, +** it is needed by "mutex_w32.c" (when debugging), "os_win.c" (when +** OSTRACE is enabled), and by several "test*.c" files (which are +** compiled using SQLITE_TEST). +*/ +#if defined(SQLITE_HAVE_OS_TRACE) || defined(SQLITE_TEST) || \ + (defined(SQLITE_DEBUG) && SQLITE_OS_WIN) +# define SQLITE_NEED_ERR_NAME +#else +# undef SQLITE_NEED_ERR_NAME +#endif + /* ** Return true (non-zero) if the input is an integer that is too large ** to fit in 32-bits. This macro is used inside of various testcase() ** macros to verify that we have tested SQLite for large-file support. */ @@ -1555,38 +1581,12 @@ int iSavepoint; /* Depth of the SAVEPOINT stack */ VTable *pNext; /* Next in linked list (see above) */ }; /* -** Each SQL table is represented in memory by an instance of the -** following structure. -** -** Table.zName is the name of the table. The case of the original -** CREATE TABLE statement is stored, but case is not significant for -** comparisons. -** -** Table.nCol is the number of columns in this table. Table.aCol is a -** pointer to an array of Column structures, one for each column. -** -** If the table has an INTEGER PRIMARY KEY, then Table.iPKey is the index of -** the column that is that key. Otherwise Table.iPKey is negative. Note -** that the datatype of the PRIMARY KEY must be INTEGER for this field to -** be set. An INTEGER PRIMARY KEY is used as the rowid for each row of -** the table. If a table has no INTEGER PRIMARY KEY, then a random rowid -** is generated for each row of the table. TF_HasPrimaryKey is set if -** the table has any PRIMARY KEY, INTEGER or otherwise. -** -** Table.tnum is the page number for the root BTree page of the table in the -** database file. If Table.iDb is the index of the database table backend -** in sqlite.aDb[]. 0 is for the main database and 1 is for the file that -** holds temporary tables and indices. If TF_Ephemeral is set -** then the table is stored in a file that is automatically deleted -** when the VDBE cursor to the table is closed. In this case Table.tnum -** refers VDBE cursor number that holds the table open, not to the root -** page number. Transient tables are used to hold the results of a -** sub-query that appears instead of a real table name in the FROM clause -** of a SELECT statement. +** The schema for each SQL table and view is represented in memory +** by an instance of the following structure. */ struct Table { char *zName; /* Name of the table or view */ Column *aCol; /* Information about each column */ Index *pIndex; /* List of SQL indexes on this table. */ @@ -1594,15 +1594,15 @@ FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ #ifndef SQLITE_OMIT_CHECK ExprList *pCheck; /* All CHECK constraints */ #endif - LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ - int tnum; /* Root BTree node for this table (see note above) */ - i16 iPKey; /* If not negative, use aCol[iPKey] as the primary key */ + int tnum; /* Root BTree page for this table */ + i16 iPKey; /* If not negative, use aCol[iPKey] as the rowid */ i16 nCol; /* Number of columns in this table */ u16 nRef; /* Number of pointers to this Table */ + LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ LogEst szTabRow; /* Estimated size of each table row in bytes */ #ifdef SQLITE_ENABLE_COSTMULT LogEst costMult; /* Cost multiplier for using this table */ #endif u8 tabFlags; /* Mask of TF_* values */ @@ -2833,12 +2833,11 @@ sqlite3 *db; /* Optional database for lookaside. Can be NULL */ char *zBase; /* A base allocation. Not from malloc. */ char *zText; /* The string collected so far */ int nChar; /* Length of the string so far */ int nAlloc; /* Amount of space allocated in zText */ - int mxAlloc; /* Maximum allowed string length */ - u8 useMalloc; /* 0: none, 1: sqlite3DbMalloc, 2: sqlite3_malloc */ + int mxAlloc; /* Maximum allowed allocation. 0 for no malloc usage */ u8 accError; /* STRACCUM_NOMEM or STRACCUM_TOOBIG */ }; #define STRACCUM_NOMEM 1 #define STRACCUM_TOOBIG 2 @@ -3151,11 +3150,11 @@ void sqlite3VXPrintf(StrAccum*, u32, const char*, va_list); void sqlite3XPrintf(StrAccum*, u32, const char*, ...); char *sqlite3MPrintf(sqlite3*,const char*, ...); char *sqlite3VMPrintf(sqlite3*,const char*, va_list); char *sqlite3MAppendf(sqlite3*,char*,const char*,...); -#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) void sqlite3DebugPrintf(const char*, ...); #endif #if defined(SQLITE_TEST) void *sqlite3TestTextToPtr(const char*); #endif @@ -3498,11 +3497,11 @@ void sqlite3Error(sqlite3*,int); void *sqlite3HexToBlob(sqlite3*, const char *z, int n); u8 sqlite3HexToInt(int h); int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); -#if defined(SQLITE_TEST) +#if defined(SQLITE_NEED_ERR_NAME) const char *sqlite3ErrName(int); #endif const char *sqlite3ErrStr(int); int sqlite3ReadSchema(Parse *pParse); @@ -3592,11 +3591,11 @@ FuncDestructor *pDestructor ); int sqlite3ApiExit(sqlite3 *db, int); int sqlite3OpenTempDatabase(Parse *); -void sqlite3StrAccumInit(StrAccum*, char*, int, int); +void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); void sqlite3StrAccumAppend(StrAccum*,const char*,int); void sqlite3StrAccumAppendAll(StrAccum*,const char*); void sqlite3AppendChar(StrAccum*,int,char); char *sqlite3StrAccumFinish(StrAccum*); void sqlite3StrAccumReset(StrAccum*); @@ -3865,7 +3864,11 @@ */ #if SQLITE_MAX_WORKER_THREADS>0 int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); int sqlite3ThreadJoin(SQLiteThread*, void**); #endif + +#if defined(SQLITE_ENABLE_DBSTAT_VTAB) || defined(SQLITE_TEST) +int sqlite3DbstatRegister(sqlite3*); +#endif #endif /* _SQLITEINT_H_ */ Index: src/table.c ================================================================== --- src/table.c +++ src/table.c @@ -88,11 +88,11 @@ for(i=0; iazResult[p->nData++] = z; } @@ -137,11 +137,11 @@ res.nRow = 0; res.nColumn = 0; res.nData = 1; res.nAlloc = 20; res.rc = SQLITE_OK; - res.azResult = sqlite3_malloc(sizeof(char*)*res.nAlloc ); + res.azResult = sqlite3_malloc64(sizeof(char*)*res.nAlloc ); if( res.azResult==0 ){ db->errCode = SQLITE_NOMEM; return SQLITE_NOMEM; } res.azResult[0] = 0; @@ -165,11 +165,11 @@ sqlite3_free_table(&res.azResult[1]); return rc; } if( res.nAlloc>res.nData ){ char **azNew; - azNew = sqlite3_realloc( res.azResult, sizeof(char*)*res.nData ); + azNew = sqlite3_realloc64( res.azResult, sizeof(char*)*res.nData ); if( azNew==0 ){ sqlite3_free_table(&res.azResult[1]); db->errCode = SQLITE_NOMEM; return SQLITE_NOMEM; } Index: src/tclsqlite.c ================================================================== --- src/tclsqlite.c +++ src/tclsqlite.c @@ -3702,11 +3702,11 @@ } Tcl_SetResult(interp, zBuf, TCL_VOLATILE); return TCL_OK; } -#endif +#endif /* SQLITE_TEST */ /* ** Configure the interpreter passed as the first argument to have access ** to the commands and linked variables that make up: ** @@ -3720,21 +3720,10 @@ static void init_all(Tcl_Interp *interp){ Sqlite3_Init(interp); #if defined(SQLITE_TEST) || defined(SQLITE_TCLMD5) Md5_Init(interp); -#endif - - /* Install the [register_dbstat_vtab] command to access the implementation - ** of virtual table dbstat (source file test_stat.c). This command is - ** required for testfixture and sqlite3_analyzer, but not by the production - ** Tcl extension. */ -#if defined(SQLITE_TEST) || TCLSH==2 - { - extern int SqlitetestStat_Init(Tcl_Interp*); - SqlitetestStat_Init(interp); - } #endif #ifdef SQLITE_TEST { extern int Sqliteconfig_Init(Tcl_Interp*); Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -6678,11 +6678,44 @@ break; } } return TCL_OK; } - + +/* +** tclcmd: register_dbstat_vtab DB +** +** Cause the dbstat virtual table to be available on the connection DB +*/ +static int test_register_dbstat_vtab( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ +#ifdef SQLITE_OMIT_VIRTUALTABLE + Tcl_AppendResult(interp, "dbstat not available because of " + "SQLITE_OMIT_VIRTUALTABLE", (void*)0); + return TCL_ERROR; +#else + struct SqliteDb { sqlite3 *db; }; + char *zDb; + Tcl_CmdInfo cmdInfo; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + + zDb = Tcl_GetString(objv[1]); + if( Tcl_GetCommandInfo(interp, zDb, &cmdInfo) ){ + sqlite3* db = ((struct SqliteDb*)cmdInfo.objClientData)->db; + sqlite3DbstatRegister(db); + } + return TCL_OK; +#endif /* SQLITE_OMIT_VIRTUALTABLE */ +} /* ** Register commands with the TCL interpreter. */ int Sqlitetest1_Init(Tcl_Interp *interp){ @@ -6750,10 +6783,11 @@ char *zName; Tcl_ObjCmdProc *xProc; void *clientData; } aObjCmd[] = { { "bad_behavior", test_bad_behavior, (void*)&iZero }, + { "register_dbstat_vtab", test_register_dbstat_vtab }, { "sqlite3_connection_pointer", get_sqlite_pointer, 0 }, { "sqlite3_bind_int", test_bind_int, 0 }, { "sqlite3_bind_zeroblob", test_bind_zeroblob, 0 }, { "sqlite3_bind_int64", test_bind_int64, 0 }, { "sqlite3_bind_double", test_bind_double, 0 }, Index: src/test_blob.c ================================================================== --- src/test_blob.c +++ src/test_blob.c @@ -14,10 +14,11 @@ #include "sqliteInt.h" #include "tcl.h" #include #include #include +#ifndef SQLITE_OMIT_INCRBLOB /* These functions are implemented in main.c. */ extern const char *sqlite3ErrName(int); /* From test1.c: */ @@ -293,16 +294,17 @@ Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); } return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); } - +#endif /* SQLITE_OMIT_INCRBLOB */ /* ** Register commands with the TCL interpreter. */ int Sqlitetest_blob_Init(Tcl_Interp *interp){ +#ifndef SQLITE_OMIT_INCRBLOB static struct { char *zName; Tcl_ObjCmdProc *xProc; } aObjCmd[] = { { "sqlite3_blob_open", test_blob_open }, @@ -313,7 +315,8 @@ }; int i; for(i=0; ipContent = (sqlite3_intarray*)pAux; rc = sqlite3_declare_vtab(db, "CREATE TABLE x(value INTEGER PRIMARY KEY)"); @@ -100,11 +100,11 @@ ** Open a new cursor on the intarray table. */ static int intarrayOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ int rc = SQLITE_NOMEM; intarray_cursor *pCur; - pCur = sqlite3_malloc(sizeof(intarray_cursor)); + pCur = sqlite3_malloc64(sizeof(intarray_cursor)); if( pCur ){ memset(pCur, 0, sizeof(intarray_cursor)); *ppCursor = (sqlite3_vtab_cursor *)pCur; rc = SQLITE_OK; } @@ -223,11 +223,11 @@ ){ int rc = SQLITE_OK; #ifndef SQLITE_OMIT_VIRTUALTABLE sqlite3_intarray *p; - *ppReturn = p = sqlite3_malloc( sizeof(*p) ); + *ppReturn = p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ){ return SQLITE_NOMEM; } memset(p, 0, sizeof(*p)); rc = sqlite3_create_module_v2(db, zName, &intarrayModule, p, @@ -338,11 +338,11 @@ return TCL_ERROR; } pArray = (sqlite3_intarray*)sqlite3TestTextToPtr(Tcl_GetString(objv[1])); n = objc - 2; #ifndef SQLITE_OMIT_VIRTUALTABLE - a = sqlite3_malloc( sizeof(a[0])*n ); + a = sqlite3_malloc64( sizeof(a[0])*n ); if( a==0 ){ Tcl_AppendResult(interp, "SQLITE_NOMEM", (char*)0); return TCL_ERROR; } for(i=0; i=pGroup->nReal ){ struct multiplexReal *p; - p = sqlite3_realloc(pGroup->aReal, (iChunk+1)*sizeof(*p)); + p = sqlite3_realloc64(pGroup->aReal, (iChunk+1)*sizeof(*p)); if( p==0 ){ return SQLITE_NOMEM; } memset(&p[pGroup->nReal], 0, sizeof(p[0])*(iChunk+1-pGroup->nReal)); pGroup->aReal = p; @@ -295,11 +295,11 @@ pGroup->nReal = iChunk+1; } if( pGroup->zName && pGroup->aReal[iChunk].z==0 ){ char *z; int n = pGroup->nName; - pGroup->aReal[iChunk].z = z = sqlite3_malloc( n+5 ); + pGroup->aReal[iChunk].z = z = sqlite3_malloc64( n+5 ); if( z==0 ){ return SQLITE_NOMEM; } multiplexFilename(pGroup->zName, pGroup->nName, pGroup->flags, iChunk, z); } @@ -355,11 +355,11 @@ } return 0; } flags &= ~SQLITE_OPEN_CREATE; } - pSubOpen = sqlite3_malloc( pOrigVfs->szOsFile ); + pSubOpen = sqlite3_malloc64( pOrigVfs->szOsFile ); if( pSubOpen==0 ){ *rc = SQLITE_IOERR_NOMEM; return 0; } pGroup->aReal[iChunk].p = pSubOpen; @@ -522,11 +522,11 @@ if( rc==SQLITE_OK ){ /* allocate space for group */ nName = zName ? multiplexStrlen30(zName) : 0; sz = sizeof(multiplexGroup) /* multiplexGroup */ + nName + 1; /* zName */ - pGroup = sqlite3_malloc( sz ); + pGroup = sqlite3_malloc64( sz ); if( pGroup==0 ){ rc = SQLITE_NOMEM; } } @@ -653,11 +653,11 @@ /* If the main chunk was deleted successfully, also delete any subsequent ** chunks - starting with the last (highest numbered). */ int nName = (int)strlen(zName); char *z; - z = sqlite3_malloc(nName + 5); + z = sqlite3_malloc64(nName + 5); if( z==0 ){ rc = SQLITE_IOERR_NOMEM; }else{ int iChunk = 0; int bExists; DELETED src/test_stat.c Index: src/test_stat.c ================================================================== --- src/test_stat.c +++ /dev/null @@ -1,647 +0,0 @@ -/* -** 2010 July 12 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains an implementation of the "dbstat" virtual table. -** -** The dbstat virtual table is used to extract low-level formatting -** information from an SQLite database in order to implement the -** "sqlite3_analyzer" utility. See the ../tool/spaceanal.tcl script -** for an example implementation. -*/ - -#ifndef SQLITE_AMALGAMATION -# include "sqliteInt.h" -#endif - -#ifndef SQLITE_OMIT_VIRTUALTABLE - -/* -** Page paths: -** -** The value of the 'path' column describes the path taken from the -** root-node of the b-tree structure to each page. The value of the -** root-node path is '/'. -** -** The value of the path for the left-most child page of the root of -** a b-tree is '/000/'. (Btrees store content ordered from left to right -** so the pages to the left have smaller keys than the pages to the right.) -** The next to left-most child of the root page is -** '/001', and so on, each sibling page identified by a 3-digit hex -** value. The children of the 451st left-most sibling have paths such -** as '/1c2/000/, '/1c2/001/' etc. -** -** Overflow pages are specified by appending a '+' character and a -** six-digit hexadecimal value to the path to the cell they are linked -** from. For example, the three overflow pages in a chain linked from -** the left-most cell of the 450th child of the root page are identified -** by the paths: -** -** '/1c2/000+000000' // First page in overflow chain -** '/1c2/000+000001' // Second page in overflow chain -** '/1c2/000+000002' // Third page in overflow chain -** -** If the paths are sorted using the BINARY collation sequence, then -** the overflow pages associated with a cell will appear earlier in the -** sort-order than its child page: -** -** '/1c2/000/' // Left-most child of 451st child of root -*/ -#define VTAB_SCHEMA \ - "CREATE TABLE xx( " \ - " name STRING, /* Name of table or index */" \ - " path INTEGER, /* Path to page from root */" \ - " pageno INTEGER, /* Page number */" \ - " pagetype STRING, /* 'internal', 'leaf' or 'overflow' */" \ - " ncell INTEGER, /* Cells on page (0 for overflow) */" \ - " payload INTEGER, /* Bytes of payload on this page */" \ - " unused INTEGER, /* Bytes of unused space on this page */" \ - " mx_payload INTEGER, /* Largest payload size of all cells */" \ - " pgoffset INTEGER, /* Offset of page in file */" \ - " pgsize INTEGER /* Size of the page */" \ - ");" - - -typedef struct StatTable StatTable; -typedef struct StatCursor StatCursor; -typedef struct StatPage StatPage; -typedef struct StatCell StatCell; - -struct StatCell { - int nLocal; /* Bytes of local payload */ - u32 iChildPg; /* Child node (or 0 if this is a leaf) */ - int nOvfl; /* Entries in aOvfl[] */ - u32 *aOvfl; /* Array of overflow page numbers */ - int nLastOvfl; /* Bytes of payload on final overflow page */ - int iOvfl; /* Iterates through aOvfl[] */ -}; - -struct StatPage { - u32 iPgno; - DbPage *pPg; - int iCell; - - char *zPath; /* Path to this page */ - - /* Variables populated by statDecodePage(): */ - u8 flags; /* Copy of flags byte */ - int nCell; /* Number of cells on page */ - int nUnused; /* Number of unused bytes on page */ - StatCell *aCell; /* Array of parsed cells */ - u32 iRightChildPg; /* Right-child page number (or 0) */ - int nMxPayload; /* Largest payload of any cell on this page */ -}; - -struct StatCursor { - sqlite3_vtab_cursor base; - sqlite3_stmt *pStmt; /* Iterates through set of root pages */ - int isEof; /* After pStmt has returned SQLITE_DONE */ - - StatPage aPage[32]; - int iPage; /* Current entry in aPage[] */ - - /* Values to return. */ - char *zName; /* Value of 'name' column */ - char *zPath; /* Value of 'path' column */ - u32 iPageno; /* Value of 'pageno' column */ - char *zPagetype; /* Value of 'pagetype' column */ - int nCell; /* Value of 'ncell' column */ - int nPayload; /* Value of 'payload' column */ - int nUnused; /* Value of 'unused' column */ - int nMxPayload; /* Value of 'mx_payload' column */ - i64 iOffset; /* Value of 'pgOffset' column */ - int szPage; /* Value of 'pgSize' column */ -}; - -struct StatTable { - sqlite3_vtab base; - sqlite3 *db; -}; - -#ifndef get2byte -# define get2byte(x) ((x)[0]<<8 | (x)[1]) -#endif - -/* -** Connect to or create a statvfs virtual table. -*/ -static int statConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr -){ - StatTable *pTab; - - pTab = (StatTable *)sqlite3_malloc(sizeof(StatTable)); - memset(pTab, 0, sizeof(StatTable)); - pTab->db = db; - - sqlite3_declare_vtab(db, VTAB_SCHEMA); - *ppVtab = &pTab->base; - return SQLITE_OK; -} - -/* -** Disconnect from or destroy a statvfs virtual table. -*/ -static int statDisconnect(sqlite3_vtab *pVtab){ - sqlite3_free(pVtab); - return SQLITE_OK; -} - -/* -** There is no "best-index". This virtual table always does a linear -** scan of the binary VFS log file. -*/ -static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ - - /* Records are always returned in ascending order of (name, path). - ** If this will satisfy the client, set the orderByConsumed flag so that - ** SQLite does not do an external sort. - */ - if( ( pIdxInfo->nOrderBy==1 - && pIdxInfo->aOrderBy[0].iColumn==0 - && pIdxInfo->aOrderBy[0].desc==0 - ) || - ( pIdxInfo->nOrderBy==2 - && pIdxInfo->aOrderBy[0].iColumn==0 - && pIdxInfo->aOrderBy[0].desc==0 - && pIdxInfo->aOrderBy[1].iColumn==1 - && pIdxInfo->aOrderBy[1].desc==0 - ) - ){ - pIdxInfo->orderByConsumed = 1; - } - - pIdxInfo->estimatedCost = 10.0; - return SQLITE_OK; -} - -/* -** Open a new statvfs cursor. -*/ -static int statOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ - StatTable *pTab = (StatTable *)pVTab; - StatCursor *pCsr; - int rc; - - pCsr = (StatCursor *)sqlite3_malloc(sizeof(StatCursor)); - memset(pCsr, 0, sizeof(StatCursor)); - pCsr->base.pVtab = pVTab; - - rc = sqlite3_prepare_v2(pTab->db, - "SELECT 'sqlite_master' AS name, 1 AS rootpage, 'table' AS type" - " UNION ALL " - "SELECT name, rootpage, type FROM sqlite_master WHERE rootpage!=0" - " ORDER BY name", -1, - &pCsr->pStmt, 0 - ); - if( rc!=SQLITE_OK ){ - sqlite3_free(pCsr); - return rc; - } - - *ppCursor = (sqlite3_vtab_cursor *)pCsr; - return SQLITE_OK; -} - -static void statClearPage(StatPage *p){ - int i; - for(i=0; inCell; i++){ - sqlite3_free(p->aCell[i].aOvfl); - } - sqlite3PagerUnref(p->pPg); - sqlite3_free(p->aCell); - sqlite3_free(p->zPath); - memset(p, 0, sizeof(StatPage)); -} - -static void statResetCsr(StatCursor *pCsr){ - int i; - sqlite3_reset(pCsr->pStmt); - for(i=0; iaPage); i++){ - statClearPage(&pCsr->aPage[i]); - } - pCsr->iPage = 0; - sqlite3_free(pCsr->zPath); - pCsr->zPath = 0; -} - -/* -** Close a statvfs cursor. -*/ -static int statClose(sqlite3_vtab_cursor *pCursor){ - StatCursor *pCsr = (StatCursor *)pCursor; - statResetCsr(pCsr); - sqlite3_finalize(pCsr->pStmt); - sqlite3_free(pCsr); - return SQLITE_OK; -} - -static void getLocalPayload( - int nUsable, /* Usable bytes per page */ - u8 flags, /* Page flags */ - int nTotal, /* Total record (payload) size */ - int *pnLocal /* OUT: Bytes stored locally */ -){ - int nLocal; - int nMinLocal; - int nMaxLocal; - - if( flags==0x0D ){ /* Table leaf node */ - nMinLocal = (nUsable - 12) * 32 / 255 - 23; - nMaxLocal = nUsable - 35; - }else{ /* Index interior and leaf nodes */ - nMinLocal = (nUsable - 12) * 32 / 255 - 23; - nMaxLocal = (nUsable - 12) * 64 / 255 - 23; - } - - nLocal = nMinLocal + (nTotal - nMinLocal) % (nUsable - 4); - if( nLocal>nMaxLocal ) nLocal = nMinLocal; - *pnLocal = nLocal; -} - -static int statDecodePage(Btree *pBt, StatPage *p){ - int nUnused; - int iOff; - int nHdr; - int isLeaf; - int szPage; - - u8 *aData = sqlite3PagerGetData(p->pPg); - u8 *aHdr = &aData[p->iPgno==1 ? 100 : 0]; - - p->flags = aHdr[0]; - p->nCell = get2byte(&aHdr[3]); - p->nMxPayload = 0; - - isLeaf = (p->flags==0x0A || p->flags==0x0D); - nHdr = 12 - isLeaf*4 + (p->iPgno==1)*100; - - nUnused = get2byte(&aHdr[5]) - nHdr - 2*p->nCell; - nUnused += (int)aHdr[7]; - iOff = get2byte(&aHdr[1]); - while( iOff ){ - nUnused += get2byte(&aData[iOff+2]); - iOff = get2byte(&aData[iOff]); - } - p->nUnused = nUnused; - p->iRightChildPg = isLeaf ? 0 : sqlite3Get4byte(&aHdr[8]); - szPage = sqlite3BtreeGetPageSize(pBt); - - if( p->nCell ){ - int i; /* Used to iterate through cells */ - int nUsable; /* Usable bytes per page */ - - sqlite3BtreeEnter(pBt); - nUsable = szPage - sqlite3BtreeGetReserveNoMutex(pBt); - sqlite3BtreeLeave(pBt); - p->aCell = sqlite3_malloc((p->nCell+1) * sizeof(StatCell)); - memset(p->aCell, 0, (p->nCell+1) * sizeof(StatCell)); - - for(i=0; inCell; i++){ - StatCell *pCell = &p->aCell[i]; - - iOff = get2byte(&aData[nHdr+i*2]); - if( !isLeaf ){ - pCell->iChildPg = sqlite3Get4byte(&aData[iOff]); - iOff += 4; - } - if( p->flags==0x05 ){ - /* A table interior node. nPayload==0. */ - }else{ - u32 nPayload; /* Bytes of payload total (local+overflow) */ - int nLocal; /* Bytes of payload stored locally */ - iOff += getVarint32(&aData[iOff], nPayload); - if( p->flags==0x0D ){ - u64 dummy; - iOff += sqlite3GetVarint(&aData[iOff], &dummy); - } - if( nPayload>(u32)p->nMxPayload ) p->nMxPayload = nPayload; - getLocalPayload(nUsable, p->flags, nPayload, &nLocal); - pCell->nLocal = nLocal; - assert( nLocal>=0 ); - assert( nPayload>=(u32)nLocal ); - assert( nLocal<=(nUsable-35) ); - if( nPayload>(u32)nLocal ){ - int j; - int nOvfl = ((nPayload - nLocal) + nUsable-4 - 1) / (nUsable - 4); - pCell->nLastOvfl = (nPayload-nLocal) - (nOvfl-1) * (nUsable-4); - pCell->nOvfl = nOvfl; - pCell->aOvfl = sqlite3_malloc(sizeof(u32)*nOvfl); - pCell->aOvfl[0] = sqlite3Get4byte(&aData[iOff+nLocal]); - for(j=1; jaOvfl[j-1]; - DbPage *pPg = 0; - rc = sqlite3PagerGet(sqlite3BtreePager(pBt), iPrev, &pPg); - if( rc!=SQLITE_OK ){ - assert( pPg==0 ); - return rc; - } - pCell->aOvfl[j] = sqlite3Get4byte(sqlite3PagerGetData(pPg)); - sqlite3PagerUnref(pPg); - } - } - } - } - } - - return SQLITE_OK; -} - -/* -** Populate the pCsr->iOffset and pCsr->szPage member variables. Based on -** the current value of pCsr->iPageno. -*/ -static void statSizeAndOffset(StatCursor *pCsr){ - StatTable *pTab = (StatTable *)((sqlite3_vtab_cursor *)pCsr)->pVtab; - Btree *pBt = pTab->db->aDb[0].pBt; - Pager *pPager = sqlite3BtreePager(pBt); - sqlite3_file *fd; - sqlite3_int64 x[2]; - - /* The default page size and offset */ - pCsr->szPage = sqlite3BtreeGetPageSize(pBt); - pCsr->iOffset = (i64)pCsr->szPage * (pCsr->iPageno - 1); - - /* If connected to a ZIPVFS backend, override the page size and - ** offset with actual values obtained from ZIPVFS. - */ - fd = sqlite3PagerFile(pPager); - x[0] = pCsr->iPageno; - if( sqlite3OsFileControl(fd, 230440, &x)==SQLITE_OK ){ - pCsr->iOffset = x[0]; - pCsr->szPage = (int)x[1]; - } -} - -/* -** Move a statvfs cursor to the next entry in the file. -*/ -static int statNext(sqlite3_vtab_cursor *pCursor){ - int rc; - int nPayload; - StatCursor *pCsr = (StatCursor *)pCursor; - StatTable *pTab = (StatTable *)pCursor->pVtab; - Btree *pBt = pTab->db->aDb[0].pBt; - Pager *pPager = sqlite3BtreePager(pBt); - - sqlite3_free(pCsr->zPath); - pCsr->zPath = 0; - -statNextRestart: - if( pCsr->aPage[0].pPg==0 ){ - rc = sqlite3_step(pCsr->pStmt); - if( rc==SQLITE_ROW ){ - int nPage; - u32 iRoot = (u32)sqlite3_column_int64(pCsr->pStmt, 1); - sqlite3PagerPagecount(pPager, &nPage); - if( nPage==0 ){ - pCsr->isEof = 1; - return sqlite3_reset(pCsr->pStmt); - } - rc = sqlite3PagerGet(pPager, iRoot, &pCsr->aPage[0].pPg); - pCsr->aPage[0].iPgno = iRoot; - pCsr->aPage[0].iCell = 0; - pCsr->aPage[0].zPath = sqlite3_mprintf("/"); - pCsr->iPage = 0; - }else{ - pCsr->isEof = 1; - return sqlite3_reset(pCsr->pStmt); - } - }else{ - - /* Page p itself has already been visited. */ - StatPage *p = &pCsr->aPage[pCsr->iPage]; - - while( p->iCellnCell ){ - StatCell *pCell = &p->aCell[p->iCell]; - if( pCell->iOvflnOvfl ){ - int nUsable; - sqlite3BtreeEnter(pBt); - nUsable = sqlite3BtreeGetPageSize(pBt) - - sqlite3BtreeGetReserveNoMutex(pBt); - sqlite3BtreeLeave(pBt); - pCsr->zName = (char *)sqlite3_column_text(pCsr->pStmt, 0); - pCsr->iPageno = pCell->aOvfl[pCell->iOvfl]; - pCsr->zPagetype = "overflow"; - pCsr->nCell = 0; - pCsr->nMxPayload = 0; - pCsr->zPath = sqlite3_mprintf( - "%s%.3x+%.6x", p->zPath, p->iCell, pCell->iOvfl - ); - if( pCell->iOvflnOvfl-1 ){ - pCsr->nUnused = 0; - pCsr->nPayload = nUsable - 4; - }else{ - pCsr->nPayload = pCell->nLastOvfl; - pCsr->nUnused = nUsable - 4 - pCsr->nPayload; - } - pCell->iOvfl++; - statSizeAndOffset(pCsr); - return SQLITE_OK; - } - if( p->iRightChildPg ) break; - p->iCell++; - } - - if( !p->iRightChildPg || p->iCell>p->nCell ){ - statClearPage(p); - if( pCsr->iPage==0 ) return statNext(pCursor); - pCsr->iPage--; - goto statNextRestart; /* Tail recursion */ - } - pCsr->iPage++; - assert( p==&pCsr->aPage[pCsr->iPage-1] ); - - if( p->iCell==p->nCell ){ - p[1].iPgno = p->iRightChildPg; - }else{ - p[1].iPgno = p->aCell[p->iCell].iChildPg; - } - rc = sqlite3PagerGet(pPager, p[1].iPgno, &p[1].pPg); - p[1].iCell = 0; - p[1].zPath = sqlite3_mprintf("%s%.3x/", p->zPath, p->iCell); - p->iCell++; - } - - - /* Populate the StatCursor fields with the values to be returned - ** by the xColumn() and xRowid() methods. - */ - if( rc==SQLITE_OK ){ - int i; - StatPage *p = &pCsr->aPage[pCsr->iPage]; - pCsr->zName = (char *)sqlite3_column_text(pCsr->pStmt, 0); - pCsr->iPageno = p->iPgno; - - statDecodePage(pBt, p); - statSizeAndOffset(pCsr); - - switch( p->flags ){ - case 0x05: /* table internal */ - case 0x02: /* index internal */ - pCsr->zPagetype = "internal"; - break; - case 0x0D: /* table leaf */ - case 0x0A: /* index leaf */ - pCsr->zPagetype = "leaf"; - break; - default: - pCsr->zPagetype = "corrupted"; - break; - } - pCsr->nCell = p->nCell; - pCsr->nUnused = p->nUnused; - pCsr->nMxPayload = p->nMxPayload; - pCsr->zPath = sqlite3_mprintf("%s", p->zPath); - nPayload = 0; - for(i=0; inCell; i++){ - nPayload += p->aCell[i].nLocal; - } - pCsr->nPayload = nPayload; - } - - return rc; -} - -static int statEof(sqlite3_vtab_cursor *pCursor){ - StatCursor *pCsr = (StatCursor *)pCursor; - return pCsr->isEof; -} - -static int statFilter( - sqlite3_vtab_cursor *pCursor, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv -){ - StatCursor *pCsr = (StatCursor *)pCursor; - - statResetCsr(pCsr); - return statNext(pCursor); -} - -static int statColumn( - sqlite3_vtab_cursor *pCursor, - sqlite3_context *ctx, - int i -){ - StatCursor *pCsr = (StatCursor *)pCursor; - switch( i ){ - case 0: /* name */ - sqlite3_result_text(ctx, pCsr->zName, -1, SQLITE_STATIC); - break; - case 1: /* path */ - sqlite3_result_text(ctx, pCsr->zPath, -1, SQLITE_TRANSIENT); - break; - case 2: /* pageno */ - sqlite3_result_int64(ctx, pCsr->iPageno); - break; - case 3: /* pagetype */ - sqlite3_result_text(ctx, pCsr->zPagetype, -1, SQLITE_STATIC); - break; - case 4: /* ncell */ - sqlite3_result_int(ctx, pCsr->nCell); - break; - case 5: /* payload */ - sqlite3_result_int(ctx, pCsr->nPayload); - break; - case 6: /* unused */ - sqlite3_result_int(ctx, pCsr->nUnused); - break; - case 7: /* mx_payload */ - sqlite3_result_int(ctx, pCsr->nMxPayload); - break; - case 8: /* pgoffset */ - sqlite3_result_int64(ctx, pCsr->iOffset); - break; - case 9: /* pgsize */ - sqlite3_result_int(ctx, pCsr->szPage); - break; - } - return SQLITE_OK; -} - -static int statRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ - StatCursor *pCsr = (StatCursor *)pCursor; - *pRowid = pCsr->iPageno; - return SQLITE_OK; -} - -int sqlite3_dbstat_register(sqlite3 *db){ - static sqlite3_module dbstat_module = { - 0, /* iVersion */ - statConnect, /* xCreate */ - statConnect, /* xConnect */ - statBestIndex, /* xBestIndex */ - statDisconnect, /* xDisconnect */ - statDisconnect, /* xDestroy */ - statOpen, /* xOpen - open a cursor */ - statClose, /* xClose - close a cursor */ - statFilter, /* xFilter - configure scan constraints */ - statNext, /* xNext - advance a cursor */ - statEof, /* xEof - check for end of scan */ - statColumn, /* xColumn - read data */ - statRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ - }; - sqlite3_create_module(db, "dbstat", &dbstat_module, 0); - return SQLITE_OK; -} - -#endif - -#if defined(SQLITE_TEST) || TCLSH==2 -#include - -static int test_dbstat( - void *clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ -#ifdef SQLITE_OMIT_VIRTUALTABLE - Tcl_AppendResult(interp, "dbstat not available because of " - "SQLITE_OMIT_VIRTUALTABLE", (void*)0); - return TCL_ERROR; -#else - struct SqliteDb { sqlite3 *db; }; - char *zDb; - Tcl_CmdInfo cmdInfo; - - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); - return TCL_ERROR; - } - - zDb = Tcl_GetString(objv[1]); - if( Tcl_GetCommandInfo(interp, zDb, &cmdInfo) ){ - sqlite3* db = ((struct SqliteDb*)cmdInfo.objClientData)->db; - sqlite3_dbstat_register(db); - } - return TCL_OK; -#endif -} - -int SqlitetestStat_Init(Tcl_Interp *interp){ - Tcl_CreateObjCommand(interp, "register_dbstat_vtab", test_dbstat, 0, 0); - return TCL_OK; -} -#endif /* if defined(SQLITE_TEST) || TCLSH==2 */ Index: src/tokenize.c ================================================================== --- src/tokenize.c +++ src/tokenize.c @@ -448,16 +448,18 @@ } } } abort_parse: assert( nErr==0 ); - if( zSql[i]==0 && pParse->rc==SQLITE_OK ){ + if( zSql[i]==0 && pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ if( lastTokenParsed!=TK_SEMI ){ sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); pParse->zTail = &zSql[i]; } - sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); + if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ + sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); + } } #ifdef YYTRACKMAXSTACKDEPTH sqlite3_mutex_enter(sqlite3MallocMutex()); sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK, sqlite3ParserStackPeak(pEngine) Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -983,11 +983,10 @@ }else{ assert( rc==SQLITE_OK || (p->rc&0xff)==SQLITE_CONSTRAINT ); assert( rc==SQLITE_OK || db->nDeferredCons>0 || db->nDeferredImmCons>0 ); rc = p->rc ? SQLITE_ERROR : SQLITE_DONE; } - pOp = &aOp[pcx]; goto vdbe_return; } /* Opcode: Integer P1 P2 * * * ** Synopsis: r[P2]=P1 @@ -1203,14 +1202,15 @@ assert( pIn1<=&aMem[(p->nMem-p->nCursor)] ); assert( memIsValid(pIn1) ); memAboutToChange(p, pOut); sqlite3VdbeMemMove(pOut, pIn1); #ifdef SQLITE_DEBUG - if( pOut->pScopyFrom>=&aMem[p1] && pOut->pScopyFrom<&aMem[p1+pOp->p3] ){ - pOut->pScopyFrom += p1 - pOp->p2; + if( pOut->pScopyFrom>=&aMem[p1] && pOut->pScopyFrompScopyFrom += pOp->p2 - p1; } #endif + Deephemeralize(pOut); REGISTER_TRACE(p2++, pOut); pIn1++; pOut++; }while( --n ); break; @@ -2475,11 +2475,11 @@ rc = SQLITE_CORRUPT_BKPT; goto op_column_error; } } - /* If after trying to extra new entries from the header, nHdrParsed is + /* If after trying to extract new entries from the header, nHdrParsed is ** still not up to p2, that means that the record has fewer than p2 ** columns. So the result will be either the default value or a NULL. */ if( pC->nHdrParsed<=p2 ){ if( pOp->p4type==P4_MEM ){ @@ -5910,11 +5910,10 @@ if( rc ){ eNew = eOld; } eNew = sqlite3PagerSetJournalMode(pPager, eNew); - pOut = &aMem[pOp->p2]; pOut->flags = MEM_Str|MEM_Static|MEM_Term; pOut->z = (char *)sqlite3JournalModename(eNew); pOut->n = sqlite3Strlen30(pOut->z); pOut->enc = SQLITE_UTF8; sqlite3VdbeChangeEncoding(pOut, encoding); @@ -6105,12 +6104,13 @@ pCur = allocateCursor(p, pOp->p1, 0, -1, 0); if( pCur ){ pCur->pVtabCursor = pVtabCursor; pVtab->nRef++; }else{ - db->mallocFailed = 1; + assert( db->mallocFailed ); pModule->xClose(pVtabCursor); + goto no_mem; } } break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ Index: src/vdbemem.c ================================================================== --- src/vdbemem.c +++ src/vdbemem.c @@ -198,14 +198,15 @@ return SQLITE_NOMEM; } pMem->z[pMem->n] = 0; pMem->z[pMem->n+1] = 0; pMem->flags |= MEM_Term; + } + pMem->flags &= ~MEM_Ephem; #ifdef SQLITE_DEBUG - pMem->pScopyFrom = 0; + pMem->pScopyFrom = 0; #endif - } return SQLITE_OK; } /* Index: src/vdbesort.c ================================================================== --- src/vdbesort.c +++ src/vdbesort.c @@ -2061,15 +2061,16 @@ */ #define INCRINIT_NORMAL 0 #define INCRINIT_TASK 1 #define INCRINIT_ROOT 2 -/* Forward reference. -** The vdbeIncrMergeInit() and vdbePmaReaderIncrMergeInit() routines call each -** other (when building a merge tree). +/* +** Forward reference required as the vdbeIncrMergeInit() and +** vdbePmaReaderIncrInit() routines are called mutually recursively when +** building a merge tree. */ -static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode); +static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode); /* ** Initialize the MergeEngine object passed as the second argument. Once this ** function returns, the first key of merged data may be read from the ** MergeEngine object in the usual fashion. @@ -2112,11 +2113,11 @@ ** the main thread to fill its buffer. So calling PmaReaderNext() ** on this PmaReader before any of the multi-threaded PmaReaders takes ** better advantage of multi-processor hardware. */ rc = vdbePmaReaderNext(&pMerger->aReadr[nTree-i-1]); }else{ - rc = vdbePmaReaderIncrMergeInit(&pMerger->aReadr[i], INCRINIT_NORMAL); + rc = vdbePmaReaderIncrInit(&pMerger->aReadr[i], INCRINIT_NORMAL); } if( rc!=SQLITE_OK ) return rc; } for(i=pMerger->nTree-1; i>0; i--){ @@ -2124,21 +2125,19 @@ } return pTask->pUnpacked->errCode; } /* -** Initialize the IncrMerge field of a PmaReader. -** -** If the PmaReader passed as the first argument is not an incremental-reader -** (if pReadr->pIncr==0), then this function is a no-op. Otherwise, it serves -** to open and/or initialize the temp file related fields of the IncrMerge +** The PmaReader passed as the first argument is guaranteed to be an +** incremental-reader (pReadr->pIncr!=0). This function serves to open +** and/or initialize the temp file related fields of the IncrMerge ** object at (pReadr->pIncr). ** ** If argument eMode is set to INCRINIT_NORMAL, then all PmaReaders -** in the sub-tree headed by pReadr are also initialized. Data is then loaded -** into the buffers belonging to pReadr and it is set to -** point to the first key in its range. +** in the sub-tree headed by pReadr are also initialized. Data is then +** loaded into the buffers belonging to pReadr and it is set to point to +** the first key in its range. ** ** If argument eMode is set to INCRINIT_TASK, then pReadr is guaranteed ** to be a multi-threaded PmaReader and this function is being called in a ** background thread. In this case all PmaReaders in the sub-tree are ** initialized as for INCRINIT_NORMAL and the aFile[1] buffer belonging to @@ -2161,93 +2160,112 @@ ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ int rc = SQLITE_OK; IncrMerger *pIncr = pReadr->pIncr; + SortSubtask *pTask = pIncr->pTask; + sqlite3 *db = pTask->pSorter->db; /* eMode is always INCRINIT_NORMAL in single-threaded mode */ assert( SQLITE_MAX_WORKER_THREADS>0 || eMode==INCRINIT_NORMAL ); - if( pIncr ){ - SortSubtask *pTask = pIncr->pTask; - sqlite3 *db = pTask->pSorter->db; - - rc = vdbeMergeEngineInit(pTask, pIncr->pMerger, eMode); - - /* Set up the required files for pIncr. A multi-theaded IncrMerge object - ** requires two temp files to itself, whereas a single-threaded object - ** only requires a region of pTask->file2. */ - if( rc==SQLITE_OK ){ - int mxSz = pIncr->mxSz; -#if SQLITE_MAX_WORKER_THREADS>0 - if( pIncr->bUseThread ){ - rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[0].pFd); - if( rc==SQLITE_OK ){ - rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[1].pFd); - } - }else -#endif - /*if( !pIncr->bUseThread )*/{ - if( pTask->file2.pFd==0 ){ - assert( pTask->file2.iEof>0 ); - rc = vdbeSorterOpenTempFile(db, pTask->file2.iEof, &pTask->file2.pFd); - pTask->file2.iEof = 0; - } - if( rc==SQLITE_OK ){ - pIncr->aFile[1].pFd = pTask->file2.pFd; - pIncr->iStartOff = pTask->file2.iEof; - pTask->file2.iEof += mxSz; - } - } - } - -#if SQLITE_MAX_WORKER_THREADS>0 - if( rc==SQLITE_OK && pIncr->bUseThread ){ - /* Use the current thread to populate aFile[1], even though this - ** PmaReader is multi-threaded. The reason being that this function - ** is already running in background thread pIncr->pTask->thread. */ - assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK ); - rc = vdbeIncrPopulate(pIncr); - } -#endif - - if( rc==SQLITE_OK - && (SQLITE_MAX_WORKER_THREADS==0 || eMode!=INCRINIT_TASK) - ){ - rc = vdbePmaReaderNext(pReadr); - } - } + rc = vdbeMergeEngineInit(pTask, pIncr->pMerger, eMode); + + /* Set up the required files for pIncr. A multi-theaded IncrMerge object + ** requires two temp files to itself, whereas a single-threaded object + ** only requires a region of pTask->file2. */ + if( rc==SQLITE_OK ){ + int mxSz = pIncr->mxSz; +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->bUseThread ){ + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[0].pFd); + if( rc==SQLITE_OK ){ + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[1].pFd); + } + }else +#endif + /*if( !pIncr->bUseThread )*/{ + if( pTask->file2.pFd==0 ){ + assert( pTask->file2.iEof>0 ); + rc = vdbeSorterOpenTempFile(db, pTask->file2.iEof, &pTask->file2.pFd); + pTask->file2.iEof = 0; + } + if( rc==SQLITE_OK ){ + pIncr->aFile[1].pFd = pTask->file2.pFd; + pIncr->iStartOff = pTask->file2.iEof; + pTask->file2.iEof += mxSz; + } + } + } + +#if SQLITE_MAX_WORKER_THREADS>0 + if( rc==SQLITE_OK && pIncr->bUseThread ){ + /* Use the current thread to populate aFile[1], even though this + ** PmaReader is multi-threaded. If this is an INCRINIT_TASK object, + ** then this function is already running in background thread + ** pIncr->pTask->thread. + ** + ** If this is the INCRINIT_ROOT object, then it is running in the + ** main VDBE thread. But that is Ok, as that thread cannot return + ** control to the VDBE or proceed with anything useful until the + ** first results are ready from this merger object anyway. + */ + assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK ); + rc = vdbeIncrPopulate(pIncr); + } +#endif + + if( rc==SQLITE_OK && (SQLITE_MAX_WORKER_THREADS==0 || eMode!=INCRINIT_TASK) ){ + rc = vdbePmaReaderNext(pReadr); + } + return rc; } #if SQLITE_MAX_WORKER_THREADS>0 /* ** The main routine for vdbePmaReaderIncrMergeInit() operations run in ** background threads. */ -static void *vdbePmaReaderBgInit(void *pCtx){ +static void *vdbePmaReaderBgIncrInit(void *pCtx){ PmaReader *pReader = (PmaReader*)pCtx; void *pRet = SQLITE_INT_TO_PTR( vdbePmaReaderIncrMergeInit(pReader,INCRINIT_TASK) ); pReader->pIncr->pTask->bDone = 1; return pRet; } +#endif /* -** Use a background thread to invoke vdbePmaReaderIncrMergeInit(INCRINIT_TASK) -** on the PmaReader object passed as the first argument. -** -** This call will initialize the various fields of the pReadr->pIncr -** structure and, if it is a multi-threaded IncrMerger, launch a -** background thread to populate aFile[1]. +** If the PmaReader passed as the first argument is not an incremental-reader +** (if pReadr->pIncr==0), then this function is a no-op. Otherwise, it invokes +** the vdbePmaReaderIncrMergeInit() function with the parameters passed to +** this routine to initialize the incremental merge. +** +** If the IncrMerger object is multi-threaded (IncrMerger.bUseThread==1), +** then a background thread is launched to call vdbePmaReaderIncrMergeInit(). +** Or, if the IncrMerger is single threaded, the same function is called +** using the current thread. */ -static int vdbePmaReaderBgIncrInit(PmaReader *pReadr){ - void *pCtx = (void*)pReadr; - return vdbeSorterCreateThread(pReadr->pIncr->pTask, vdbePmaReaderBgInit, pCtx); -} +static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode){ + IncrMerger *pIncr = pReadr->pIncr; /* Incremental merger */ + int rc = SQLITE_OK; /* Return code */ + if( pIncr ){ +#if SQLITE_MAX_WORKER_THREADS>0 + assert( pIncr->bUseThread==0 || eMode==INCRINIT_TASK ); + if( pIncr->bUseThread ){ + void *pCtx = (void*)pReadr; + rc = vdbeSorterCreateThread(pIncr->pTask, vdbePmaReaderBgIncrInit, pCtx); + }else #endif + { + rc = vdbePmaReaderIncrMergeInit(pReadr, eMode); + } + } + return rc; +} /* ** Allocate a new MergeEngine object to merge the contents of nPMA level-0 ** PMAs from pTask->file. If no error occurs, set *ppOut to point to ** the new object and return SQLITE_OK. Or, if an error does occur, set *ppOut @@ -2488,19 +2506,25 @@ vdbeIncrMergerSetThreads(pIncr); assert( pIncr->pTask!=pLast ); } } for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ + /* Check that: + ** + ** a) The incremental merge object is configured to use the + ** right task, and + ** b) If it is using task (nTask-1), it is configured to run + ** in single-threaded mode. This is important, as the + ** root merge (INCRINIT_ROOT) will be using the same task + ** object. + */ PmaReader *p = &pMain->aReadr[iTask]; - assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); - if( p->pIncr ){ - if( iTask==pSorter->nTask-1 ){ - rc = vdbePmaReaderIncrMergeInit(p, INCRINIT_TASK); - }else{ - rc = vdbePmaReaderBgIncrInit(p); - } - } + assert( p->pIncr==0 || ( + (p->pIncr->pTask==&pSorter->aTask[iTask]) /* a */ + && (iTask!=pSorter->nTask-1 || p->pIncr->bUseThread==0) /* b */ + )); + rc = vdbePmaReaderIncrInit(p, INCRINIT_TASK); } } pMain = 0; } if( rc==SQLITE_OK ){ Index: src/vdbetrace.c ================================================================== --- src/vdbetrace.c +++ src/vdbetrace.c @@ -82,13 +82,12 @@ Mem *pVar; /* Value of a host parameter */ StrAccum out; /* Accumulate the output here */ char zBase[100]; /* Initial working space */ db = p->db; - sqlite3StrAccumInit(&out, zBase, sizeof(zBase), + sqlite3StrAccumInit(&out, db, zBase, sizeof(zBase), db->aLimit[SQLITE_LIMIT_LENGTH]); - out.db = db; if( db->nVdbeExec>1 ){ while( *zRawSql ){ const char *zStart = zRawSql; while( *(zRawSql++)!='\n' && *zRawSql ); sqlite3StrAccumAppend(&out, "-- ", 3); Index: src/vtab.c ================================================================== --- src/vtab.c +++ src/vtab.c @@ -1081,11 +1081,11 @@ assert( IsVirtual(pTab) ); for(i=0; inVtabLock; i++){ if( pTab==pToplevel->apVtabLock[i] ) return; } n = (pToplevel->nVtabLock+1)*sizeof(pToplevel->apVtabLock[0]); - apVtabLock = sqlite3_realloc(pToplevel->apVtabLock, n); + apVtabLock = sqlite3_realloc64(pToplevel->apVtabLock, n); if( apVtabLock ){ pToplevel->apVtabLock = apVtabLock; pToplevel->apVtabLock[pToplevel->nVtabLock++] = pTab; }else{ pToplevel->db->mallocFailed = 1; Index: src/wal.c ================================================================== --- src/wal.c +++ src/wal.c @@ -520,11 +520,11 @@ /* Enlarge the pWal->apWiData[] array if required */ if( pWal->nWiData<=iPage ){ int nByte = sizeof(u32*)*(iPage+1); volatile u32 **apNew; - apNew = (volatile u32 **)sqlite3_realloc((void *)pWal->apWiData, nByte); + apNew = (volatile u32 **)sqlite3_realloc64((void *)pWal->apWiData, nByte); if( !apNew ){ *ppPage = 0; return SQLITE_NOMEM; } memset((void*)&apNew[pWal->nWiData], 0, @@ -1145,11 +1145,11 @@ goto finished; } /* Malloc a buffer to read frames into. */ szFrame = szPage + WAL_FRAME_HDRSIZE; - aFrame = (u8 *)sqlite3_malloc(szFrame); + aFrame = (u8 *)sqlite3_malloc64(szFrame); if( !aFrame ){ rc = SQLITE_NOMEM; goto recovery_error; } aData = &aFrame[WAL_FRAME_HDRSIZE]; @@ -1538,21 +1538,21 @@ /* Allocate space for the WalIterator object. */ nSegment = walFramePage(iLast) + 1; nByte = sizeof(WalIterator) + (nSegment-1)*sizeof(struct WalSegment) + iLast*sizeof(ht_slot); - p = (WalIterator *)sqlite3_malloc(nByte); + p = (WalIterator *)sqlite3_malloc64(nByte); if( !p ){ return SQLITE_NOMEM; } memset(p, 0, nByte); p->nSegment = nSegment; /* Allocate temporary space used by the merge-sort routine. This block ** of memory will be freed before this function returns. */ - aTmp = (ht_slot *)sqlite3_malloc( + aTmp = (ht_slot *)sqlite3_malloc64( sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) ); if( !aTmp ){ rc = SQLITE_NOMEM; } Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -3101,12 +3101,11 @@ isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); - sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); - str.db = db; + sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); if( pItem->pSelect ){ sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId); }else{ sqlite3XPrintf(&str, 0, " TABLE %s", pItem->zName); @@ -4301,10 +4300,17 @@ /* ** Free a WhereInfo structure */ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ if( ALWAYS(pWInfo) ){ + int i; + for(i=0; inLevel; i++){ + WhereLevel *pLevel = &pWInfo->a[i]; + if( pLevel->pWLoop && (pLevel->pWLoop->wsFlags & WHERE_IN_ABLE) ){ + sqlite3DbFree(db, pLevel->u.in.aInLoop); + } + } whereClauseClear(&pWInfo->sWC); while( pWInfo->pLoops ){ WhereLoop *p = pWInfo->pLoops; pWInfo->pLoops = p->pNextLoop; whereLoopDelete(db, p); @@ -6897,11 +6903,10 @@ VdbeCoverage(v); VdbeCoverageIf(v, pIn->eEndLoopOp==OP_PrevIfOpen); VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen); sqlite3VdbeJumpHere(v, pIn->addrInTop-1); } - sqlite3DbFree(db, pLevel->u.in.aInLoop); } sqlite3VdbeResolveLabel(v, pLevel->addrBrk); if( pLevel->addrSkip ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrSkip); VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName)); Index: test/alter.test ================================================================== --- test/alter.test +++ test/alter.test @@ -910,7 +910,16 @@ NULL, 'lmnop'); } {{}} do_execsql_test alter-17.10 { SELECT sqlite_rename_parent(NULL,'abc','xyz'); } {{}} +do_execsql_test alter-17.11 { + SELECT sqlite_rename_parent('create references ''','abc','xyz'); +} {{create references '}} +do_execsql_test alter-17.12 { + SELECT sqlite_rename_parent('create references "abc"123" ','abc','xyz'); +} {{create references "xyz"123" }} +do_execsql_test alter-17.13 { + SELECT sqlite_rename_parent("references '''",'abc','xyz'); +} {{references '''}} finish_test ADDED test/analyzer1.test Index: test/analyzer1.test ================================================================== --- /dev/null +++ test/analyzer1.test @@ -0,0 +1,47 @@ +# 2015-05-11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Quick tests for the sqlite3_analyzer tool +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +ifcapable !vtab { + finish_test + return +} + +if {$tcl_platform(platform)=="windows"} { + set PROG "sqlite3_analyzer.exe" +} else { + set PROG "./sqlite3_analyzer" +} +db close +forcedelete test.db test.db-journal test.db-wal +sqlite3 db test.db + +do_test analyzer1-1.0 { + db eval { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INT PRIMARY KEY, b) WITHOUT ROWID; + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<250) + INSERT INTO t1(a,b) SELECT x, randomblob(200) FROM c; + INSERT INTO t2(a,b) SELECT a, b FROM t1; + } + set line "exec $PROG test.db" + unset -nocomplain ::MSG + catch {eval $line} ::MSG +} {0} +do_test analyzer1-1.1 { + regexp {^/\*\* Disk-Space Utilization.*COMMIT;\W*$} $::MSG +} {1} + +finish_test Index: test/fts3matchinfo.test ================================================================== --- test/fts3matchinfo.test +++ test/fts3matchinfo.test @@ -447,7 +447,109 @@ AND matchinfo(t10) not null GROUP BY docId ORDER BY 1; } {1 1 one 2 2 two 3 3 three} +#--------------------------------------------------------------------------- +# Test the 'y' matchinfo flag +# +set sqlite_fts3_enable_parentheses 1 +reset_db +do_execsql_test 11.0 { + CREATE VIRTUAL TABLE tt USING fts3(x, y); + INSERT INTO tt VALUES('c d a c d d', 'e a g b d a'); -- 1 + INSERT INTO tt VALUES('c c g a e b', 'c g d g e c'); -- 2 + INSERT INTO tt VALUES('b e f d e g', 'b a c b c g'); -- 3 + INSERT INTO tt VALUES('a c f f g d', 'd b f d e g'); -- 4 + INSERT INTO tt VALUES('g a c f c f', 'd g g b c c'); -- 5 + INSERT INTO tt VALUES('g a c e b b', 'd b f b g g'); -- 6 + INSERT INTO tt VALUES('f d a a f c', 'e e a d c f'); -- 7 + INSERT INTO tt VALUES('a c b b g f', 'a b a e d f'); -- 8 + INSERT INTO tt VALUES('b a f e c c', 'f d b b a b'); -- 9 + INSERT INTO tt VALUES('f d c e a c', 'f a f a a f'); -- 10 +} + +db func mit mit +foreach {tn expr res} { + 1 "a" { + 1 {1 2} 2 {1 0} 3 {0 1} 4 {1 0} 5 {1 0} + 6 {1 0} 7 {2 1} 8 {1 2} 9 {1 1} 10 {1 3} + } + + 2 "b" { + 1 {0 1} 2 {1 0} 3 {1 2} 4 {0 1} 5 {0 1} + 6 {2 2} 8 {2 1} 9 {1 3} + } + + 3 "y:a" { + 1 {0 2} 3 {0 1} + 7 {0 1} 8 {0 2} 9 {0 1} 10 {0 3} + } + + 4 "x:a" { + 1 {1 0} 2 {1 0} 4 {1 0} 5 {1 0} + 6 {1 0} 7 {2 0} 8 {1 0} 9 {1 0} 10 {1 0} + } + + 5 "a OR b" { + 1 {1 2 0 1} 2 {1 0 1 0} 3 {0 1 1 2} 4 {1 0 0 1} 5 {1 0 0 1} + 6 {1 0 2 2} 7 {2 1 0 0} 8 {1 2 2 1} 9 {1 1 1 3} 10 {1 3 0 0} + } + + 6 "a AND b" { + 1 {1 2 0 1} 2 {1 0 1 0} 3 {0 1 1 2} 4 {1 0 0 1} 5 {1 0 0 1} + 6 {1 0 2 2} 8 {1 2 2 1} 9 {1 1 1 3} + } + + 7 "a OR (a AND b)" { + 1 {1 2 1 2 0 1} 2 {1 0 1 0 1 0} 3 {0 1 0 1 1 2} 4 {1 0 1 0 0 1} + 5 {1 0 1 0 0 1} 6 {1 0 1 0 2 2} 7 {2 1 0 0 0 0} 8 {1 2 1 2 2 1} + 9 {1 1 1 1 1 3} 10 {1 3 0 0 0 0} + } + +} { + do_execsql_test 11.1.$tn.1 { + SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr + } $res + + set r2 [list] + foreach {rowid L} $res { + lappend r2 $rowid + set M [list] + foreach {a b} $L { + lappend M [expr ($a ? 1 : 0) + ($b ? 2 : 0)] + } + lappend r2 $M + } + + do_execsql_test 11.1.$tn.2 { + SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr + } $r2 + breakpoint + + do_execsql_test 11.1.$tn.2 { + SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr + } $r2 +} +set sqlite_fts3_enable_parentheses 0 + +#--------------------------------------------------------------------------- +# Test the 'b' matchinfo flag +# +set sqlite_fts3_enable_parentheses 1 +reset_db +db func mit mit + +do_test 12.0 { + set cols [list] + for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" } + execsql "CREATE VIRTUAL TABLE tt USING fts3([join $cols ,])" +} {} + +do_execsql_test 12.1 { + INSERT INTO tt (rowid, c4, c45) VALUES(1, 'abc', 'abc'); + SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc'; +} [list [list [expr 1<<4] [expr 1<<(45-32)]]] +set sqlite_fts3_enable_parentheses 0 finish_test + Index: test/fts3query.test ================================================================== --- test/fts3query.test +++ test/fts3query.test @@ -171,12 +171,12 @@ 2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'" offsets 3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'" snippet 4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'" optimize } do_catchsql_test 5.5.1 { - SELECT matchinfo(t2, 'abc') FROM t2 WHERE t2 MATCH 'history' -} {1 {unrecognized matchinfo request: b}} + SELECT matchinfo(t2, 'abcd') FROM t2 WHERE t2 MATCH 'history' +} {1 {unrecognized matchinfo request: d}} do_execsql_test 5.5 { DROP TABLE t2 } # Test the snippet() function with 1 to 6 arguments. Index: test/fuzzdata1.txt ================================================================== --- test/fuzzdata1.txt +++ test/fuzzdata1.txt cannot compute difference between binary files ADDED test/fuzzdata2.txt Index: test/fuzzdata2.txt ================================================================== --- /dev/null +++ test/fuzzdata2.txt cannot compute difference between binary files Index: test/jrnlmode.test ================================================================== --- test/jrnlmode.test +++ test/jrnlmode.test @@ -552,7 +552,21 @@ do_execsql_test jrnlmode-8.26 { CREATE TABLE t4(w) } {} do_execsql_test jrnlmode-8.27 { BEGIN IMMEDIATE } {} do_execsql_test jrnlmode-8.28 { PRAGMA journal_mode=DELETE } {delete} do_execsql_test jrnlmode-8.29 { COMMIT } {} do_execsql_test jrnlmode-8.30 { PRAGMA journal_mode=DELETE } {delete} + +# Assertion fault on 2015-05-01 +do_test jrnlmode-9.1 { + forcedelete test2.db + sqlite3 db2 test2.db + db2 eval {CREATE TEMP TABLE t(l); PRAGMA journal_mode=off;} + db2 close +} {} +do_execsql_test jrnlmode-9.2 { + PRAGMA locking_mode = exclusive; + CREATE TABLE tx(a); + PRAGMA journal_mode = off; +} {exclusive off} + finish_test Index: test/malloc.test ================================================================== --- test/malloc.test +++ test/malloc.test @@ -920,10 +920,31 @@ execsql { SELECT * FROM t1 WHERE a = ('abcde' || $::big)} } -test { faultsim_test_result [list 0 "abcde$::big"] faultsim_integrity_check } + +reset_db +do_execsql_test 42.0 { + CREATE TABLE t1(x INTEGER PRIMARY KEY, y, z); + CREATE TABLE t2(a, b); + CREATE VIEW a002 AS SELECT *, sum(b) AS m FROM t2 GROUP BY a; +} +faultsim_save_and_close +do_faultsim_test 42 -faults oom-tran* -prep { + faultsim_restore_and_reopen + execsql { SELECT * FROM sqlite_master } +} -body { + execsql { + SELECT t1.z, a002.m + FROM t1 JOIN a002 ON t1.y=a002.m + WHERE t1.x IN (1,2,3); + } +} -test { + faultsim_test_result {0 {}} +} + # Ensure that no file descriptors were leaked. do_test malloc-99.X { catch {db close} set sqlite_open_file_count ADDED test/mkfuzzdata1.tcl Index: test/mkfuzzdata1.tcl ================================================================== --- /dev/null +++ test/mkfuzzdata1.tcl @@ -0,0 +1,113 @@ +#!/usr/bin/tclsh +# +# Run this script in order to rebuild the fuzzdata1.txt file containing +# fuzzer data for the fuzzershell utility that is create by afl-fuzz. +# +# This script gathers all of the test cases identified by afl-fuzz and +# runs afl-cmin and afl-tmin over them all to try to generate a mimimum +# set of tests that cover all observed behavior. +# +# Options: +# +# --afl-bin DIR1 DIR1 contains the AFL binaries +# --fuzzershell PATH Full pathname of instrumented fuzzershell +# --afl-data DIR3 DIR3 is the "-o" directory from afl-fuzz +# -o FILE Write results into FILE +# +set AFLBIN {} +set FUZZERSHELL {} +set AFLDATA {} +set OUTFILE {} + +proc usage {} { + puts stderr "Usage: $::argv0 --afl-bin DIR --fuzzershell PATH\ + --afl-data DIR -o FILE" + exit 1 +} +proc cmdlineerr {msg} { + puts stderr $msg + usage +} + +for {set i 0} {$i<[llength $argv]} {incr i} { + set x [lindex $argv $i] + if {[string index $x 0]!="-"} {cmdlineerr "illegal argument: $x"} + set x [string trimleft $x -] + incr i + if {$i>=[llength $argv]} {cmdlineerr "no argument on --$x"} + set a [lindex $argv $i] + switch -- $x { + afl-bin {set AFLBIN $a} + afl-data {set AFLDATA $a} + fuzzershell {set FUZZERSHELL $a} + o {set OUTFILE $a} + default {cmdlineerr "unknown option: --$x"} + } +} +proc checkarg {varname option} { + set val [set ::$varname] + if {$val==""} {cmdlineerr "required option missing: --$option"} +} +checkarg AFLBIN afl-bin +checkarg AFLDATA afl-data +checkarg FUZZERSHELL fuzzershell +checkarg OUTFILE o +proc checkexec {x} { + if {![file exec $x]} {cmdlineerr "cannot find $x"} +} +checkexec $AFLBIN/afl-cmin +checkexec $AFLBIN/afl-tmin +checkexec $FUZZERSHELL +proc checkdir {x} { + if {![file isdir $x]} {cmdlineerr "no such directory: $x"} +} +checkdir $AFLDATA/queue + +proc progress {msg} { + puts "******** $msg" + flush stdout +} +progress "mkdir tmp1 tmp2" +file mkdir tmp1 tmp2 +progress "copying test cases from $AFLDATA into tmp1..." +set n 0 +foreach file [glob -nocomplain $AFLDATA/queue/id:*] { + incr n + file copy $file tmp1/$n +} +foreach file [glob -nocomplain $AFLDATA/crash*/id:*] { + incr n + file copy $file tmp1/$n +} +progress "total $n files copied." +progress "running: $AFLBIN/afl-cmin -i tmp1 -o tmp2 $FUZZERSHELL" +exec $AFLBIN/afl-cmin -i tmp1 -o tmp2 $FUZZERSHELL >&@ stdout +progress "afl-cmin complete." +# +# Experiments show that running afl-tmin is too slow for this application. +# And it doesn't really make the test cases that much smaller. So let's +# just skip it. +# +# foreach file [glob tmp2/*] { +# progress "$AFLBIN/afl-tmin -i $file -o tmp3/[file tail $file] $FUZZERSHELL" +# exec $AFLBIN/afl-tmin -i $file -o tmp3/[file tail $file] \ +# $FUZZERSHELL >&@ stdout +# } +progress "generating final output into $OUTFILE" +set out [open $OUTFILE wb] +puts $out "# Test data for use with fuzzershell. Automatically +# generated using $argv0. This file contains binary data +#" +set n 0 +foreach file [glob tmp2/*] { + incr n + puts -nonewline $out "/****<$n>****/" + set in [open $file rb] + puts -nonewline $out [read $in] + close $in +} +close $out +progress "done. $n test cases written to $OUTFILE" +progress "clean-up..." +file delete -force tmp1 +progress "culled test cases left in the tmp2 directory" Index: test/pagesize.test ================================================================== --- test/pagesize.test +++ test/pagesize.test @@ -213,7 +213,25 @@ PRAGMA main.page_size; PRAGMA temp.page_size; " } [list $PGSZ $PGSZ] } + +reset_db +do_execsql_test pagesize-3.1 { + BEGIN; + SELECT * FROM sqlite_master; + PRAGMA page_size=2048; + PRAGMA main.page_size; +} {1024} +do_execsql_test pagesize-3.2 { + CREATE TABLE t1(x); + COMMIT; +} +do_execsql_test pagesize-3.3 { + BEGIN; + PRAGMA page_size = 2048; + COMMIT; + PRAGMA main.page_size; +} {1024} finish_test Index: test/releasetest.tcl ================================================================== --- test/releasetest.tcl +++ test/releasetest.tcl @@ -109,10 +109,17 @@ -DSQLITE_ENABLE_MEMSYS3=1 -DSQLITE_ENABLE_COLUMN_METADATA=1 -DSQLITE_ENABLE_STAT4 -DSQLITE_MAX_ATTACHED=125 } + "Fast-One" { + -O6 + -DSQLITE_ENABLE_FTS4=1 + -DSQLITE_ENABLE_RTREE=1 + -DSQLITE_ENABLE_STAT4 + -DSQLITE_MAX_ATTACHED=125 + } "Device-One" { -O2 -DSQLITE_DEBUG=1 -DSQLITE_DEFAULT_AUTOVACUUM=1 -DSQLITE_DEFAULT_CACHE_SIZE=64 @@ -215,10 +222,11 @@ "Extra-Robustness" test "Device-Two" test "No-lookaside" test "Devkit" test "Sanitize" {QUICKTEST_OMIT=func4.test,nan.test test} + "Fast-One" fuzzoomtest "Valgrind" valgrindtest "Default" "threadtest fulltest" "Device-One" fulltest } Linux-i686 { @@ -651,14 +659,15 @@ # If the configuration included the SQLITE_DEBUG option, then remove # it and run veryquick.test. If it did not include the SQLITE_DEBUG option # add it and run veryquick.test. if {$target!="checksymbols" && $target!="valgrindtest" - && !$::BUILDONLY && $::QUICK<2} { + && $target!="fuzzoomtest" && !$::BUILDONLY && $::QUICK<2} { set debug_idx [lsearch -glob $config_options -DSQLITE_DEBUG*] set xtarget $target regsub -all {fulltest[a-z]*} $xtarget test xtarget + regsub -all {fuzzoomtest} $xtarget fuzztest xtarget if {$debug_idx < 0} { incr NTEST append config_options " -DSQLITE_DEBUG=1" run_test_suite "${zConfig}_debug" $xtarget $config_options } else { Index: test/select4.test ================================================================== --- test/select4.test +++ test/select4.test @@ -156,10 +156,13 @@ set v [catch {execsql { SELECT 0 ORDER BY (SELECT 0) UNION SELECT 0; }} msg] lappend v $msg } {1 {ORDER BY clause should come after UNION not before}} +do_execsql_test select4-2.5 { + SELECT 123 AS x ORDER BY (SELECT x ORDER BY 1); +} {123} # Except operator # do_test select4-3.1.1 { execsql { @@ -271,10 +274,20 @@ INTERSECT SELECT n FROM t1 WHERE log=3 ORDER BY log; }} msg] lappend v $msg +} {1 {ORDER BY clause should come after INTERSECT not before}} +do_catchsql_test select4-4.4 { + SELECT 3 IN ( + SELECT 0 ORDER BY 1 + INTERSECT + SELECT 1 + INTERSECT + SELECT 2 + ORDER BY 1 + ); } {1 {ORDER BY clause should come after INTERSECT not before}} # Various error messages while processing UNION or INTERSECT # do_test select4-5.1 { Index: test/shell1.test ================================================================== --- test/shell1.test +++ test/shell1.test @@ -736,10 +736,13 @@ sqlite3 db test.db db eval { PRAGMA encoding=UTF16; CREATE TABLE t1(x); INSERT INTO t1 VALUES(null), (''), (1), (2.25), ('hello'), (x'807f'); + CREATE TABLE t3(x,y); + INSERT INTO t3 VALUES(1,null), (2,''), (3,1), + (4,2.25), (5,'hello'), (6,x'807f'); } catchcmd test.db {.dump} } {0 {PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE t1(x); @@ -747,22 +750,62 @@ INSERT INTO "t1" VALUES(''); INSERT INTO "t1" VALUES(1); INSERT INTO "t1" VALUES(2.25); INSERT INTO "t1" VALUES('hello'); INSERT INTO "t1" VALUES(X'807F'); +CREATE TABLE t3(x,y); +INSERT INTO "t3" VALUES(1,NULL); +INSERT INTO "t3" VALUES(2,''); +INSERT INTO "t3" VALUES(3,1); +INSERT INTO "t3" VALUES(4,2.25); +INSERT INTO "t3" VALUES(5,'hello'); +INSERT INTO "t3" VALUES(6,X'807F'); COMMIT;}} # Test the output of ".mode insert" # -do_test shell1-4.2 { +do_test shell1-4.2.1 { catchcmd test.db ".mode insert t1\nselect * from t1;" } {0 {INSERT INTO t1 VALUES(NULL); INSERT INTO t1 VALUES(''); INSERT INTO t1 VALUES(1); INSERT INTO t1 VALUES(2.25); INSERT INTO t1 VALUES('hello'); INSERT INTO t1 VALUES(X'807f');}} + +# Test the output of ".mode insert" with headers +# +do_test shell1-4.2.2 { + catchcmd test.db ".mode insert t1\n.headers on\nselect * from t1;" +} {0 {INSERT INTO t1(x) VALUES(NULL); +INSERT INTO t1(x) VALUES(''); +INSERT INTO t1(x) VALUES(1); +INSERT INTO t1(x) VALUES(2.25); +INSERT INTO t1(x) VALUES('hello'); +INSERT INTO t1(x) VALUES(X'807f');}} + +# Test the output of ".mode insert" +# +do_test shell1-4.2.3 { + catchcmd test.db ".mode insert t3\nselect * from t3;" +} {0 {INSERT INTO t3 VALUES(1,NULL); +INSERT INTO t3 VALUES(2,''); +INSERT INTO t3 VALUES(3,1); +INSERT INTO t3 VALUES(4,2.25); +INSERT INTO t3 VALUES(5,'hello'); +INSERT INTO t3 VALUES(6,X'807f');}} + +# Test the output of ".mode insert" with headers +# +do_test shell1-4.2.4 { + catchcmd test.db ".mode insert t3\n.headers on\nselect * from t3;" +} {0 {INSERT INTO t3(x,y) VALUES(1,NULL); +INSERT INTO t3(x,y) VALUES(2,''); +INSERT INTO t3(x,y) VALUES(3,1); +INSERT INTO t3(x,y) VALUES(4,2.25); +INSERT INTO t3(x,y) VALUES(5,'hello'); +INSERT INTO t3(x,y) VALUES(6,X'807f');}} # Test the output of ".mode tcl" # do_test shell1-4.3 { db close @@ -815,7 +858,64 @@ "]" "\\{" "\\}" ";" "$"} 7} + +# Test using arbitrary byte data with the shell via standard input/output. +# +do_test shell1-5.0 { + # + # NOTE: Skip NUL byte because it appears to be incompatible with command + # shell argument parsing. + # + for {set i 1} {$i < 256} {incr i} { + # + # NOTE: Due to how the Tcl [exec] command works (i.e. where it treats + # command channels opened for it as textual ones), the carriage + # return character (and on Windows, the end-of-file character) + # cannot be used here. + # + if {$i==0x0D || ($tcl_platform(platform)=="windows" && $i==0x1A)} { + continue + } + set hex [format %02X $i] + set char [subst \\x$hex]; set oldChar $char + set escapes [list] + if {$tcl_platform(platform)=="windows"} { + # + # NOTE: On Windows, we need to escape all the whitespace characters, + # the alarm (\a) character, and those with special meaning to + # the SQLite shell itself. + # + set escapes [list \ + \a \\a \b \\b \t \\t \n \\n \v \\v \f \\f \r \\r \ + " " "\" \"" \" \\\" ' \"'\" \\ \\\\] + } else { + # + # NOTE: On Unix, we need to escape most of the whitespace characters + # and those with special meaning to the SQLite shell itself. + # The alarm (\a), backspace (\b), and carriage-return (\r) + # characters do not appear to require escaping on Unix. For + # the alarm and backspace characters, this is probably due to + # differences in the command shell. For the carriage-return, + # it is probably due to differences in how Tcl handles command + # channel end-of-line translations. + # + set escapes [list \ + \t \\t \n \\n \v \\v \f \\f \ + " " "\" \"" \" \\\" ' \"'\" \\ \\\\] + } + set char [string map $escapes $char] + set x [catchcmdex test.db ".print $char\n"] + set code [lindex $x 0] + set res [lindex $x 1] + if {$code ne "0"} { + error "failed with error: $res" + } + if {$res ne "$oldChar\n"} { + error "failed with byte $hex mismatch" + } + } +} {} finish_test ADDED test/sqldiff1.test Index: test/sqldiff1.test ================================================================== --- /dev/null +++ test/sqldiff1.test @@ -0,0 +1,60 @@ +# 2015-05-11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Quick tests for the sqldiff tool +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +if {$tcl_platform(platform)=="windows"} { + set PROG "sqldiff.exe" +} else { + set PROG "./sqldiff" +} +db close +forcedelete test.db test2.db +sqlite3 db test.db + +do_test sqldiff-1.0 { + db eval { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INT PRIMARY KEY, b) WITHOUT ROWID; + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) + INSERT INTO t1(a,b) SELECT x, printf('abc-%d-xyz',x) FROM c; + INSERT INTO t2(a,b) SELECT a, b FROM t1; + } + db backup test2.db + db eval { + ATTACH 'test2.db' AS x2; + DELETE FROM x2.t1 WHERE a=49; + DELETE FROM x2.t2 WHERE a=48; + INSERT INTO x2.t1(a,b) VALUES(1234,'hello'); + INSERT INTO x2.t2(a,b) VALUES(50.5,'xyzzy'); + CREATE TABLE x2.t3(a,b,c); + INSERT INTO x2.t3 VALUES(111,222,333); + CREATE TABLE main.t4(x,y,z); + INSERT INTO t4 SELECT * FROM t3; + } + set line "exec $PROG test.db test2.db" + unset -nocomplain ::MSG + catch {eval $line} ::MSG +} {0} +do_test sqldiff-1.1 { + set ::MSG +} {DELETE FROM t1 WHERE a=49; +INSERT INTO t1(a,b) VALUES(1234,'hello'); +DELETE FROM t2 WHERE a=48; +INSERT INTO t2(a,b) VALUES(50.5,'xyzzy'); +CREATE TABLE t3(a,b,c); +INSERT INTO t3(rowid,a,b,c) VALUES(1,111,222,333); +DROP TABLE t4;} + +finish_test Index: test/stat.test ================================================================== --- test/stat.test +++ test/stat.test @@ -164,12 +164,14 @@ forcedelete test.db sqlite3 db test.db register_dbstat_vtab db do_execsql_test stat-5.1 { PRAGMA auto_vacuum = OFF; - CREATE VIRTUAL TABLE temp.stat USING dbstat; - CREATE TABLE t1(x); + CREATE TABLE tx(y); + ATTACH ':memory:' AS aux1; + CREATE VIRTUAL TABLE temp.stat USING dbstat(aux1); + CREATE TABLE aux1.t1(x); INSERT INTO t1 VALUES(zeroblob(1513)); INSERT INTO t1 VALUES(zeroblob(1514)); SELECT name, path, pageno, pagetype, ncell, payload, unused, mx_payload FROM stat WHERE name = 't1'; } [list \ @@ -176,6 +178,10 @@ t1 / 2 leaf 2 993 5 1517 \ t1 /000+000000 3 overflow 0 1020 0 0 \ t1 /001+000000 4 overflow 0 1020 0 0 \ ] +do_catchsql_test stat-6.1 { + CREATE VIRTUAL TABLE temp.s2 USING dbstat(mainx); +} {1 {no such database: mainx}} + finish_test ADDED test/statfault.test Index: test/statfault.test ================================================================== --- /dev/null +++ test/statfault.test @@ -0,0 +1,45 @@ +# 2015 April 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix statfault + +ifcapable !vtab||!compound { + finish_test + return +} + +register_dbstat_vtab db +do_execsql_test statfault-1 { + CREATE TABLE t1(a, b UNIQUE); + INSERT INTO t1 VALUES(1, randomblob(500)); + INSERT INTO t1 VALUES(randomblob(500), 1); + INSERT INTO t1 VALUES(2, randomblob(250)); + INSERT INTO t1 VALUES(randomblob(250), 2); + CREATE VIRTUAL TABLE sss USING dbstat; +} {} +faultsim_save_and_close + +do_faultsim_test 1 -faults * -prep { + faultsim_restore_and_reopen + register_dbstat_vtab db + execsql { SELECT 1 FROM sqlite_master LIMIT 1 } +} -body { + execsql { SELECT count(*) FROM sss } +} -test { + faultsim_test_result {0 8} +} + + +finish_test Index: test/tester.tcl ================================================================== --- test/tester.tcl +++ test/tester.tcl @@ -663,10 +663,19 @@ puts " Omitted" omit_test $name "pattern mismatch" 0 } flush stdout } + +proc dumpbytes {s} { + set r "" + for {set i 0} {$i < [string length $s]} {incr i} { + if {$i > 0} {append r " "} + append r [format %02X [scan [string index $s $i] %c]] + } + return $r +} proc catchcmd {db {cmd ""}} { global CLI set out [open cmds.txt w] puts $out $cmd @@ -673,10 +682,34 @@ close $out set line "exec $CLI $db < cmds.txt" set rc [catch { eval $line } msg] list $rc $msg } + +proc catchcmdex {db {cmd ""}} { + global CLI + set out [open cmds.txt w] + fconfigure $out -encoding binary -translation binary + puts -nonewline $out $cmd + close $out + set line "exec -keepnewline -- $CLI $db < cmds.txt" + set chans [list stdin stdout stderr] + foreach chan $chans { + catch { + set modes($chan) [fconfigure $chan] + fconfigure $chan -encoding binary -translation binary -buffering none + } + } + set rc [catch { eval $line } msg] + foreach chan $chans { + catch { + eval fconfigure [list $chan] $modes($chan) + } + } + # puts [dumpbytes $msg] + list $rc $msg +} proc filepath_normalize {p} { # test cases should be written to assume "unix"-like file paths if {$::tcl_platform(platform)!="unix"} { # lreverse*2 as a hack to remove any unneeded {} after the string map Index: tool/fuzzershell.c ================================================================== --- tool/fuzzershell.c +++ tool/fuzzershell.c @@ -30,18 +30,20 @@ ** content. ** ** (4) The eval() SQL function is added, allowing the fuzzer to do ** interesting recursive operations. ** -** 2015-04-20: The input text can be divided into separate SQL chunks using -** lines of the form: +** (5) An error is raised if there is a memory leak. +** +** The input text can be divided into separate test cases using comments +** of the form: ** ** |****<...>****| ** -** where the "..." is arbitrary text, except the "|" should really be "/". -** ("|" is used here to avoid compiler warnings about nested comments.) -** A separate in-memory SQLite database is created to run each chunk of SQL. +** where the "..." is arbitrary text. (Except the "|" should really be "/". +** "|" is used here to avoid compiler errors about nested comments.) +** A separate in-memory SQLite database is created to run each test case. ** This feature allows the "queue" of AFL to be captured into a single big ** file using a command like this: ** ** (for i in id:*; do echo '|****<'$i'>****|'; cat $i; done) >~/all-queue.txt ** @@ -52,15 +54,14 @@ ** ** After running each chunk of SQL, the database connection is closed. The ** program aborts if the close fails or if there is any unfreed memory after ** the close. ** -** New cases can be appended to all-queue.txt at any time. If redundant cases -** are added, that can be eliminated by running: +** New test cases can be appended to all-queue.txt at any time. If redundant +** test cases are added, they can be eliminated by running: ** ** fuzzershell -f ~/all-queue.txt --unique-cases ~/unique-cases.txt -** */ #include #include #include #include @@ -69,22 +70,74 @@ /* ** All global variables are gathered into the "g" singleton. */ struct GlobalVars { - const char *zArgv0; /* Name of program */ + const char *zArgv0; /* Name of program */ + sqlite3_mem_methods sOrigMem; /* Original memory methods */ + sqlite3_mem_methods sOomMem; /* Memory methods with OOM simulator */ + int iOomCntdown; /* Memory fails on 1 to 0 transition */ + int nOomFault; /* Increments for each OOM fault */ + int bOomOnce; /* Fail just once if true */ + int bOomEnable; /* True to enable OOM simulation */ + int nOomBrkpt; /* Number of calls to oomFault() */ + char zTestName[100]; /* Name of current test */ } g; +/* +** Maximum number of iterations for an OOM test +*/ +#ifndef OOM_MAX +# define OOM_MAX 625 +#endif + +/* +** This routine is called when a simulated OOM occurs. It exists as a +** convenient place to set a debugger breakpoint. +*/ +static void oomFault(void){ + g.nOomBrkpt++; /* Prevent oomFault() from being optimized out */ +} + +/* Versions of malloc() and realloc() that simulate OOM conditions */ +static void *oomMalloc(int nByte){ + if( nByte>0 && g.bOomEnable && g.iOomCntdown>0 ){ + g.iOomCntdown--; + if( g.iOomCntdown==0 ){ + if( g.nOomFault==0 ) oomFault(); + g.nOomFault++; + if( !g.bOomOnce ) g.iOomCntdown = 1; + return 0; + } + } + return g.sOrigMem.xMalloc(nByte); +} +static void *oomRealloc(void *pOld, int nByte){ + if( nByte>0 && g.bOomEnable && g.iOomCntdown>0 ){ + g.iOomCntdown--; + if( g.iOomCntdown==0 ){ + if( g.nOomFault==0 ) oomFault(); + g.nOomFault++; + if( !g.bOomOnce ) g.iOomCntdown = 1; + return 0; + } + } + return g.sOrigMem.xRealloc(pOld, nByte); +} /* ** Print an error message and abort in such a way to indicate to the ** fuzzer that this counts as a crash. */ static void abendError(const char *zFormat, ...){ va_list ap; - fprintf(stderr, "%s: ", g.zArgv0); + if( g.zTestName[0] ){ + fprintf(stderr, "%s (%s): ", g.zArgv0, g.zTestName); + }else{ + fprintf(stderr, "%s: ", g.zArgv0); + } va_start(ap, zFormat); vfprintf(stderr, zFormat, ap); va_end(ap); fprintf(stderr, "\n"); abort(); @@ -93,11 +146,15 @@ ** Print an error message and quit, but not in a way that would look ** like a crash. */ static void fatalError(const char *zFormat, ...){ va_list ap; - fprintf(stderr, "%s: ", g.zArgv0); + if( g.zTestName[0] ){ + fprintf(stderr, "%s (%s): ", g.zArgv0, g.zTestName); + }else{ + fprintf(stderr, "%s: ", g.zArgv0); + } va_start(ap, zFormat); vfprintf(stderr, zFormat, ap); va_end(ap); fprintf(stderr, "\n"); exit(1); @@ -122,10 +179,14 @@ /* ** This callback is invoked by sqlite3_log(). */ static void shellLog(void *pNotUsed, int iErrCode, const char *zMsg){ printf("LOG: (%d) %s\n", iErrCode, zMsg); + fflush(stdout); +} +static void shellLogNoop(void *pNotUsed, int iErrCode, const char *zMsg){ + return; } /* ** This callback is invoked by sqlite3_exec() to return query results. */ @@ -139,10 +200,11 @@ printf("[%s]\n", argv[i]); }else{ printf("NULL\n"); } } + fflush(stdout); return 0; } static int execNoop(void *NotUsed, int argc, char **argv, char **colv){ return 0; } @@ -152,10 +214,14 @@ ** This callback is invoked by sqlite3_trace() as each SQL statement ** starts. */ static void traceCallback(void *NotUsed, const char *zMsg){ printf("TRACE: %s\n", zMsg); + fflush(stdout); +} +static void traceNoop(void *NotUsed, const char *zMsg){ + return; } #endif /*************************************************************************** ** eval() implementation copied from ../ext/misc/eval.c @@ -247,20 +313,20 @@ /* ** Print sketchy documentation for this utility program */ static void showHelp(void){ - printf("Usage: %s [options]\n", g.zArgv0); + printf("Usage: %s [options] ?FILE...?\n", g.zArgv0); printf( -"Read SQL text from standard input and evaluate it.\n" +"Read SQL text from FILE... (or from standard input if FILE... is omitted)\n" +"and then evaluate each block of SQL contained therein.\n" "Options:\n" " --autovacuum Enable AUTOVACUUM mode\n" -" -f FILE Read SQL text from FILE instead of standard input\n" " --heap SZ MIN Memory allocator uses SZ bytes & min allocation MIN\n" " --help Show this help text\n" -" --initdb DBFILE Initialize the in-memory database using template DBFILE\n" " --lookaside N SZ Configure lookaside for N slots of SZ bytes each\n" +" --oom Run each test multiple times in a simulated OOM loop\n" " --pagesize N Set the page size to N\n" " --pcache N SZ Configure N pages of pagecache each of size SZ bytes\n" " -q Reduced output\n" " --quiet Reduced output\n" " --scratch N SZ Configure scratch memory for N slots of SZ bytes each\n" @@ -328,32 +394,35 @@ } if( v>0x7fffffff ) abendError("parameter too large - max 2147483648"); return (int)(isNeg? -v : v); } -/* -** Various operating modes -*/ -#define FZMODE_Generic 1 -#define FZMODE_Strftime 2 -#define FZMODE_Printf 3 -#define FZMODE_Glob 4 - +/* Return the current wall-clock time */ +static sqlite3_int64 timeOfDay(void){ + static sqlite3_vfs *clockVfs = 0; + sqlite3_int64 t; + if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); + if( clockVfs->iVersion>=1 && clockVfs->xCurrentTimeInt64!=0 ){ + clockVfs->xCurrentTimeInt64(clockVfs, &t); + }else{ + double r; + clockVfs->xCurrentTime(clockVfs, &r); + t = (sqlite3_int64)(r*86400000.0); + } + return t; +} int main(int argc, char **argv){ - char *zIn = 0; /* Input text */ - int nAlloc = 0; /* Number of bytes allocated for zIn[] */ - int nIn = 0; /* Number of bytes of zIn[] used */ - size_t got; /* Bytes read from input */ - FILE *in = stdin; /* Where to read SQL text from */ - int rc = SQLITE_OK; /* Result codes from API functions */ - int i; /* Loop counter */ - int iNext; /* Next block of SQL */ - sqlite3 *db; /* Open database */ - sqlite3 *dbInit = 0; /* On-disk database used to initialize the in-memory db */ - const char *zInitDb = 0;/* Name of the initialization database file */ - char *zErrMsg = 0; /* Error message returned from sqlite3_exec() */ + char *zIn = 0; /* Input text */ + int nAlloc = 0; /* Number of bytes allocated for zIn[] */ + int nIn = 0; /* Number of bytes of zIn[] used */ + size_t got; /* Bytes read from input */ + int rc = SQLITE_OK; /* Result codes from API functions */ + int i; /* Loop counter */ + int iNext; /* Next block of SQL */ + sqlite3 *db; /* Open database */ + char *zErrMsg = 0; /* Error message returned from sqlite3_exec() */ const char *zEncoding = 0; /* --utf16be or --utf16le */ int nHeap = 0, mnHeap = 0; /* Heap size from --heap */ int nLook = 0, szLook = 0; /* --lookaside configuration */ int nPCache = 0, szPCache = 0;/* --pcache configuration */ int nScratch = 0, szScratch=0;/* --scratch configuration */ @@ -363,36 +432,45 @@ void *pPCache = 0; /* Allocated storage for pcache */ void *pScratch = 0; /* Allocated storage for scratch */ int doAutovac = 0; /* True for --autovacuum */ char *zSql; /* SQL to run */ char *zToFree = 0; /* Call sqlite3_free() on this afte running zSql */ - int iMode = FZMODE_Generic; /* Operating mode */ - const char *zCkGlob = 0; /* Inputs must match this glob */ int verboseFlag = 0; /* --verbose or -v flag */ int quietFlag = 0; /* --quiet or -q flag */ int nTest = 0; /* Number of test cases run */ int multiTest = 0; /* True if there will be multiple test cases */ int lastPct = -1; /* Previous percentage done output */ sqlite3 *dataDb = 0; /* Database holding compacted input data */ sqlite3_stmt *pStmt = 0; /* Statement to insert testcase into dataDb */ const char *zDataOut = 0; /* Write compacted data to this output file */ int nHeader = 0; /* Bytes of header comment text on input file */ - + int oomFlag = 0; /* --oom */ + int oomCnt = 0; /* Counter for the OOM loop */ + char zErrBuf[200]; /* Space for the error message */ + const char *zFailCode; /* Value of the TEST_FAILURE environment var */ + const char *zPrompt; /* Initial prompt when large-file fuzzing */ + int nInFile = 0; /* Number of input files to read */ + char **azInFile = 0; /* Array of input file names */ + int jj; /* Loop counter for azInFile[] */ + sqlite3_int64 iBegin; /* Start time for the whole program */ + sqlite3_int64 iStart, iEnd; /* Start and end-times for a test case */ + iBegin = timeOfDay(); + zFailCode = getenv("TEST_FAILURE"); g.zArgv0 = argv[0]; + zPrompt = ""; for(i=1; i=argc-2 ) abendError("missing arguments on %s\n", argv[i]); nHeap = integerValue(argv[i+1]); mnHeap = integerValue(argv[i+2]); @@ -400,38 +478,18 @@ }else if( strcmp(z,"help")==0 ){ showHelp(); return 0; }else - if( strcmp(z, "initdb")==0 && i+1=argc-2 ) abendError("missing arguments on %s", argv[i]); nLook = integerValue(argv[i+1]); szLook = integerValue(argv[i+2]); i += 2; }else - if( strcmp(z,"mode")==0 ){ - if( i>=argc-1 ) abendError("missing argument on %s", argv[i]); - z = argv[++i]; - if( strcmp(z,"generic")==0 ){ - iMode = FZMODE_Printf; - zCkGlob = 0; - }else if( strcmp(z, "glob")==0 ){ - iMode = FZMODE_Glob; - zCkGlob = "'*','*'"; - }else if( strcmp(z, "printf")==0 ){ - iMode = FZMODE_Printf; - zCkGlob = "'*',*"; - }else if( strcmp(z, "strftime")==0 ){ - iMode = FZMODE_Strftime; - zCkGlob = "'*',*"; - }else{ - abendError("unknown --mode: %s", z); - } + if( strcmp(z,"oom")==0 ){ + oomFlag = 1; }else if( strcmp(z,"pagesize")==0 ){ if( i>=argc-1 ) abendError("missing argument on %s", argv[i]); pageSize = integerValue(argv[++i]); }else @@ -468,20 +526,33 @@ }else { abendError("unknown option: %s", argv[i]); } }else{ - abendError("unknown argument: %s", argv[i]); + addNewInFile: + nInFile++; + azInFile = realloc(azInFile, sizeof(azInFile[0])*nInFile); + if( azInFile==0 ) abendError("out of memory"); + azInFile[nInFile-1] = argv[i]; } } - if( verboseFlag ) sqlite3_config(SQLITE_CONFIG_LOG, shellLog, 0); + + /* Do global SQLite initialization */ + sqlite3_config(SQLITE_CONFIG_LOG, verboseFlag ? shellLog : shellLogNoop, 0); if( nHeap>0 ){ pHeap = malloc( nHeap ); if( pHeap==0 ) fatalError("cannot allocate %d-byte heap\n", nHeap); rc = sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nHeap, mnHeap); if( rc ) abendError("heap configuration failed: %d\n", rc); } + if( oomFlag ){ + sqlite3_config(SQLITE_CONFIG_GETMALLOC, &g.sOrigMem); + g.sOomMem = g.sOrigMem; + g.sOomMem.xMalloc = oomMalloc; + g.sOomMem.xRealloc = oomRealloc; + sqlite3_config(SQLITE_CONFIG_MALLOC, &g.sOomMem); + } if( nLook>0 ){ sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0); if( szLook>0 ){ pLook = malloc( nLook*szLook ); if( pLook==0 ) fatalError("out of memory"); @@ -499,144 +570,229 @@ if( pPCache==0 ) fatalError("cannot allocate %lld-byte pcache", nPCache*(sqlite3_int64)szPCache); rc = sqlite3_config(SQLITE_CONFIG_PAGECACHE, pPCache, szPCache, nPCache); if( rc ) abendError("pcache configuration failed: %d", rc); } - while( !feof(in) ){ - nAlloc += nAlloc+1000; - zIn = realloc(zIn, nAlloc); - if( zIn==0 ) fatalError("out of memory"); - got = fread(zIn+nIn, 1, nAlloc-nIn-1, in); - nIn += (int)got; - zIn[nIn] = 0; - if( got==0 ) break; - } - if( in!=stdin ) fclose(in); + + /* If the --unique-cases option was supplied, open the database that will + ** be used to gather unique test cases. + */ if( zDataOut ){ rc = sqlite3_open(":memory:", &dataDb); if( rc ) abendError("cannot open :memory: database"); rc = sqlite3_exec(dataDb, - "CREATE TABLE testcase(sql BLOB PRIMARY KEY) WITHOUT ROWID;",0,0,0); - if( rc ) abendError("%s", sqlite3_errmsg(dataDb)); - rc = sqlite3_prepare_v2(dataDb, "INSERT OR IGNORE INTO testcase(sql)VALUES(?1)", - -1, &pStmt, 0); - if( rc ) abendError("%s", sqlite3_errmsg(dataDb)); - } - if( zInitDb ){ - rc = sqlite3_open_v2(zInitDb, &dbInit, SQLITE_OPEN_READONLY, 0); - if( rc!=SQLITE_OK ){ - abendError("unable to open initialization database \"%s\"", zInitDb); - } - } - for(i=0; i****/"); - if( z ){ - z += 6; - if( verboseFlag ) printf("%.*s\n", (int)(z-&zIn[i]), &zIn[i]); - i += (int)(z-&zIn[i]); - multiTest = 1; - } - } - for(iNext=i; iNext0 ){ - abendError("memory in use after close: %lld bytes", sqlite3_memory_used()); - } - if( nTest==1 ){ - /* Simulate an error if the TEST_FAILURE environment variable is "5" */ - char *zFailCode = getenv("TEST_FAILURE"); + "CREATE TABLE testcase(sql BLOB PRIMARY KEY, tm) WITHOUT ROWID;",0,0,0); + if( rc ) abendError("%s", sqlite3_errmsg(dataDb)); + rc = sqlite3_prepare_v2(dataDb, + "INSERT OR IGNORE INTO testcase(sql,tm)VALUES(?1,?2)", + -1, &pStmt, 0); + if( rc ) abendError("%s", sqlite3_errmsg(dataDb)); + } + + /* Initialize the input buffer used to hold SQL text */ + if( nInFile==0 ) nInFile = 1; + nAlloc = 1000; + zIn = malloc(nAlloc); + if( zIn==0 ) fatalError("out of memory"); + + /* Loop over all input files */ + for(jj=0; jj****/"); + if( z ){ + z += 6; + sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "%.*s", + (int)(z-&zIn[i]) - 12, &zIn[i+6]); + if( verboseFlag ){ + printf("%.*s\n", (int)(z-&zIn[i]), &zIn[i]); + fflush(stdout); + } + i += (int)(z-&zIn[i]); + multiTest = 1; + } + } + for(iNext=i; iNext1 ){ + printf("%s\n", zPrompt); + } + fflush(stdout); + + /* Run the next test case. Run it multiple times in --oom mode + */ + if( oomFlag ){ + oomCnt = g.iOomCntdown = 1; + g.nOomFault = 0; + g.bOomOnce = 1; + if( verboseFlag ){ + printf("Once.%d\n", oomCnt); + fflush(stdout); + } + }else{ + oomCnt = 0; + } + do{ + rc = sqlite3_open_v2( + "main.db", &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY, + 0); + if( rc!=SQLITE_OK ){ + abendError("Unable to open the in-memory database"); + } + if( pLook ){ + rc = sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE,pLook,szLook,nLook); + if( rc!=SQLITE_OK ) abendError("lookaside configuration filed: %d", rc); + } + #ifndef SQLITE_OMIT_TRACE + sqlite3_trace(db, verboseFlag ? traceCallback : traceNoop, 0); + #endif + sqlite3_create_function(db, "eval", 1, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0); + sqlite3_create_function(db, "eval", 2, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0); + sqlite3_limit(db, SQLITE_LIMIT_LENGTH, 1000000); + if( zEncoding ) sqlexec(db, "PRAGMA encoding=%s", zEncoding); + if( pageSize ) sqlexec(db, "PRAGMA pagesize=%d", pageSize); + if( doAutovac ) sqlexec(db, "PRAGMA auto_vacuum=FULL"); + iStart = timeOfDay(); + g.bOomEnable = 1; + if( verboseFlag ){ + zErrMsg = 0; + rc = sqlite3_exec(db, zSql, execCallback, 0, &zErrMsg); + if( zErrMsg ){ + sqlite3_snprintf(sizeof(zErrBuf),zErrBuf,"%z", zErrMsg); + zErrMsg = 0; + } + }else { + rc = sqlite3_exec(db, zSql, execNoop, 0, 0); + } + g.bOomEnable = 0; + iEnd = timeOfDay(); + rc = sqlite3_close(db); + if( rc ){ + abendError("sqlite3_close() failed with rc=%d", rc); + } + if( !zDataOut && sqlite3_memory_used()>0 ){ + abendError("memory in use after close: %lld bytes",sqlite3_memory_used()); + } + if( oomFlag ){ + /* Limit the number of iterations of the OOM loop to OOM_MAX. If the + ** first pass (single failure) exceeds 2/3rds of OOM_MAX this skip the + ** second pass (continuous failure after first) completely. */ + if( g.nOomFault==0 || oomCnt>OOM_MAX ){ + if( g.bOomOnce && oomCnt<=(OOM_MAX*2/3) ){ + oomCnt = g.iOomCntdown = 1; + g.bOomOnce = 0; + }else{ + oomCnt = 0; + } + }else{ + g.iOomCntdown = ++oomCnt; + g.nOomFault = 0; + } + if( oomCnt ){ + if( verboseFlag ){ + printf("%s.%d\n", g.bOomOnce ? "Once" : "Multi", oomCnt); + fflush(stdout); + } + nTest++; + } + } + }while( oomCnt>0 ); + + /* Store unique test cases in the in the dataDb database if the + ** --unique-cases flag is present + */ + if( zDataOut ){ + sqlite3_bind_blob(pStmt, 1, &zIn[i], iNext-i, SQLITE_STATIC); + sqlite3_bind_int64(pStmt, 2, iEnd - iStart); + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_DONE ) abendError("%s", sqlite3_errmsg(dataDb)); + sqlite3_reset(pStmt); + } + + /* Free the SQL from the current test case + */ + if( zToFree ){ + sqlite3_free(zToFree); + zToFree = 0; + } + zIn[iNext] = cSaved; + + /* Show test-case results in --verbose mode + */ + if( verboseFlag ){ + printf("RESULT-CODE: %d\n", rc); + if( zErrMsg ){ + printf("ERROR-MSG: [%s]\n", zErrBuf); + } + fflush(stdout); + } + + /* Simulate an error if the TEST_FAILURE environment variable is "5". + ** This is used to verify that automated test script really do spot + ** errors that occur in this test program. + */ if( zFailCode ){ if( zFailCode[0]=='5' && zFailCode[1]==0 ){ abendError("simulated failure"); }else if( zFailCode[0]!=0 ){ /* If TEST_FAILURE is something other than 5, just exit the test @@ -644,34 +800,47 @@ printf("\nExit early due to TEST_FAILURE being set"); break; } } } + if( !verboseFlag && multiTest && !quietFlag && !oomFlag ) printf("\n"); } - if( !verboseFlag && multiTest && !quietFlag ) printf("\n"); + + /* Report total number of tests run + */ if( nTest>1 && !quietFlag ){ - printf("%d fuzz tests with no errors\nSQLite %s %s\n", - nTest, sqlite3_libversion(), sqlite3_sourceid()); + sqlite3_int64 iElapse = timeOfDay() - iBegin; + printf("%s: 0 errors out of %d tests in %d.%03d seconds\nSQLite %s %s\n", + g.zArgv0, nTest, (int)(iElapse/1000), (int)(iElapse%1000), + sqlite3_libversion(), sqlite3_sourceid()); } + + /* Write the unique test cases if the --unique-cases flag was used + */ if( zDataOut ){ int n = 0; FILE *out = fopen(zDataOut, "wb"); if( out==0 ) abendError("cannot open %s for writing", zDataOut); if( nHeader>0 ) fwrite(zIn, nHeader, 1, out); sqlite3_finalize(pStmt); - rc = sqlite3_prepare_v2(dataDb, "SELECT sql FROM testcase", -1, &pStmt, 0); + rc = sqlite3_prepare_v2(dataDb, "SELECT sql, tm FROM testcase ORDER BY tm, sql", + -1, &pStmt, 0); if( rc ) abendError("%s", sqlite3_errmsg(dataDb)); while( sqlite3_step(pStmt)==SQLITE_ROW ){ - fprintf(out,"/****<%d>****/", ++n); + fprintf(out,"/****<%d:%dms>****/", ++n, sqlite3_column_int(pStmt,1)); fwrite(sqlite3_column_blob(pStmt,0),sqlite3_column_bytes(pStmt,0),1,out); } fclose(out); sqlite3_finalize(pStmt); sqlite3_close(dataDb); } + + /* Clean up and exit. + */ + free(azInFile); free(zIn); free(pHeap); free(pLook); free(pScratch); free(pPCache); return 0; } Index: tool/mksqlite3c.tcl ================================================================== --- tool/mksqlite3c.tcl +++ tool/mksqlite3c.tcl @@ -15,28 +15,30 @@ # "keywordhash.h" files is generated by a program named "mkkeywordhash". # # After the "tsrc" directory has been created and populated, run # this script: # -# tclsh mksqlite3c.tcl +# tclsh mksqlite3c.tcl --srcdir $SRC # # The amalgamated SQLite code will be written into sqlite3.c # # Begin by reading the "sqlite3.h" header file. Extract the version number # from in this file. The version number is needed to generate the header # comment of the amalgamation. # -if {[lsearch $argv --nostatic]>=0} { - set addstatic 0 -} else { - set addstatic 1 -} -if {[lsearch $argv --linemacros]>=0} { - set linemacros 1 -} else { - set linemacros 0 +set addstatic 1 +set linemacros 0 +for {set i 0} {$i<[llength $argv]} {incr i} { + set x [lindex $argv $i] + if {[regexp {^-+nostatic$} $x]} { + set addstatic 0 + } elseif {[regexp {^-+linemacros} $x]} { + set linemacros 1 + } else { + error "unknown command-line option: $x" + } } set in [open tsrc/sqlite3.h] set cnt 0 set VERSION ????? while {![eof $in]} { @@ -380,10 +382,11 @@ fts5_vocab.c rtree.c icu.c fts3_icu.c + dbstat.c } { copy_file tsrc/$file } close $out Index: tool/spaceanal.tcl ================================================================== --- tool/spaceanal.tcl +++ tool/spaceanal.tcl @@ -86,11 +86,10 @@ # if {[catch {sqlite3 db $file_to_analyze -uri 1} msg]} { puts stderr "error trying to open $file_to_analyze: $msg" exit 1 } -register_dbstat_vtab db db eval {SELECT count(*) FROM sqlite_master} set pageSize [expr {wide([db one {PRAGMA page_size}])}] if {$flags(-pageinfo)} { Index: tool/sqldiff.c ================================================================== --- tool/sqldiff.c +++ tool/sqldiff.c @@ -495,11 +495,11 @@ char *zId = safeId(zTab); /* Name of table (translated for us in SQL) */ char **az = 0; /* Columns in main */ char **az2 = 0; /* Columns in aux */ int nPk; /* Primary key columns in main */ int nPk2; /* Primary key columns in aux */ - int n; /* Number of columns in main */ + int n = 0; /* Number of columns in main */ int n2; /* Number of columns in aux */ int nQ; /* Number of output columns in the diff query */ int i; /* Loop counter */ const char *zSep; /* Separator string */ Str sql; /* Comparison query */ @@ -732,11 +732,11 @@ char *zId = safeId(zTab); /* Name of table (translated for us in SQL) */ char **az = 0; /* Columns in main */ char **az2 = 0; /* Columns in aux */ int nPk; /* Primary key columns in main */ int nPk2; /* Primary key columns in aux */ - int n; /* Number of columns in main */ + int n = 0; /* Number of columns in main */ int n2; /* Number of columns in aux */ int i; /* Loop counter */ const char *zSep; /* Separator string */ Str sql; /* Comparison query */ sqlite3_stmt *pStmt; /* Query statement to do the diff */ @@ -895,23 +895,23 @@ iX = sqlite3_value_int64(pVal); memcpy(&uX, &iX, 8); for(j=56; j>=0; j-=8) putc((uX>>j)&0xff, out); break; case SQLITE_FLOAT: - rX = sqlite3_value_int64(pVal); + rX = sqlite3_value_double(pVal); memcpy(&uX, &rX, 8); for(j=56; j>=0; j-=8) putc((uX>>j)&0xff, out); break; case SQLITE_TEXT: iX = sqlite3_value_bytes(pVal); putsVarint(out, (sqlite3_uint64)iX); - fwrite(sqlite3_value_text(pVal),1,iX,out); + fwrite(sqlite3_value_text(pVal),1,(size_t)iX,out); break; case SQLITE_BLOB: iX = sqlite3_value_bytes(pVal); putsVarint(out, (sqlite3_uint64)iX); - fwrite(sqlite3_value_blob(pVal),1,iX,out); + fwrite(sqlite3_value_blob(pVal),1,(size_t)iX,out); break; case SQLITE_NULL: break; } } @@ -1114,10 +1114,11 @@ printf("Usage: %s [options] DB1 DB2\n", g.zArgv0); printf( "Output SQL text that would transform DB1 into DB2.\n" "Options:\n" " --changeset FILE Write a CHANGESET into FILE\n" +" -L|--lib LIBRARY Load an SQLite extension library\n" " --primarykey Use schema-defined PRIMARY KEYs\n" " --schema Show only differences in the schema\n" " --summary Show only a summary of the differences\n" " --table TAB Show only differences in table TAB\n" ); @@ -1132,29 +1133,42 @@ char *zSql; sqlite3_stmt *pStmt; char *zTab = 0; FILE *out = stdout; void (*xDiff)(const char*,FILE*) = diff_one_table; + int nExt = 0; + char **azExt = 0; g.zArgv0 = argv[0]; + sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); for(i=1; i