Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -8,10 +8,27 @@ TOP = . # Set this non-0 to create and use the SQLite amalgamation file. # USE_AMALGAMATION = 1 + +# Set this to non-0 to create and use PDBs. +# +SYMBOLS = 1 + +# Set this to one of the following values to enable various debugging +# features. Each level includes the debugging options from the previous +# levels. Currently, the recognized values for DEBUG are: +# +# 0 == NDEBUG: Disables assert() and other runtime diagnostics. +# 1 == Disables NDEBUG and all optimizations and then enables PDBs. +# 2 == SQLITE_DEBUG: Enables various diagnostics messages and code. +# 3 == SQLITE_WIN32_MALLOC_VALIDATE: Validate the Win32 native heap per call. +# 4 == SQLITE_DEBUG_OS_TRACE: Enables output from the OSTRACE() macros. +# 5 == SQLITE_ENABLE_IOTRACE: Enables output from the IOTRACE() macros. +# +DEBUG = 0 # Version numbers and release number for the SQLite being compiled. # VERSION = 3.7 VERSION_NUMBER = 3007007 @@ -18,17 +35,17 @@ RELEASE = 3.7.7 # C Compiler and options for use in building executables that # will run on the platform that is doing the build. # -BCC = cl.exe -O2 +BCC = cl.exe # C Compile and options for use in building executables that # will run on the target platform. (BCC and TCC are usually the # same unless your are cross-compiling.) # -TCC = cl.exe -W3 -O2 -DSQLITE_OS_WIN=1 -I. -I$(TOP)\src -fp:precise +TCC = cl.exe -W3 -DSQLITE_OS_WIN=1 -I. -I$(TOP)\src -fp:precise # The mksqlite3c.tcl and mksqlite3h.tcl scripts will pull in # any extension header files by default. For non-amalgamation # builds, we need to make sure the compiler can find these. # @@ -39,17 +56,43 @@ # Define -DNDEBUG to compile without debugging (i.e., for production usage) # Omitting the define will cause extra debugging code to be inserted and # includes extra comments when "EXPLAIN stmt" is used. # +!IF $(DEBUG)==0 TCC = $(TCC) -DNDEBUG +!ENDIF + +!IF $(DEBUG)>1 +TCC = $(TCC) -DSQLITE_DEBUG +!ENDIF + +!IF $(DEBUG)>3 +TCC = $(TCC) -DSQLITE_DEBUG_OS_TRACE=1 +!ENDIF + +!IF $(DEBUG)>4 +TCC = $(TCC) -DSQLITE_ENABLE_IOTRACE +!ENDIF # # Prevent warnings about "insecure" runtime library functions being used. # TCC = $(TCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS +# +# Use native Win32 heap instead of malloc/free? +# +# TCC = $(TCC) -DSQLITE_WIN32_MALLOC=1 + +# +# Validate the heap on every call into the native Win32 heap subsystem? +# +!IF $(DEBUG)>2 +TCC = $(TCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 +!ENDIF + # The locations of the Tcl header and library files. Also, the library that # non-stubs enabled programs using Tcl must link against. These variables # (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment # prior to running nmake in order to match the actual installed location and # version on this machine. @@ -121,10 +164,22 @@ TCC = $(TCC) $(OPT_FEATURE_FLAGS) # Add in any optional parameters specified on the make commane line # ie. make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1". TCC = $(TCC) $(OPTS) + +# If symbols are enabled, enable PDBs. +# If debugging is enabled, disable all optimizations and enable PDBs. +!IF $(DEBUG)>0 +TCC = $(TCC) -Od -D_DEBUG +!ELSE +TCC = $(TCC) -O2 +!ENDIF + +!IF $(DEBUG)>0 || $(SYMBOLS)!=0 +TCC = $(TCC) -Zi +!ENDIF # libtool compile/link LTCOMPILE = $(TCC) -Fo$@ LTLIB = lib.exe LTLINK = $(TCC) -Fe$@ @@ -135,10 +190,15 @@ # to deduce the binary type based on the object files. !IF "$(PLATFORM)"!="" LTLINKOPTS = /MACHINE:$(PLATFORM) LTLIBOPTS = /MACHINE:$(PLATFORM) !ENDIF + +# If debugging is enabled, enable PDBs. +!IF $(DEBUG)>0 || $(SYMBOLS)!=0 +LTLINKOPTS = $(LTLINKOPTS) /DEBUG +!ENDIF # nawk compatible awk. NAWK = gawk.exe # You should not have to change anything below this line Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -386,11 +386,11 @@ echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c cat sqlite3.c >>tclsqlite3.c echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c -sqlite3-debug.c: target_source $(TOP)/tool/mksqlite3c.tcl +sqlite3.c-debug: target_source $(TOP)/tool/mksqlite3c.tcl tclsh $(TOP)/tool/mksqlite3c.tcl --linemacros echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c cat sqlite3.c >>tclsqlite3.c echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c echo '#line 1 "tclsqlite.c"' >>tclsqlite3.c Index: src/backup.c ================================================================== --- src/backup.c +++ src/backup.c @@ -408,115 +408,110 @@ /* Update the schema version field in the destination database. This ** is to make sure that the schema-version really does change in ** the case where the source and destination databases have the ** same schema version. */ - if( rc==SQLITE_DONE - && (rc = sqlite3BtreeUpdateMeta(p->pDest,1,p->iDestSchema+1))==SQLITE_OK - ){ - int nDestTruncate; - - if( p->pDestDb ){ - sqlite3ResetInternalSchema(p->pDestDb, -1); - } - - if( destMode==PAGER_JOURNALMODE_WAL ){ - /* This call cannot fail. The success of the BtreeUpdateMeta() - ** method above indicates that a write transaction has been opened - ** and page 1 is already dirty. Therefore this always succeeds. - */ - TESTONLY(int rc2 =) sqlite3BtreeSetVersion(p->pDest, 2); - assert( rc2==SQLITE_OK ); + if( rc==SQLITE_DONE ){ + rc = sqlite3BtreeUpdateMeta(p->pDest,1,p->iDestSchema+1); + if( rc==SQLITE_OK ){ + if( p->pDestDb ){ + sqlite3ResetInternalSchema(p->pDestDb, -1); + } + if( destMode==PAGER_JOURNALMODE_WAL ){ + rc = sqlite3BtreeSetVersion(p->pDest, 2); + } } - - /* Set nDestTruncate to the final number of pages in the destination - ** database. The complication here is that the destination page - ** size may be different to the source page size. - ** - ** If the source page size is smaller than the destination page size, - ** round up. In this case the call to sqlite3OsTruncate() below will - ** fix the size of the file. However it is important to call - ** sqlite3PagerTruncateImage() here so that any pages in the - ** destination file that lie beyond the nDestTruncate page mark are - ** journalled by PagerCommitPhaseOne() before they are destroyed - ** by the file truncation. - */ - assert( pgszSrc==sqlite3BtreeGetPageSize(p->pSrc) ); - assert( pgszDest==sqlite3BtreeGetPageSize(p->pDest) ); - if( pgszSrcpDest->pBt) ){ - nDestTruncate--; - } - }else{ - nDestTruncate = nSrcPage * (pgszSrc/pgszDest); - } - sqlite3PagerTruncateImage(pDestPager, nDestTruncate); - - if( pgszSrc= iSize || ( - nDestTruncate==(int)(PENDING_BYTE_PAGE(p->pDest->pBt)-1) - && iSize>=PENDING_BYTE && iSize<=PENDING_BYTE+pgszDest - )); - - /* This call ensures that all data required to recreate the original - ** database has been stored in the journal for pDestPager and the - ** journal synced to disk. So at this point we may safely modify - ** the database file in any way, knowing that if a power failure - ** occurs, the original database will be reconstructed from the - ** journal file. */ - rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); - - /* Write the extra pages and truncate the database file as required. */ - iEnd = MIN(PENDING_BYTE + pgszDest, iSize); - for( - iOff=PENDING_BYTE+pgszSrc; - rc==SQLITE_OK && iOffpSrc) ); + assert( pgszDest==sqlite3BtreeGetPageSize(p->pDest) ); + if( pgszSrcpDest->pBt) ){ + nDestTruncate--; + } + }else{ + nDestTruncate = nSrcPage * (pgszSrc/pgszDest); + } + sqlite3PagerTruncateImage(pDestPager, nDestTruncate); + + if( pgszSrc= iSize || ( + nDestTruncate==(int)(PENDING_BYTE_PAGE(p->pDest->pBt)-1) + && iSize>=PENDING_BYTE && iSize<=PENDING_BYTE+pgszDest + )); + + /* This call ensures that all data required to recreate the original + ** database has been stored in the journal for pDestPager and the + ** journal synced to disk. So at this point we may safely modify + ** the database file in any way, knowing that if a power failure + ** occurs, the original database will be reconstructed from the + ** journal file. */ + rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); + + /* Write the extra pages and truncate the database file as required */ + iEnd = MIN(PENDING_BYTE + pgszDest, iSize); + for( + iOff=PENDING_BYTE+pgszSrc; + rc==SQLITE_OK && iOffpDest, 0)) + ){ + rc = SQLITE_DONE; } - - /* Sync the database file to disk. */ - if( rc==SQLITE_OK ){ - rc = sqlite3PagerSync(pDestPager); - } - }else{ - rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 0); - } - - /* Finish committing the transaction to the destination database. */ - if( SQLITE_OK==rc - && SQLITE_OK==(rc = sqlite3BtreeCommitPhaseTwo(p->pDest, 0)) - ){ - rc = SQLITE_DONE; } } /* If bCloseTrans is true, then this function opened a read transaction ** on the source database. Close the read transaction here. There is Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -1738,15 +1738,26 @@ assert( (flags & BTREE_UNORDERED)==0 || (flags & BTREE_SINGLE)!=0 ); /* A BTREE_SINGLE database is always a temporary and/or ephemeral */ assert( (flags & BTREE_SINGLE)==0 || isTempDb ); + /* The BTREE_SORTER flag is only used if SQLITE_OMIT_MERGE_SORT is undef */ +#ifdef SQLITE_OMIT_MERGE_SORT + assert( (flags & BTREE_SORTER)==0 ); +#endif + + /* BTREE_SORTER is always on a BTREE_SINGLE, BTREE_OMIT_JOURNAL */ + assert( (flags & BTREE_SORTER)==0 || + (flags & (BTREE_SINGLE|BTREE_OMIT_JOURNAL)) + ==(BTREE_SINGLE|BTREE_OMIT_JOURNAL) ); + if( db->flags & SQLITE_NoReadlock ){ flags |= BTREE_NO_READLOCK; } if( isMemdb ){ flags |= BTREE_MEMORY; + flags &= ~BTREE_SORTER; } if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){ vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; } p = sqlite3MallocZero(sizeof(Btree)); @@ -8199,7 +8210,5 @@ } pBt->doNotUseWAL = 0; return rc; } - - Index: src/btree.h ================================================================== --- src/btree.h +++ src/btree.h @@ -59,10 +59,11 @@ #define BTREE_OMIT_JOURNAL 1 /* Do not create or use a rollback journal */ #define BTREE_NO_READLOCK 2 /* Omit readlocks on readonly files */ #define BTREE_MEMORY 4 /* This is an in-memory DB */ #define BTREE_SINGLE 8 /* The file contains at most 1 b-tree */ #define BTREE_UNORDERED 16 /* Use of a hash implementation is OK */ +#define BTREE_SORTER 32 /* Used as accumulator in external merge sort */ int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int); int sqlite3BtreeSyncDisabled(Btree*); Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -1672,11 +1672,11 @@ Table *p; int n; const char *z; Token sEnd; DbFixer sFix; - Token *pName; + Token *pName = 0; int iDb; sqlite3 *db = pParse->db; if( pParse->nVar>0 ){ sqlite3ErrorMsg(pParse, "parameters are not allowed in views"); @@ -2370,10 +2370,11 @@ /* Open the sorter cursor if we are to use one. */ if( bUseSorter ){ iSorter = pParse->nTab++; sqlite3VdbeAddOp4(v, OP_OpenSorter, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); + sqlite3VdbeChangeP5(v, BTREE_SORTER); } /* Open the table. Loop through all rows of the table, inserting index ** records into the sorter. */ sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -234,10 +234,20 @@ memcpy(&y, &x, 8); assert( sqlite3IsNaN(y) ); } #endif #endif + + /* Do extra initialization steps requested by the SQLITE_EXTRA_INIT + ** compile-time option. + */ +#ifdef SQLITE_EXTRA_INIT + if( rc==SQLITE_OK && sqlite3GlobalConfig.isInit ){ + int SQLITE_EXTRA_INIT(void); + rc = SQLITE_EXTRA_INIT(); + } +#endif return rc; } /* Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -258,11 +258,15 @@ ** Allowed values for the unixFile.ctrlFlags bitmask: */ #define UNIXFILE_EXCL 0x01 /* Connections from one process only */ #define UNIXFILE_RDONLY 0x02 /* Connection is read only */ #define UNIXFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ -#define UNIXFILE_DIRSYNC 0x08 /* Directory sync needed */ +#ifndef SQLITE_DISABLE_DIRSYNC +# define UNIXFILE_DIRSYNC 0x08 /* Directory sync needed */ +#else +# define UNIXFILE_DIRSYNC 0x00 +#endif /* ** Include code that is common to all os_*.c files */ #include "os_common.h" @@ -3737,10 +3741,12 @@ rc = SQLITE_IOERR_DIR_CLOSE; } #else robust_close(pFile, dirfd, __LINE__); #endif + }else if( rc==SQLITE_CANTOPEN ){ + rc = SQLITE_OK; } pFile->ctrlFlags &= ~UNIXFILE_DIRSYNC; } return rc; @@ -5898,10 +5904,12 @@ rc = SQLITE_IOERR_DIR_CLOSE; } #else robust_close(0, fd, __LINE__); #endif + }else if( rc==SQLITE_CANTOPEN ){ + rc = SQLITE_OK; } } #endif return rc; } Index: src/os_win.c ================================================================== --- src/os_win.c +++ src/os_win.c @@ -117,10 +117,80 @@ winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ #endif }; +/* + * If compiled with SQLITE_WIN32_MALLOC on Windows, we will use the + * various Win32 API heap functions instead of our own. + */ +#ifdef SQLITE_WIN32_MALLOC +/* + * The initial size of the Win32-specific heap. This value may be zero. + */ +#ifndef SQLITE_WIN32_HEAP_INIT_SIZE +# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_DEFAULT_CACHE_SIZE) * \ + (SQLITE_DEFAULT_PAGE_SIZE) + 4194304) +#endif + +/* + * The maximum size of the Win32-specific heap. This value may be zero. + */ +#ifndef SQLITE_WIN32_HEAP_MAX_SIZE +# define SQLITE_WIN32_HEAP_MAX_SIZE (0) +#endif + +/* + * The extra flags to use in calls to the Win32 heap APIs. This value may be + * zero for the default behavior. + */ +#ifndef SQLITE_WIN32_HEAP_FLAGS +# define SQLITE_WIN32_HEAP_FLAGS (0) +#endif + +/* +** The winMemData structure stores information required by the Win32-specific +** sqlite3_mem_methods implementation. +*/ +typedef struct winMemData winMemData; +struct winMemData { +#ifndef NDEBUG + u32 magic; /* Magic number to detect structure corruption. */ +#endif + HANDLE hHeap; /* The handle to our heap. */ + BOOL bOwned; /* Do we own the heap (i.e. destroy it on shutdown)? */ +}; + +#ifndef NDEBUG +#define WINMEM_MAGIC 0x42b2830b +#endif + +static struct winMemData win_mem_data = { +#ifndef NDEBUG + WINMEM_MAGIC, +#endif + NULL, FALSE +}; + +#ifndef NDEBUG +#define winMemAssertMagic() assert( win_mem_data.magic==WINMEM_MAGIC ) +#else +#define winMemAssertMagic() +#endif + +#define winMemGetHeap() win_mem_data.hHeap + +static void *winMemMalloc(int nBytes); +static void winMemFree(void *pPrior); +static void *winMemRealloc(void *pPrior, int nBytes); +static int winMemSize(void *p); +static int winMemRoundup(int n); +static int winMemInit(void *pAppData); +static void winMemShutdown(void *pAppData); + +const sqlite3_mem_methods *sqlite3MemGetWin32(void); +#endif /* SQLITE_WIN32_MALLOC */ /* ** Forward prototypes. */ static int getSectorSize( @@ -169,10 +239,192 @@ } return sqlite3_os_type==2; } #endif /* SQLITE_OS_WINCE */ +#ifdef SQLITE_WIN32_MALLOC +/* +** Allocate nBytes of memory. +*/ +static void *winMemMalloc(int nBytes){ + HANDLE hHeap; + void *p; + + winMemAssertMagic(); + hHeap = winMemGetHeap(); + assert( hHeap!=0 ); + assert( hHeap!=INVALID_HANDLE_VALUE ); +#ifdef SQLITE_WIN32_MALLOC_VALIDATE + assert ( HeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); +#endif + assert( nBytes>=0 ); + p = HeapAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, (SIZE_T)nBytes); + if( !p ){ + sqlite3_log(SQLITE_NOMEM, "failed to HeapAlloc %u bytes (%d), heap=%p", + nBytes, GetLastError(), (void*)hHeap); + } + return p; +} + +/* +** Free memory. +*/ +static void winMemFree(void *pPrior){ + HANDLE hHeap; + + winMemAssertMagic(); + hHeap = winMemGetHeap(); + assert( hHeap!=0 ); + assert( hHeap!=INVALID_HANDLE_VALUE ); +#ifdef SQLITE_WIN32_MALLOC_VALIDATE + assert ( HeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); +#endif + if( !pPrior ) return; /* Passing NULL to HeapFree is undefined. */ + if( !HeapFree(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ){ + sqlite3_log(SQLITE_NOMEM, "failed to HeapFree block %p (%d), heap=%p", + pPrior, GetLastError(), (void*)hHeap); + } +} + +/* +** Change the size of an existing memory allocation +*/ +static void *winMemRealloc(void *pPrior, int nBytes){ + HANDLE hHeap; + void *p; + + winMemAssertMagic(); + hHeap = winMemGetHeap(); + assert( hHeap!=0 ); + assert( hHeap!=INVALID_HANDLE_VALUE ); +#ifdef SQLITE_WIN32_MALLOC_VALIDATE + assert ( HeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); +#endif + assert( nBytes>=0 ); + if( !pPrior ){ + p = HeapAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, (SIZE_T)nBytes); + }else{ + p = HeapReAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior, (SIZE_T)nBytes); + } + if( !p ){ + sqlite3_log(SQLITE_NOMEM, "failed to %s %u bytes (%d), heap=%p", + pPrior ? "HeapReAlloc" : "HeapAlloc", nBytes, GetLastError(), + (void*)hHeap); + } + return p; +} + +/* +** Return the size of an outstanding allocation, in bytes. +*/ +static int winMemSize(void *p){ + HANDLE hHeap; + SIZE_T n; + + winMemAssertMagic(); + hHeap = winMemGetHeap(); + assert( hHeap!=0 ); + assert( hHeap!=INVALID_HANDLE_VALUE ); +#ifdef SQLITE_WIN32_MALLOC_VALIDATE + assert ( HeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); +#endif + if( !p ) return 0; + n = HeapSize(hHeap, SQLITE_WIN32_HEAP_FLAGS, p); + if( n==(SIZE_T)-1 ){ + sqlite3_log(SQLITE_NOMEM, "failed to HeapSize block %p (%d), heap=%p", + p, GetLastError(), (void*)hHeap); + return 0; + } + return (int)n; +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int winMemRoundup(int n){ + return n; +} + +/* +** Initialize this module. +*/ +static int winMemInit(void *pAppData){ + winMemData *pWinMemData = (winMemData *)pAppData; + + if( !pWinMemData ) return SQLITE_ERROR; + assert( pWinMemData->magic==WINMEM_MAGIC ); + if( !pWinMemData->hHeap ){ + pWinMemData->hHeap = HeapCreate(SQLITE_WIN32_HEAP_FLAGS, + SQLITE_WIN32_HEAP_INIT_SIZE, + SQLITE_WIN32_HEAP_MAX_SIZE); + if( !pWinMemData->hHeap ){ + sqlite3_log(SQLITE_NOMEM, + "failed to HeapCreate (%d), flags=%u, initSize=%u, maxSize=%u", + GetLastError(), SQLITE_WIN32_HEAP_FLAGS, SQLITE_WIN32_HEAP_INIT_SIZE, + SQLITE_WIN32_HEAP_MAX_SIZE); + return SQLITE_NOMEM; + } + pWinMemData->bOwned = TRUE; + } + assert( pWinMemData->hHeap!=0 ); + assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); +#ifdef SQLITE_WIN32_MALLOC_VALIDATE + assert( HeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); +#endif + return SQLITE_OK; +} + +/* +** Deinitialize this module. +*/ +static void winMemShutdown(void *pAppData){ + winMemData *pWinMemData = (winMemData *)pAppData; + + if( !pWinMemData ) return; + if( pWinMemData->hHeap ){ + assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); +#ifdef SQLITE_WIN32_MALLOC_VALIDATE + assert( HeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); +#endif + if( pWinMemData->bOwned ){ + if( !HeapDestroy(pWinMemData->hHeap) ){ + sqlite3_log(SQLITE_NOMEM, "failed to HeapDestroy (%d), heap=%p", + GetLastError(), (void*)pWinMemData->hHeap); + } + pWinMemData->bOwned = FALSE; + } + pWinMemData->hHeap = NULL; + } +} + +/* +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. The +** arguments specify the block of memory to manage. +** +** This routine is only called by sqlite3_config(), and therefore +** is not required to be threadsafe (it is not). +*/ +const sqlite3_mem_methods *sqlite3MemGetWin32(void){ + static const sqlite3_mem_methods winMemMethods = { + winMemMalloc, + winMemFree, + winMemRealloc, + winMemSize, + winMemRoundup, + winMemInit, + winMemShutdown, + &win_mem_data + }; + return &winMemMethods; +} + +void sqlite3MemSetDefault(void){ + sqlite3_config(SQLITE_CONFIG_MALLOC, sqlite3MemGetWin32()); +} +#endif /* SQLITE_WIN32_MALLOC */ + /* ** Convert a UTF-8 string to microsoft unicode (UTF-16?). ** ** Space to hold the returned string is obtained from malloc. */ @@ -1349,15 +1601,22 @@ case SQLITE_FCNTL_CHUNK_SIZE: { pFile->szChunk = *(int *)pArg; return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { - sqlite3_int64 sz = *(sqlite3_int64*)pArg; - SimulateIOErrorBenign(1); - winTruncate(id, sz); - SimulateIOErrorBenign(0); - return SQLITE_OK; + winFile *pFile = (winFile*)id; + sqlite3_int64 oldSz; + int rc = winFileSize(id, &oldSz); + if( rc==SQLITE_OK ){ + sqlite3_int64 newSz = *(sqlite3_int64*)pArg; + if( newSz>oldSz ){ + SimulateIOErrorBenign(1); + rc = winTruncate(id, newSz); + SimulateIOErrorBenign(0); + } + } + return rc; } case SQLITE_FCNTL_PERSIST_WAL: { int bPersist = *(int*)pArg; if( bPersist<0 ){ *(int*)pArg = pFile->bPersistWal; Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -618,10 +618,12 @@ u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */ u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ u8 tempFile; /* zFilename is a temporary file */ u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ + u8 hasSeenStress; /* pagerStress() called one or more times */ + u8 isSorter; /* True for a PAGER_SORTER */ /************************************************************************** ** The following block contains those class members that change during ** routine opertion. Class members not in this block are either fixed ** when the pager is first created or else only change when there is a @@ -840,10 +842,19 @@ || p->journalMode==PAGER_JOURNALMODE_MEMORY ); assert( p->eState!=PAGER_ERROR && p->eState!=PAGER_OPEN ); assert( pagerUseWal(p)==0 ); } + + /* A sorter is a temp file that never spills to disk and always has + ** the doNotSpill flag set + */ + if( p->isSorter ){ + assert( p->tempFile ); + assert( p->doNotSpill ); + assert( p->fd->pMethods==0 ); + } /* If changeCountDone is set, a RESERVED lock or greater must be held ** on the file. */ assert( pPager->changeCountDone==0 || pPager->eLock>=RESERVED_LOCK ); @@ -3749,10 +3760,11 @@ ** to the caller. */ int sqlite3PagerClose(Pager *pPager){ u8 *pTmp = (u8 *)pPager->pTmpSpace; + assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL @@ -4183,10 +4195,11 @@ ** is impossible for sqlite3PCacheFetch() to be called with createFlag==1 ** while in the error state, hence it is impossible for this routine to ** be called in the error state. Nevertheless, we include a NEVER() ** test for the error state as a safeguard against future changes. */ + pPager->hasSeenStress = 1; if( NEVER(pPager->errCode) ) return SQLITE_OK; if( pPager->doNotSpill ) return SQLITE_OK; if( pPager->doNotSyncSpill && (pPg->flags & PGHDR_NEED_SYNC)!=0 ){ return SQLITE_OK; } @@ -4558,10 +4571,16 @@ } /* pPager->xBusyHandler = 0; */ /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ +#ifndef SQLITE_OMIT_MERGE_SORT + if( flags & PAGER_SORTER ){ + pPager->doNotSpill = 1; + pPager->isSorter = 1; + } +#endif *ppPager = pPager; return SQLITE_OK; } @@ -6112,10 +6131,21 @@ ** Return true if this is an in-memory pager. */ int sqlite3PagerIsMemdb(Pager *pPager){ return MEMDB; } + +#ifndef SQLITE_OMIT_MERGE_SORT +/* +** Return true if the pager has seen a pagerStress callback. +*/ +int sqlite3PagerUnderStress(Pager *pPager){ + assert( pPager->isSorter ); + assert( pPager->doNotSpill ); + return pPager->hasSeenStress; +} +#endif /* ** Check that there are at least nSavepoint savepoints open. If there are ** currently less than nSavepoints open, then open one or more savepoints ** to make up the difference. If the number of savepoints is already Index: src/pager.h ================================================================== --- src/pager.h +++ src/pager.h @@ -58,10 +58,11 @@ ** NOTE: These values must match the corresponding BTREE_ values in btree.h. */ #define PAGER_OMIT_JOURNAL 0x0001 /* Do not use a rollback journal */ #define PAGER_NO_READLOCK 0x0002 /* Omit readlocks on readonly files */ #define PAGER_MEMORY 0x0004 /* In-memory database */ +#define PAGER_SORTER 0x0020 /* Accumulator in external merge sort */ /* ** Valid values for the second argument to sqlite3PagerLockingMode(). */ #define PAGER_LOCKINGMODE_QUERY -1 @@ -153,10 +154,13 @@ sqlite3_file *sqlite3PagerFile(Pager*); const char *sqlite3PagerJournalname(Pager*); int sqlite3PagerNosync(Pager*); void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); +#ifndef SQLITE_OMIT_MERGE_SORT +int sqlite3PagerUnderStress(Pager*); +#endif /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) Index: src/pcache1.c ================================================================== --- src/pcache1.c +++ src/pcache1.c @@ -54,11 +54,14 @@ int nMaxPage; /* Sum of nMax for purgeable caches */ int nMinPage; /* Sum of nMin for purgeable caches */ int mxPinned; /* nMaxpage + 10 - nMinPage */ int nCurrentPage; /* Number of purgeable pages allocated */ PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */ +#ifdef SQLITE_PAGECACHE_BLOCKALLOC + int isBusy; /* Do not run ReleaseMemory() if true */ PGroupBlockList *pBlockList; /* List of block-lists for this group */ +#endif }; /* ** If SQLITE_PAGECACHE_BLOCKALLOC is defined when the library is built, ** each PGroup structure has a linked list of the the following starting @@ -399,45 +402,44 @@ int i; nByte += sizeof(PGroupBlockList *); nByte = ROUND8(nByte); - do{ + for(pList=pGroup->pBlockList; pList; pList=pList->pNext){ + if( pList->nByte==nByte ) break; + } + if( pList==0 ){ + PGroupBlockList *pNew; + assert( pGroup->isBusy==0 ); + assert( sqlite3_mutex_held(pGroup->mutex) ); + pGroup->isBusy = 1; /* Disable sqlite3PcacheReleaseMemory() */ + pNew = (PGroupBlockList *)sqlite3MallocZero(sizeof(PGroupBlockList)); + pGroup->isBusy = 0; /* Reenable sqlite3PcacheReleaseMemory() */ + if( pNew==0 ){ + /* malloc() failure. Return early. */ + return 0; + } +#ifdef SQLITE_DEBUG for(pList=pGroup->pBlockList; pList; pList=pList->pNext){ - if( pList->nByte==nByte ) break; + assert( pList->nByte!=nByte ); } - if( pList==0 ){ - PGroupBlockList *pNew; - pcache1LeaveMutex(pCache->pGroup); - pNew = (PGroupBlockList *)sqlite3MallocZero(sizeof(PGroupBlockList)); - pcache1EnterMutex(pCache->pGroup); - if( pNew==0 ){ - /* malloc() failure. Return early. */ - return 0; - } - for(pList=pGroup->pBlockList; pList; pList=pList->pNext){ - if( pList->nByte==nByte ) break; - } - if( pList ){ - sqlite3_free(pNew); - }else{ - pNew->nByte = nByte; - pNew->pNext = pGroup->pBlockList; - pGroup->pBlockList = pNew; - pList = pNew; - } - } - }while( pList==0 ); +#endif + pNew->nByte = nByte; + pNew->pNext = pGroup->pBlockList; + pGroup->pBlockList = pNew; + pList = pNew; + } pBlock = pList->pFirst; if( pBlock==0 || pBlock->mUsed==(((Bitmask)1<nEntry)-1) ){ int sz; /* Allocate a new block. Try to allocate enough space for the PGroupBlock ** structure and MINENTRY allocations of nByte bytes each. If the ** allocator returns more memory than requested, then more than MINENTRY ** allocations may fit in it. */ + assert( sqlite3_mutex_held(pGroup->mutex) ); pcache1LeaveMutex(pCache->pGroup); sz = sizeof(PGroupBlock) + PAGECACHE_BLOCKALLOC_MINENTRY * nByte; pBlock = (PGroupBlock *)sqlite3Malloc(sz); pcache1EnterMutex(pCache->pGroup); @@ -479,28 +481,31 @@ pBlock->pPrev = pList->pLast; pBlock->pNext = 0; pList->pLast->pNext = pBlock; pList->pLast = pBlock; } + p = PAGE_TO_PGHDR1(pCache, pPg); + if( pCache->bPurgeable ){ + pCache->pGroup->nCurrentPage++; + } #else /* The group mutex must be released before pcache1Alloc() is called. This ** is because it may call sqlite3_release_memory(), which assumes that ** this mutex is not held. */ assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); pcache1LeaveMutex(pCache->pGroup); pPg = pcache1Alloc(nByte); pcache1EnterMutex(pCache->pGroup); -#endif - if( pPg ){ p = PAGE_TO_PGHDR1(pCache, pPg); if( pCache->bPurgeable ){ pCache->pGroup->nCurrentPage++; } }else{ p = 0; } +#endif return p; } /* ** Free a page object allocated by pcache1AllocPage(). @@ -1163,10 +1168,13 @@ ** been released, the function returns. The return value is the total number ** of bytes of memory released. */ int sqlite3PcacheReleaseMemory(int nReq){ int nFree = 0; +#ifdef SQLITE_PAGECACHE_BLOCKALLOC + if( pcache1.grp.isBusy ) return 0; +#endif assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); assert( sqlite3_mutex_notheld(pcache1.mutex) ); if( pcache1.pStart==0 ){ PgHdr1 *p; pcache1EnterMutex(&pcache1.grp); Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -1204,20 +1204,14 @@ ** also used during testing of SQLite in order to specify an alternative ** memory allocator that simulates memory out-of-memory conditions in ** order to verify that SQLite recovers gracefully from such ** conditions. ** -** The xMalloc and xFree methods must work like the -** malloc() and free() functions from the standard C library. -** The xRealloc method must work like realloc() from the standard C library -** with the exception that if the second argument to xRealloc is zero, -** xRealloc must be a no-op - it must not perform any allocation or -** deallocation. ^SQLite guarantees that the second argument to +** The xMalloc, xRealloc, and xFree methods must work like the +** malloc(), realloc() and free() functions from the standard C library. +** ^SQLite guarantees that the second argument to ** xRealloc is always a value returned by a prior call to xRoundup. -** And so in cases where xRoundup always returns a positive number, -** xRealloc can perform exactly as the standard library realloc() and -** still be in compliance with this specification. ** ** xSize should return the allocated size of a memory allocation ** previously obtained from xMalloc or xRealloc. The allocated size ** is always at least as big as the requested size but may be larger. ** Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -145,23 +145,29 @@ /* ** Exactly one of the following macros must be defined in order to ** specify which memory allocation subsystem to use. ** ** SQLITE_SYSTEM_MALLOC // Use normal system malloc() +** SQLITE_WIN32_MALLOC // Use Win32 native heap API ** SQLITE_MEMDEBUG // Debugging version of system malloc() +** +** On Windows, if the SQLITE_WIN32_MALLOC_VALIDATE macro is defined and the +** assert() macro is enabled, each call into the Win32 native heap subsystem +** will cause HeapValidate to be called. If heap validation should fail, an +** assertion will be triggered. ** ** (Historical note: There used to be several other options, but we've ** pared it down to just these two.) ** ** If none of the above are defined, then set SQLITE_SYSTEM_MALLOC as ** the default. */ -#if defined(SQLITE_SYSTEM_MALLOC)+defined(SQLITE_MEMDEBUG)>1 +#if defined(SQLITE_SYSTEM_MALLOC)+defined(SQLITE_WIN32_MALLOC)+defined(SQLITE_MEMDEBUG)>1 # error "At most one of the following compile-time configuration options\ - is allows: SQLITE_SYSTEM_MALLOC, SQLITE_MEMDEBUG" + is allows: SQLITE_SYSTEM_MALLOC, SQLITE_WIN32_MALLOC, SQLITE_MEMDEBUG" #endif -#if defined(SQLITE_SYSTEM_MALLOC)+defined(SQLITE_MEMDEBUG)==0 +#if defined(SQLITE_SYSTEM_MALLOC)+defined(SQLITE_WIN32_MALLOC)+defined(SQLITE_MEMDEBUG)==0 # define SQLITE_SYSTEM_MALLOC 1 #endif /* ** If SQLITE_MALLOC_SOFT_LIMIT is not zero, then try to keep the @@ -364,10 +370,18 @@ */ #ifndef SQLITE_TEMP_STORE # define SQLITE_TEMP_STORE 1 #endif +/* +** If all temporary storage is in-memory, then omit the external merge-sort +** logic since it is superfluous. +*/ +#if SQLITE_TEMP_STORE==3 && !defined(SQLITE_OMIT_MERGE_SORT) +# define SQLITE_OMIT_MERGE_SORT +#endif + /* ** GCC does not define the offsetof() macro so we'll have to do it ** ourselves. */ #ifndef offsetof Index: src/test6.c ================================================================== --- src/test6.c +++ src/test6.c @@ -507,12 +507,15 @@ static int cfFileControl(sqlite3_file *pFile, int op, void *pArg){ if( op==SQLITE_FCNTL_SIZE_HINT ){ CrashFile *pCrash = (CrashFile *)pFile; i64 nByte = *(i64 *)pArg; if( nByte>pCrash->iSize ){ - return cfWrite(pFile, "", 1, nByte-1); + if( SQLITE_OK==writeListAppend(pFile, nByte, 0, 0) ){ + pCrash->iSize = nByte; + } } + return SQLITE_OK; } return sqlite3OsFileControl(((CrashFile *)pFile)->pRealFile, op, pArg); } /* Index: src/test_malloc.c ================================================================== --- src/test_malloc.c +++ src/test_malloc.c @@ -1220,11 +1220,11 @@ if( objc!=2 ){ Tcl_WrongNumArgs(interp, 1, objv, "FILENAME"); return TCL_ERROR; } - switch( (int)clientData ){ + switch( SQLITE_PTR_TO_INT(clientData) ){ case 3: { #ifdef SQLITE_ENABLE_MEMSYS3 extern void sqlite3Memsys3Dump(const char*); sqlite3Memsys3Dump(Tcl_GetString(objv[1])); break; @@ -1458,11 +1458,11 @@ { "sqlite3_install_memsys3", test_install_memsys3 ,0 }, { "sqlite3_memdebug_vfs_oom_test", test_vfs_oom_test ,0 }, }; int i; for(i=0; ipFiles; + while( pFile ){ + if( pFile->nRef ) N++; + pFile = pFile->pNext; + } + return N; +} + +/* Remove a file from a quota group. +*/ +static void quotaRemoveFile(quotaFile *pFile){ + quotaGroup *pGroup = pFile->pGroup; + pGroup->iSize -= pFile->iSize; + *pFile->ppPrev = pFile->pNext; + if( pFile->pNext ) pFile->pNext->ppPrev = pFile->ppPrev; + sqlite3_free(pFile); +} + +/* Remove all files from a quota group. It is always the case that +** all files will be closed when this routine is called. +*/ +static void quotaRemoveAllFiles(quotaGroup *pGroup){ + while( pGroup->pFiles ){ + assert( pGroup->pFiles->nRef==0 ); + quotaRemoveFile(pGroup->pFiles); + } +} + /* If the reference count and threshold for a quotaGroup are both ** zero, then destroy the quotaGroup. */ static void quotaGroupDeref(quotaGroup *pGroup){ - if( pGroup->pFiles==0 && pGroup->iLimit==0 ){ + if( pGroup->iLimit==0 && quotaGroupOpenFileCount(pGroup)==0 ){ + quotaRemoveAllFiles(pGroup); *pGroup->ppPrev = pGroup->pNext; if( pGroup->pNext ) pGroup->pNext->ppPrev = pGroup->ppPrev; if( pGroup->xDestroy ) pGroup->xDestroy(pGroup->pArg); sqlite3_free(pGroup); } @@ -273,10 +307,21 @@ */ static sqlite3_file *quotaSubOpen(sqlite3_file *pConn){ quotaConn *p = (quotaConn*)pConn; return (sqlite3_file*)&p[1]; } + +/* Find a file in a quota group and return a pointer to that file. +** Return NULL if the file is not in the group. +*/ +static quotaFile *quotaFindFile(quotaGroup *pGroup, const char *zName){ + quotaFile *pFile = pGroup->pFiles; + while( pFile && strcmp(pFile->zFilename, zName)!=0 ){ + pFile = pFile->pNext; + } + return pFile; +} /************************* VFS Method Wrappers *****************************/ /* ** This is the xOpen method used for the "quota" VFS. ** @@ -317,12 +362,11 @@ */ pQuotaOpen = (quotaConn*)pConn; pSubOpen = quotaSubOpen(pConn); rc = pOrigVfs->xOpen(pOrigVfs, zName, pSubOpen, flags, pOutFlags); if( rc==SQLITE_OK ){ - for(pFile=pGroup->pFiles; pFile && strcmp(pFile->zFilename, zName); - pFile=pFile->pNext){} + pFile = quotaFindFile(pGroup, zName); if( pFile==0 ){ int nName = strlen(zName); pFile = (quotaFile *)sqlite3_malloc( sizeof(*pFile) + nName + 1 ); if( pFile==0 ){ quotaLeave(); @@ -335,10 +379,11 @@ pFile->pNext = pGroup->pFiles; if( pGroup->pFiles ) pGroup->pFiles->ppPrev = &pFile->pNext; pFile->ppPrev = &pGroup->pFiles; pGroup->pFiles = pFile; pFile->pGroup = pGroup; + pFile->deleteOnClose = (flags & SQLITE_OPEN_DELETEONCLOSE)!=0; } pFile->nRef++; pQuotaOpen->pFile = pFile; if( pSubOpen->pMethods->iVersion==1 ){ pQuotaOpen->base.pMethods = &gQuota.sIoMethodsV1; @@ -348,10 +393,53 @@ } } quotaLeave(); return rc; } + +/* +** This is the xDelete method used for the "quota" VFS. +** +** If the file being deleted is part of the quota group, then reduce +** the size of the quota group accordingly. And remove the file from +** the set of files in the quota group. +*/ +static int quotaDelete( + sqlite3_vfs *pVfs, /* The quota VFS */ + const char *zName, /* Name of file to be deleted */ + int syncDir /* Do a directory sync after deleting */ +){ + int rc; /* Result code */ + quotaFile *pFile; /* Files in the quota */ + quotaGroup *pGroup; /* The group file belongs to */ + sqlite3_vfs *pOrigVfs = gQuota.pOrigVfs; /* Real VFS */ + + /* Do the actual file delete */ + rc = pOrigVfs->xDelete(pOrigVfs, zName, syncDir); + + /* If the file just deleted is a member of a quota group, then remove + ** it from that quota group. + */ + if( rc==SQLITE_OK ){ + quotaEnter(); + pGroup = quotaGroupFind(zName); + if( pGroup ){ + pFile = quotaFindFile(pGroup, zName); + if( pFile ){ + if( pFile->nRef ){ + pFile->deleteOnClose = 1; + }else{ + quotaRemoveFile(pFile); + quotaGroupDeref(pGroup); + } + } + } + quotaLeave(); + } + return rc; +} + /************************ I/O Method Wrappers *******************************/ /* xClose requests get passed through to the original VFS. But we ** also have to unlink the quotaConn from the quotaFile and quotaGroup. @@ -365,15 +453,12 @@ rc = pSubOpen->pMethods->xClose(pSubOpen); quotaEnter(); pFile->nRef--; if( pFile->nRef==0 ){ quotaGroup *pGroup = pFile->pGroup; - pGroup->iSize -= pFile->iSize; - if( pFile->pNext ) pFile->pNext->ppPrev = pFile->ppPrev; - *pFile->ppPrev = pFile->pNext; + if( pFile->deleteOnClose ) quotaRemoveFile(pFile); quotaGroupDeref(pGroup); - sqlite3_free(pFile); } quotaLeave(); return rc; } @@ -584,10 +669,11 @@ } gQuota.isInitialized = 1; gQuota.pOrigVfs = pOrigVfs; gQuota.sThisVfs = *pOrigVfs; gQuota.sThisVfs.xOpen = quotaOpen; + gQuota.sThisVfs.xDelete = quotaDelete; gQuota.sThisVfs.szOsFile += sizeof(quotaConn); gQuota.sThisVfs.zName = "quota"; gQuota.sIoMethodsV1.iVersion = 1; gQuota.sIoMethodsV1.xClose = quotaClose; gQuota.sIoMethodsV1.xRead = quotaRead; @@ -615,23 +701,24 @@ ** Shutdown the quota system. ** ** All SQLite database connections must be closed before calling this ** routine. ** -** THIS ROUTINE IS NOT THREADSAFE. Call this routine exactly one while +** THIS ROUTINE IS NOT THREADSAFE. Call this routine exactly once while ** shutting down in order to free all remaining quota groups. */ int sqlite3_quota_shutdown(void){ quotaGroup *pGroup; if( gQuota.isInitialized==0 ) return SQLITE_MISUSE; for(pGroup=gQuota.pGroup; pGroup; pGroup=pGroup->pNext){ - if( pGroup->pFiles ) return SQLITE_MISUSE; + if( quotaGroupOpenFileCount(pGroup)>0 ) return SQLITE_MISUSE; } while( gQuota.pGroup ){ pGroup = gQuota.pGroup; gQuota.pGroup = pGroup->pNext; pGroup->iLimit = 0; + assert( quotaGroupOpenFileCount(pGroup)==0 ); quotaGroupDeref(pGroup); } gQuota.isInitialized = 0; sqlite3_mutex_free(gQuota.pMutex); sqlite3_vfs_unregister(&gQuota.sThisVfs); @@ -705,10 +792,47 @@ pGroup->xDestroy = xDestroy; quotaGroupDeref(pGroup); quotaLeave(); return SQLITE_OK; } + +/* +** Bring the named file under quota management. Or if it is already under +** management, update its size. +*/ +int sqlite3_quota_file(const char *zFilename){ + char *zFull; + sqlite3_file *fd; + int rc; + int outFlags = 0; + sqlite3_int64 iSize; + fd = sqlite3_malloc(gQuota.sThisVfs.szOsFile + gQuota.sThisVfs.mxPathname+1); + if( fd==0 ) return SQLITE_NOMEM; + zFull = gQuota.sThisVfs.szOsFile + (char*)fd; + rc = gQuota.pOrigVfs->xFullPathname(gQuota.pOrigVfs, zFilename, + gQuota.sThisVfs.mxPathname+1, zFull); + if( rc==SQLITE_OK ){ + rc = quotaOpen(&gQuota.sThisVfs, zFull, fd, + SQLITE_OPEN_READONLY | SQLITE_OPEN_MAIN_DB, &outFlags); + } + if( rc==SQLITE_OK ){ + fd->pMethods->xFileSize(fd, &iSize); + fd->pMethods->xClose(fd); + }else if( rc==SQLITE_CANTOPEN ){ + quotaGroup *pGroup; + quotaFile *pFile; + quotaEnter(); + pGroup = quotaGroupFind(zFull); + if( pGroup ){ + pFile = quotaFindFile(pGroup, zFull); + if( pFile ) quotaRemoveFile(pFile); + } + quotaLeave(); + } + sqlite3_free(fd); + return rc; +} /***************************** Test Code ***********************************/ #ifdef SQLITE_TEST #include @@ -881,10 +1005,36 @@ rc = sqlite3_quota_set(zPattern, iLimit, xCallback, (void*)p, xDestroy); Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); return TCL_OK; } + +/* +** tclcmd: sqlite3_quota_file FILENAME +*/ +static int test_quota_file( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zFilename; /* File pattern to configure */ + int rc; /* Value returned by quota_file() */ + + /* Process arguments */ + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "FILENAME"); + return TCL_ERROR; + } + zFilename = Tcl_GetString(objv[1]); + + /* Invoke sqlite3_quota_file() */ + rc = sqlite3_quota_file(zFilename); + + Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + return TCL_OK; +} /* ** tclcmd: sqlite3_quota_dump */ static int test_quota_dump( @@ -915,10 +1065,12 @@ Tcl_NewStringObj(pFile->zFilename, -1)); Tcl_ListObjAppendElement(interp, pFileTerm, Tcl_NewWideIntObj(pFile->iSize)); Tcl_ListObjAppendElement(interp, pFileTerm, Tcl_NewWideIntObj(pFile->nRef)); + Tcl_ListObjAppendElement(interp, pFileTerm, + Tcl_NewWideIntObj(pFile->deleteOnClose)); Tcl_ListObjAppendElement(interp, pGroupTerm, pFileTerm); } Tcl_ListObjAppendElement(interp, pResult, pGroupTerm); } quotaLeave(); @@ -937,10 +1089,11 @@ Tcl_ObjCmdProc *xProc; } aCmd[] = { { "sqlite3_quota_initialize", test_quota_initialize }, { "sqlite3_quota_shutdown", test_quota_shutdown }, { "sqlite3_quota_set", test_quota_set }, + { "sqlite3_quota_file", test_quota_file }, { "sqlite3_quota_dump", test_quota_dump }, }; int i; for(i=0; iisTable = pOp->p4type!=P4_KEYINFO; pCur->isIndex = !pCur->isTable; break; } -/* Opcode: OpenEphemeral P1 P2 * P4 * +/* Opcode: OpenEphemeral P1 P2 * P4 P5 ** ** Open a new cursor P1 to a transient table. ** The cursor is always opened read/write even if ** the main database is read-only. The ephemeral ** table is deleted automatically when the cursor is closed. @@ -3146,18 +3146,29 @@ ** This opcode was once called OpenTemp. But that created ** confusion because the term "temp table", might refer either ** to a TEMP table at the SQL level, or to a table opened by ** this opcode. Then this opcode was call OpenVirtual. But ** that created confusion with the whole virtual-table idea. +** +** The P5 parameter can be a mask of the BTREE_* flags defined +** in btree.h. These flags control aspects of the operation of +** the btree. The BTREE_OMIT_JOURNAL and BTREE_SINGLE flags are +** added automatically. */ /* Opcode: OpenAutoindex P1 P2 * P4 * ** ** This opcode works the same as OP_OpenEphemeral. It has a ** different name to distinguish its use. Tables created using ** by this opcode will be used for automatically created transient ** indices in joins. */ +/* Opcode: OpenSorter P1 P2 * P4 * +** +** This opcode works like OP_OpenEphemeral except that it opens +** a transient index that is specifically designed to sort large +** tables using an external merge-sort algorithm. +*/ case OP_OpenSorter: case OP_OpenAutoindex: case OP_OpenEphemeral: { VdbeCursor *pCx; static const int vfsFlags = @@ -3166,10 +3177,11 @@ SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE | SQLITE_OPEN_TRANSIENT_DB; assert( pOp->p1>=0 ); + assert( (pOp->opcode==OP_OpenSorter)==((pOp->p5 & BTREE_SORTER)!=0) ); pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBt, BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); Index: src/vdbe.h ================================================================== --- src/vdbe.h +++ src/vdbe.h @@ -171,13 +171,13 @@ int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp); void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); -void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); -void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); -void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); +void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1); +void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); +void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); void sqlite3VdbeChangeP5(Vdbe*, u8 P5); void sqlite3VdbeJumpHere(Vdbe*, int addr); void sqlite3VdbeChangeToNoop(Vdbe*, int addr, int N); void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N); void sqlite3VdbeUsesBtree(Vdbe*, int); Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -522,37 +522,34 @@ ** Change the value of the P1 operand for a specific instruction. ** This routine is useful when a large program is loaded from a ** static array using sqlite3VdbeAddOpList but we want to make a ** few minor changes to the program. */ -void sqlite3VdbeChangeP1(Vdbe *p, int addr, int val){ +void sqlite3VdbeChangeP1(Vdbe *p, u32 addr, int val){ assert( p!=0 ); - assert( addr>=0 ); - if( p->nOp>addr ){ + if( ((u32)p->nOp)>addr ){ p->aOp[addr].p1 = val; } } /* ** Change the value of the P2 operand for a specific instruction. ** This routine is useful for setting a jump destination. */ -void sqlite3VdbeChangeP2(Vdbe *p, int addr, int val){ +void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){ assert( p!=0 ); - assert( addr>=0 ); - if( p->nOp>addr ){ + if( ((u32)p->nOp)>addr ){ p->aOp[addr].p2 = val; } } /* ** Change the value of the P3 operand for a specific instruction. */ -void sqlite3VdbeChangeP3(Vdbe *p, int addr, int val){ +void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){ assert( p!=0 ); - assert( addr>=0 ); - if( p->nOp>addr ){ + if( ((u32)p->nOp)>addr ){ p->aOp[addr].p3 = val; } } /* Index: src/vdbesort.c ================================================================== --- src/vdbesort.c +++ src/vdbesort.c @@ -386,12 +386,14 @@ static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){ int rc = SQLITE_OK; /* Return code */ VdbeSorter *pSorter = pCsr->pSorter; int res = 0; + /* sqlite3BtreeFirst() cannot fail because sorter btrees are always held + ** in memory and so an I/O error is not possible. */ rc = sqlite3BtreeFirst(pCsr->pCursor, &res); - if( rc!=SQLITE_OK || res ) return rc; + if( NEVER(rc!=SQLITE_OK) || res ) return rc; assert( pSorter->nBtree>0 ); /* If the first temporary PMA file has not been opened, open it now. */ if( pSorter->pTemp1==0 ){ rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1); @@ -427,12 +429,13 @@ } } /* Write the record itself to the output file */ if( rc==SQLITE_OK ){ + /* sqlite3BtreeKey() cannot fail because sorter btrees held in memory */ rc = sqlite3BtreeKey(pCsr->pCursor, 0, nKey, aMalloc); - if( rc==SQLITE_OK ){ + if( ALWAYS(rc==SQLITE_OK) ){ rc = sqlite3OsWrite(pSorter->pTemp1, aMalloc, nKey, iWriteOff); iWriteOff += nKey; } } @@ -472,20 +475,23 @@ VdbeSorter *pSorter = pCsr->pSorter; if( pSorter ){ Pager *pPager = sqlite3BtreePager(pCsr->pBt); int nPage; /* Current size of temporary file in pages */ + /* Sorters never spill to disk */ + assert( sqlite3PagerFile(pPager)->pMethods==0 ); + /* Determine how many pages the temporary b-tree has grown to */ sqlite3PagerPagecount(pPager, &nPage); /* If pSorter->nWorking is still zero, but the temporary file has been ** created in the file-system, then the most recent insert into the ** current b-tree segment probably caused the cache to overflow (it is ** also possible that sqlite3_release_memory() was called). So set the ** size of the working set to a little less than the current size of the ** file in pages. */ - if( pSorter->nWorking==0 && sqlite3PagerFile(pPager)->pMethods ){ + if( pSorter->nWorking==0 && sqlite3PagerUnderStress(pPager) ){ pSorter->nWorking = nPage-5; if( pSorter->nWorkingnWorking = SORTER_MIN_WORKING; } } @@ -624,16 +630,16 @@ } if( rc==SQLITE_OK ){ int bEof = 0; while( rc==SQLITE_OK && bEof==0 ){ - int nByte; + int nToWrite; VdbeSorterIter *pIter = &pSorter->aIter[ pSorter->aTree[1] ]; assert( pIter->pFile ); - nByte = pIter->nKey + sqlite3VarintLen(pIter->nKey); - rc = sqlite3OsWrite(pTemp2, pIter->aAlloc, nByte, iWrite2); - iWrite2 += nByte; + nToWrite = pIter->nKey + sqlite3VarintLen(pIter->nKey); + rc = sqlite3OsWrite(pTemp2, pIter->aAlloc, nToWrite, iWrite2); + iWrite2 += nToWrite; if( rc==SQLITE_OK ){ rc = sqlite3VdbeSorterNext(db, pCsr, &bEof); } } } Index: test/pager1.test ================================================================== --- test/pager1.test +++ test/pager1.test @@ -2424,7 +2424,44 @@ sqlite3 db2 test.db2 execsql { PRAGMA integrity_check } db2 } {ok} } +#------------------------------------------------------------------------- +# Test that a database file can be "pre-hinted" to a certain size and that +# subsequent spilling of the pager cache does not result in the database +# file being shrunk. +# +catch {db close} +forcedelete test.db + +do_test pager1-32.1 { + sqlite3 db test.db + execsql { + CREATE TABLE t1(x, y); + } + db close + sqlite3 db test.db + execsql { + BEGIN; + INSERT INTO t1 VALUES(1, randomblob(10000)); + } + file_control_sizehint_test db main 20971520; # 20MB + execsql { + PRAGMA cache_size = 10; + INSERT INTO t1 VALUES(1, randomblob(10000)); + INSERT INTO t1 VALUES(2, randomblob(10000)); + INSERT INTO t1 SELECT x+2, randomblob(10000) from t1; + INSERT INTO t1 SELECT x+4, randomblob(10000) from t1; + INSERT INTO t1 SELECT x+8, randomblob(10000) from t1; + INSERT INTO t1 SELECT x+16, randomblob(10000) from t1; + SELECT count(*) FROM t1; + COMMIT; + } + db close + file size test.db +} {20971520} + +# Cleanup 20MB file left by the previous test. +forcedelete test.db finish_test Index: test/quota.test ================================================================== --- test/quota.test +++ test/quota.test @@ -221,11 +221,11 @@ foreach db {db1a db2a db2b db1b} { catch { $db close } } sqlite3_quota_set * 0 {} } {SQLITE_OK} #------------------------------------------------------------------------- -# Quotas are deleted when unused and when there limit is set to zero +# Quotas are deleted when unused and when their limit is set to zero # # Return a list of all currently defined quotas. Each quota is identified # by its pattern. proc quota_list {} { @@ -232,10 +232,17 @@ set allq {} foreach q [sqlite3_quota_dump] { lappend allq [lindex $q 0] } return [lsort $allq] +} +proc quota_size {name} { + set allq {} + foreach q [sqlite3_quota_dump] { + if {[lindex $q 0]==$name} {return [lindex $q 2]} + } + return 0 } do_test quota-4.1.1 { sqlite3_quota_set *test.db 0 {} quota_list @@ -327,15 +334,125 @@ sqlite3_quota_set A 0 quota_callback db close quota_list } {} +unset -nocomplain quotagroup +if {$tcl_platform(platform)=="windows"} { + set quotagroup *\\quota-test-A?.db +} else { + set quotagroup */quota-test-A?.db +} +foreach file [glob -nocomplain quota-test-A*] { + forcedelete $file +} do_test quota-4.4.1 { + set ::quota {} + sqlite3_quota_set $::quotagroup 10000 quota_callback + file delete -force ./quota-test-A1.db ./quota-test-A2.db + sqlite3 db ./quota-test-A1.db + db eval { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(randomblob(5000)); + } + quota_list +} [list $quotagroup] +do_test quota-4.4.2 { + expr {$::quota==""} +} {1} +do_test quota-4.4.3 { + db close + sqlite3 db ./quota-test-A2.db + db eval { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(randomblob(5000)); + } + quota_list +} [list $quotagroup] +do_test quota-4.4.4 { + expr {$::quota!=""} +} {1} +do_test quota-4.4.5 { + db close + sqlite3_quota_set $::quotagroup 0 {} + sqlite3_quota_dump +} {} +do_test quota-4.4.6 { + sqlite3_quota_set $quotagroup 10000 quota_callback + sqlite3 db quota-test-A1.db + db eval {SELECT count(*) FROM sqlite_master} + quota_size $quotagroup +} [file size quota-test-A1.db] +do_test quota-4.4.7 { + sqlite3_quota_file quota-test-A2.db + quota_size $::quotagroup +} [expr {[file size quota-test-A1.db]+[file size quota-test-A2.db]}] + +unset -nocomplain quotagroup +if {$tcl_platform(platform)=="windows"} { + set quotagroup *\\quota-test-B* +} else { + set quotagroup */quota-test-B* +} +foreach file [glob -nocomplain quota-test-B*] { + forcedelete $file +} +do_test quota-4.5.1 { + sqlite3_quota_set $::quotagroup 100000 quota_callback + quota_size $::quotagroup +} {0} +do_test quota-4.5.2 { + sqlite3_quota_file quota-test-B1.txt + quota_size $::quotagroup +} {0} +proc add_to_file {name n} { + set out [open $name a] + fconfigure $out -translation binary + puts -nonewline $out [string repeat x $n] + close $out +} +do_test quota-4.5.3 { + add_to_file quota-test-B1.txt 123 + sqlite3_quota_file quota-test-B1.txt + quota_size $::quotagroup +} {123} +do_test quota-4.5.4 { + add_to_file quota-test-B2.txt 234 + sqlite3_quota_file quota-test-B2.txt + quota_size $::quotagroup +} {357} +do_test quota-4.5.5 { + add_to_file quota-test-B1.txt 2000 + sqlite3_quota_file quota-test-B1.txt + quota_size $::quotagroup +} {2357} +do_test quota-4.5.6 { + forcedelete quota-test-B1.txt + sqlite3_quota_file quota-test-B1.txt + quota_size $::quotagroup +} {234} +do_test quota-4.5.7 { + forcedelete quota-test-B2.txt + sqlite3_quota_file quota-test-B2.txt + quota_size $::quotagroup +} {0} +do_test quota-4.5.8 { + add_to_file quota-test-B3.txt 1234 + sqlite3_quota_file quota-test-B3.txt + quota_size $::quotagroup +} {1234} +do_test quota-4.5.9 { + sqlite3_quota_set $quotagroup 0 {} + quota_size $::quotagroup +} {0} + +do_test quota-4.9.1 { + db close sqlite3_quota_set A 1000 quota_callback sqlite3_quota_shutdown } {SQLITE_OK} -do_test quota-4.4.2 { +do_test quota-4.9.2 { quota_list } {} #------------------------------------------------------------------------- # The following tests test that the quota VFS handles malloc and IO Index: test/win32lock.test ================================================================== --- test/win32lock.test +++ test/win32lock.test @@ -67,10 +67,14 @@ } {{delayed #ms for lock/sharing conflict}} } } if {[llength $win32_lock_ok] && [llength $win32_lock_error]} break incr delay1 25 + if {$delay1 > 12500} { + puts "Timed out waiting for \"ok\" and \"error\" results." + break + } sqlite3_sleep 10 } do_test win32lock-2.0 { file_control_win32_av_retry db -1 -1 @@ -111,14 +115,19 @@ } {{delayed #ms for lock/sharing conflict}} } } if {[llength $win32_lock_ok] && [llength $win32_lock_error]} break incr delay1 1 + if {$delay1 > 500} { + puts "Timed out waiting for \"ok\" and \"error\" results." + break + } sqlite3_sleep 10 } file_control_win32_av_retry db 10 25 sqlite3_test_control_pending_byte $old_pending_byte +db close sqlite3_shutdown test_sqlite3_log sqlite3_initialize finish_test Index: tool/warnings.sh ================================================================== --- tool/warnings.sh +++ tool/warnings.sh @@ -1,11 +1,12 @@ #/bin/sh # # Run this script in a directory with a working makefile to check for # compiler warnings in SQLite. # -make sqlite3.c +rm -f sqlite3.c +make sqlite3.c-debug echo '********** No optimizations. Includes FTS4 and RTREE *********' gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ sqlite3.c echo '********** No optimizations. ENABLE_STAT2. THREADSAFE=0 *******'