Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -357,10 +357,11 @@ $(TOP)/src/test8.c \ $(TOP)/src/test9.c \ $(TOP)/src/test_autoext.c \ $(TOP)/src/test_async.c \ $(TOP)/src/test_backup.c \ + $(TOP)/src/test_blob.c \ $(TOP)/src/test_btree.c \ $(TOP)/src/test_config.c \ $(TOP)/src/test_demovfs.c \ $(TOP)/src/test_devsym.c \ $(TOP)/src/test_fs.c \ Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -826,10 +826,11 @@ $(TOP)\src\test8.c \ $(TOP)\src\test9.c \ $(TOP)\src\test_autoext.c \ $(TOP)\src\test_async.c \ $(TOP)\src\test_backup.c \ + $(TOP)\src\test_blob.c \ $(TOP)\src\test_btree.c \ $(TOP)\src\test_config.c \ $(TOP)\src\test_demovfs.c \ $(TOP)\src\test_devsym.c \ $(TOP)\src\test_fs.c \ Index: ext/fts3/fts3_porter.c ================================================================== --- ext/fts3/fts3_porter.c +++ ext/fts3/fts3_porter.c @@ -181,11 +181,11 @@ ** Return true if the m-value for z is 1 or more. In other words, ** return true if z contains at least one vowel that is followed ** by a consonant. ** ** In this routine z[] is in reverse order. So we are really looking -** for an instance of of a consonant followed by a vowel. +** for an instance of a consonant followed by a vowel. */ static int m_gt_0(const char *z){ while( isVowel(z) ){ z++; } if( *z==0 ) return 0; while( isConsonant(z) ){ z++; } Index: ext/misc/eval.c ================================================================== --- ext/misc/eval.c +++ ext/misc/eval.c @@ -19,15 +19,15 @@ /* ** Structure used to accumulate the output */ struct EvalResult { - char *z; /* Accumulated output */ - const char *zSep; /* Separator */ - int szSep; /* Size of the separator string */ - int nAlloc; /* Number of bytes allocated for z[] */ - int nUsed; /* Number of bytes of z[] actually used */ + char *z; /* Accumulated output */ + const char *zSep; /* Separator */ + int szSep; /* Size of the separator string */ + sqlite3_int64 nAlloc; /* Number of bytes allocated for z[] */ + sqlite3_int64 nUsed; /* Number of bytes of z[] actually used */ }; /* ** Callback from sqlite_exec() for the eval() function. */ @@ -35,14 +35,17 @@ struct EvalResult *p = (struct EvalResult*)pCtx; int i; for(i=0; inUsed+p->szSep+1 > p->nAlloc ){ + if( (sqlite3_int64)sz+p->nUsed+p->szSep+1 > p->nAlloc ){ char *zNew; p->nAlloc = p->nAlloc*2 + sz + p->szSep + 1; - zNew = sqlite3_realloc(p->z, p->nAlloc); + /* Using sqlite3_realloc64() would be better, but it is a recent + ** addition and will cause a segfault if loaded by an older version + ** of SQLite. */ + zNew = p->nAlloc<=0x7fffffff ? sqlite3_realloc(p->z, (int)p->nAlloc) : 0; if( zNew==0 ){ sqlite3_free(p->z); memset(p, 0, sizeof(*p)); return 1; } @@ -91,11 +94,11 @@ sqlite3_free(zErr); }else if( x.zSep==0 ){ sqlite3_result_error_nomem(context); sqlite3_free(x.z); }else{ - sqlite3_result_text(context, x.z, x.nUsed, sqlite3_free); + sqlite3_result_text(context, x.z, (int)x.nUsed, sqlite3_free); } } #ifdef _WIN32 Index: src/analyze.c ================================================================== --- src/analyze.c +++ src/analyze.c @@ -1436,12 +1436,10 @@ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( z==0 ) z = ""; #else assert( z!=0 ); - pIndex->bUnordered = 0; - pIndex->noSkipScan = 0; #endif for(i=0; *z && i='0' && c<='9' ){ v = v*10 + c - '0'; @@ -1457,29 +1455,32 @@ aLog[i] = sqlite3LogEst(v); #endif if( *z==' ' ) z++; } #ifndef SQLITE_ENABLE_STAT3_OR_STAT4 - assert( pIndex!=0 ); + assert( pIndex!=0 ); { #else - if( pIndex ) + if( pIndex ){ #endif - while( z[0] ){ - if( sqlite3_strglob("unordered*", z)==0 ){ - pIndex->bUnordered = 1; - }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ - pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); - }else if( sqlite3_strglob("noskipscan*", z)==0 ){ - pIndex->noSkipScan = 1; - } + pIndex->bUnordered = 0; + pIndex->noSkipScan = 0; + while( z[0] ){ + if( sqlite3_strglob("unordered*", z)==0 ){ + pIndex->bUnordered = 1; + }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ + pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); + }else if( sqlite3_strglob("noskipscan*", z)==0 ){ + pIndex->noSkipScan = 1; + } #ifdef SQLITE_ENABLE_COSTMULT - else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ - pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); - } + else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ + pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); + } #endif - while( z[0]!=0 && z[0]!=' ' ) z++; - while( z[0]==' ' ) z++; + while( z[0]!=0 && z[0]!=' ' ) z++; + while( z[0]==' ' ) z++; + } } } /* ** This callback is invoked once for each index when reading the @@ -1593,11 +1594,11 @@ tRowcnt avgEq = 0; tRowcnt nRow; /* Number of rows in index */ i64 nSum100 = 0; /* Number of terms contributing to sumEq */ i64 nDist100; /* Number of distinct values in index */ - if( pIdx->aiRowEst==0 || pIdx->aiRowEst[iCol+1]==0 ){ + if( !pIdx->aiRowEst || iCol>=pIdx->nKeyCol || pIdx->aiRowEst[iCol+1]==0 ){ nRow = pFinal->anLt[iCol]; nDist100 = (i64)100 * pFinal->anDLt[iCol]; nSample--; }else{ nRow = pIdx->aiRowEst[0]; Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -1355,11 +1355,11 @@ ** to see if defragmentation is necessary. */ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ defragment_page: - testcase( pPage->nCell==0 ); + assert( pPage->nCell>0 || CORRUPT_DB ); rc = defragmentPage(pPage); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); assert( gap+nByte<=top ); } @@ -6256,10 +6256,18 @@ } return nRet; } /* +** apCell[] and szCell[] contains pointers to and sizes of all cells in the +** pages being balanced. The current page, pPg, has pPg->nCell cells starting +** with apCell[iOld]. After balancing, this page should hold nNew cells +** starting at apCell[iNew]. +** +** This routine makes the necessary adjustments to pPg so that it contains +** the correct cells after being balanced. +** ** The pPg->nFree field is invalid when this function returns. It is the ** responsibility of the caller to set it correctly. */ static void editPage( MemPage *pPg, /* Edit this page */ @@ -6296,16 +6304,17 @@ nCell -= pageFreeArray( pPg, iOldEnd-iNewEnd, &apCell[iNewEnd], &szCell[iNewEnd] ); } - pData = &aData[get2byte(&aData[hdr+5])]; + pData = &aData[get2byteNotZero(&aData[hdr+5])]; if( pDataaCellIdx; memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); if( pageInsertArray( pPg, pBegin, &pData, pCellptr, nAdd, &apCell[iNew], &szCell[iNew] @@ -6406,11 +6415,11 @@ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); assert( pPage->nOverflow==1 ); /* This error condition is now caught prior to reaching this function */ - if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT; + if( NEVER(pPage->nCell==0) ) return SQLITE_CORRUPT_BKPT; /* Allocate a new page. This page will become the right-sibling of ** pPage. Make the parent page writable, so that the new divider cell ** may be inserted. If both these operations are successful, proceed. */ @@ -6849,11 +6858,15 @@ ** pointer of the divider cell */ memcpy(apCell[nCell], &pOld->aData[8], 4); }else{ assert( leafCorrection==4 ); if( szCell[nCell]<4 ){ - /* Do not allow any cells smaller than 4 bytes. */ + /* Do not allow any cells smaller than 4 bytes. If a smaller cell + ** does exist, pad it with 0x00 bytes. */ + assert( szCell[nCell]==3 ); + assert( apCell[nCell]==&pTemp[iSpace1-3] ); + pTemp[iSpace1++] = 0x00; szCell[nCell] = 4; } } nCell++; } @@ -7084,11 +7097,15 @@ ** was either part of sibling page iOld (possibly an overflow cell), ** or else the divider cell to the left of sibling page iOld. So, ** if sibling page iOld had the same page number as pNew, and if ** pCell really was a part of sibling page iOld (not a divider or ** overflow cell), we can skip updating the pointer map entries. */ - if( pNew->pgno!=aPgno[iOld] || pCell=&aOld[usableSize] ){ + if( iOld>=nNew + || pNew->pgno!=aPgno[iOld] + || pCell=&aOld[usableSize] + ){ if( !leafCorrection ){ ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); } if( szCell[i]>pNew->minLocal ){ ptrmapPutOvflPtr(pNew, pCell, &rc); Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -1412,11 +1412,12 @@ case TK_FLOAT: case TK_BLOB: return 0; case TK_COLUMN: assert( p->pTab!=0 ); - return p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0; + return ExprHasProperty(p, EP_CanBeNull) || + (p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0); default: return 1; } } Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -1030,10 +1030,18 @@ /* Close all database connections */ for(j=0; jnDb; j++){ struct Db *pDb = &db->aDb[j]; if( pDb->pBt ){ + if( pDb->pSchema ){ + /* Must clear the KeyInfo cache. See ticket [e4a18565a36884b00edf] */ + for(i=sqliteHashFirst(&pDb->pSchema->idxHash); i; i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + sqlite3KeyInfoUnref(pIdx->pKeyInfo); + pIdx->pKeyInfo = 0; + } + } sqlite3BtreeClose(pDb->pBt); pDb->pBt = 0; if( j!=1 ){ pDb->pSchema = 0; } @@ -1934,14 +1942,17 @@ /* Initialize the output variables to -1 in case an error occurs. */ if( pnLog ) *pnLog = -1; if( pnCkpt ) *pnCkpt = -1; - assert( SQLITE_CHECKPOINT_FULL>SQLITE_CHECKPOINT_PASSIVE ); - assert( SQLITE_CHECKPOINT_FULLSQLITE_CHECKPOINT_RESTART ){ + assert( SQLITE_CHECKPOINT_PASSIVE==0 ); + assert( SQLITE_CHECKPOINT_FULL==1 ); + assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); + if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ + /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint + ** mode: */ return SQLITE_MISUSE; } sqlite3_mutex_enter(db->mutex); if( zDb && zDb[0] ){ @@ -1992,11 +2003,13 @@ ** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points ** to contains a zero-length string, all attached databases are ** checkpointed. */ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ - return sqlite3_wal_checkpoint_v2(db, zDb, SQLITE_CHECKPOINT_PASSIVE, 0, 0); + /* EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is equivalent to + ** sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). */ + return sqlite3_wal_checkpoint_v2(db,zDb,SQLITE_CHECKPOINT_PASSIVE,0,0); } #ifndef SQLITE_OMIT_WAL /* ** Run a checkpoint on database iDb. This is a no-op if database iDb is Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -7119,11 +7119,12 @@ */ int sqlite3PagerCheckpoint(Pager *pPager, int eMode, int *pnLog, int *pnCkpt){ int rc = SQLITE_OK; if( pPager->pWal && PagerOtaMode(pPager)==0 ){ rc = sqlite3WalCheckpoint(pPager->pWal, eMode, - pPager->xBusyHandler, pPager->pBusyHandlerArg, + (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), + pPager->pBusyHandlerArg, pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, pnLog, pnCkpt ); } return rc; Index: src/pragma.c ================================================================== --- src/pragma.c +++ src/pragma.c @@ -2255,11 +2255,11 @@ break; #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ #ifndef SQLITE_OMIT_WAL /* - ** PRAGMA [database.]wal_checkpoint = passive|full|restart + ** PRAGMA [database.]wal_checkpoint = passive|full|restart|truncate ** ** Checkpoint the database. */ case PragTyp_WAL_CHECKPOINT: { int iBt = (pId2->z?iDb:SQLITE_MAX_ATTACHED); @@ -2267,10 +2267,12 @@ if( zRight ){ if( sqlite3StrICmp(zRight, "full")==0 ){ eMode = SQLITE_CHECKPOINT_FULL; }else if( sqlite3StrICmp(zRight, "restart")==0 ){ eMode = SQLITE_CHECKPOINT_RESTART; + }else if( sqlite3StrICmp(zRight, "truncate")==0 ){ + eMode = SQLITE_CHECKPOINT_TRUNCATE; } } sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "busy", SQLITE_STATIC); Index: src/resolve.c ================================================================== --- src/resolve.c +++ src/resolve.c @@ -318,10 +318,14 @@ } } if( pMatch ){ pExpr->iTable = pMatch->iCursor; pExpr->pTab = pMatch->pTab; + assert( (pMatch->jointype & JT_RIGHT)==0 ); /* RIGHT JOIN not (yet) supported */ + if( (pMatch->jointype & JT_LEFT)!=0 ){ + ExprSetProperty(pExpr, EP_CanBeNull); + } pSchema = pExpr->pTab->pSchema; } } /* if( pSrcList ) */ #ifndef SQLITE_OMIT_TRIGGER Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -4827,11 +4827,11 @@ ** ** SELECT DISTINCT xyz FROM ... ORDER BY xyz ** ** is transformed to: ** - ** SELECT xyz FROM ... GROUP BY xyz + ** SELECT xyz FROM ... GROUP BY xyz ORDER BY xyz ** ** The second form is preferred as a single index (or temp-table) may be ** used for both the ORDER BY and DISTINCT processing. As originally ** written the query must use a temp-table for at least one of the ORDER ** BY and DISTINCT, and an index or separate temp-table for the other. @@ -4840,11 +4840,10 @@ && sqlite3ExprListCompare(sSort.pOrderBy, p->pEList, -1)==0 ){ p->selFlags &= ~SF_Distinct; p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); pGroupBy = p->pGroupBy; - sSort.pOrderBy = 0; /* Notice that even thought SF_Distinct has been cleared from p->selFlags, ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ assert( sDistinct.isTnct ); } Index: src/shell.c ================================================================== --- src/shell.c +++ src/shell.c @@ -4036,14 +4036,16 @@ int main(int argc, char **argv){ char *zErrMsg = 0; ShellState data; const char *zInitFile = 0; - char *zFirstCmd = 0; int i; int rc = 0; int warnInmemoryDb = 0; + int readStdin = 1; + int nCmd = 0; + char **azCmd = 0; #if USE_SYSTEM_SQLITE+0!=1 if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){ fprintf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", sqlite3_sourceid(), SQLITE_SOURCE_ID); @@ -4058,10 +4060,22 @@ ** else is done. */ #ifdef SIGINT signal(SIGINT, interrupt_handler); #endif + +#ifdef SQLITE_SHELL_DBNAME_PROC + { + /* If the SQLITE_SHELL_DBNAME_PROC macro is defined, then it is the name + ** of a C-function that will provide the name of the database file. Use + ** this compile-time option to embed this shell program in larger + ** applications. */ + extern void SQLITE_SHELL_DBNAME_PROC(const char**); + SQLITE_SHELL_DBNAME_PROC(&data.zDbFilename); + warnInmemoryDb = 0; + } +#endif /* Do an initial pass through the command-line argument to locate ** the name of the database file, the name of the initialization file, ** the size of the alternative malloc heap, ** and the first command to execute. @@ -4070,19 +4084,22 @@ char *z; z = argv[i]; if( z[0]!='-' ){ if( data.zDbFilename==0 ){ data.zDbFilename = z; - continue; - } - if( zFirstCmd==0 ){ - zFirstCmd = z; - continue; - } - fprintf(stderr,"%s: Error: too many options: \"%s\"\n", Argv0, argv[i]); - fprintf(stderr,"Use -help for a list of options.\n"); - return 1; + }else{ + /* Excesss arguments are interpreted as SQL (or dot-commands) and + ** mean that nothing is read from stdin */ + readStdin = 0; + nCmd++; + azCmd = realloc(azCmd, sizeof(azCmd[0])*nCmd); + if( azCmd==0 ){ + fprintf(stderr, "out of memory\n"); + exit(1); + } + azCmd[nCmd-1] = z; + } } if( z[1]=='-' ) z++; if( strcmp(z,"-separator")==0 || strcmp(z,"-nullvalue")==0 || strcmp(z,"-newline")==0 @@ -4169,15 +4186,10 @@ warnInmemoryDb = argc==1; #else fprintf(stderr,"%s: Error: no database filename specified\n", Argv0); return 1; #endif -#ifdef SQLITE_SHELL_DBNAME_PROC - { extern void SQLITE_SHELL_DBNAME_PROC(const char**); - SQLITE_SHELL_DBNAME_PROC(&data.zDbFilename); - warnInmemoryDb = 0; } -#endif } data.out = stdout; /* Go ahead and open the database file if it already exists. If the ** file does not exist, delay opening it. This prevents empty database @@ -4270,10 +4282,14 @@ i++; #endif }else if( strcmp(z,"-help")==0 ){ usage(1); }else if( strcmp(z,"-cmd")==0 ){ + /* Run commands that follow -cmd first and separately from commands + ** that simply appear on the command-line. This seems goofy. It would + ** be better if all commands ran in the order that they appear. But + ** we retain the goofy behavior for historical compatibility. */ if( i==argc-1 ) break; z = cmdline_option_value(argc,argv,++i); if( z[0]=='.' ){ rc = do_meta_command(z, &data); if( rc && bail_on_error ) return rc==2 ? 0 : rc; @@ -4293,27 +4309,32 @@ fprintf(stderr,"Use -help for a list of options.\n"); return 1; } } - if( zFirstCmd ){ - /* Run just the command that follows the database name - */ - if( zFirstCmd[0]=='.' ){ - rc = do_meta_command(zFirstCmd, &data); - if( rc==2 ) rc = 0; - }else{ - open_db(&data, 0); - rc = shell_exec(data.db, zFirstCmd, shell_callback, &data, &zErrMsg); - if( zErrMsg!=0 ){ - fprintf(stderr,"Error: %s\n", zErrMsg); - return rc!=0 ? rc : 1; - }else if( rc!=0 ){ - fprintf(stderr,"Error: unable to process SQL \"%s\"\n", zFirstCmd); - return rc; - } - } + if( !readStdin ){ + /* Run all arguments that do not begin with '-' as if they were separate + ** command-line inputs, except for the argToSkip argument which contains + ** the database filename. + */ + for(i=0; i SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED **
  • SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE ** ** ** When unlocking, the same SHARED or EXCLUSIVE flag must be supplied as -** was given no the corresponding lock. +** was given on the corresponding lock. ** ** The xShmLock method can transition between unlocked and SHARED or ** between unlocked and EXCLUSIVE. It cannot transition between SHARED ** and EXCLUSIVE. */ @@ -1522,12 +1522,12 @@ ** tracks memory usage, for example. ** ** [[SQLITE_CONFIG_MEMSTATUS]]
    SQLITE_CONFIG_MEMSTATUS
    **
    ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, ** interpreted as a boolean, which enables or disables the collection of -** memory allocation statistics. ^(When memory allocation statistics are disabled, the -** following SQLite interfaces become non-operational: +** memory allocation statistics. ^(When memory allocation statistics are +** disabled, the following SQLite interfaces become non-operational: **
      **
    • [sqlite3_memory_used()] **
    • [sqlite3_memory_highwater()] **
    • [sqlite3_soft_heap_limit64()] **
    • [sqlite3_status()] @@ -1564,11 +1564,12 @@ ** that SQLite can use for the database page cache with the default page ** cache implementation. ** This configuration should not be used if an application-define page ** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2] ** configuration option. -** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned +** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to +** 8-byte aligned ** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page ** (a power of two between 512 and 32768) plus some extra bytes for each ** page header. ^The number of extra bytes needed by the page header ** can be determined using the [SQLITE_CONFIG_PCACHE_HDRSZ] option @@ -1584,11 +1585,12 @@ ** SQLite goes to [sqlite3_malloc()] for the additional storage space.
    ** ** [[SQLITE_CONFIG_HEAP]]
    SQLITE_CONFIG_HEAP
    **
    ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer ** that SQLite will use for all of its dynamic memory allocation needs -** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. +** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and +** [SQLITE_CONFIG_PAGECACHE]. ** ^The SQLITE_CONFIG_HEAP option is only available if SQLite is compiled ** with either [SQLITE_ENABLE_MEMSYS3] or [SQLITE_ENABLE_MEMSYS5] and returns ** [SQLITE_ERROR] if invoked otherwise. ** ^There are three arguments to SQLITE_CONFIG_HEAP: ** An 8-byte aligned pointer to the memory, @@ -1604,13 +1606,13 @@ ** for the minimum allocation size are 2**5 through 2**8.
    ** ** [[SQLITE_CONFIG_MUTEX]]
    SQLITE_CONFIG_MUTEX
    **
    ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a ** pointer to an instance of the [sqlite3_mutex_methods] structure. -** The argument specifies alternative low-level mutex routines to be used in place -** the mutex routines built into SQLite.)^ ^SQLite makes a copy of the -** content of the [sqlite3_mutex_methods] structure before the call to +** The argument specifies alternative low-level mutex routines to be used +** in place the mutex routines built into SQLite.)^ ^SQLite makes a copy of +** the content of the [sqlite3_mutex_methods] structure before the call to ** [sqlite3_config()] returns. ^If SQLite is compiled with ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then ** the entire mutexing subsystem is omitted from the build and hence calls to ** [sqlite3_config()] with the SQLITE_CONFIG_MUTEX configuration option will ** return [SQLITE_ERROR].
    @@ -1644,12 +1646,12 @@ ** the interface to a custom page cache implementation.)^ ** ^SQLite makes a copy of the [sqlite3_pcache_methods2] object. ** ** [[SQLITE_CONFIG_GETPCACHE2]]
    SQLITE_CONFIG_GETPCACHE2
    **
    ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which -** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of the current -** page cache implementation into that object.)^
    +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of +** the current page cache implementation into that object.)^ ** ** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    **
    The SQLITE_CONFIG_LOG option is used to configure the SQLite ** global [error log]. ** (^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a @@ -1670,12 +1672,13 @@ ** function must be threadsafe.
    ** ** [[SQLITE_CONFIG_URI]]
    SQLITE_CONFIG_URI **
    ^(The SQLITE_CONFIG_URI option takes a single argument of type int. ** If non-zero, then URI handling is globally enabled. If the parameter is zero, -** then URI handling is globally disabled.)^ ^If URI handling is globally enabled, -** all filenames passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or +** then URI handling is globally disabled.)^ ^If URI handling is globally +** enabled, all filenames passed to [sqlite3_open()], [sqlite3_open_v2()], +** [sqlite3_open16()] or ** specified as part of [ATTACH] commands are interpreted as URIs, regardless ** of whether or not the [SQLITE_OPEN_URI] flag is set when the database ** connection is opened. ^If it is globally disabled, filenames are ** only interpreted as URIs if the SQLITE_OPEN_URI flag is set when the ** database connection is opened. ^(By default, URI handling is globally @@ -1733,21 +1736,21 @@ ** changed to its compile-time default. ** ** [[SQLITE_CONFIG_WIN32_HEAPSIZE]] **
    SQLITE_CONFIG_WIN32_HEAPSIZE **
    ^The SQLITE_CONFIG_WIN32_HEAPSIZE option is only available if SQLite is -** compiled for Windows with the [SQLITE_WIN32_MALLOC] pre-processor macro defined. -** ^SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value +** compiled for Windows with the [SQLITE_WIN32_MALLOC] pre-processor macro +** defined. ^SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value ** that specifies the maximum size of the created heap. ** ** ** [[SQLITE_CONFIG_PCACHE_HDRSZ]] **
    SQLITE_CONFIG_PCACHE_HDRSZ **
    ^The SQLITE_CONFIG_PCACHE_HDRSZ option takes a single parameter which ** is a pointer to an integer and writes into that integer the number of extra -** bytes per page required for each page in [SQLITE_CONFIG_PAGECACHE]. The amount of -** extra space required can change depending on the compiler, +** bytes per page required for each page in [SQLITE_CONFIG_PAGECACHE]. +** The amount of extra space required can change depending on the compiler, ** target platform, and SQLite version. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ #define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ @@ -2047,10 +2050,11 @@ int sqlite3_complete(const char *sql); int sqlite3_complete16(const void *sql); /* ** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors +** KEYWORDS: {busy-handler callback} {busy handler} ** ** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X ** that might be invoked with argument P whenever ** an attempt is made to access a database table associated with ** [database connection] D when another thread @@ -2063,11 +2067,11 @@ ** is not NULL, then the callback might be invoked with two arguments. ** ** ^The first argument to the busy handler is a copy of the void* pointer which ** is the third argument to sqlite3_busy_handler(). ^The second argument to ** the busy handler callback is the number of times that the busy handler has -** been invoked for the same locking event. ^If the +** been invoked previously for the same locking event. ^If the ** busy callback returns 0, then no additional attempts are made to ** access the database and [SQLITE_BUSY] is returned ** to the application. ** ^If the callback returns non-zero, then another attempt ** is made to access the database and the cycle repeats. @@ -4518,11 +4522,12 @@ ** If these routines are called from within the different thread ** than the one containing the application-defined function that received ** the [sqlite3_context] pointer, the results are undefined. */ void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); -void sqlite3_result_blob64(sqlite3_context*,const void*,sqlite3_uint64,void(*)(void*)); +void sqlite3_result_blob64(sqlite3_context*,const void*, + sqlite3_uint64,void(*)(void*)); void sqlite3_result_double(sqlite3_context*, double); void sqlite3_result_error(sqlite3_context*, const char*, int); void sqlite3_result_error16(sqlite3_context*, const void*, int); void sqlite3_result_error_toobig(sqlite3_context*); void sqlite3_result_error_nomem(sqlite3_context*); @@ -7244,101 +7249,118 @@ int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); /* ** CAPI3REF: Checkpoint a database ** -** ^The [sqlite3_wal_checkpoint(D,X)] interface causes database named X -** on [database connection] D to be [checkpointed]. ^If X is NULL or an -** empty string, then a checkpoint is run on all databases of -** connection D. ^If the database connection D is not in -** [WAL | write-ahead log mode] then this interface is a harmless no-op. -** ^The [sqlite3_wal_checkpoint(D,X)] interface initiates a -** [sqlite3_wal_checkpoint_v2|PASSIVE] checkpoint. -** Use the [sqlite3_wal_checkpoint_v2()] interface to get a FULL -** or RESET checkpoint. -** -** ^The [wal_checkpoint pragma] can be used to invoke this interface -** from SQL. ^The [sqlite3_wal_autocheckpoint()] interface and the -** [wal_autocheckpoint pragma] can be used to cause this interface to be -** run whenever the WAL reaches a certain size threshold. -** -** See also: [sqlite3_wal_checkpoint_v2()] +** ^(The sqlite3_wal_checkpoint(D,X) is equivalent to +** [sqlite3_wal_checkpoint_v2](D,X,[SQLITE_CHECKPOINT_PASSIVE],0,0).)^ +** +** In brief, sqlite3_wal_checkpoint(D,X) causes the content in the +** [write-ahead log] for database X on [database connection] D to be +** transferred into the database file and for the write-ahead log to +** be reset. See the [checkpointing] documentation for addition +** information. +** +** This interface used to be the only way to cause a checkpoint to +** occur. But then the newer and more powerful [sqlite3_wal_checkpoint_v2()] +** interface was added. This interface is retained for backwards +** compatibility and as a convenience for applications that need to manually +** start a callback but which do not need the full power (and corresponding +** complication) of [sqlite3_wal_checkpoint_v2()]. */ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); /* ** CAPI3REF: Checkpoint a database ** -** Run a checkpoint operation on WAL database zDb attached to database -** handle db. The specific operation is determined by the value of the -** eMode parameter: +** ^(The sqlite3_wal_checkpoint_v2(D,X,M,L,C) interface runs a checkpoint +** operation on database X of [database connection] D in mode M. Status +** information is written back into integers pointed to by L and C.)^ +** ^(The M parameter must be a valid [checkpoint mode]:)^ ** **
    **
    SQLITE_CHECKPOINT_PASSIVE
    -** Checkpoint as many frames as possible without waiting for any database -** readers or writers to finish. Sync the db file if all frames in the log -** are checkpointed. This mode is the same as calling -** sqlite3_wal_checkpoint(). The [sqlite3_busy_handler|busy-handler callback] -** is never invoked. +** ^Checkpoint as many frames as possible without waiting for any database +** readers or writers to finish, then sync the database file if all frames +** in the log were checkpointed. ^The [busy-handler callback] +** is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode. +** ^On the other hand, passive mode might leave the checkpoint unfinished +** if there are concurrent readers or writers. ** **
    SQLITE_CHECKPOINT_FULL
    -** This mode blocks (it invokes the +** ^This mode blocks (it invokes the ** [sqlite3_busy_handler|busy-handler callback]) until there is no ** database writer and all readers are reading from the most recent database -** snapshot. It then checkpoints all frames in the log file and syncs the -** database file. This call blocks database writers while it is running, -** but not database readers. +** snapshot. ^It then checkpoints all frames in the log file and syncs the +** database file. ^This mode blocks new database writers while it is pending, +** but new database readers are allowed to continue unimpeded. ** **
    SQLITE_CHECKPOINT_RESTART
    -** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after -** checkpointing the log file it blocks (calls the -** [sqlite3_busy_handler|busy-handler callback]) -** until all readers are reading from the database file only. This ensures -** that the next client to write to the database file restarts the log file -** from the beginning. This call blocks database writers while it is running, -** but not database readers. +** ^This mode works the same way as SQLITE_CHECKPOINT_FULL with the addition +** that after checkpointing the log file it blocks (calls the +** [busy-handler callback]) +** until all readers are reading from the database file only. ^This ensures +** that the next writer will restart the log file from the beginning. +** ^Like SQLITE_CHECKPOINT_FULL, this mode blocks new +** database writer attempts while it is pending, but does not impede readers. +** +**
    SQLITE_CHECKPOINT_TRUNCATE
    +** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the +** addition that it also truncates the log file to zero bytes just prior +** to a successful return. **
    ** -** If pnLog is not NULL, then *pnLog is set to the total number of frames in -** the log file before returning. If pnCkpt is not NULL, then *pnCkpt is set to -** the total number of checkpointed frames (including any that were already -** checkpointed when this function is called). *pnLog and *pnCkpt may be -** populated even if sqlite3_wal_checkpoint_v2() returns other than SQLITE_OK. -** If no values are available because of an error, they are both set to -1 -** before returning to communicate this to the caller. +** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in +** the log file or to -1 if the checkpoint could not run because +** of an error or because the database is not in [WAL mode]. ^If pnCkpt is not +** NULL,then *pnCkpt is set to the total number of checkpointed frames in the +** log file (including any that were already checkpointed before the function +** was called) or to -1 if the checkpoint could not run due to an error or +** because the database is not in WAL mode. ^Note that upon successful +** completion of an SQLITE_CHECKPOINT_TRUNCATE, the log file will have been +** truncated to zero bytes and so both *pnLog and *pnCkpt will be set to zero. ** -** All calls obtain an exclusive "checkpoint" lock on the database file. If +** ^All calls obtain an exclusive "checkpoint" lock on the database file. ^If ** any other process is running a checkpoint operation at the same time, the -** lock cannot be obtained and SQLITE_BUSY is returned. Even if there is a +** lock cannot be obtained and SQLITE_BUSY is returned. ^Even if there is a ** busy-handler configured, it will not be invoked in this case. ** -** The SQLITE_CHECKPOINT_FULL and RESTART modes also obtain the exclusive -** "writer" lock on the database file. If the writer lock cannot be obtained -** immediately, and a busy-handler is configured, it is invoked and the writer -** lock retried until either the busy-handler returns 0 or the lock is -** successfully obtained. The busy-handler is also invoked while waiting for -** database readers as described above. If the busy-handler returns 0 before +** ^The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the +** exclusive "writer" lock on the database file. ^If the writer lock cannot be +** obtained immediately, and a busy-handler is configured, it is invoked and +** the writer lock retried until either the busy-handler returns 0 or the lock +** is successfully obtained. ^The busy-handler is also invoked while waiting for +** database readers as described above. ^If the busy-handler returns 0 before ** the writer lock is obtained or while waiting for database readers, the ** checkpoint operation proceeds from that point in the same way as ** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible -** without blocking any further. SQLITE_BUSY is returned in this case. +** without blocking any further. ^SQLITE_BUSY is returned in this case. ** -** If parameter zDb is NULL or points to a zero length string, then the -** specified operation is attempted on all WAL databases. In this case the -** values written to output parameters *pnLog and *pnCkpt are undefined. If +** ^If parameter zDb is NULL or points to a zero length string, then the +** specified operation is attempted on all WAL databases [attached] to +** [database connection] db. In this case the +** values written to output parameters *pnLog and *pnCkpt are undefined. ^If ** an SQLITE_BUSY error is encountered when processing one or more of the ** attached WAL databases, the operation is still attempted on any remaining -** attached databases and SQLITE_BUSY is returned to the caller. If any other +** attached databases and SQLITE_BUSY is returned at the end. ^If any other ** error occurs while processing an attached database, processing is abandoned -** and the error code returned to the caller immediately. If no error +** and the error code is returned to the caller immediately. ^If no error ** (SQLITE_BUSY or otherwise) is encountered while processing the attached ** databases, SQLITE_OK is returned. ** -** If database zDb is the name of an attached database that is not in WAL -** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. If +** ^If database zDb is the name of an attached database that is not in WAL +** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. ^If ** zDb is not NULL (or a zero length string) and is not the name of any ** attached database, SQLITE_ERROR is returned to the caller. +** +** ^Unless it returns SQLITE_MISUSE, +** the sqlite3_wal_checkpoint_v2() interface +** sets the error information that is queried by +** [sqlite3_errcode()] and [sqlite3_errmsg()]. +** +** ^The [PRAGMA wal_checkpoint] command can be used to invoke this interface +** from SQL. */ int sqlite3_wal_checkpoint_v2( sqlite3 *db, /* Database handle */ const char *zDb, /* Name of attached database (or NULL) */ int eMode, /* SQLITE_CHECKPOINT_* value */ @@ -7345,20 +7367,22 @@ int *pnLog, /* OUT: Size of WAL log in frames */ int *pnCkpt /* OUT: Total number of frames checkpointed */ ); /* -** CAPI3REF: Checkpoint operation parameters +** CAPI3REF: Checkpoint Mode Values +** KEYWORDS: {checkpoint mode} ** -** These constants can be used as the 3rd parameter to -** [sqlite3_wal_checkpoint_v2()]. See the [sqlite3_wal_checkpoint_v2()] -** documentation for additional information about the meaning and use of -** each of these values. +** These constants define all valid values for the "checkpoint mode" passed +** as the third parameter to the [sqlite3_wal_checkpoint_v2()] interface. +** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the +** meaning of each of these checkpoint modes. */ -#define SQLITE_CHECKPOINT_PASSIVE 0 -#define SQLITE_CHECKPOINT_FULL 1 -#define SQLITE_CHECKPOINT_RESTART 2 +#define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ +#define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */ +#define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ /* ** CAPI3REF: Virtual Table Interface Configuration ** ** This function may be called by either the [xConnect] or [xCreate] method @@ -7453,16 +7477,16 @@ ** [sqlite3_stmt_scanstatus(S,X,T,V)] interface. Each constant designates a ** different metric for sqlite3_stmt_scanstatus() to return. ** **
    ** [[SQLITE_SCANSTAT_NLOOP]]
    SQLITE_SCANSTAT_NLOOP
    -**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set to the -** total number of times that the X-th loop has run.
    +**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be +** set to the total number of times that the X-th loop has run.
    ** ** [[SQLITE_SCANSTAT_NVISIT]]
    SQLITE_SCANSTAT_NVISIT
    -**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set to the -** total number of rows examined by all iterations of the X-th loop.
    +**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set +** to the total number of rows examined by all iterations of the X-th loop.
    ** ** [[SQLITE_SCANSTAT_EST]]
    SQLITE_SCANSTAT_EST
    **
    ^The "double" variable pointed to by the T parameter will be set to the ** query planner's estimate for the average number of rows output from each ** iteration of the X-th loop. If the query planner's estimates was accurate, @@ -7469,18 +7493,18 @@ ** then this value will approximate the quotient NVISIT/NLOOP and the ** product of this value for all prior loops with the same SELECTID will ** be the NLOOP value for the current loop. ** ** [[SQLITE_SCANSTAT_NAME]]
    SQLITE_SCANSTAT_NAME
    -**
    ^The "const char *" variable pointed to by the T parameter will be set to -** a zero-terminated UTF-8 string containing the name of the index or table used -** for the X-th loop. +**
    ^The "const char *" variable pointed to by the T parameter will be set +** to a zero-terminated UTF-8 string containing the name of the index or table +** used for the X-th loop. ** ** [[SQLITE_SCANSTAT_EXPLAIN]]
    SQLITE_SCANSTAT_EXPLAIN
    -**
    ^The "const char *" variable pointed to by the T parameter will be set to -** a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] description -** for the X-th loop. +**
    ^The "const char *" variable pointed to by the T parameter will be set +** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] +** description for the X-th loop. ** ** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECT
    **
    ^The "int" variable pointed to by the T parameter will be set to the ** "select-id" for the X-th loop. The select-id identifies which query or ** subquery the loop is part of. The main query has a select-id of zero. @@ -7499,12 +7523,12 @@ ** CAPI3REF: Prepared Statement Scan Status ** ** Return status data for a single loop within query pStmt. ** ** The "iScanStatusOp" parameter determines which status information to return. -** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior of -** this interface is undefined. +** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior +** of this interface is undefined. ** ^The requested measurement is written into a variable pointed to by ** the "pOut" parameter. ** Parameter "idx" identifies the specific loop to retrieve statistics for. ** Loops are numbered starting from zero. ^If idx is out of range - less than ** zero or greater than or equal to the total number of loops used to implement Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -2018,11 +2018,11 @@ }; /* ** The following are the meanings of bits in the Expr.flags field. */ -#define EP_FromJoin 0x000001 /* Originated in ON or USING clause of a join */ +#define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ #define EP_Agg 0x000002 /* Contains one or more aggregate functions */ #define EP_Resolved 0x000004 /* IDs have been resolved to COLUMNs */ #define EP_Error 0x000008 /* Expression contains one or more errors */ #define EP_Distinct 0x000010 /* Aggregate function with DISTINCT keyword */ #define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ @@ -2038,10 +2038,11 @@ #define EP_Static 0x008000 /* Held in memory not obtained from malloc() */ #define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ #define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ #define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ #define EP_Constant 0x080000 /* Node is a constant */ +#define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ /* ** These macros can be used to test, set, or clear bits in the ** Expr.flags field. */ Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -3636,10 +3636,11 @@ int objc, Tcl_Obj *CONST objv[] ){ sqlite3 *db; const char *zSql; + char *zCopy = 0; /* malloc() copy of zSql */ int bytes; const char *zTail = 0; sqlite3_stmt *pStmt = 0; char zBuf[50]; int rc; @@ -3651,11 +3652,25 @@ } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; zSql = Tcl_GetString(objv[2]); if( Tcl_GetIntFromObj(interp, objv[3], &bytes) ) return TCL_ERROR; - rc = sqlite3_prepare_v2(db, zSql, bytes, &pStmt, objc>=5 ? &zTail : 0); + /* Instead of using zSql directly, make a copy into a buffer obtained + ** directly from malloc(). The idea is to make it easier for valgrind + ** to spot buffer overreads. */ + if( bytes>=0 ){ + zCopy = malloc(bytes); + memcpy(zCopy, zSql, bytes); + }else{ + int n = (int)strlen(zSql) + 1; + zCopy = malloc(n); + memcpy(zCopy, zSql, n); + } + rc = sqlite3_prepare_v2(db, zCopy, bytes, &pStmt, objc>=5 ? &zTail : 0); + free(zCopy); + zTail = &zSql[(zTail - zCopy)]; + assert(rc==SQLITE_OK || pStmt==0); Tcl_ResetResult(interp); if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; if( zTail && objc>=5 ){ if( bytes>=0 ){ @@ -5673,32 +5688,35 @@ int eMode; int nLog = -555; int nCkpt = -555; Tcl_Obj *pRet; - const char * aMode[] = { "passive", "full", "restart", 0 }; + const char * aMode[] = { "passive", "full", "restart", "truncate", 0 }; assert( SQLITE_CHECKPOINT_PASSIVE==0 ); assert( SQLITE_CHECKPOINT_FULL==1 ); assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); if( objc!=3 && objc!=4 ){ Tcl_WrongNumArgs(interp, 1, objv, "DB MODE ?NAME?"); return TCL_ERROR; } if( objc==4 ){ zDb = Tcl_GetString(objv[3]); } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) - || Tcl_GetIndexFromObj(interp, objv[2], aMode, "mode", 0, &eMode) - ){ + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) || ( + TCL_OK!=Tcl_GetIntFromObj(0, objv[2], &eMode) + && TCL_OK!=Tcl_GetIndexFromObj(interp, objv[2], aMode, "mode", 0, &eMode) + )){ return TCL_ERROR; } rc = sqlite3_wal_checkpoint_v2(db, zDb, eMode, &nLog, &nCkpt); if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ - Tcl_SetResult(interp, (char *)sqlite3_errmsg(db), TCL_VOLATILE); + const char *zErrCode = sqlite3ErrName(rc); + Tcl_AppendResult(interp, zErrCode, " - ", (char *)sqlite3_errmsg(db), 0); return TCL_ERROR; } pRet = Tcl_NewObj(); Tcl_ListObjAppendElement(interp, pRet, Tcl_NewIntObj(rc==SQLITE_BUSY?1:0)); Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -5703,12 +5703,12 @@ #ifndef SQLITE_OMIT_WAL /* Opcode: Checkpoint P1 P2 P3 * * ** ** Checkpoint database P1. This is a no-op if P1 is not currently in -** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL -** or RESTART. Write 1 or 0 into mem[P3] if the checkpoint returns +** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL, +** RESTART, or TRUNCATE. Write 1 or 0 into mem[P3] if the checkpoint returns ** SQLITE_BUSY or not, respectively. Write the number of pages in the ** WAL after the checkpoint into mem[P3+1] and the number of pages ** in the WAL that have been checkpointed after the checkpoint ** completes into mem[P3+2]. However on an error, mem[P3+1] and ** mem[P3+2] are initialized to -1. @@ -5722,10 +5722,11 @@ aRes[0] = 0; aRes[1] = aRes[2] = -1; assert( pOp->p2==SQLITE_CHECKPOINT_PASSIVE || pOp->p2==SQLITE_CHECKPOINT_FULL || pOp->p2==SQLITE_CHECKPOINT_RESTART + || pOp->p2==SQLITE_CHECKPOINT_TRUNCATE ); rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]); if( rc==SQLITE_BUSY ){ rc = SQLITE_OK; aRes[0] = 1; Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -1740,11 +1740,11 @@ for(n=0; naVar[n].flags = MEM_Null; p->aVar[n].db = db; } } - if( p->azVar ){ + if( p->azVar && pParse->nzVar>0 ){ p->nzVar = pParse->nzVar; memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0])); memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0])); } if( p->aMem ){ Index: src/vdbesort.c ================================================================== --- src/vdbesort.c +++ src/vdbesort.c @@ -145,10 +145,17 @@ */ #if 0 # define SQLITE_DEBUG_SORTER_THREADS 1 #endif +/* +** Hard-coded maximum amount of data to accumulate in memory before flushing +** to a level 0 PMA. The purpose of this limit is to prevent various integer +** overflows. 512MiB. +*/ +#define SQLITE_MAX_MXPMASIZE (1<<29) + /* ** Private objects used by the sorter */ typedef struct MergeEngine MergeEngine; /* Merge PMAs together */ typedef struct PmaReader PmaReader; /* Incrementally read one PMA */ @@ -843,11 +850,11 @@ if( !sqlite3TempInMemory(db) ){ pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; mxCache = db->aDb[0].pSchema->cache_size; if( mxCachemxPmaSize = mxCache * pgsz; + pSorter->mxPmaSize = MIN((i64)mxCache*pgsz, SQLITE_MAX_MXPMASIZE); /* EVIDENCE-OF: R-26747-61719 When the application provides any amount of ** scratch memory using SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary ** large heap allocations. */ Index: src/vtab.c ================================================================== --- src/vtab.c +++ src/vtab.c @@ -330,11 +330,16 @@ pTable->tabFlags |= TF_Virtual; pTable->nModuleArg = 0; addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); addModuleArgument(db, pTable, 0); addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); - pParse->sNameToken.n = (int)(&pModuleName->z[pModuleName->n] - pName1->z); + assert( (pParse->sNameToken.z==pName2->z && pName2->z!=0) + || (pParse->sNameToken.z==pName1->z && pName2->z==0) + ); + pParse->sNameToken.n = (int)( + &pModuleName->z[pModuleName->n] - pParse->sNameToken.z + ); #ifndef SQLITE_OMIT_AUTHORIZATION /* Creating a virtual table invokes the authorization callback twice. ** The first invocation, to obtain permission to INSERT a row into the ** sqlite_master table, has already been made by sqlite3StartTable(). Index: src/wal.c ================================================================== --- src/wal.c +++ src/wal.c @@ -1826,10 +1826,42 @@ p->pIter = 0; } return p->rc; } + +/* +** The following is guaranteed when this function is called: +** +** a) the WRITER lock is held, +** b) the entire log file has been checkpointed, and +** c) any existing readers are reading exclusively from the database +** file - there are no readers that may attempt to read a frame from +** the log file. +** +** This function updates the shared-memory structures so that the next +** client to write to the database (which may be this one) does so by +** writing frames into the start of the log file. +** +** The value of parameter salt1 is used as the aSalt[1] value in the +** new wal-index header. It should be passed a pseudo-random value (i.e. +** one obtained from sqlite3_randomness()). +*/ +static void walRestartHdr(Wal *pWal, u32 salt1){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + int i; /* Loop counter */ + u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ + pWal->nCkpt++; + pWal->hdr.mxFrame = 0; + sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); + memcpy(&pWal->hdr.aSalt[1], &salt1, 4); + walIndexWriteHdr(pWal); + pInfo->nBackfill = 0; + pInfo->aReadMark[1] = 0; + for(i=2; iaReadMark[i] = READMARK_NOT_USED; + assert( pInfo->aReadMark[0]==0 ); +} /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. ** @@ -1861,43 +1893,64 @@ ** time. */ static int walCheckpoint( Wal *pWal, /* Wal connection */ int eMode, /* One of PASSIVE, FULL or RESTART */ - int (*xBusyCall)(void*), /* Function to call when busy */ + int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags for OsSync() (or 0) */ u8 *zBuf, /* Temporary buffer to use */ int nBuf /* Size of zBuf in bytes */ ){ int rc; /* Return code */ - int (*xBusy)(void*) = 0; /* Function to call when waiting for locks */ WalCkpt sC; - if( eMode!=SQLITE_CHECKPOINT_PASSIVE ) xBusy = xBusyCall; + /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked + ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ + assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); + rc = walCheckpointStart(pWal, zBuf, nBuf, xBusy, pBusyArg, sync_flags, &sC); if( sC.pIter==0 ) goto walcheckpoint_out; assert( rc==SQLITE_OK ); /* Step the checkpoint object until it reports something other than ** SQLITE_OK. */ while( SQLITE_OK==(rc = walCheckpointStep(&sC)) ); rc = walCheckpointFinalize(&sC); - /* If this is an SQLITE_CHECKPOINT_RESTART operation, and the entire wal - ** file has been copied into the database file, then block until all - ** readers have finished using the wal file. This ensures that the next - ** process to write to the database restarts the wal file. + /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the + ** entire wal file has been copied into the database file, then block + ** until all readers have finished using the wal file. This ensures that + ** the next process to write to the database restarts the wal file. */ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); if( sC.pInfo->nBackfillhdr.mxFrame ){ rc = SQLITE_BUSY; - }else if( eMode==SQLITE_CHECKPOINT_RESTART ){ + }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ + u32 salt1; + sqlite3_randomness(4, &salt1); assert( sC.mxSafeFrame==pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ + if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){ + /* IMPLEMENTATION-OF: R-44699-57140 This mode works the same way as + ** SQLITE_CHECKPOINT_RESTART with the addition that it also + ** truncates the log file to zero bytes just prior to a + ** successful return. + ** + ** In theory, it might be safe to do this without updating the + ** wal-index header in shared memory, as all subsequent reader or + ** writer clients should see that the entire log file has been + ** checkpointed and behave accordingly. This seems unsafe though, + ** as it would leave the system in a state where the contents of + ** the wal-index header do not match the contents of the + ** file-system. To avoid this, update the wal-index header to + ** indicate that the log file contains zero valid frames. */ + walRestartHdr(pWal, salt1); + rc = sqlite3OsTruncate(pWal->pWalFd, 0); + } walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); } } } @@ -2687,11 +2740,10 @@ } return rc; } - /* ** This function is called just before writing a set of frames to the log ** file (see sqlite3WalFrames()). It checks to see if, instead of appending ** to the current log file, it is possible to overwrite the start of the ** existing log file with the new frames (i.e. "reset" the log). If so, @@ -2720,24 +2772,12 @@ ** wal-index header to reflect this. ** ** In theory it would be Ok to update the cache of the header only ** at this point. But updating the actual wal-index header is also ** safe and means there is no special case for sqlite3WalUndo() - ** to handle if this transaction is rolled back. - */ - int i; /* Loop counter */ - u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ - - pWal->nCkpt++; - pWal->hdr.mxFrame = 0; - sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); - aSalt[1] = salt1; - walIndexWriteHdr(pWal); - pInfo->nBackfill = 0; - pInfo->aReadMark[1] = 0; - for(i=2; iaReadMark[i] = READMARK_NOT_USED; - assert( pInfo->aReadMark[0]==0 ); + ** to handle if this transaction is rolled back. */ + walRestartHdr(pWal, salt1); walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); }else if( rc!=SQLITE_BUSY ){ return rc; } } @@ -3023,11 +3063,11 @@ ** If parameter xBusy is not NULL, it is a pointer to a busy-handler ** callback. In this case this function runs a blocking checkpoint. */ int sqlite3WalCheckpoint( Wal *pWal, /* Wal connection */ - int eMode, /* PASSIVE, FULL or RESTART */ + int eMode, /* PASSIVE, FULL, RESTART, or TRUNCATE */ int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags to sync db file with (or 0) */ int nBuf, /* Size of temporary buffer */ u8 *zBuf, /* Temporary buffer to use */ @@ -3035,40 +3075,54 @@ int *pnCkpt /* OUT: Number of backfilled frames in WAL */ ){ int rc; /* Return code */ int isChanged = 0; /* True if a new wal-index header is loaded */ int eMode2 = eMode; /* Mode to pass to walCheckpoint() */ + int (*xBusy2)(void*) = xBusy; /* Busy handler for eMode2 */ assert( pWal->ckptLock==0 ); assert( pWal->writeLock==0 ); + /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked + ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ + assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); + if( pWal->readOnly ) return SQLITE_READONLY; WALTRACE(("WAL%p: checkpoint begins\n", pWal)); + + /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive + ** "checkpoint" lock on the database file. */ rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); if( rc ){ - /* Usually this is SQLITE_BUSY meaning that another thread or process - ** is already running a checkpoint, or maybe a recovery. But it might - ** also be SQLITE_IOERR. */ + /* EVIDENCE-OF: R-10421-19736 If any other process is running a + ** checkpoint operation at the same time, the lock cannot be obtained and + ** SQLITE_BUSY is returned. + ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, + ** it will not be invoked in this case. + */ + testcase( rc==SQLITE_BUSY ); + testcase( xBusy!=0 ); return rc; } pWal->ckptLock = 1; - /* If this is a blocking-checkpoint, then obtain the write-lock as well - ** to prevent any writers from running while the checkpoint is underway. - ** This has to be done before the call to walIndexReadHdr() below. + /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and + ** TRUNCATE modes also obtain the exclusive "writer" lock on the database + ** file. ** - ** If the writer lock cannot be obtained, then a passive checkpoint is - ** run instead. Since the checkpointer is not holding the writer lock, - ** there is no point in blocking waiting for any readers. Assuming no - ** other error occurs, this function will return SQLITE_BUSY to the caller. + ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained + ** immediately, and a busy-handler is configured, it is invoked and the + ** writer lock retried until either the busy-handler returns 0 or the + ** lock is successfully obtained. */ if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1); if( rc==SQLITE_OK ){ pWal->writeLock = 1; }else if( rc==SQLITE_BUSY ){ eMode2 = SQLITE_CHECKPOINT_PASSIVE; + xBusy2 = 0; rc = SQLITE_OK; } } /* Read the wal-index header. */ @@ -3079,11 +3133,11 @@ } } /* Copy data from the log to the database file. */ if( rc==SQLITE_OK ){ - rc = walCheckpoint(pWal, eMode2, xBusy, pBusyArg, sync_flags, zBuf, nBuf); + rc = walCheckpoint(pWal, eMode2, xBusy2, pBusyArg, sync_flags, zBuf, nBuf); /* If no error occurred, set the output variables. */ if( rc==SQLITE_OK || rc==SQLITE_BUSY ){ if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; if( pnCkpt ) *pnCkpt = (int)(walCkptInfo(pWal)->nBackfill); Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -220,10 +220,11 @@ memcpy(pWC->a, pOld, sizeof(pWC->a[0])*pWC->nTerm); if( pOld!=pWC->aStatic ){ sqlite3DbFree(db, pOld); } pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); + memset(&pWC->a[pWC->nTerm], 0, sizeof(pWC->a[0])*(pWC->nSlot-pWC->nTerm)); } pTerm = &pWC->a[idx = pWC->nTerm++]; if( p && ExprHasProperty(p, EP_Unlikely) ){ pTerm->truthProb = sqlite3LogEst(p->iTable) - 270; }else{ @@ -2945,11 +2946,11 @@ WhereLevel *pLvl, /* Level to add scanstatus() entry for */ int addrExplain /* Address of OP_Explain (or 0) */ ){ const char *zObj = 0; WhereLoop *pLoop = pLvl->pWLoop; - if( (pLoop->wsFlags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ zObj = pLoop->u.btree.pIndex->zName; }else{ zObj = pSrclist->a[pLvl->iFrom].zName; } sqlite3VdbeScanStatus( @@ -3589,14 +3590,13 @@ int iTerm; for(iTerm=0; iTermnTerm; iTerm++){ Expr *pExpr = pWC->a[iTerm].pExpr; if( &pWC->a[iTerm] == pTerm ) continue; if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; - testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); - testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL ); - if( pWC->a[iTerm].wtFlags & (TERM_ORINFO|TERM_VIRTUAL) ) continue; + if( (pWC->a[iTerm].wtFlags & TERM_VIRTUAL)!=0 ) continue; if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; + testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); pExpr = sqlite3ExprDup(db, pExpr, 0); pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); } if( pAndExpr ){ pAndExpr = sqlite3PExpr(pParse, TK_AND, 0, pAndExpr, 0); @@ -4289,25 +4289,46 @@ /* ** Adjust the WhereLoop.nOut value downward to account for terms of the ** WHERE clause that reference the loop but which are not used by an ** index. +* +** For every WHERE clause term that is not used by the index +** and which has a truth probability assigned by one of the likelihood(), +** likely(), or unlikely() SQL functions, reduce the estimated number +** of output rows by the probability specified. +** +** TUNING: For every WHERE clause term that is not used by the index +** and which does not have an assigned truth probability, heuristics +** described below are used to try to estimate the truth probability. +** TODO --> Perhaps this is something that could be improved by better +** table statistics. +** +** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75% +** value corresponds to -1 in LogEst notation, so this means decrement +** the WhereLoop.nOut field for every such WHERE clause term. ** -** In the current implementation, the first extra WHERE clause term reduces -** the number of output rows by a factor of 10 and each additional term -** reduces the number of output rows by sqrt(2). +** Heuristic 2: If there exists one or more WHERE clause terms of the +** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the +** final output row estimate is no greater than 1/4 of the total number +** of rows in the table. In other words, assume that x==EXPR will filter +** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the +** "x" column is boolean or else -1 or 0 or 1 is a common default value +** on the "x" column and so in that case only cap the output row estimate +** at 1/2 instead of 1/4. */ static void whereLoopOutputAdjust( WhereClause *pWC, /* The WHERE clause */ WhereLoop *pLoop, /* The loop to adjust downward */ LogEst nRow /* Number of rows in the entire table */ ){ WhereTerm *pTerm, *pX; Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); - int i, j; - int nEq = 0; /* Number of = constraints not within likely()/unlikely() */ + int i, j, k; + LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ + assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; if( (pTerm->prereqAll & notAllowed)!=0 ) continue; for(j=pLoop->nLTerm-1; j>=0; j--){ @@ -4316,24 +4337,30 @@ if( pX==pTerm ) break; if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; } if( j<0 ){ if( pTerm->truthProb<=0 ){ + /* If a truth probability is specified using the likelihood() hints, + ** then use the probability provided by the application. */ pLoop->nOut += pTerm->truthProb; }else{ + /* In the absence of explicit truth probabilities, use heuristics to + ** guess a reasonable truth probability. */ pLoop->nOut--; - if( pTerm->eOperator&WO_EQ ) nEq++; + if( pTerm->eOperator&WO_EQ ){ + Expr *pRight = pTerm->pExpr->pRight; + if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ + k = 10; + }else{ + k = 20; + } + if( iReducenOut>nRow-10 ){ - pLoop->nOut = nRow - 10; - } + if( pLoop->nOut > nRow-iReduce ) pLoop->nOut = nRow - iReduce; } /* ** Adjust the cost C by the costMult facter T. This only occurs if ** compiled with -DSQLITE_ENABLE_COSTMULT Index: test/autoindex2.test ================================================================== --- test/autoindex2.test +++ test/autoindex2.test @@ -222,50 +222,6 @@ # # ^^^--- Before being fixed, the above was using an automatic covering # on t3 and reordering the tables so that t3 was in the outer loop and # implementing the ORDER BY clause using a B-Tree. -do_execsql_test autoindex2-120 { - EXPLAIN QUERY PLAN - SELECT - t1_id, - t1.did, - param2, - param3, - t1.ptime, - t1.trange, - t1.exmass, - t1.mass, - t1.vstatus, - type, - subtype, - t1.deviation, - t1.formula, - dparam1, - reserve1, - reserve2, - param4, - t1.last_operation, - t1.admin_uuid, - t1.previous_value, - t1.job_id, - client_did, - t1.last_t1, - t1.data_t1, - t1.previous_date, - param5, - param6, - mgr_uuid - FROM - t3, - t2, - t1 - WHERE - t1.ptime > 1393520400 - AND param3<>9001 - AND t3.flg7 = 1 - AND t1.did = t2.did - AND t2.uid = t3.uid - ORDER BY t1.ptime desc LIMIT 500; -} {0 0 2 {SEARCH TABLE t1 USING INDEX t1x1 (ptime>?)} 0 1 1 {SEARCH TABLE t2 USING INDEX t2x0 (did=?)} 0 2 0 {SEARCH TABLE t3 USING INDEX t3x0 (uid=?)}} - finish_test ADDED test/bigsort.test Index: test/bigsort.test ================================================================== --- /dev/null +++ test/bigsort.test @@ -0,0 +1,43 @@ +# 2014 November 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix bigsort + +#-------------------------------------------------------------------- +# At one point there was an overflow problem if the product of the +# cache-size and page-size was larger than 2^31. Causing an infinite +# loop if the product was also an integer multiple of 2^32, or +# inefficiency otherwise. +# +do_execsql_test 1.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b); + BEGIN; + WITH data(x,y) AS ( + SELECT 1, zeroblob(10000) + UNION ALL + SELECT x+1, y FROM data WHERE x < 300000 + ) + INSERT INTO t1 SELECT * FROM data; + COMMIT; +} +do_execsql_test 1.1 { + PRAGMA cache_size = 4194304; + CREATE INDEX i1 ON t1(a, b); +} + + +finish_test + + ADDED test/btree01.test Index: test/btree01.test ================================================================== --- /dev/null +++ test/btree01.test @@ -0,0 +1,132 @@ +# 2014-11-27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains test cases for b-tree logic. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix btree01 + +# The refactoring on the b-tree balance() routine in check-in +# http://www.sqlite.org/src/info/face33bea1ba3a (2014-10-27) +# caused the integrity_check on the following SQL to fail. +# +do_execsql_test btree01-1.1 { + PRAGMA page_size=65536; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB); + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(3000); + UPDATE t1 SET b=zeroblob(64000) WHERE a=2; + PRAGMA integrity_check; +} {ok} + +# The previous test is sufficient to prevent a regression. But we +# add a number of additional tests to stress the balancer in similar +# ways, looking for related problems. +# +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.2.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(3000); + UPDATE t1 SET b=zeroblob(64000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.3.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(2000); + UPDATE t1 SET b=zeroblob(64000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.4.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(6499) WHERE (a%3)==0; + UPDATE t1 SET b=zeroblob(6499) WHERE (a%3)==1; + UPDATE t1 SET b=zeroblob(6499) WHERE (a%3)==2; + UPDATE t1 SET b=zeroblob(64000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.5.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6542) FROM c; + UPDATE t1 SET b=zeroblob(2331); + UPDATE t1 SET b=zeroblob(65496) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.6.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6542) FROM c; + UPDATE t1 SET b=zeroblob(2332); + UPDATE t1 SET b=zeroblob(65496) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.7.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(1); + UPDATE t1 SET b=zeroblob(65000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=31} {incr i} { + do_test btree01-1.8.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<31) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(4000); + UPDATE t1 SET b=zeroblob(65000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} + +finish_test Index: test/distinct.test ================================================================== --- test/distinct.test +++ test/distinct.test @@ -219,7 +219,37 @@ WHEN 2 THEN zeroblob(5) ELSE 'xyzzy' END FROM t1; SELECT quote(x) FROM t2 ORDER BY 1; } {'xyzzy' X'0000000000'} + +#---------------------------------------------------------------------------- +# Ticket [c5ea805691bfc4204b1cb9e9aa0103bd48bc7d34] (2014-12-04) +# Make sure that DISTINCT works together with ORDER BY and descending +# indexes. +# +do_execsql_test 5.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x); + INSERT INTO t1(x) VALUES(3),(1),(5),(2),(6),(4),(5),(1),(3); + CREATE INDEX t1x ON t1(x DESC); + SELECT DISTINCT x FROM t1 ORDER BY x ASC; +} {1 2 3 4 5 6} +do_execsql_test 5.2 { + SELECT DISTINCT x FROM t1 ORDER BY x DESC; +} {6 5 4 3 2 1} +do_execsql_test 5.3 { + SELECT DISTINCT x FROM t1 ORDER BY x; +} {1 2 3 4 5 6} +do_execsql_test 5.4 { + DROP INDEX t1x; + CREATE INDEX t1x ON t1(x ASC); + SELECT DISTINCT x FROM t1 ORDER BY x ASC; +} {1 2 3 4 5 6} +do_execsql_test 5.5 { + SELECT DISTINCT x FROM t1 ORDER BY x DESC; +} {6 5 4 3 2 1} +do_execsql_test 5.6 { + SELECT DISTINCT x FROM t1 ORDER BY x; +} {1 2 3 4 5 6} finish_test ADDED test/e_walckpt.test Index: test/e_walckpt.test ================================================================== --- /dev/null +++ test/e_walckpt.test @@ -0,0 +1,290 @@ +# 2014 December 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/wal_common.tcl +set testprefix e_walckpt + +# The following two commands are used to determine if any of the files +# "test.db", "test.db2" and "test.db3" are modified by a test case. +# +# The [save_db_hashes] command saves a hash of the current contents of +# all three files in global variables. The [compare_db_hashes] compares +# the current contents with the saved hashes and returns a list of the +# files that have changed. +# +proc save_db_hashes {} { + global H + foreach f {test.db test.db2 test.db3} { + set H($f) 0 + catch { set H($f) [md5file $f] } + } +} +proc compare_db_hashes {} { + global H + set ret [list] + foreach f {test.db test.db2 test.db3} { + set expect 0 + catch { set expect [md5file $f] } + if {$H($f) != $expect} { lappend ret $f } + } + set ret +} + + +# The following tests are run 3 times, each using a different method of +# invoking a checkpoint: +# +# 1) Using sqlite3_wal_checkpoint_v2() +# 2) Using "PRAGMA wal_checkpoint" +# 3) Using sqlite3_wal_checkpoint() in place of checkpoint_v2(PASSIVE) +# +# Cases (2) and (3) are to show that the following statements are +# correct, respectively: +# +# EVIDENCE-OF: R-36706-10507 The PRAGMA wal_checkpoint command can be +# used to invoke this interface from SQL. +# +# EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is +# equivalent to +# sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). +# +foreach {tn script} { + 1 { + proc checkpoint {db mode args} { + eval sqlite3_wal_checkpoint_v2 [list $db] [list $mode] $args + } + } + + 2 { + proc checkpoint {db mode args} { + set sql "PRAGMA wal_checkpoint" + if {[llength $args] && [lindex $args 0]!=""} { + set sql "PRAGMA [lindex $args 0].wal_checkpoint" + } + set rc [catch { $db eval $sql } msg] + if {$rc} { + regsub {database} $msg {database:} msg + error "[sqlite3_errcode $db] - $msg" + } + set msg + } + } + + 3 { + proc checkpoint {db mode args} { + if {$mode == "passive"} { + set rc [eval sqlite3_wal_checkpoint [list $db] $args] + if {$rc != "SQLITE_OK"} { + error "$rc - [sqlite3_errmsg $db]" + } + } else { + eval sqlite3_wal_checkpoint_v2 [list $db] [list $mode] $args + } + } + } + +} { + + eval $script + + reset_db + forcedelete test.db2 test.db3 test.db4 + execsql { + ATTACH 'test.db2' AS aux; + ATTACH 'test.db3' AS aux2; + ATTACH 'test.db4' AS aux3; + CREATE TABLE t1(x); + CREATE TABLE aux.t2(x); + CREATE TABLE aux2.t3(x); + CREATE TABLE aux3.t4(x); + PRAGMA main.journal_mode = WAL; + PRAGMA aux.journal_mode = WAL; + PRAGMA aux2.journal_mode = WAL; + /* Leave aux4 in rollback mode */ + } + + # EVIDENCE-OF: R-49787-09095 The sqlite3_wal_checkpoint_v2(D,X,M,L,C) + # interface runs a checkpoint operation on database X of database + # connection D in mode M. Status information is written back into + # integers pointed to by L and C. + # + # Tests 1, 2 and 3 below verify the "on database X" part of the + # above. Other parts of this requirement are tested below. + # + # EVIDENCE-OF: R-00653-06026 If parameter zDb is NULL or points to a + # zero length string, then the specified operation is attempted on all + # WAL databases attached to database connection db. + # + # Tests 4 and 5 below test this. + # + foreach {tn2 zDb dblist} { + 1 main test.db + 2 aux test.db2 + 3 aux2 test.db3 + 4 "" {test.db test.db2 test.db3} + 5 - {test.db test.db2 test.db3} + 6 temp {} + } { + do_test $tn.1.$tn2 { + execsql { + INSERT INTO t1 VALUES(1); + INSERT INTO t2 VALUES(2); + INSERT INTO t3 VALUES(3); + } + save_db_hashes + + if {$zDb == "-"} { + checkpoint db passive + } else { + checkpoint db passive $zDb + } + + compare_db_hashes + } $dblist + } + + # EVIDENCE-OF: R-38207-48996 If zDb is not NULL (or a zero length + # string) and is not the name of any attached database, SQLITE_ERROR is + # returned to the caller. + do_test $tn.2.1 { + list [catch { checkpoint db passive notadb } msg] $msg + } {1 {SQLITE_ERROR - unknown database: notadb}} + + # EVIDENCE-OF: R-14303-42483 If database zDb is the name of an attached + # database that is not in WAL mode, SQLITE_OK is returned and both + # *pnLog and *pnCkpt set to -1. + # + if {$tn==3} { + # With sqlite3_wal_checkpoint() the two output variables cannot be + # tested. So just test that no error is returned when attempting to + # checkpoint a db in rollback mode. + do_test $tn.2.2.a { checkpoint db passive aux3 } {} + } else { + do_test $tn.2.2.b { checkpoint db passive aux3 } {0 -1 -1} + } + + # EVIDENCE-OF: R-62028-47212 All calls obtain an exclusive "checkpoint" + # lock on the database file. + db close + testvfs tvfs + tvfs filter xShmLock + tvfs script filelock + proc filelock {method file handle details} { + # Test for an exclusive checkpoint lock. A checkpoint lock locks a + # single byte starting at offset 1. + if {$details == "1 1 lock exclusive"} { set ::seen_checkpoint_lock 1 } + } + sqlite3 db test.db -vfs tvfs + do_test $tn.3.1 { + execsql { INSERT INTO t1 VALUES('xyz') } + unset -nocomplain ::seen_checkpoint_lock + checkpoint db passive + set ::seen_checkpoint_lock + } {1} + db close + tvfs delete + reset_db + + + + + #----------------------------------------------------------------------- + # EVIDENCE-OF: R-10421-19736 If any other process is running a + # checkpoint operation at the same time, the lock cannot be obtained and + # SQLITE_BUSY is returned. + # + # EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, + # it will not be invoked in this case. + # + testvfs tvfs + tvfs filter xWrite + sqlite3 db test.db -vfs tvfs + sqlite3 db2 test.db -vfs tvfs + + do_test $tn.3.2.1 { + db2 eval { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1,2); + INSERT INTO t1 VALUES(3,4); + INSERT INTO t1 VALUES(5,6); + } + file size test.db-wal + } [wal_file_size 5 1024] + + + # Connection [db] runs a checkpoint. During this checkpoint, each + # time it calls xWrite() to write a page into the database file, we + # attempt to start a checkpoint using [db2]. According to the + # first requirement being tested, this should return SQLITE_BUSY. According + # to the second, the busy-handler belonging to [db2] should not be + # invoked. + # + set ::write_count 0 + set ::write_errors [list] + proc busy_callback {args} { + lappend ::write_errors "busy handler called!" + } + proc write_callback {args} { + set rc [catch {checkpoint db2 passive} msg] + if {0==[regexp "database is locked" $msg] && $msg!="1 -1 -1"} { + lappend ::write_errors "$rc $msg" + } + incr ::write_count + } + db2 busy busy_callback + tvfs script write_callback + + do_test $tn.3.2.2 { + db eval {SELECT * FROM sqlite_master} + checkpoint db full + set ::write_count + } {2} + + do_test $tn.3.2.3 { + set ::write_errors + } {} + + db close + db2 close + tvfs delete + +} + +#----------------------------------------------------------------------- +# EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint +# mode: +# +# Valid checkpoint modes are 0, 1, 2 and 3. +# +sqlite3 db test.db +foreach {tn mode res} { + 0 -1001 {1 {SQLITE_MISUSE - not an error}} + 1 -1 {1 {SQLITE_MISUSE - not an error}} + 2 0 {0 {0 -1 -1}} + 3 1 {0 {0 -1 -1}} + 4 2 {0 {0 -1 -1}} + 5 3 {0 {0 -1 -1}} + 6 4 {1 {SQLITE_MISUSE - not an error}} + 7 114 {1 {SQLITE_MISUSE - not an error}} + 8 1000000 {1 {SQLITE_MISUSE - not an error}} +} { + do_test 4.$tn { + list [catch "sqlite3_wal_checkpoint_v2 db $mode" msg] $msg + } $res +} + + +finish_test Index: test/join5.test ================================================================== --- test/join5.test +++ test/join5.test @@ -104,7 +104,61 @@ } {} do_test join5-2.12 { execsql {SELECT * FROM xy LEFT JOIN ab ON NULL WHERE NULL} } {} +# Ticket https://www.sqlite.org/src/tktview/6f2222d550f5b0ee7ed37601 +# Incorrect output on a LEFT JOIN. +# +do_execsql_test join5-3.1 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + DROP TABLE IF EXISTS t3; + CREATE TABLE x1(a); + INSERT INTO x1 VALUES(1); + CREATE TABLE x2(b NOT NULL); + CREATE TABLE x3(c, d); + INSERT INTO x3 VALUES('a', NULL); + INSERT INTO x3 VALUES('b', NULL); + INSERT INTO x3 VALUES('c', NULL); + SELECT * FROM x1 LEFT JOIN x2 LEFT JOIN x3 ON x3.d = x2.b; +} {1 {} {} {}} +do_execsql_test join5-3.2 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + DROP TABLE IF EXISTS t3; + DROP TABLE IF EXISTS t4; + DROP TABLE IF EXISTS t5; + CREATE TABLE t1(x text NOT NULL, y text); + CREATE TABLE t2(u text NOT NULL, x text NOT NULL); + CREATE TABLE t3(w text NOT NULL, v text); + CREATE TABLE t4(w text NOT NULL, z text NOT NULL); + CREATE TABLE t5(z text NOT NULL, m text); + INSERT INTO t1 VALUES('f6d7661f-4efe-4c90-87b5-858e61cd178b',NULL); + INSERT INTO t1 VALUES('f6ea82c3-2cad-45ce-ae8f-3ddca4fb2f48',NULL); + INSERT INTO t1 VALUES('f6f47499-ecb4-474b-9a02-35be73c235e5',NULL); + INSERT INTO t1 VALUES('56f47499-ecb4-474b-9a02-35be73c235e5',NULL); + INSERT INTO t3 VALUES('007f2033-cb20-494c-b135-a1e4eb66130c', + 'f6d7661f-4efe-4c90-87b5-858e61cd178b'); + SELECT * + FROM t3 + INNER JOIN t1 ON t1.x= t3.v AND t1.y IS NULL + LEFT JOIN t4 ON t4.w = t3.w + LEFT JOIN t5 ON t5.z = t4.z + LEFT JOIN t2 ON t2.u = t5.m + LEFT JOIN t1 xyz ON xyz.y = t2.x; +} {007f2033-cb20-494c-b135-a1e4eb66130c f6d7661f-4efe-4c90-87b5-858e61cd178b f6d7661f-4efe-4c90-87b5-858e61cd178b {} {} {} {} {} {} {} {} {}} +do_execsql_test join5-3.3 { + DROP TABLE IF EXISTS x1; + DROP TABLE IF EXISTS x2; + DROP TABLE IF EXISTS x3; + CREATE TABLE x1(a); + INSERT INTO x1 VALUES(1); + CREATE TABLE x2(b NOT NULL); + CREATE TABLE x3(c, d); + INSERT INTO x3 VALUES('a', NULL); + INSERT INTO x3 VALUES('b', NULL); + INSERT INTO x3 VALUES('c', NULL); + SELECT * FROM x1 LEFT JOIN x2 JOIN x3 WHERE x3.d = x2.b; +} {} finish_test Index: test/permutations.test ================================================================== --- test/permutations.test +++ test/permutations.test @@ -111,10 +111,11 @@ thread003.test thread004.test thread005.test trans2.test vacuum3.test incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test sort3.test sort4.test fts4growth.test fts4growth2.test + bigsort.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] } Index: test/scanstatus.test ================================================================== --- test/scanstatus.test +++ test/scanstatus.test @@ -266,11 +266,11 @@ do_execsql_test 4.2.1 { DELETE FROM p1 WHERE x=4 } do_scanstatus_test 4.2.2 { nLoop 1 nVisit 1 nEst 1.0 zName sqlite_autoindex_p1_1 zExplain {SEARCH TABLE p1 USING INDEX sqlite_autoindex_p1_1 (x=?)} - nLoop 1 nVisit 3 nEst 524288.0 zName c1 zExplain {SCAN TABLE c1} + nLoop 1 nVisit 3 nEst 262144.0 zName c1 zExplain {SCAN TABLE c1} } #------------------------------------------------------------------------- # Further tests of different scan types. # ADDED test/sharedB.test Index: test/sharedB.test ================================================================== --- /dev/null +++ test/sharedB.test @@ -0,0 +1,60 @@ +# 2014-12-05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Open two database connections on the same database in shared cache +# mode. Hold one open while repeatedly closing, reopening, and using +# the second. +# +# This test is designed to demostrate that the fix for ticket +# [e4a18565a36884b00edf66541f38c693827968ab] works. +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +if {[run_thread_tests]==0} { finish_test ; return } +db close +set ::testprefix sharedB + +set ::enable_shared_cache [sqlite3_enable_shared_cache 1] + +#------------------------------------------------------------------------- +# +do_test 1.1 { + sqlite3 db1 test.db + sqlite3 db2 test.db + + db1 eval { + CREATE TABLE t1(x,y TEXT COLLATE nocase); + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<100) + INSERT INTO t1(x,y) SELECT i, printf('x%03dy',i) FROM c; + CREATE INDEX t1yx ON t1(y,x); + } + db2 eval { + SELECT x FROM t1 WHERE y='X014Y'; + } +} {14} + +for {set j 1} {$j<=100} {incr j} { + do_test 1.2.$j { + db2 close + sqlite3 db2 test.db + db2 eval { + SELECT x FROM t1 WHERE y='X014Y'; + } + } {14} +} + +db1 close +db2 close +sqlite3_enable_shared_cache $::enable_shared_cache +finish_test Index: test/shell1.test ================================================================== --- test/shell1.test +++ test/shell1.test @@ -43,24 +43,25 @@ set res [catchcmd "-bad test.db" ""] set rc [lindex $res 0] list $rc \ [regexp {Error: unknown option: -bad} $res] } {1 1} -# error on extra options -do_test shell1-1.1.2 { - set res [catchcmd "-bad test.db \"select 3\" \"select 4\"" ""] +do_test shell1-1.1.1b { + set res [catchcmd "test.db -bad" ""] set rc [lindex $res 0] list $rc \ - [regexp {Error: too many options: "select 4"} $res] + [regexp {Error: unknown option: -bad} $res] } {1 1} +# error on extra options +do_test shell1-1.1.2 { + catchcmd "test.db \"select 3\" \"select 4\"" "" +} {0 {3 +4}} # error on extra options do_test shell1-1.1.3 { - set res [catchcmd "-bad FOO test.db BAD" ".quit"] - set rc [lindex $res 0] - list $rc \ - [regexp {Error: too many options: "BAD"} $res] -} {1 1} + catchcmd "test.db FOO test.db BAD" ".quit" +} {1 {Error: near "FOO": syntax error}} # -help do_test shell1-1.2.1 { set res [catchcmd "-help test.db" ""] set rc [lindex $res 0] @@ -73,15 +74,15 @@ # -init filename read/process named file do_test shell1-1.3.1 { catchcmd "-init FOO test.db" "" } {0 {}} do_test shell1-1.3.2 { - set res [catchcmd "-init FOO test.db .quit BAD" ""] - set rc [lindex $res 0] - list $rc \ - [regexp {Error: too many options: "BAD"} $res] -} {1 1} + catchcmd "-init FOO test.db .quit BAD" "" +} {0 {}} +do_test shell1-1.3.3 { + catchcmd "-init FOO test.db BAD .quit" "" +} {1 {Error: near "BAD": syntax error}} # -echo print commands before execution do_test shell1-1.4.1 { catchcmd "-echo test.db" "" } {0 {}} Index: test/shell2.test ================================================================== --- test/shell2.test +++ test/shell2.test @@ -50,13 +50,13 @@ # Shell silently ignores extra parameters. # Ticket [f5cb008a65]. do_test shell2-1.2.1 { set rc [catch { eval exec $CLI \":memory:\" \"select 3\" \"select 4\" } msg] - list $rc \ - [regexp {Error: too many options: "select 4"} $msg] -} {1 1} + list $rc $msg +} {0 {3 +4}} # Test a problem reported on the mailing list. The shell was at one point # returning the generic SQLITE_ERROR message ("SQL error or missing database") # instead of the "too many levels..." message in the test below. # Index: test/vtab1.test ================================================================== --- test/vtab1.test +++ test/vtab1.test @@ -1392,7 +1392,49 @@ } {1 2 3} do_execsql_test 21.3 { SELECT * FROM t9v WHERE a=b; } {2 2 2} + +#------------------------------------------------------------------------- +# At one point executing a CREATE VIRTUAL TABLE statement that specified +# a database name but no virtual table arguments was causing an internal +# buffer overread. Valgrind would report errors while running the following +# tests. Specifically: +# +# CREATE VIRTUAL TABLE t1 USING fts4; -- Ok - no db name. +# CREATE VIRTUAL TABLE main.t1 USING fts4(x); -- Ok - has vtab arguments. +# CREATE VIRTUAL TABLE main.t1 USING fts4; -- Had the problem. +# +ifcapable fts3 { + forcedelete test.db2 + set nm [string repeat abcdefghij 100] + do_execsql_test 22.1 { + ATTACH 'test.db2' AS $nm + } + + execsql "SELECT * FROM sqlite_master" + do_execsql_test 22.2 "CREATE VIRTUAL TABLE ${nm}.t1 USING fts4" + + do_test 22.3.1 { + set sql "CREATE VIRTUAL TABLE ${nm}.t2 USING fts4" + set stmt [sqlite3_prepare_v2 db $sql -1 dummy] + sqlite3_step $stmt + } {SQLITE_DONE} + + do_test 22.3.2 { + sqlite3_finalize $stmt + } {SQLITE_OK} + + do_test 22.4.1 { + set sql "CREATE VIRTUAL TABLE ${nm}.t3 USING fts4" + set n [string length $sql] + set stmt [sqlite3_prepare db "${sql}xyz" $n dummy] + sqlite3_step $stmt + } {SQLITE_DONE} + + do_test 22.4.2 { + sqlite3_finalize $stmt + } {SQLITE_OK} +} finish_test Index: test/wal5.test ================================================================== --- test/wal5.test +++ test/wal5.test @@ -53,11 +53,12 @@ array set a $args foreach key [array names a] { if {[lsearch {-mode -db} $key]<0} { error "unknown switch: $key" } } - if {$a(-mode)!="restart" && $a(-mode)!="full"} { set a(-mode) passive } + set vals {restart full truncate} + if {[lsearch -exact $vals $a(-mode)]<0} { set a(-mode) passive } set cmd [list sqlite3_wal_checkpoint_v2 $dbhandle $a(-mode)] if {[info exists a(-db)]} { lappend sql $a(-db) } uplevel $cmd @@ -276,10 +277,15 @@ 7 RESTART - {0 4 4} 3 8 RESTART 1 {1 3 3} 1 9 RESTART 2 {1 4 3} 2 10 RESTART 3 {1 4 4} 3 + 11 TRUNCATE - {0 0 0} 3 + 12 TRUNCATE 1 {1 3 3} 1 + 13 TRUNCATE 2 {1 4 3} 2 + 14 TRUNCATE 3 {1 4 4} 3 + } { do_multiclient_test tn { setup_and_attach_aux proc busyhandler {x} { @@ -345,10 +351,45 @@ code3 {sqlite3 db3 test.db} do_test 3.$tn.5 { sql3 { PRAGMA journal_mode } } {wal} do_test 3.$tn.6 { code3 { do_wal_checkpoint db3 } } {0 0 0} + } + + # Test SQLITE_CHECKPOINT_TRUNCATE. + # + do_multiclient_test tn { + + code1 $do_wal_checkpoint + code2 $do_wal_checkpoint + code3 $do_wal_checkpoint + + do_test 3.$tn.1 { + sql1 { + PRAGMA page_size = 1024; + PRAGMA journal_mode = WAL; + PRAGMA synchronous = normal; + CREATE TABLE t1(x, y); + CREATE INDEX i1 ON t1(x, y); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + } + file size test.db-wal + } [wal_file_size 8 1024] + + do_test 3.$tn.2 { do_wal_checkpoint db -mode truncate } {0 0 0} + do_test 3.$tn.3 { file size test.db-wal } 0 + + do_test 3.$tn.4 { + sql2 { SELECT * FROM t1 } + } {1 2 3 4} + + do_test 3.$tn.5 { + sql2 { INSERT INTO t1 VALUES('a', 'b') } + file size test.db-wal + } [wal_file_size 2 1024] + } } finish_test Index: test/whereJ.test ================================================================== --- test/whereJ.test +++ test/whereJ.test @@ -636,8 +636,43 @@ AND cx.type=2 AND px.cx_id = cx.cx_id AND px.px_tid = 0 AND px.le_id = le.le_id; } {/.*SCAN TABLE cx.*SEARCH TABLE px.*SEARCH TABLE le.*/} + + +# The following test is derived from a performance problem reported from +# the field. Notice the multiple indexes with the same initial tables, +# and the unusual WHERE clause terms. +# +do_test 5.1 { + set res [db eval { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a,b,c,d,e,f,g,h); + CREATE INDEX t1abc ON t1(a,b,c); + CREATE INDEX t1abe ON t1(a,b,e); + CREATE INDEX t1abf ON t1(a,b,f); + ANALYZE; + DROP TABLE IF EXISTS sqlite_stat4; + DROP TABLE IF EXISTS sqlite_stat3; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl,idx,stat) + VALUES('t1','t1abc','2000000 8000 1600 800'), + ('t1','t1abe','2000000 8000 1600 150'), + ('t1','t1abf','2000000 8000 1600 150'); + ANALYZE sqlite_master; + + EXPLAIN QUERY PLAN + SELECT * FROM t1 + WHERE (a=1 OR a=2) + AND (b=3 OR b=4) + AND (d>=5 AND d<=5) + AND ((e>=7 AND e<=7) OR (f>=8 AND f<=8)) + AND g>0; + }] +} {~/ANY/} +do_test 5.2 {set res} {/USING INDEX t1abe/} +do_test 5.3 {set res} {/USING INDEX t1abf/} + finish_test