Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -543,10 +543,13 @@ sqlite3$(TEXE): $(TOP)/src/shell.c libsqlite3.la sqlite3.h $(LTLINK) $(READLINE_FLAGS) \ -o $@ $(TOP)/src/shell.c libsqlite3.la \ $(LIBREADLINE) $(TLIBS) -rpath "$(libdir)" +sqldiff$(EXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h + $(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) + mptester$(EXE): sqlite3.c $(TOP)/mptest/mptest.c $(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \ $(TLIBS) -rpath "$(libdir)" MPTEST1=./mptester$(EXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20 Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -40,12 +40,12 @@ # compiling the various tools. For the SQLite source code itself, warnings, # if any, will be disabled from within it. # !IFNDEF NO_WARN !IF $(USE_FULLWARN)!=0 -NO_WARN = -wd4054 -wd4055 -wd4100 -wd4127 -wd4152 -wd4189 -wd4206 -wd4210 -NO_WARN = $(NO_WARN) -wd4232 -wd4244 -wd4305 -wd4306 -wd4702 -wd4706 +NO_WARN = -wd4054 -wd4055 -wd4100 -wd4127 -wd4130 -wd4152 -wd4189 -wd4206 +NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4244 -wd4305 -wd4306 -wd4702 -wd4706 !ENDIF !ENDIF # Set this non-0 to use the library paths and other options necessary for # Windows Phone 8.1. @@ -453,14 +453,16 @@ # The mksqlite3c.tcl script accepts some options on the command # line. When compiling with debugging enabled, some of these # options are necessary in order to allow debugging symbols to # work correctly with Visual Studio when using the amalgamation. # +!IFNDEF MKSQLITE3C_ARGS !IF $(DEBUG)>1 MKSQLITE3C_ARGS = --linemacros !ELSE MKSQLITE3C_ARGS = +!ENDIF !ENDIF # Define -DNDEBUG to compile without debugging (i.e., for production usage) # Omitting the define will cause extra debugging code to be inserted and # includes extra comments when "EXPLAIN stmt" is used. @@ -1175,10 +1177,13 @@ $(LTLIB) $(LTLIBOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCL:tcl=tclstub) $(TLIBS) sqlite3.exe: $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c \ /link /pdb:sqlite3sh.pdb $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) + +sqldiff.exe: $(TOP)\tool\sqldiff.c sqlite3.c sqlite3.h + $(LTLINK) $(TOP)\tool\sqldiff.c sqlite3.c mptester.exe: $(TOP)\mptest\mptest.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h $(LTLINK) $(SHELL_COMPILE_OPTS) $(TOP)\mptest\mptest.c \ /link $(LTLINKOPTS) $(LTLIBPATHS) $(SHELL_LINK_OPTS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -1017,11 +1017,12 @@ sqlite3 *db, /* Database handle */ const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */ const char *zTbl, /* Name of content table */ const char ***pazCol, /* OUT: Malloc'd array of column names */ int *pnCol, /* OUT: Size of array *pazCol */ - int *pnStr /* OUT: Bytes of string content */ + int *pnStr, /* OUT: Bytes of string content */ + char **pzErr /* OUT: error message */ ){ int rc = SQLITE_OK; /* Return code */ char *zSql; /* "SELECT *" statement on zTbl */ sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ @@ -1028,10 +1029,13 @@ zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); if( !zSql ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } } sqlite3_free(zSql); if( rc==SQLITE_OK ){ const char **azCol; /* Output array */ @@ -1279,11 +1283,11 @@ zCompress = 0; zUncompress = 0; if( nCol==0 ){ sqlite3_free((void*)aCol); aCol = 0; - rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString); + rc = fts3ContentColumns(db, argv[1], zContent,&aCol,&nCol,&nString,pzErr); /* If a languageid= option was specified, remove the language id ** column from the aCol[] array. */ if( rc==SQLITE_OK && zLanguageid ){ int j; Index: ext/session/sqlite3session.c ================================================================== --- ext/session/sqlite3session.c +++ ext/session/sqlite3session.c @@ -777,11 +777,10 @@ sqlite3_session *pSession, /* Session object that owns SessionTable */ SessionTable *pTab, /* Table associated with change */ SessionChange *pChange, /* Change to compare to */ int op /* Current pre-update operation */ ){ - sqlite3 *db = pSession->db; int iCol; /* Used to iterate through columns */ u8 *a = pChange->aRecord; /* Cursor used to scan change record */ assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE ); for(iCol=0; iColnCol; iCol++){ Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -409,10 +409,14 @@ sqlite3$(EXE): $(TOP)/src/shell.c libsqlite3.a sqlite3.h $(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) \ $(TOP)/src/shell.c \ libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB) + +sqldiff$(EXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h + $(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \ + $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB) mptester$(EXE): sqlite3.c $(TOP)/mptest/mptest.c $(TCCX) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \ $(TLIBS) $(THREADLIB) Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -2761,11 +2761,12 @@ sqlite3UniqueConstraint(pParse, OE_Abort, pIndex); }else{ addr2 = sqlite3VdbeCurrentAddr(v); } sqlite3VdbeAddOp3(v, OP_SorterData, iSorter, regRecord, iIdx); - sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); + sqlite3VdbeAddOp3(v, OP_Last, iIdx, 0, -1); + sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 0); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); @@ -3774,11 +3775,10 @@ ** operator with A. This routine shifts that operator over to B. */ void sqlite3SrcListShiftJoinType(SrcList *p){ if( p ){ int i; - assert( p->a || p->nSrc==0 ); for(i=p->nSrc-1; i>0; i--){ p->a[i].jointype = p->a[i-1].jointype; } p->a[0].jointype = 0; } Index: src/complete.c ================================================================== --- src/complete.c +++ src/complete.c @@ -267,11 +267,11 @@ ** UTF-8. */ int sqlite3_complete16(const void *zSql){ sqlite3_value *pVal; char const *zSql8; - int rc = SQLITE_NOMEM; + int rc; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); if( rc ) return rc; #endif Index: src/fkey.c ================================================================== --- src/fkey.c +++ src/fkey.c @@ -1182,11 +1182,12 @@ int iFromCol; /* Idx of column in child table */ Expr *pEq; /* tFromCol = OLD.tToCol */ iFromCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; assert( iFromCol>=0 ); - tToCol.z = pIdx ? pTab->aCol[pIdx->aiColumn[i]].zName : "oid"; + assert( pIdx!=0 || (pTab->iPKey>=0 && pTab->iPKeynCol) ); + tToCol.z = pTab->aCol[pIdx ? pIdx->aiColumn[i] : pTab->iPKey].zName; tFromCol.z = pFKey->pFrom->aCol[iFromCol].zName; tToCol.n = sqlite3Strlen30(tToCol.z); tFromCol.n = sqlite3Strlen30(tFromCol.z); Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -1775,10 +1775,11 @@ Table *pDest, /* The table we are inserting into */ Select *pSelect, /* A SELECT statement to use as the data source */ int onError, /* How to handle constraint errors */ int iDbDest /* The database of pDest */ ){ + sqlite3 *db = pParse->db; ExprList *pEList; /* The result set of the SELECT */ Table *pSrc; /* The table in the FROM clause of SELECT */ Index *pSrcIdx, *pDestIdx; /* Source and destination indices */ struct SrcList_item *pItem; /* An element of pSelect->pSrc */ int i; /* Loop counter */ @@ -1922,15 +1923,15 @@ ** But the main beneficiary of the transfer optimization is the VACUUM ** command, and the VACUUM command disables foreign key constraints. So ** the extra complication to make this rule less restrictive is probably ** not worth the effort. Ticket [6284df89debdfa61db8073e062908af0c9b6118e] */ - if( (pParse->db->flags & SQLITE_ForeignKeys)!=0 && pDest->pFKey!=0 ){ + if( (db->flags & SQLITE_ForeignKeys)!=0 && pDest->pFKey!=0 ){ return 0; } #endif - if( (pParse->db->flags & SQLITE_CountRows)!=0 ){ + if( (db->flags & SQLITE_CountRows)!=0 ){ return 0; /* xfer opt does not play well with PRAGMA count_changes */ } /* If we get this far, it means that the xfer optimization is at ** least a possibility, though it might only work if the destination @@ -1937,28 +1938,32 @@ ** table (tab1) is initially empty. */ #ifdef SQLITE_TEST sqlite3_xferopt_count++; #endif - iDbSrc = sqlite3SchemaToIndex(pParse->db, pSrc->pSchema); + iDbSrc = sqlite3SchemaToIndex(db, pSrc->pSchema); v = sqlite3GetVdbe(pParse); sqlite3CodeVerifySchema(pParse, iDbSrc); iSrc = pParse->nTab++; iDest = pParse->nTab++; regAutoinc = autoIncBegin(pParse, iDbDest, pDest); regData = sqlite3GetTempReg(pParse); regRowid = sqlite3GetTempReg(pParse); sqlite3OpenTable(pParse, iDest, iDbDest, pDest, OP_OpenWrite); assert( HasRowid(pDest) || destHasUniqueIdx ); - if( (pDest->iPKey<0 && pDest->pIndex!=0) /* (1) */ + if( (db->flags & SQLITE_Vacuum)==0 && ( + (pDest->iPKey<0 && pDest->pIndex!=0) /* (1) */ || destHasUniqueIdx /* (2) */ || (onError!=OE_Abort && onError!=OE_Rollback) /* (3) */ - ){ + )){ /* In some circumstances, we are able to run the xfer optimization - ** only if the destination table is initially empty. This code makes - ** that determination. Conditions under which the destination must - ** be empty: + ** only if the destination table is initially empty. Unless the + ** SQLITE_Vacuum flag is set, this block generates code to make + ** that determination. If SQLITE_Vacuum is set, then the destination + ** table is always empty. + ** + ** Conditions under which the destination must be empty: ** ** (1) There is no INTEGER PRIMARY KEY but there are indices. ** (If the destination is not initially empty, the rowid fields ** of index entries might need to change.) ** @@ -1997,10 +2002,11 @@ }else{ sqlite3TableLock(pParse, iDbDest, pDest->tnum, 1, pDest->zName); sqlite3TableLock(pParse, iDbSrc, pSrc->tnum, 0, pSrc->zName); } for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ + u8 useSeekResult = 0; for(pSrcIdx=pSrc->pIndex; ALWAYS(pSrcIdx); pSrcIdx=pSrcIdx->pNext){ if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; } assert( pSrcIdx ); sqlite3VdbeAddOp3(v, OP_OpenRead, iSrc, pSrcIdx->tnum, iDbSrc); @@ -2010,11 +2016,37 @@ sqlite3VdbeSetP4KeyInfo(pParse, pDestIdx); sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR); VdbeComment((v, "%s", pDestIdx->zName)); addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_RowKey, iSrc, regData); + if( db->flags & SQLITE_Vacuum ){ + /* This INSERT command is part of a VACUUM operation, which guarantees + ** that the destination table is empty. If all indexed columns use + ** collation sequence BINARY, then it can also be assumed that the + ** index will be populated by inserting keys in strictly sorted + ** order. In this case, instead of seeking within the b-tree as part + ** of every OP_IdxInsert opcode, an OP_Last is added before the + ** OP_IdxInsert to seek to the point within the b-tree where each key + ** should be inserted. This is faster. + ** + ** If any of the indexed columns use a collation sequence other than + ** BINARY, this optimization is disabled. This is because the user + ** might change the definition of a collation sequence and then run + ** a VACUUM command. In that case keys may not be written in strictly + ** sorted order. */ + int i; + for(i=0; inColumn; i++){ + char *zColl = pSrcIdx->azColl[i]; + if( zColl && sqlite3_stricmp("BINARY", zColl) ) break; + } + if( i==pSrcIdx->nColumn ){ + useSeekResult = OPFLAG_USESEEKRESULT; + sqlite3VdbeAddOp3(v, OP_Last, iDest, 0, -1); + } + } sqlite3VdbeAddOp3(v, OP_IdxInsert, iDest, regData, 1); + sqlite3VdbeChangeP5(v, useSeekResult); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); } Index: src/msvc.h ================================================================== --- src/msvc.h +++ src/msvc.h @@ -18,10 +18,11 @@ #if defined(_MSC_VER) #pragma warning(disable : 4054) #pragma warning(disable : 4055) #pragma warning(disable : 4100) #pragma warning(disable : 4127) +#pragma warning(disable : 4130) #pragma warning(disable : 4152) #pragma warning(disable : 4189) #pragma warning(disable : 4206) #pragma warning(disable : 4210) #pragma warning(disable : 4232) Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -1232,10 +1232,11 @@ #define SQLITE_LoadExtension 0x00400000 /* Enable load_extension */ #define SQLITE_EnableTrigger 0x00800000 /* True to enable triggers */ #define SQLITE_DeferFKs 0x01000000 /* Defer all FK constraints */ #define SQLITE_QueryOnly 0x02000000 /* Disable database changes */ #define SQLITE_VdbeEQP 0x04000000 /* Debug EXPLAIN QUERY PLAN */ +#define SQLITE_Vacuum 0x08000000 /* Currently in a VACUUM */ /* ** Bits of the sqlite3.dbOptFlags field that are used by the ** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to Index: src/trigger.c ================================================================== --- src/trigger.c +++ src/trigger.c @@ -678,11 +678,10 @@ SrcList *pSrc; /* SrcList to be returned */ pSrc = sqlite3SrcListAppend(pParse->db, 0, &pStep->target, 0); if( pSrc ){ assert( pSrc->nSrc>0 ); - assert( pSrc->a!=0 ); iDb = sqlite3SchemaToIndex(pParse->db, pStep->pTrig->pSchema); if( iDb==0 || iDb>=2 ){ sqlite3 *db = pParse->db; assert( iDbdb->nDb ); pSrc->a[pSrc->nSrc-1].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); Index: src/vacuum.c ================================================================== --- src/vacuum.c +++ src/vacuum.c @@ -248,17 +248,21 @@ /* Loop through the tables in the main database. For each, do ** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy ** the contents to the temporary database. */ + assert( (db->flags & SQLITE_Vacuum)==0 ); + db->flags |= SQLITE_Vacuum; rc = execExecSql(db, pzErrMsg, "SELECT 'INSERT INTO vacuum_db.' || quote(name) " "|| ' SELECT * FROM main.' || quote(name) || ';'" "FROM main.sqlite_master " "WHERE type = 'table' AND name!='sqlite_sequence' " " AND coalesce(rootpage,1)>0" ); + assert( (db->flags & SQLITE_Vacuum)!=0 ); + db->flags &= ~SQLITE_Vacuum; if( rc!=SQLITE_OK ) goto end_of_vacuum; /* Copy over the sequence table */ rc = execExecSql(db, pzErrMsg, Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -2586,11 +2586,11 @@ u8 *zNewRecord; /* A buffer to hold the data for the new record */ Mem *pRec; /* The new record */ u64 nData; /* Number of bytes of data space */ int nHdr; /* Number of bytes of header space */ i64 nByte; /* Data space required for this record */ - int nZero; /* Number of zero bytes at the end of the record */ + i64 nZero; /* Number of zero bytes at the end of the record */ int nVarint; /* Number of bytes in a varint */ u32 serial_type; /* Type field */ Mem *pData0; /* First field to be combined into the record */ Mem *pLast; /* Last field of the record */ int nField; /* Number of fields in the record */ @@ -2678,11 +2678,11 @@ nVarint = sqlite3VarintLen(nHdr); nHdr += nVarint; if( nVarintdb->aLimit[SQLITE_LIMIT_LENGTH] ){ + if( nByte+nZero>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } /* Make sure the output register has a buffer large enough to store ** the new record. The output register (pOp->p3) is not allowed to @@ -4544,11 +4544,11 @@ sqlite3BtreeClearCursor(pC->pCursor); } break; } -/* Opcode: Last P1 P2 * * * +/* Opcode: Last P1 P2 P3 * * ** ** The next use of the Rowid or Column or Prev instruction for P1 ** will refer to the last entry in the database table or index. ** If the table or index is empty and P2>0, then jump immediately to P2. ** If P2 is 0 or if the table or index is not empty, fall through @@ -4571,10 +4571,11 @@ assert( pCrsr!=0 ); rc = sqlite3BtreeLast(pCrsr, &res); pC->nullRow = (u8)res; pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; + pC->seekResult = pOp->p3; #ifdef SQLITE_DEBUG pC->seekOp = OP_Last; #endif if( pOp->p2>0 ){ VdbeBranchTaken(res!=0,2); Index: src/vdbe.h ================================================================== --- src/vdbe.h +++ src/vdbe.h @@ -213,10 +213,11 @@ #endif int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); +int sqlite3VdbeRecordCompareWithSkip(int, const void *, UnpackedRecord *, int); UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); typedef int (*RecordCompare)(int,const void*,UnpackedRecord*); RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -3585,11 +3585,11 @@ ** If database corruption is discovered, set pPKey2->errCode to ** SQLITE_CORRUPT and return 0. If an OOM error is encountered, ** pPKey2->errCode is set to SQLITE_NOMEM and, if it is not NULL, the ** malloc-failed flag set on database handle (pPKey2->pKeyInfo->db). */ -static int vdbeRecordCompareWithSkip( +int sqlite3VdbeRecordCompareWithSkip( int nKey1, const void *pKey1, /* Left key */ UnpackedRecord *pPKey2, /* Right key */ int bSkip /* If true, skip the first field */ ){ u32 d1; /* Offset into aKey[] of next data element */ @@ -3771,11 +3771,11 @@ } int sqlite3VdbeRecordCompare( int nKey1, const void *pKey1, /* Left key */ UnpackedRecord *pPKey2 /* Right key */ ){ - return vdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 0); + return sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 0); } /* ** This function is an optimized version of sqlite3VdbeRecordCompare() @@ -3859,11 +3859,11 @@ }else if( vr2; }else if( pPKey2->nField>1 ){ /* The first fields of the two keys are equal. Compare the trailing ** fields. */ - res = vdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); + res = sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); }else{ /* The first fields of the two keys are equal and there are no trailing ** fields. Return pPKey2->default_rc in this case. */ res = pPKey2->default_rc; } @@ -3907,11 +3907,11 @@ if( res==0 ){ res = nStr - pPKey2->aMem[0].n; if( res==0 ){ if( pPKey2->nField>1 ){ - res = vdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); + res = sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); }else{ res = pPKey2->default_rc; } }else if( res>0 ){ res = pPKey2->r2; Index: src/vdbesort.c ================================================================== --- src/vdbesort.c +++ src/vdbesort.c @@ -289,20 +289,23 @@ ** ** In both cases, the effects of the main thread seeing (bDone==0) even ** after the thread has finished are not dire. So we don't worry about ** memory barriers and such here. */ +typedef int (*SorterCompare)(SortSubtask*,int*,const void*,int,const void*,int); struct SortSubtask { SQLiteThread *pThread; /* Background thread, if any */ int bDone; /* Set if thread is finished but not joined */ VdbeSorter *pSorter; /* Sorter that owns this sub-task */ UnpackedRecord *pUnpacked; /* Space to unpack a record */ SorterList list; /* List for thread to write to a PMA */ int nPMA; /* Number of PMAs currently in file */ + SorterCompare xCompare; /* Compare function to use */ SorterFile file; /* Temp file for level-0 PMAs */ SorterFile file2; /* Space for other PMAs */ }; + /* ** Main sorter structure. A single instance of this is allocated for each ** sorter cursor created by the VDBE. ** @@ -326,12 +329,16 @@ int nMemory; /* Size of list.aMemory allocation in bytes */ u8 bUsePMA; /* True if one or more PMAs created */ u8 bUseThreads; /* True to use background threads */ u8 iPrev; /* Previous thread used to flush PMA */ u8 nTask; /* Size of aTask[] array */ + u8 typeMask; SortSubtask aTask[1]; /* One or more subtasks */ }; + +#define SORTER_TYPE_INTEGER 0x01 +#define SORTER_TYPE_TEXT 0x02 /* ** An instance of the following object is used to read records out of a ** PMA, in sorted order. The next key to be read is cached in nKey/aKey. ** aKey might point into aMap or into aBuffer. If neither of those locations @@ -740,35 +747,164 @@ rc = vdbePmaReaderNext(pReadr); } return rc; } +/* +** A version of vdbeSorterCompare() that assumes that it has already been +** determined that the first field of key1 is equal to the first field of +** key2. +*/ +static int vdbeSorterCompareTail( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ +){ + UnpackedRecord *r2 = pTask->pUnpacked; + if( *pbKey2Cached==0 ){ + sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + *pbKey2Cached = 1; + } + return sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, r2, 1); +} /* ** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, ** size nKey2 bytes). Use (pTask->pKeyInfo) for the collation sequences ** used by the comparison. Return the result of the comparison. ** -** Before returning, object (pTask->pUnpacked) is populated with the -** unpacked version of key2. Or, if pKey2 is passed a NULL pointer, then it -** is assumed that the (pTask->pUnpacked) structure already contains the -** unpacked key to use as key2. +** If IN/OUT parameter *pbKey2Cached is true when this function is called, +** it is assumed that (pTask->pUnpacked) contains the unpacked version +** of key2. If it is false, (pTask->pUnpacked) is populated with the unpacked +** version of key2 and *pbKey2Cached set to true before returning. ** ** If an OOM error is encountered, (pTask->pUnpacked->error_rc) is set ** to SQLITE_NOMEM. */ static int vdbeSorterCompare( SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ const void *pKey1, int nKey1, /* Left side of comparison */ const void *pKey2, int nKey2 /* Right side of comparison */ ){ UnpackedRecord *r2 = pTask->pUnpacked; - if( pKey2 ){ + if( !*pbKey2Cached ){ sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + *pbKey2Cached = 1; } return sqlite3VdbeRecordCompare(nKey1, pKey1, r2); } + +/* +** A specially optimized version of vdbeSorterCompare() that assumes that +** the first field of each key is a TEXT value and that the collation +** sequence to compare them with is BINARY. +*/ +static int vdbeSorterCompareText( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ +){ + const u8 * const p1 = (const u8 * const)pKey1; + const u8 * const p2 = (const u8 * const)pKey2; + const u8 * const v1 = &p1[ p1[0] ]; /* Pointer to value 1 */ + const u8 * const v2 = &p2[ p2[0] ]; /* Pointer to value 2 */ + + int n1; + int n2; + int res; + + getVarint32(&p1[1], n1); n1 = (n1 - 13) / 2; + getVarint32(&p2[1], n2); n2 = (n2 - 13) / 2; + res = memcmp(v1, v2, MIN(n1, n2)); + if( res==0 ){ + res = n1 - n2; + } + + if( res==0 ){ + if( pTask->pSorter->pKeyInfo->nField>1 ){ + res = vdbeSorterCompareTail( + pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2 + ); + } + }else{ + if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){ + res = res * -1; + } + } + + return res; +} + +/* +** A specially optimized version of vdbeSorterCompare() that assumes that +** the first field of each key is an INTEGER value. +*/ +static int vdbeSorterCompareInt( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ +){ + const u8 * const p1 = (const u8 * const)pKey1; + const u8 * const p2 = (const u8 * const)pKey2; + const int s1 = p1[1]; /* Left hand serial type */ + const int s2 = p2[1]; /* Right hand serial type */ + const u8 * const v1 = &p1[ p1[0] ]; /* Pointer to value 1 */ + const u8 * const v2 = &p2[ p2[0] ]; /* Pointer to value 2 */ + int res; /* Return value */ + + assert( (s1>0 && s1<7) || s1==8 || s1==9 ); + assert( (s2>0 && s2<7) || s2==8 || s2==9 ); + + if( s1>7 && s2>7 ){ + res = s1 - s2; + }else{ + if( s1==s2 ){ + if( (*v1 ^ *v2) & 0x80 ){ + /* The two values have different signs */ + res = (*v1 & 0x80) ? -1 : +1; + }else{ + /* The two values have the same sign. Compare using memcmp(). */ + static const u8 aLen[] = {0, 1, 2, 3, 4, 6, 8 }; + int i; + res = 0; + for(i=0; i7 ){ + res = +1; + }else if( s1>7 ){ + res = -1; + }else{ + res = s1 - s2; + } + + if( res>0 ){ + if( *v1 & 0x80 ) res = -1; + }else if( res<0 ){ + if( *v2 & 0x80 ) res = +1; + } + } + } + + if( res==0 ){ + if( pTask->pSorter->pKeyInfo->nField>1 ){ + res = vdbeSorterCompareTail( + pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2 + ); + } + }else if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){ + res = res * -1; + } + + return res; +} /* ** Initialize the temporary index cursor just opened as a sorter cursor. ** ** Usually, the sorter module uses the value of (pCsr->pKeyInfo->nField) @@ -833,13 +969,17 @@ rc = SQLITE_NOMEM; }else{ pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); pKeyInfo->db = 0; - if( nField && nWorker==0 ) pKeyInfo->nField = nField; + if( nField && nWorker==0 ){ + pKeyInfo->nXField += (pKeyInfo->nField - nField); + pKeyInfo->nField = nField; + } pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); pSorter->nTask = nWorker + 1; + pSorter->iPrev = nWorker-1; pSorter->bUseThreads = (pSorter->nTask>1); pSorter->db = db; for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; pTask->pSorter = pSorter; @@ -861,10 +1001,16 @@ pSorter->nMemory = pgsz; pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz); if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM; } } + + if( (pKeyInfo->nField+pKeyInfo->nXField)<13 + && (pKeyInfo->aColl[0]==0 || pKeyInfo->aColl[0]==db->pDfltColl) + ){ + pSorter->typeMask = SORTER_TYPE_INTEGER | SORTER_TYPE_TEXT; + } } return rc; } #undef nWorker /* Defined at the top of this function */ @@ -885,34 +1031,28 @@ ** Free all resources owned by the object indicated by argument pTask. All ** fields of *pTask are zeroed before returning. */ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ sqlite3DbFree(db, pTask->pUnpacked); - pTask->pUnpacked = 0; #if SQLITE_MAX_WORKER_THREADS>0 /* pTask->list.aMemory can only be non-zero if it was handed memory ** from the main thread. That only occurs SQLITE_MAX_WORKER_THREADS>0 */ if( pTask->list.aMemory ){ sqlite3_free(pTask->list.aMemory); - pTask->list.aMemory = 0; }else #endif { assert( pTask->list.aMemory==0 ); vdbeSorterRecordFree(0, pTask->list.pList); } - pTask->list.pList = 0; if( pTask->file.pFd ){ sqlite3OsCloseFree(pTask->file.pFd); - pTask->file.pFd = 0; - pTask->file.iEof = 0; } if( pTask->file2.pFd ){ sqlite3OsCloseFree(pTask->file2.pFd); - pTask->file2.pFd = 0; - pTask->file2.iEof = 0; } + memset(pTask, 0, sizeof(SortSubtask)); } #ifdef SQLITE_DEBUG_SORTER_THREADS static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ i64 t; @@ -1088,10 +1228,11 @@ vdbeMergeEngineFree(pSorter->pMerger); pSorter->pMerger = 0; for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; vdbeSortSubtaskCleanup(db, pTask); + pTask->pSorter = pSorter; } if( pSorter->list.aMemory==0 ){ vdbeSorterRecordFree(0, pSorter->list.pList); } pSorter->list.pList = 0; @@ -1197,31 +1338,45 @@ SorterRecord *p2, /* Second list to merge */ SorterRecord **ppOut /* OUT: Head of merged list */ ){ SorterRecord *pFinal = 0; SorterRecord **pp = &pFinal; - void *pVal2 = p2 ? SRVAL(p2) : 0; + int bCached = 0; while( p1 && p2 ){ int res; - res = vdbeSorterCompare(pTask, SRVAL(p1), p1->nVal, pVal2, p2->nVal); + res = pTask->xCompare( + pTask, &bCached, SRVAL(p1), p1->nVal, SRVAL(p2), p2->nVal + ); + if( res<=0 ){ *pp = p1; pp = &p1->u.pNext; p1 = p1->u.pNext; - pVal2 = 0; }else{ *pp = p2; - pp = &p2->u.pNext; + pp = &p2->u.pNext; p2 = p2->u.pNext; - if( p2==0 ) break; - pVal2 = SRVAL(p2); + bCached = 0; } } *pp = p1 ? p1 : p2; *ppOut = pFinal; } + +/* +** Return the SorterCompare function to compare values collected by the +** sorter object passed as the only argument. +*/ +static SorterCompare vdbeSorterGetCompare(VdbeSorter *p){ + if( p->typeMask==SORTER_TYPE_INTEGER ){ + return vdbeSorterCompareInt; + }else if( p->typeMask==SORTER_TYPE_TEXT ){ + return vdbeSorterCompareText; + } + return vdbeSorterCompare; +} /* ** Sort the linked list of records headed at pTask->pList. Return ** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if ** an error occurs. @@ -1233,16 +1388,18 @@ int rc; rc = vdbeSortAllocUnpacked(pTask); if( rc!=SQLITE_OK ) return rc; + p = pList->pList; + pTask->xCompare = vdbeSorterGetCompare(pTask->pSorter); + aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); if( !aSlot ){ return SQLITE_NOMEM; } - p = pList->pList; while( p ){ SorterRecord *pNext; if( pList->aMemory ){ if( (u8*)p==pList->aMemory ){ pNext = 0; @@ -1452,28 +1609,27 @@ /* Update contents of aTree[] */ if( rc==SQLITE_OK ){ int i; /* Index of aTree[] to recalculate */ PmaReader *pReadr1; /* First PmaReader to compare */ PmaReader *pReadr2; /* Second PmaReader to compare */ - u8 *pKey2; /* To pReadr2->aKey, or 0 if record cached */ + int bCached = 0; /* Find the first two PmaReaders to compare. The one that was just ** advanced (iPrev) and the one next to it in the array. */ pReadr1 = &pMerger->aReadr[(iPrev & 0xFFFE)]; pReadr2 = &pMerger->aReadr[(iPrev | 0x0001)]; - pKey2 = pReadr2->aKey; for(i=(pMerger->nTree+iPrev)/2; i>0; i=i/2){ /* Compare pReadr1 and pReadr2. Store the result in variable iRes. */ int iRes; if( pReadr1->pFd==0 ){ iRes = +1; }else if( pReadr2->pFd==0 ){ iRes = -1; }else{ - iRes = vdbeSorterCompare(pTask, - pReadr1->aKey, pReadr1->nKey, pKey2, pReadr2->nKey + iRes = pTask->xCompare(pTask, &bCached, + pReadr1->aKey, pReadr1->nKey, pReadr2->aKey, pReadr2->nKey ); } /* If pReadr1 contained the smaller value, set aTree[i] to its index. ** Then set pReadr2 to the next PmaReader to compare to pReadr1. In this @@ -1491,13 +1647,13 @@ ** is sorted from oldest to newest, so pReadr1 contains older values ** than pReadr2 iff (pReadr1aTree[i] = (int)(pReadr1 - pMerger->aReadr); pReadr2 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; - pKey2 = pReadr2->aKey; + bCached = 0; }else{ - if( pReadr1->pFd ) pKey2 = 0; + if( pReadr1->pFd ) bCached = 0; pMerger->aTree[i] = (int)(pReadr2 - pMerger->aReadr); pReadr1 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; } } *pbEof = (pMerger->aReadr[pMerger->aTree[1]].pFd==0); @@ -1600,10 +1756,20 @@ SorterRecord *pNew; /* New list element */ int bFlush; /* True to flush contents of memory to PMA */ int nReq; /* Bytes of memory required */ int nPMA; /* Bytes of PMA space required */ + int t; /* serial type of first record field */ + + getVarint32((const u8*)&pVal->z[1], t); + if( t>0 && t<10 && t!=7 ){ + pSorter->typeMask &= SORTER_TYPE_INTEGER; + }else if( t>10 && (t & 0x01) ){ + pSorter->typeMask &= SORTER_TYPE_TEXT; + }else{ + pSorter->typeMask = 0; + } assert( pSorter ); /* Figure out whether or not the current contents of memory should be ** flushed to a PMA before continuing. If so, do so. @@ -1865,14 +2031,16 @@ if( p1->pFd==0 ){ iRes = i2; }else if( p2->pFd==0 ){ iRes = i1; }else{ + SortSubtask *pTask = pMerger->pTask; + int bCached = 0; int res; - assert( pMerger->pTask->pUnpacked!=0 ); /* from vdbeSortSubtaskMain() */ - res = vdbeSorterCompare( - pMerger->pTask, p1->aKey, p1->nKey, p2->aKey, p2->nKey + assert( pTask->pUnpacked!=0 ); /* from vdbeSortSubtaskMain() */ + res = pTask->xCompare( + pTask, &bCached, p1->aKey, p1->nKey, p2->aKey, p2->nKey ); if( res<=0 ){ iRes = i1; }else{ iRes = i2; @@ -2286,10 +2454,15 @@ int rc; /* Return code */ SortSubtask *pTask0 = &pSorter->aTask[0]; MergeEngine *pMain = 0; #if SQLITE_MAX_WORKER_THREADS sqlite3 *db = pTask0->pSorter->db; + int i; + SorterCompare xCompare = vdbeSorterGetCompare(pSorter); + for(i=0; inTask; i++){ + pSorter->aTask[i].xCompare = xCompare; + } #endif rc = vdbeSorterMergeTreeBuild(pSorter, &pMain); if( rc==SQLITE_OK ){ #if SQLITE_MAX_WORKER_THREADS Index: src/vtab.c ================================================================== --- src/vtab.c +++ src/vtab.c @@ -22,10 +22,12 @@ ** are invoked only from within xCreate and xConnect methods. */ struct VtabCtx { VTable *pVTable; /* The virtual table being constructed */ Table *pTab; /* The Table object to which the virtual table belongs */ + VtabCtx *pPrior; /* Parent context (if any) */ + int bDeclared; /* True after sqlite3_declare_vtab() is called */ }; /* ** The actual function that does the work of creating a new module. ** This function implements the sqlite3_create_module() and @@ -485,19 +487,31 @@ Table *pTab, Module *pMod, int (*xConstruct)(sqlite3*,void*,int,const char*const*,sqlite3_vtab**,char**), char **pzErr ){ - VtabCtx sCtx, *pPriorCtx; + VtabCtx sCtx; VTable *pVTable; int rc; const char *const*azArg = (const char *const*)pTab->azModuleArg; int nArg = pTab->nModuleArg; char *zErr = 0; - char *zModuleName = sqlite3MPrintf(db, "%s", pTab->zName); + char *zModuleName; int iDb; + VtabCtx *pCtx; + /* Check that the virtual-table is not already being initialized */ + for(pCtx=db->pVtabCtx; pCtx; pCtx=pCtx->pPrior){ + if( pCtx->pTab==pTab ){ + *pzErr = sqlite3MPrintf(db, + "vtable constructor called recursively: %s", pTab->zName + ); + return SQLITE_LOCKED; + } + } + + zModuleName = sqlite3MPrintf(db, "%s", pTab->zName); if( !zModuleName ){ return SQLITE_NOMEM; } pVTable = sqlite3DbMallocZero(db, sizeof(VTable)); @@ -514,15 +528,17 @@ /* Invoke the virtual table constructor */ assert( &db->pVtabCtx ); assert( xConstruct ); sCtx.pTab = pTab; sCtx.pVTable = pVTable; - pPriorCtx = db->pVtabCtx; + sCtx.pPrior = db->pVtabCtx; + sCtx.bDeclared = 0; db->pVtabCtx = &sCtx; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); - db->pVtabCtx = pPriorCtx; + db->pVtabCtx = sCtx.pPrior; if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + assert( sCtx.pTab==pTab ); if( SQLITE_OK!=rc ){ if( zErr==0 ){ *pzErr = sqlite3MPrintf(db, "vtable constructor failed: %s", zModuleName); }else { @@ -534,11 +550,11 @@ /* Justification of ALWAYS(): A correct vtab constructor must allocate ** the sqlite3_vtab object if successful. */ memset(pVTable->pVtab, 0, sizeof(pVTable->pVtab[0])); pVTable->pVtab->pModule = pMod->pModule; pVTable->nRef = 1; - if( sCtx.pTab ){ + if( sCtx.bDeclared==0 ){ const char *zFormat = "vtable constructor did not declare schema: %s"; *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); sqlite3VtabUnlock(pVTable); rc = SQLITE_ERROR; }else{ @@ -704,12 +720,12 @@ ** This function is used to set the schema of a virtual table. It is only ** valid to call this function from within the xCreate() or xConnect() of a ** virtual table module. */ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ + VtabCtx *pCtx; Parse *pParse; - int rc = SQLITE_OK; Table *pTab; char *zErr = 0; #ifdef SQLITE_ENABLE_API_ARMOR @@ -716,15 +732,17 @@ if( !sqlite3SafetyCheckOk(db) || zCreateTable==0 ){ return SQLITE_MISUSE_BKPT; } #endif sqlite3_mutex_enter(db->mutex); - if( !db->pVtabCtx || !(pTab = db->pVtabCtx->pTab) ){ + pCtx = db->pVtabCtx; + if( !pCtx || pCtx->bDeclared ){ sqlite3Error(db, SQLITE_MISUSE); sqlite3_mutex_leave(db->mutex); return SQLITE_MISUSE_BKPT; } + pTab = pCtx->pTab; assert( (pTab->tabFlags & TF_Virtual)!=0 ); pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); if( pParse==0 ){ rc = SQLITE_NOMEM; @@ -743,11 +761,11 @@ pTab->aCol = pParse->pNewTable->aCol; pTab->nCol = pParse->pNewTable->nCol; pParse->pNewTable->nCol = 0; pParse->pNewTable->aCol = 0; } - db->pVtabCtx->pTab = 0; + pCtx->bDeclared = 1; }else{ sqlite3ErrorWithMsg(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); rc = SQLITE_ERROR; } Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -4779,11 +4779,11 @@ ** changes "x IN (?)" into "x=?". */ }else if( eOp & (WO_EQ) ){ pNew->wsFlags |= WHERE_COLUMN_EQ; if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ - if( iCol>=0 && !IsUniqueIndex(pProbe) ){ + if( iCol>=0 && pProbe->uniqNotNull==0 ){ pNew->wsFlags |= WHERE_UNQ_WANTED; }else{ pNew->wsFlags |= WHERE_ONEROW; } } @@ -6239,11 +6239,11 @@ pWInfo->nOBSat = pFrom->isOrdered; if( pWInfo->nOBSat<0 ) pWInfo->nOBSat = 0; pWInfo->revMask = pFrom->revLoop; } if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP) - && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr + && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0 ){ Bitmask revMask = 0; int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask ); Index: test/fkey2.test ================================================================== --- test/fkey2.test +++ test/fkey2.test @@ -744,14 +744,14 @@ # The following tests, fkey2-11.*, test CASCADE actions. # drop_all_tables do_test fkey2-11.1.1 { execsql { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, rowid, _rowid_, oid); CREATE TABLE t2(c, d, FOREIGN KEY(c) REFERENCES t1(a) ON UPDATE CASCADE); - INSERT INTO t1 VALUES(10, 100); + INSERT INTO t1 VALUES(10, 100, 'abc', 'def', 'ghi'); INSERT INTO t2 VALUES(10, 100); UPDATE t1 SET a = 15; SELECT * FROM t2; } } {15 100} Index: test/fts4content.test ================================================================== --- test/fts4content.test +++ test/fts4content.test @@ -46,10 +46,13 @@ # 8.* - Test that if the content=xxx and prefix options are used together, # the 'rebuild' command still works. # # 9.* - Test using content=xxx where xxx is a virtual table. # +# 11.* - Test that circular references (e.g. "t1(content=t1)") are +# detected. +# do_execsql_test 1.1.1 { CREATE TABLE t1(a, b, c); INSERT INTO t1 VALUES('w x', 'x y', 'y z'); CREATE VIRTUAL TABLE ft1 USING fts4(content=t1); @@ -404,11 +407,11 @@ #------------------------------------------------------------------------- # Test cases 6.* test # do_catchsql_test 6.1.1 { CREATE VIRTUAL TABLE ft7 USING fts4(content=t7); -} {1 {vtable constructor failed: ft7}} +} {1 {no such table: main.t7}} do_execsql_test 6.2.1 { CREATE TABLE t7(one, two); CREATE VIRTUAL TABLE ft7 USING fts4(content=t7); INSERT INTO t7 VALUES('A B', 'B A'); @@ -431,11 +434,11 @@ ft7 ft7_segments ft7_segdir sqlite_autoindex_ft7_segdir_1 ft7_docsize ft7_stat } do_catchsql_test 6.2.4 { SELECT * FROM ft7; -} {1 {vtable constructor failed: ft7}} +} {1 {no such table: main.t7}} do_execsql_test 6.2.5 { CREATE TABLE t7(x, y); INSERT INTO t7 VALUES('A B', 'B A'); INSERT INTO t7 VALUES('C D', 'A A'); SELECT * FROM ft7; @@ -619,7 +622,18 @@ do_execsql_test 10.7 { SELECT snippet(ft, '[', ']', '...', -1, 5) FROM ft WHERE ft MATCH 'e' } { {...c d [e] f g...} } + +#------------------------------------------------------------------------- +# Test cases 11.* +# +reset_db + +do_catchsql_test 11.1 { + CREATE VIRTUAL TABLE x1 USING fts4(content=x1); +} {1 {vtable constructor called recursively: x1}} + finish_test + Index: test/null.test ================================================================== --- test/null.test +++ test/null.test @@ -276,8 +276,25 @@ execsql { SELECT x FROM t4 WHERE y!=33 ORDER BY x; } } {1} +do_execsql_test null-9.1 { + CREATE TABLE t5(a, b, c); + CREATE UNIQUE INDEX t5ab ON t5(a, b); + + INSERT INTO t5 VALUES(1, NULL, 'one'); + INSERT INTO t5 VALUES(1, NULL, 'i'); + INSERT INTO t5 VALUES(NULL, 'x', 'two'); + INSERT INTO t5 VALUES(NULL, 'x', 'ii'); +} + +do_execsql_test null-9.2 { + SELECT * FROM t5 WHERE a = 1 AND b IS NULL; +} {1 {} one 1 {} i} + +do_execsql_test null-9.3 { + SELECT * FROM t5 WHERE a IS NULL AND b = 'x'; +} {{} x two {} x ii} finish_test Index: test/orderby1.test ================================================================== --- test/orderby1.test +++ test/orderby1.test @@ -461,10 +461,13 @@ EXPLAIN QUERY PLAN SELECT 5 UNION ALL SELECT 3 ORDER BY 1 } {~/B-TREE/} do_execsql_test 5.2 { SELECT 5 UNION ALL SELECT 3 ORDER BY 1 } {3 5} +do_execsql_test 5.3 { + SELECT 986 AS x GROUP BY X ORDER BY X +} {986} # The following test (originally derived from a single test within fuzz.test) # verifies that a PseudoTable cursor is not closed prematurely in a deeply # nested query. This test caused a segfault on 3.8.5 beta. # @@ -493,7 +496,36 @@ CREATE INDEX t7ab ON t7(a,b); EXPLAIN QUERY PLAN SELECT * FROM t7 WHERE a=?1 ORDER BY rowid; } {~/ORDER BY/} +#------------------------------------------------------------------------- +# Test a partial sort large enough to cause the sorter to spill data +# to disk. +# +reset_db +do_execsql_test 8.0 { + PRAGMA cache_size = 5; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); +} + +do_eqp_test 8.1 { + SELECT * FROM t1 ORDER BY a, b; +} { + 0 0 0 {SCAN TABLE t1 USING INDEX i1} + 0 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} +} + +do_execsql_test 8.2 { + WITH cnt(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM cnt WHERE i<10000 + ) + INSERT INTO t1 SELECT i%2, randomblob(500) FROM cnt; +} + +do_test 8.3 { + db eval { SELECT * FROM t1 ORDER BY a, b } { incr res $a } + set res +} 5000 finish_test Index: test/whereD.test ================================================================== --- test/whereD.test +++ test/whereD.test @@ -127,41 +127,41 @@ INSERT INTO t4 VALUES('b', 'two'); } do_searchcount_test 3.1 { SELECT a, b FROM t3 WHERE (a=1 AND b='one') OR (a=2 AND b='two') -} {1 one 2 two search 2} +} {1 one 2 two search 4} do_searchcount_test 3.2 { SELECT a, c FROM t3 WHERE (a=1 AND b='one') OR (a=2 AND b='two') -} {1 i 2 ii search 4} +} {1 i 2 ii search 6} do_searchcount_test 3.4.1 { SELECT y FROM t4 WHERE x='a' } {one search 2} do_searchcount_test 3.4.2 { SELECT a, b FROM t3 WHERE (a=1 AND b=(SELECT y FROM t4 WHERE x='a')) OR (a=2 AND b='two') -} {1 one 2 two search 4} +} {1 one 2 two search 6} do_searchcount_test 3.4.3 { SELECT a, b FROM t3 WHERE (a=2 AND b='two') OR (a=1 AND b=(SELECT y FROM t4 WHERE x='a')) -} {2 two 1 one search 4} +} {2 two 1 one search 6} do_searchcount_test 3.4.4 { SELECT a, b FROM t3 WHERE (a=2 AND b=(SELECT y FROM t4 WHERE x='b')) OR (a=1 AND b=(SELECT y FROM t4 WHERE x='a')) -} {2 two 1 one search 6} +} {2 two 1 one search 8} do_searchcount_test 3.5.1 { SELECT a, b FROM t3 WHERE (a=1 AND b='one') OR rowid=4 -} {1 one 2 two search 2} +} {1 one 2 two search 3} do_searchcount_test 3.5.2 { SELECT a, c FROM t3 WHERE (a=1 AND b='one') OR rowid=4 -} {1 i 2 ii search 2} +} {1 i 2 ii search 3} # Ticket [d02e1406a58ea02d] (2012-10-04) # LEFT JOIN with an OR in the ON clause causes segfault # do_test 4.1 { Index: test/zeroblob.test ================================================================== --- test/zeroblob.test +++ test/zeroblob.test @@ -253,7 +253,16 @@ } {0} do_test zeroblob-9.8 { db eval {SELECT zeroblob(2) IN (zeroblob(2))} } {1} +# Oversized zeroblob records +# +do_test zeroblob-10.1 { + db eval { + CREATE TABLE t10(a,b,c); + } + catchsql {INSERT INTO t10 VALUES(zeroblob(1e9),zeroblob(1e9),zeroblob(1e9))} +} {1 {string or blob too big}} + finish_test ADDED tool/sqldiff.c Index: tool/sqldiff.c ================================================================== --- tool/sqldiff.c +++ tool/sqldiff.c @@ -0,0 +1,1085 @@ +/* +** 2015-04-06 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility problem that computes the differences in content +** between two SQLite databases. +*/ +#include +#include +#include +#include +#include +#include "sqlite3.h" + +/* +** All global variables are gathered into the "g" singleton. +*/ +struct GlobalVars { + const char *zArgv0; /* Name of program */ + int bSchemaOnly; /* Only show schema differences */ + int bSchemaPK; /* Use the schema-defined PK, not the true PK */ + unsigned fDebug; /* Debug flags */ + sqlite3 *db; /* The database connection */ +} g; + +/* +** Allowed values for g.fDebug +*/ +#define DEBUG_COLUMN_NAMES 0x000001 +#define DEBUG_DIFF_SQL 0x000002 + +/* +** Dynamic string object +*/ +typedef struct Str Str; +struct Str { + char *z; /* Text of the string */ + int nAlloc; /* Bytes allocated in z[] */ + int nUsed; /* Bytes actually used in z[] */ +}; + +/* +** Initialize a Str object +*/ +static void strInit(Str *p){ + p->z = 0; + p->nAlloc = 0; + p->nUsed = 0; +} + +/* +** Print an error resulting from faulting command-line arguments and +** abort the program. +*/ +static void cmdlineError(const char *zFormat, ...){ + va_list ap; + fprintf(stderr, "%s: ", g.zArgv0); + va_start(ap, zFormat); + vfprintf(stderr, zFormat, ap); + va_end(ap); + fprintf(stderr, "\n\"%s --help\" for more help\n", g.zArgv0); + exit(1); +} + +/* +** Print an error message for an error that occurs at runtime, then +** abort the program. +*/ +static void runtimeError(const char *zFormat, ...){ + va_list ap; + fprintf(stderr, "%s: ", g.zArgv0); + va_start(ap, zFormat); + vfprintf(stderr, zFormat, ap); + va_end(ap); + fprintf(stderr, "\n"); + exit(1); +} + +/* +** Free all memory held by a Str object +*/ +static void strFree(Str *p){ + sqlite3_free(p->z); + strInit(p); +} + +/* +** Add formatted text to the end of a Str object +*/ +static void strPrintf(Str *p, const char *zFormat, ...){ + int nNew; + for(;;){ + if( p->z ){ + va_list ap; + va_start(ap, zFormat); + sqlite3_vsnprintf(p->nAlloc-p->nUsed, p->z+p->nUsed, zFormat, ap); + va_end(ap); + nNew = (int)strlen(p->z + p->nUsed); + }else{ + nNew = p->nAlloc; + } + if( p->nUsed+nNew < p->nAlloc-1 ){ + p->nUsed += nNew; + break; + } + p->nAlloc = p->nAlloc*2 + 1000; + p->z = sqlite3_realloc(p->z, p->nAlloc); + if( p->z==0 ) runtimeError("out of memory"); + } +} + + + +/* Safely quote an SQL identifier. Use the minimum amount of transformation +** necessary to allow the string to be used with %s. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). The +** caller is responsible for ensuring this space is freed when no longer +** needed. +*/ +static char *safeId(const char *zId){ + /* All SQLite keywords, in alphabetical order */ + static const char *azKeywords[] = { + "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS", + "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY", + "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT", + "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE", + "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE", + "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH", + "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN", + "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF", + "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER", + "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY", + "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL", + "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA", + "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP", + "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT", + "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP", + "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE", + "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE", + "WITH", "WITHOUT", + }; + int lwr, upr, mid, c, i, x; + for(i=x=0; (c = zId[i])!=0; i++){ + if( !isalpha(c) && c!='_' ){ + if( i>0 && isdigit(c) ){ + x++; + }else{ + return sqlite3_mprintf("\"%w\"", zId); + } + } + } + if( x ) return sqlite3_mprintf("%s", zId); + lwr = 0; + upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1; + while( lwr<=upr ){ + mid = (lwr+upr)/2; + c = sqlite3_stricmp(azKeywords[mid], zId); + if( c==0 ) return sqlite3_mprintf("\"%w\"", zId); + if( c<0 ){ + lwr = mid+1; + }else{ + upr = mid-1; + } + } + return sqlite3_mprintf("%s", zId); +} + +/* +** Prepare a new SQL statement. Print an error and abort if anything +** goes wrong. +*/ +static sqlite3_stmt *db_vprepare(const char *zFormat, va_list ap){ + char *zSql; + int rc; + sqlite3_stmt *pStmt; + + zSql = sqlite3_vmprintf(zFormat, ap); + if( zSql==0 ) runtimeError("out of memory"); + rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt, 0); + if( rc ){ + runtimeError("SQL statement error: %s\n\"%s\"", sqlite3_errmsg(g.db), + zSql); + } + sqlite3_free(zSql); + return pStmt; +} +static sqlite3_stmt *db_prepare(const char *zFormat, ...){ + va_list ap; + sqlite3_stmt *pStmt; + va_start(ap, zFormat); + pStmt = db_vprepare(zFormat, ap); + va_end(ap); + return pStmt; +} + +/* +** Free a list of strings +*/ +static void namelistFree(char **az){ + if( az ){ + int i; + for(i=0; az[i]; i++) sqlite3_free(az[i]); + sqlite3_free(az); + } +} + +/* +** Return a list of column names for the table zDb.zTab. Space to +** hold the list is obtained from sqlite3_malloc() and should released +** using namelistFree() when no longer needed. +** +** Primary key columns are listed first, followed by data columns. +** The number of columns in the primary key is returned in *pnPkey. +** +** Normally, the "primary key" in the previous sentence is the true +** primary key - the rowid or INTEGER PRIMARY KEY for ordinary tables +** or the declared PRIMARY KEY for WITHOUT ROWID tables. However, if +** the g.bSchemaPK flag is set, then the schema-defined PRIMARY KEY is +** used in all cases. In that case, entries that have NULL values in +** any of their primary key fields will be excluded from the analysis. +** +** If the primary key for a table is the rowid but rowid is inaccessible, +** then this routine returns a NULL pointer. +** +** Examples: +** CREATE TABLE t1(a INT UNIQUE, b INTEGER, c TEXT, PRIMARY KEY(c)); +** *pnPKey = 1; +** az = { "rowid", "a", "b", "c", 0 } // Normal case +** az = { "c", "a", "b", 0 } // g.bSchemaPK==1 +** +** CREATE TABLE t2(a INT UNIQUE, b INTEGER, c TEXT, PRIMARY KEY(b)); +** *pnPKey = 1; +** az = { "b", "a", "c", 0 } +** +** CREATE TABLE t3(x,y,z,PRIMARY KEY(y,z)); +** *pnPKey = 1 // Normal case +** az = { "rowid", "x", "y", "z", 0 } // Normal case +** *pnPKey = 2 // g.bSchemaPK==1 +** az = { "y", "x", "z", 0 } // g.bSchemaPK==1 +** +** CREATE TABLE t4(x,y,z,PRIMARY KEY(y,z)) WITHOUT ROWID; +** *pnPKey = 2 +** az = { "y", "z", "x", 0 } +** +** CREATE TABLE t5(rowid,_rowid_,oid); +** az = 0 // The rowid is not accessible +*/ +static char **columnNames(const char *zDb, const char *zTab, int *pnPKey){ + char **az = 0; /* List of column names to be returned */ + int naz = 0; /* Number of entries in az[] */ + sqlite3_stmt *pStmt; /* SQL statement being run */ + char *zPkIdxName = 0; /* Name of the PRIMARY KEY index */ + int truePk = 0; /* PRAGMA table_info indentifies the PK to use */ + int nPK = 0; /* Number of PRIMARY KEY columns */ + int i, j; /* Loop counters */ + + if( g.bSchemaPK==0 ){ + /* Normal case: Figure out what the true primary key is for the table. + ** * For WITHOUT ROWID tables, the true primary key is the same as + ** the schema PRIMARY KEY, which is guaranteed to be present. + ** * For rowid tables with an INTEGER PRIMARY KEY, the true primary + ** key is the INTEGER PRIMARY KEY. + ** * For all other rowid tables, the rowid is the true primary key. + */ + pStmt = db_prepare("PRAGMA %s.index_list=%Q", zDb, zTab); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,3),"pk")==0 ){ + zPkIdxName = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + break; + } + } + sqlite3_finalize(pStmt); + if( zPkIdxName ){ + int nKey = 0; + int nCol = 0; + truePk = 0; + pStmt = db_prepare("PRAGMA %s.index_xinfo=%Q", zDb, zPkIdxName); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + nCol++; + if( sqlite3_column_int(pStmt,5) ){ nKey++; continue; } + if( sqlite3_column_int(pStmt,1)>=0 ) truePk = 1; + } + if( nCol==nKey ) truePk = 1; + if( truePk ){ + nPK = nKey; + }else{ + nPK = 1; + } + sqlite3_finalize(pStmt); + sqlite3_free(zPkIdxName); + }else{ + truePk = 1; + nPK = 1; + } + pStmt = db_prepare("PRAGMA %s.table_info=%Q", zDb, zTab); + }else{ + /* The g.bSchemaPK==1 case: Use whatever primary key is declared + ** in the schema. The "rowid" will still be used as the primary key + ** if the table definition does not contain a PRIMARY KEY. + */ + nPK = 0; + pStmt = db_prepare("PRAGMA %s.table_info=%Q", zDb, zTab); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_int(pStmt,5)>0 ) nPK++; + } + sqlite3_reset(pStmt); + if( nPK==0 ) nPK = 1; + truePk = 1; + } + *pnPKey = nPK; + naz = nPK; + az = sqlite3_malloc( sizeof(char*)*(nPK+1) ); + if( az==0 ) runtimeError("out of memory"); + memset(az, 0, sizeof(char*)*(nPK+1)); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + int iPKey; + if( truePk && (iPKey = sqlite3_column_int(pStmt,5))>0 ){ + az[iPKey-1] = safeId((char*)sqlite3_column_text(pStmt,1)); + }else{ + az = sqlite3_realloc(az, sizeof(char*)*(naz+2) ); + if( az==0 ) runtimeError("out of memory"); + az[naz++] = safeId((char*)sqlite3_column_text(pStmt,1)); + } + } + sqlite3_finalize(pStmt); + if( az ) az[naz] = 0; + if( az[0]==0 ){ + const char *azRowid[] = { "rowid", "_rowid_", "oid" }; + for(i=0; i=naz ){ + az[0] = sqlite3_mprintf("%s", azRowid[i]); + break; + } + } + if( az[0]==0 ){ + for(i=1; inPk2 ){ + zSep = "SELECT "; + for(i=0; i>= 8; + for(i=7; i>=0; i--){ + p[i] = (unsigned char)((v & 0x7f) | 0x80); + v >>= 7; + } + fwrite(p, 8, 1, out); + }else{ + n = 9; + do{ + p[n--] = (unsigned char)((v & 0x7f) | 0x80); + v >>= 7; + }while( v!=0 ); + p[9] &= 0x7f; + fwrite(p+n+1, 9-n, 1, out); + } +} + +/* +** Write an SQLite value onto out. +*/ +static void putValue(FILE *out, sqlite3_value *pVal){ + int iDType = sqlite3_value_type(pVal); + sqlite3_int64 iX; + double rX; + sqlite3_uint64 uX; + int j; + + putc(iDType, out); + switch( iDType ){ + case SQLITE_INTEGER: + iX = sqlite3_value_int64(pVal); + memcpy(&uX, &iX, 8); + for(j=56; j>=0; j-=8) putc((uX>>j)&0xff, out); + break; + case SQLITE_FLOAT: + rX = sqlite3_value_int64(pVal); + memcpy(&uX, &rX, 8); + for(j=56; j>=0; j-=8) putc((uX>>j)&0xff, out); + break; + case SQLITE_TEXT: + iX = sqlite3_value_bytes(pVal); + putsVarint(out, (sqlite3_uint64)iX); + fwrite(sqlite3_value_text(pVal),1,iX,out); + break; + case SQLITE_BLOB: + iX = sqlite3_value_bytes(pVal); + putsVarint(out, (sqlite3_uint64)iX); + fwrite(sqlite3_value_blob(pVal),1,iX,out); + break; + case SQLITE_NULL: + break; + } +} + +/* +** Generate a CHANGESET for all differences from main.zTab to aux.zTab. +*/ +static void changeset_one_table(const char *zTab, FILE *out){ + sqlite3_stmt *pStmt; /* SQL statment */ + char *zId = safeId(zTab); /* Escaped name of the table */ + char **azCol = 0; /* List of escaped column names */ + int nCol = 0; /* Number of columns */ + int *aiFlg = 0; /* 0 if column is not part of PK */ + int *aiPk = 0; /* Column numbers for each PK column */ + int nPk = 0; /* Number of PRIMARY KEY columns */ + Str sql; /* SQL for the diff query */ + int i, k; /* Loop counters */ + const char *zSep; /* List separator */ + + pStmt = db_prepare( + "SELECT A.sql=B.sql FROM main.sqlite_master A, aux.sqlite_master B" + " WHERE A.name=%Q AND B.name=%Q", zTab, zTab + ); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_int(pStmt,0)==0 ){ + runtimeError("schema changes for table %s", safeId(zTab)); + } + }else{ + runtimeError("table %s missing from one or both databases", safeId(zTab)); + } + sqlite3_finalize(pStmt); + pStmt = db_prepare("PRAGMA main.table_info=%Q", zTab); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + nCol++; + azCol = sqlite3_realloc(azCol, sizeof(char*)*nCol); + if( azCol==0 ) runtimeError("out of memory"); + aiFlg = sqlite3_realloc(aiFlg, sizeof(int)*nCol); + if( aiFlg==0 ) runtimeError("out of memory"); + azCol[nCol-1] = safeId((const char*)sqlite3_column_text(pStmt,1)); + aiFlg[nCol-1] = i = sqlite3_column_int(pStmt,5); + if( i>0 ){ + if( i>nPk ){ + nPk = i; + aiPk = sqlite3_realloc(aiPk, sizeof(int)*nPk); + if( aiPk==0 ) runtimeError("out of memory"); + } + aiPk[i-1] = nCol-1; + } + } + sqlite3_finalize(pStmt); + if( nPk==0 ) goto end_changeset_one_table; + strInit(&sql); + if( nCol>nPk ){ + strPrintf(&sql, "SELECT %d", SQLITE_UPDATE); + for(i=0; i0 ) sqlite3_free(azCol[--nCol]); + sqlite3_free(azCol); + sqlite3_free(aiPk); + sqlite3_free(zId); +} + +/* +** Print sketchy documentation for this utility program +*/ +static void showHelp(void){ + printf("Usage: %s [options] DB1 DB2\n", g.zArgv0); + printf( +"Output SQL text that would transform DB1 into DB2.\n" +"Options:\n" +" --changeset FILE Write a CHANGESET into FILE\n" +" --primarykey Use schema-defined PRIMARY KEYs\n" +" --schema Show only differences in the schema\n" +" --table TAB Show only differences in table TAB\n" + ); +} + +int main(int argc, char **argv){ + const char *zDb1 = 0; + const char *zDb2 = 0; + int i; + int rc; + char *zErrMsg = 0; + char *zSql; + sqlite3_stmt *pStmt; + char *zTab = 0; + FILE *out = 0; + + g.zArgv0 = argv[0]; + for(i=1; i