Index: src/delete.c ================================================================== --- src/delete.c +++ src/delete.c @@ -110,10 +110,11 @@ assert( pFrom->a[0].pUsing==0 ); }else{ sqlite3SelectDelete(db, pDup); } pDup = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); + if( pDup ) pDup->selFlags |= SF_Materialize; } sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); sqlite3Select(pParse, pDup, &dest); sqlite3SelectDelete(db, pDup); } Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -937,10 +937,11 @@ pNewItem->jointype = pOldItem->jointype; pNewItem->iCursor = pOldItem->iCursor; pNewItem->addrFillSub = pOldItem->addrFillSub; pNewItem->regReturn = pOldItem->regReturn; pNewItem->isCorrelated = pOldItem->isCorrelated; + pNewItem->viaCoroutine = pOldItem->viaCoroutine; pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); pNewItem->notIndexed = pOldItem->notIndexed; pNewItem->pIndex = pOldItem->pIndex; pTab = pNewItem->pTab = pOldItem->pTab; if( pTab ){ Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -3017,11 +3017,11 @@ ** with various optimizations disabled to verify that the same answer ** is obtained in every case. */ case SQLITE_TESTCTRL_OPTIMIZATIONS: { sqlite3 *db = va_arg(ap, sqlite3*); - db->dbOptFlags = (u8)(va_arg(ap, int) & 0xff); + db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); break; } #ifdef SQLITE_N_KEYWORD /* sqlite3_test_control(SQLITE_TESTCTRL_ISKEYWORD, const char *zWord) Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -3907,12 +3907,21 @@ SelectDest dest; Select *pSub = pItem->pSelect; int isAggSub; if( pSub==0 ) continue; + + /* Sometimes the code for a subquery will be generated more than + ** once, if the subquery is part of the WHERE clause in a LEFT JOIN, + ** for example. In that case, do not regenerate the code to manifest + ** a view or the co-routine to implement a view. The first instance + ** is sufficient, though the subroutine to manifest the view does need + ** to be invoked again. */ if( pItem->addrFillSub ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); + if( pItem->viaCoroutine==0 ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); + } continue; } /* Increment Parse.nHeight by the height of the largest expression ** tree refered to by this, the parent select. The child select @@ -3929,10 +3938,39 @@ if( isAggSub ){ isAgg = 1; p->selFlags |= SF_Aggregate; } i = -1; + }else if( pTabList->nSrc==1 && (p->selFlags & SF_Materialize)==0 + && OptimizationEnabled(db, SQLITE_SubqCoroutine) + ){ + /* Implement a co-routine that will return a single row of the result + ** set on each invocation. + */ + int addrTop; + int addrEof; + pItem->regReturn = ++pParse->nMem; + addrEof = ++pParse->nMem; + sqlite3VdbeAddOp0(v, OP_Goto); + addrTop = sqlite3VdbeAddOp1(v, OP_OpenPseudo, pItem->iCursor); + sqlite3VdbeChangeP5(v, 1); + VdbeComment((v, "coroutine for %s", pItem->pTab->zName)); + pItem->addrFillSub = addrTop; + sqlite3VdbeAddOp2(v, OP_Integer, 0, addrEof); + sqlite3VdbeChangeP5(v, 1); + sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); + explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); + sqlite3Select(pParse, pSub, &dest); + pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->viaCoroutine = 1; + sqlite3VdbeChangeP2(v, addrTop, dest.iSdst); + sqlite3VdbeChangeP3(v, addrTop, dest.nSdst); + sqlite3VdbeAddOp2(v, OP_Integer, 1, addrEof); + sqlite3VdbeAddOp1(v, OP_Yield, pItem->regReturn); + VdbeComment((v, "end %s", pItem->pTab->zName)); + sqlite3VdbeJumpHere(v, addrTop-1); + sqlite3ClearTempRegCache(pParse); }else{ /* Generate a subroutine that will fill an ephemeral table with ** the content of this subquery. pItem->addrFillSub will point ** to the address of the generated subroutine. pItem->regReturn ** is a register allocated to hold the subroutine return address Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -825,11 +825,11 @@ int flags; /* Miscellaneous flags. See below */ i64 lastRowid; /* ROWID of most recent insert (see above) */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ - u8 dbOptFlags; /* Flags to enable/disable optimizations */ + u16 dbOptFlags; /* Flags to enable/disable optimizations */ u8 autoCommit; /* The auto-commit flag. */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ u8 dfltLockMode; /* Default locking-mode for attached dbs */ signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */ @@ -970,11 +970,12 @@ #define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ #define SQLITE_IdxRealAsInt 0x0010 /* Store REAL as INT in indices */ #define SQLITE_DistinctOpt 0x0020 /* DISTINCT using indexes */ #define SQLITE_CoverIdxScan 0x0040 /* Covering index scans */ #define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ -#define SQLITE_AllOpts 0x00ff /* All optimizations */ +#define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ +#define SQLITE_AllOpts 0xffff /* All optimizations */ /* ** Macros for testing whether or not optimizations are enabled or disabled. */ #ifndef SQLITE_OMIT_BUILTIN_TEST @@ -1876,12 +1877,13 @@ Table *pTab; /* An SQL table corresponding to zName */ Select *pSelect; /* A SELECT statement used in place of a table name */ int addrFillSub; /* Address of subroutine to manifest a subquery */ int regReturn; /* Register holding return address of addrFillSub */ u8 jointype; /* Type of join between this able and the previous */ - u8 notIndexed; /* True if there is a NOT INDEXED clause */ - u8 isCorrelated; /* True if sub-query is correlated */ + unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ + unsigned isCorrelated :1; /* True if sub-query is correlated */ + unsigned viaCoroutine :1; /* Implemented as a co-routine */ #ifndef SQLITE_OMIT_EXPLAIN u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ #endif int iCursor; /* The VDBE cursor number used to access this table */ Expr *pOn; /* The ON clause of a join */ @@ -2101,18 +2103,19 @@ /* ** Allowed values for Select.selFlags. The "SF" prefix stands for ** "Select Flag". */ -#define SF_Distinct 0x01 /* Output should be DISTINCT */ -#define SF_Resolved 0x02 /* Identifiers have been resolved */ -#define SF_Aggregate 0x04 /* Contains aggregate functions */ -#define SF_UsesEphemeral 0x08 /* Uses the OpenEphemeral opcode */ -#define SF_Expanded 0x10 /* sqlite3SelectExpand() called on this */ -#define SF_HasTypeInfo 0x20 /* FROM subqueries have Table metadata */ -#define SF_UseSorter 0x40 /* Sort using a sorter */ -#define SF_Values 0x80 /* Synthesized from VALUES clause */ +#define SF_Distinct 0x0001 /* Output should be DISTINCT */ +#define SF_Resolved 0x0002 /* Identifiers have been resolved */ +#define SF_Aggregate 0x0004 /* Contains aggregate functions */ +#define SF_UsesEphemeral 0x0008 /* Uses the OpenEphemeral opcode */ +#define SF_Expanded 0x0010 /* sqlite3SelectExpand() called on this */ +#define SF_HasTypeInfo 0x0020 /* FROM subqueries have Table metadata */ +#define SF_UseSorter 0x0040 /* Sort using a sorter */ +#define SF_Values 0x0080 /* Synthesized from VALUES clause */ +#define SF_Materialize 0x0100 /* Force materialization of views */ /* ** The results of a select can be distributed in several ways. The ** "SRT" prefix means "SELECT Result Type". Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -2235,10 +2235,15 @@ VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &payloadSize); assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ } }else if( ALWAYS(pC->pseudoTableReg>0) ){ pReg = &aMem[pC->pseudoTableReg]; + if( pC->multiPseudo ){ + sqlite3VdbeMemShallowCopy(pDest, pReg+p2, MEM_Ephem); + Deephemeralize(pDest); + goto op_column_out; + } assert( pReg->flags & MEM_Blob ); assert( memIsValid(pReg) ); payloadSize = pReg->n; zRec = pReg->z; pC->cacheStatus = (pOp->p5&OPFLAG_CLEARCACHE) ? CACHE_STALE : p->cacheCtr; @@ -3316,16 +3321,17 @@ pc--; #endif break; } -/* Opcode: OpenPseudo P1 P2 P3 * * +/* Opcode: OpenPseudo P1 P2 P3 * P5 ** ** Open a new cursor that points to a fake table that contains a single ** row of data. The content of that one row in the content of memory -** register P2. In other words, cursor P1 becomes an alias for the -** MEM_Blob content contained in register P2. +** register P2 when P5==0. In other words, cursor P1 becomes an alias for the +** MEM_Blob content contained in register P2. When P5==1, then the +** row is represented by P3 consecutive registers beginning with P2. ** ** A pseudo-table created by this opcode is used to hold a single ** row output from the sorter so that the row can be decomposed into ** individual columns using the OP_Column opcode. The OP_Column opcode ** is the only cursor opcode that works with a pseudo-table. @@ -3341,10 +3347,11 @@ if( pCx==0 ) goto no_mem; pCx->nullRow = 1; pCx->pseudoTableReg = pOp->p2; pCx->isTable = 1; pCx->isIndex = 0; + pCx->multiPseudo = pOp->p5; break; } /* Opcode: Close P1 * * * * ** @@ -4305,11 +4312,11 @@ const sqlite3_module *pModule; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->pseudoTableReg==0 ); + assert( pC->pseudoTableReg==0 || pC->nullRow ); if( pC->nullRow ){ pOut->flags = MEM_Null; break; }else if( pC->deferredMoveto ){ v = pC->movetoTarget; Index: src/vdbeInt.h ================================================================== --- src/vdbeInt.h +++ src/vdbeInt.h @@ -61,10 +61,11 @@ Bool deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ Bool isTable; /* True if a table requiring integer keys */ Bool isIndex; /* True if an index containing keys only - no data */ Bool isOrdered; /* True if the underlying table is BTREE_UNORDERED */ Bool isSorter; /* True if a new-style sorter */ + Bool multiPseudo; /* Multi-register pseudo-cursor */ sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ const sqlite3_module *pModule; /* Module for cursor pVtabCursor */ i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ i64 lastRowid; /* Last rowid from a Next or NextIdx operation */ Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -1818,10 +1818,14 @@ if( (p->cost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 && (p->cost.plan.wsFlags & WHERE_COVER_SCAN)==0 ){ /* We already have some kind of index in use for this query. */ return; + } + if( pSrc->viaCoroutine ){ + /* Cannot index a co-routine */ + return; } if( pSrc->notIndexed ){ /* The NOT INDEXED clause appears in the SQL. */ return; } @@ -2996,11 +3000,11 @@ ** the SQL statement, then this function only considers plans using the ** named index. If no such plan is found, then the returned cost is ** SQLITE_BIG_DBL. If a plan is found that uses the named index, ** then the cost is calculated in the usual way. ** -** If a NOT INDEXED clause (pSrc->notIndexed!=0) was attached to the table +** If a NOT INDEXED clause was attached to the table ** in the SELECT statement, then no indexes are considered. However, the ** selected plan may still take advantage of the built-in rowid primary key ** index. */ static void bestBtreeIndex(WhereBestIdx *p){ @@ -4023,10 +4027,20 @@ if( pLevel->iFrom>0 && (pTabItem[0].jointype & JT_LEFT)!=0 ){ pLevel->iLeftJoin = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); VdbeComment((v, "init LEFT JOIN no-match flag")); } + + /* Special case of a FROM clause subquery implemented as a co-routine */ + if( pTabItem->viaCoroutine ){ + int regYield = pTabItem->regReturn; + sqlite3VdbeAddOp2(v, OP_Integer, pTabItem->addrFillSub-1, regYield); + pLevel->p2 = sqlite3VdbeAddOp1(v, OP_Yield, regYield); + VdbeComment((v, "next row of co-routine %s", pTabItem->pTab->zName)); + sqlite3VdbeAddOp2(v, OP_If, regYield+1, addrBrk); + pLevel->op = OP_Goto; + }else #ifndef SQLITE_OMIT_VIRTUALTABLE if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ /* Case 0: The table is a virtual-table. Use the VFilter and VNext ** to access the data. Index: test/minmax.test ================================================================== --- test/minmax.test +++ test/minmax.test @@ -297,11 +297,11 @@ execsql { SELECT max(rowid) FROM ( SELECT max(rowid) FROM t4 UNION SELECT max(rowid) FROM t5 ) } - } {1} + } {{}} do_test minmax-9.2 { execsql { SELECT max(rowid) FROM ( SELECT max(rowid) FROM t4 EXCEPT SELECT max(rowid) FROM t5 ) Index: test/minmax2.test ================================================================== --- test/minmax2.test +++ test/minmax2.test @@ -287,11 +287,11 @@ execsql { SELECT max(rowid) FROM ( SELECT max(rowid) FROM t4 UNION SELECT max(rowid) FROM t5 ) } - } {1} + } {{}} do_test minmax2-9.2 { execsql { SELECT max(rowid) FROM ( SELECT max(rowid) FROM t4 EXCEPT SELECT max(rowid) FROM t5 ) Index: test/tkt-31338dca7e.test ================================================================== --- test/tkt-31338dca7e.test +++ test/tkt-31338dca7e.test @@ -89,11 +89,11 @@ CREATE INDEX t1b ON t1(b); CREATE TABLE t3(g); INSERT INTO t3 VALUES(4); CREATE TABLE t4(h); INSERT INTO t4 VALUES(5); - + SELECT * FROM t3 LEFT JOIN t1 ON d=g LEFT JOIN t4 ON c=h WHERE (a=1 AND h=4) OR (b IN ( SELECT x FROM (SELECT e+f AS x, e FROM t2 ORDER BY 1 LIMIT 2) GROUP BY e Index: test/tkt3527.test ================================================================== --- test/tkt3527.test +++ test/tkt3527.test @@ -50,12 +50,12 @@ INSERT INTO Element VALUES(3,'Elem3'); INSERT INTO Element VALUES(4,'Elem4'); INSERT INTO Element VALUES(5,'Elem5'); INSERT INTO ElemOr Values(3,4); INSERT INTO ElemOr Values(3,5); - INSERT INTO ElemAnd VALUES(1,3,1,1,1); - INSERT INTO ElemAnd VALUES(1,2,1,1,1); + INSERT INTO ElemAnd VALUES(1,3,'a','b','c'); + INSERT INTO ElemAnd VALUES(1,2,'x','y','z'); CREATE VIEW ElemView1 AS SELECT CAST(Element.Code AS VARCHAR(50)) AS ElemId, Element.Code AS ElemCode, @@ -110,14 +110,14 @@ ON Element.Level=0 AND Element.InnerCode=InnerElem.ElemCode ORDER BY ElemId, InnerCode; SELECT * FROM ElemView1; } -} {1 1 Elem1 2 1 1 1 0 0 1 1 Elem1 3 1 1 1 0 0 3 3 Elem3 4 {} {} {} 0 1 3 3 Elem3 5 {} {} {} 0 1} +} {1 1 Elem1 2 x y z 0 0 1 1 Elem1 3 a b c 0 0 3 3 Elem3 4 {} {} {} 0 1 3 3 Elem3 5 {} {} {} 0 1} do_test tkt3527-1.2 { db eval { SELECT * FROM ElemView2; } -} {1 1 Elem1 2 1 1 1 0 0 1 1 Elem1 3 1 1 1 0 0 1.3 3 Elem3 4 {} {} {} 1 1 1.3 3 Elem3 5 {} {} {} 1 1 3 3 Elem3 4 {} {} {} 0 1 3 3 Elem3 5 {} {} {} 0 1} +} {1 1 Elem1 2 x y z 0 0 1 1 Elem1 3 a b c 0 0 1.3 3 Elem3 4 {} {} {} 1 1 1.3 3 Elem3 5 {} {} {} 1 1 3 3 Elem3 4 {} {} {} 0 1 3 3 Elem3 5 {} {} {} 0 1} finish_test