Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -3296,13 +3296,15 @@ ** exist on the table t1, a complete scan of the data might be ** avoided. ** ** Flattening is subject to the following constraints: ** -** (1) The subquery and the outer query cannot both be aggregates. +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** The subquery and the outer query cannot both be aggregates. ** -** (2) If the subquery is an aggregate then +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** (2) If the subquery is an aggregate then ** (2a) the outer query must not be a join and ** (2b) the outer query must not use subqueries ** other than the one FROM-clause subquery that is a candidate ** for flattening. (This is due to ticket [2f7170d73bf9abf80] ** from 2015-02-09.) @@ -3317,11 +3319,12 @@ ** ** (**) At one point restrictions (4) and (5) defined a subset of DISTINCT ** sub-queries that were excluded from this optimization. Restriction ** (4) has since been expanded to exclude all DISTINCT subqueries. ** -** (6) If the subquery is aggregate, the outer query may not be DISTINCT. +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** If the subquery is aggregate, the outer query may not be DISTINCT. ** ** (7) The subquery must have a FROM clause. TODO: For subqueries without ** A FROM clause, consider adding a FROM clause with the special ** table sqlite_once that consists of a single row containing a ** single NULL. @@ -3404,11 +3407,11 @@ ** return the value X for which Y was maximal.) ** ** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query -** uses aggregates and subqueryIsAgg is true if the subquery uses aggregates. +** uses aggregates. ** ** If flattening is not attempted, this routine is a no-op and returns 0. ** If flattening is attempted this routine returns 1. ** ** All of the expression analysis must occur on both the outer query and @@ -3416,12 +3419,11 @@ */ static int flattenSubquery( Parse *pParse, /* Parsing context */ Select *p, /* The parent or outer SELECT statement */ int iFrom, /* Index in p->pSrc->a[] of the inner subquery */ - int isAgg, /* True if outer SELECT uses aggregate functions */ - int subqueryIsAgg /* True if the subquery uses aggregate functions */ + int isAgg /* True if outer SELECT uses aggregate functions */ ){ const char *zSavedAuthContext = pParse->zAuthContext; Select *pParent; /* Current UNION ALL term of the other query */ Select *pSub; /* The inner query or "subquery" */ Select *pSub1; /* Pointer to the rightmost select in sub-query */ @@ -3444,20 +3446,10 @@ assert( pSrc && iFrom>=0 && iFromnSrc ); pSubitem = &pSrc->a[iFrom]; iParent = pSubitem->iCursor; pSub = pSubitem->pSelect; assert( pSub!=0 ); - if( subqueryIsAgg ){ - if( isAgg ) return 0; /* Restriction (1) */ - if( pSrc->nSrc>1 ) return 0; /* Restriction (2a) */ - if( (p->pWhere && ExprHasProperty(p->pWhere,EP_Subquery)) - || (sqlite3ExprListFlags(p->pEList) & EP_Subquery)!=0 - || (sqlite3ExprListFlags(p->pOrderBy) & EP_Subquery)!=0 - ){ - return 0; /* Restriction (2b) */ - } - } pSubSrc = pSub->pSrc; assert( pSubSrc ); /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants, ** not arbitrary expressions, we allowed some combining of LIMIT and OFFSET @@ -3472,13 +3464,10 @@ if( pSubSrc->nSrc==0 ) return 0; /* Restriction (7) */ if( pSub->selFlags & SF_Distinct ) return 0; /* Restriction (4) */ if( pSub->pLimit && (pSrc->nSrc>1 || isAgg) ){ return 0; /* Restrictions (8)(9) */ } - if( (p->selFlags & SF_Distinct)!=0 && subqueryIsAgg ){ - return 0; /* Restriction (6) */ - } if( p->pOrderBy && pSub->pOrderBy ){ return 0; /* Restriction (11) */ } if( isAgg && pSub->pOrderBy ) return 0; /* Restriction (16) */ if( pSub->pLimit && p->pWhere ) return 0; /* Restriction (19) */ @@ -3776,22 +3765,11 @@ } pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); if( isLeftJoin>0 ){ setJoinExpr(pWhere, iNewParent); } - if( subqueryIsAgg ){ - assert( pParent->pHaving==0 ); - pParent->pHaving = pParent->pWhere; - pParent->pWhere = pWhere; - pParent->pHaving = sqlite3ExprAnd(db, - sqlite3ExprDup(db, pSub->pHaving, 0), pParent->pHaving - ); - assert( pParent->pGroupBy==0 ); - pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0); - }else{ - pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere); - } + pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere); if( db->mallocFailed==0 ){ SubstContext x; x.pParse = pParse; x.iTable = iParent; x.iNewTable = iNewParent; @@ -3850,13 +3828,13 @@ ** The hope is that the terms added to the inner query will make it more ** efficient. ** ** Do not attempt this optimization if: ** -** (1) The inner query is an aggregate. (In that case, we'd really want -** to copy the outer WHERE-clause terms onto the HAVING clause of the -** inner query. But they probably won't help there so do not bother.) +** (1) (** This restriction was removed on 2017-09-29. We used to +** disallow this optimization for aggregate subqueries, but now +** it is allowed by putting the extra terms on the HAVING clause **) ** ** (2) The inner query is the recursive part of a common table expression. ** ** (3) The inner query has a LIMIT clause (since the changes to the WHERE ** close would change the meaning of the LIMIT). @@ -3880,25 +3858,23 @@ Expr *pNew; int nChng = 0; Select *pX; /* For looping over compound SELECTs in pSubq */ if( pWhere==0 ) return 0; for(pX=pSubq; pX; pX=pX->pPrior){ - if( (pX->selFlags & (SF_Aggregate|SF_Recursive))!=0 ){ - testcase( pX->selFlags & SF_Aggregate ); - testcase( pX->selFlags & SF_Recursive ); + if( (pX->selFlags & (SF_Recursive))!=0 ){ testcase( pX!=pSubq ); - return 0; /* restrictions (1) and (2) */ + return 0; /* restriction (2) */ } } if( pSubq->pLimit!=0 ){ return 0; /* restriction (3) */ } while( pWhere->op==TK_AND ){ nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, iCursor); pWhere = pWhere->pLeft; } - if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction 5 */ + if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction (5) */ if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){ nChng++; while( pSubq ){ SubstContext x; pNew = sqlite3ExprDup(pParse->db, pWhere, 0); @@ -3906,11 +3882,15 @@ x.iTable = iCursor; x.iNewTable = iCursor; x.isLeftJoin = 0; x.pEList = pSubq->pEList; pNew = substExpr(&x, pNew); - pSubq->pWhere = sqlite3ExprAnd(pParse->db, pSubq->pWhere, pNew); + if( pSubq->selFlags & SF_Aggregate ){ + pSubq->pHaving = sqlite3ExprAnd(pParse->db, pSubq->pHaving, pNew); + }else{ + pSubq->pWhere = sqlite3ExprAnd(pParse->db, pSubq->pWhere, pNew); + } pSubq = pSubq->pPrior; } } return nChng; } @@ -5200,11 +5180,10 @@ */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) for(i=0; !p->pPrior && inSrc; i++){ struct SrcList_item *pItem = &pTabList->a[i]; Select *pSub = pItem->pSelect; - int isAggSub; Table *pTab = pItem->pTab; if( pSub==0 ) continue; /* Catch mismatch in the declared columns of a view and the number of ** columns in the SELECT on the RHS */ @@ -5212,36 +5191,40 @@ sqlite3ErrorMsg(pParse, "expected %d columns for '%s' but got %d", pTab->nCol, pTab->zName, pSub->pEList->nExpr); goto select_end; } - /* If the subquery contains an ORDER BY or GROUP BY clause and if + /* Do not try to flatten an aggregate subquery. + ** + ** Flattening an aggregate subquery is only possible if the outer query + ** is not a join. But if the outer query is not a join, then the subquery + ** will be implemented as a co-routine and there is no advantage to + ** flattening in that case. + */ + if( (pSub->selFlags & SF_Aggregate)!=0 ) continue; + assert( pSub->pGroupBy==0 ); + + /* If the subquery contains an ORDER BY clause and if ** it will be implemented as a co-routine, then do not flatten. This ** restriction allows SQL constructs like this: ** ** SELECT expensive_function(x) ** FROM (SELECT x FROM tab ORDER BY y LIMIT 10); ** ** The expensive_function() is only computed on the 10 rows that ** are output, rather than every row of the table. */ - if( (pSub->pOrderBy!=0 || pSub->pGroupBy!=0) + if( pSub->pOrderBy!=0 && i==0 && (pTabList->nSrc==1 || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) - && OptimizationEnabled(db, SQLITE_SubqCoroutine) ){ continue; } - isAggSub = (pSub->selFlags & SF_Aggregate)!=0; - if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ + if( flattenSubquery(pParse, p, i, isAgg) ){ /* This subquery can be absorbed into its parent. */ - if( isAggSub ){ - isAgg = 1; - p->selFlags |= SF_Aggregate; - } i = -1; } pTabList = p->pSrc; if( db->mallocFailed ) goto select_end; if( !IgnorableOrderby(pDest) ){ @@ -5346,25 +5329,20 @@ zSavedAuthContext = pParse->zAuthContext; pParse->zAuthContext = pItem->zName; /* Generate code to implement the subquery ** - ** The subquery is implemented as a co-routine if all of these are true: - ** (1) The subquery is guaranteed to be the outer loop (so that it - ** does not need to be computed more than once) - ** (2) REMOVED (2017-09-28): The ALL keyword after SELECT is omitted. - ** (3) Co-routines are not disabled using sqlite3_test_control() - ** with SQLITE_TESTCTRL_OPTIMIZATIONS. + ** The subquery is implemented as a co-routine if the subquery is + ** guaranteed to be the outer loop (so that it does not need to be + ** computed more than once) ** ** TODO: Are there other reasons beside (1) to use a co-routine ** implementation? */ if( i==0 && (pTabList->nSrc==1 || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) /* (1) */ - /*** constraint removed: && (p->selFlags & SF_All)==0 (2) */ - && OptimizationEnabled(db, SQLITE_SubqCoroutine) /* (3) */ ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. */ int addrTop = sqlite3VdbeCurrentAddr(v)+1; Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -1504,20 +1504,18 @@ */ #define SQLITE_QueryFlattener 0x0001 /* Query flattening */ #define SQLITE_ColumnCache 0x0002 /* Column cache */ #define SQLITE_GroupByOrder 0x0004 /* GROUPBY cover of ORDERBY */ #define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ -/* not used 0x0010 // Was: SQLITE_IdxRealAsInt */ -#define SQLITE_DistinctOpt 0x0020 /* DISTINCT using indexes */ -#define SQLITE_CoverIdxScan 0x0040 /* Covering index scans */ -#define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ -#define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ -#define SQLITE_Transitive 0x0200 /* Transitive constraints */ -#define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ -#define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ -#define SQLITE_CountOfView 0x1000 /* The count-of-view optimization */ -#define SQLITE_CursorHints 0x2000 /* Add OP_CursorHint opcodes */ +#define SQLITE_DistinctOpt 0x0010 /* DISTINCT using indexes */ +#define SQLITE_CoverIdxScan 0x0020 /* Covering index scans */ +#define SQLITE_OrderByIdxJoin 0x0040 /* ORDER BY of joins via index */ +#define SQLITE_Transitive 0x0080 /* Transitive constraints */ +#define SQLITE_OmitNoopJoin 0x0100 /* Omit unused tables in joins */ +#define SQLITE_Stat34 0x0200 /* Use STAT3 or STAT4 data */ +#define SQLITE_CountOfView 0x0400 /* The count-of-view optimization */ +#define SQLITE_CursorHints 0x0800 /* Add OP_CursorHint opcodes */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* ** Macros for testing whether or not optimizations are enabled or disabled. */ Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -6899,11 +6899,10 @@ { "factor-constants", SQLITE_FactorOutConst }, { "distinct-opt", SQLITE_DistinctOpt }, { "cover-idx-scan", SQLITE_CoverIdxScan }, { "order-by-idx-join", SQLITE_OrderByIdxJoin }, { "transitive", SQLITE_Transitive }, - { "subquery-coroutine", SQLITE_SubqCoroutine }, { "omit-noop-join", SQLITE_OmitNoopJoin }, { "stat3", SQLITE_Stat34 }, { "stat4", SQLITE_Stat34 }, };