Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Enhance aggregate order-by so that it transmits subtype information through the sorter. Fix for the deficiency reported by forum post 87347ad2fb5a8f76. |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA3-256: |
d302a389460d0c15775a8b5f5afbac2c |
User & Date: | drh 2023-12-14 15:11:39 |
Context
2023-12-14
| ||
15:31 | Improve the error message returned by an fts5 'rebuild' command on an external content table if there is a problem with the content table or view. (check-in: 0fbf4b8a user: dan tags: trunk) | |
15:11 | Enhance aggregate order-by so that it transmits subtype information through the sorter. Fix for the deficiency reported by forum post 87347ad2fb5a8f76. (check-in: d302a389 user: drh tags: trunk) | |
13:58 | Pass subtype information through the aggregate ORDER BY sorter for aggregate functions that use subtype information. (Closed-Leaf check-in: 3536f403 user: drh tags: agg-orderby-subtype) | |
2023-12-13
| ||
20:37 | In CLI, fix .read inability to open 2GB+ files on WIN32. (check-in: 56c80a62 user: larrybr tags: trunk) | |
Changes
Changes to src/expr.c.
︙ | ︙ | |||
6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 | pExpr->x.pList->a[0].pExpr,0)==0 ){ pItem->bOBPayload = 0; pItem->bOBUnique = ExprHasProperty(pExpr, EP_Distinct); }else{ pItem->bOBPayload = 1; } }else{ pItem->iOBTab = -1; } if( ExprHasProperty(pExpr, EP_Distinct) && !pItem->bOBUnique ){ pItem->iDistinct = pParse->nTab++; }else{ pItem->iDistinct = -1; | > > | 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 | pExpr->x.pList->a[0].pExpr,0)==0 ){ pItem->bOBPayload = 0; pItem->bOBUnique = ExprHasProperty(pExpr, EP_Distinct); }else{ pItem->bOBPayload = 1; } pItem->bUseSubtype = (pItem->pFunc->funcFlags & SQLITE_SUBTYPE)!=0; }else{ pItem->iOBTab = -1; } if( ExprHasProperty(pExpr, EP_Distinct) && !pItem->bOBUnique ){ pItem->iDistinct = pParse->nTab++; }else{ pItem->iDistinct = -1; |
︙ | ︙ |
Changes to src/select.c.
︙ | ︙ | |||
6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 | if( pFunc->iOBTab>=0 ){ ExprList *pOBList; KeyInfo *pKeyInfo; int nExtra = 0; assert( pFunc->pFExpr->pLeft!=0 ); assert( pFunc->pFExpr->pLeft->op==TK_ORDER ); assert( ExprUseXList(pFunc->pFExpr->pLeft) ); pOBList = pFunc->pFExpr->pLeft->x.pList; if( !pFunc->bOBUnique ){ nExtra++; /* One extra column for the OP_Sequence */ } if( pFunc->bOBPayload ){ /* extra columns for the function arguments */ assert( ExprUseXList(pFunc->pFExpr) ); nExtra += pFunc->pFExpr->x.pList->nExpr; } pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOBList, 0, nExtra); if( !pFunc->bOBUnique && pParse->nErr==0 ){ pKeyInfo->nKeyField++; } sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iOBTab, pOBList->nExpr+nExtra, 0, (char*)pKeyInfo, P4_KEYINFO); | > > > > | 6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 | if( pFunc->iOBTab>=0 ){ ExprList *pOBList; KeyInfo *pKeyInfo; int nExtra = 0; assert( pFunc->pFExpr->pLeft!=0 ); assert( pFunc->pFExpr->pLeft->op==TK_ORDER ); assert( ExprUseXList(pFunc->pFExpr->pLeft) ); assert( pFunc->pFunc!=0 ); pOBList = pFunc->pFExpr->pLeft->x.pList; if( !pFunc->bOBUnique ){ nExtra++; /* One extra column for the OP_Sequence */ } if( pFunc->bOBPayload ){ /* extra columns for the function arguments */ assert( ExprUseXList(pFunc->pFExpr) ); nExtra += pFunc->pFExpr->x.pList->nExpr; } if( pFunc->bUseSubtype ){ nExtra += pFunc->pFExpr->x.pList->nExpr; } pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOBList, 0, nExtra); if( !pFunc->bOBUnique && pParse->nErr==0 ){ pKeyInfo->nKeyField++; } sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iOBTab, pOBList->nExpr+nExtra, 0, (char*)pKeyInfo, P4_KEYINFO); |
︙ | ︙ | |||
6690 6691 6692 6693 6694 6695 6696 | int i; struct AggInfo_func *pF; for(i=0, pF=pAggInfo->aFunc; i<pAggInfo->nFunc; i++, pF++){ ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); pList = pF->pFExpr->x.pList; if( pF->iOBTab>=0 ){ | | | | | > > > > > > > > > > | 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721 6722 6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739 6740 6741 6742 6743 | int i; struct AggInfo_func *pF; for(i=0, pF=pAggInfo->aFunc; i<pAggInfo->nFunc; i++, pF++){ ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); pList = pF->pFExpr->x.pList; if( pF->iOBTab>=0 ){ /* For an ORDER BY aggregate, calls to OP_AggStep were deferred. Inputs ** were stored in emphermal table pF->iOBTab. Here, we extract those ** inputs (in ORDER BY order) and make all calls to OP_AggStep ** before doing the OP_AggFinal call. */ int iTop; /* Start of loop for extracting columns */ int nArg; /* Number of columns to extract */ int nKey; /* Key columns to be skipped */ int regAgg; /* Extract into this array */ int j; /* Loop counter */ assert( pF->pFunc!=0 ); nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); if( pF->bOBPayload==0 ){ nKey = 0; }else{ assert( pF->pFExpr->pLeft!=0 ); assert( ExprUseXList(pF->pFExpr->pLeft) ); assert( pF->pFExpr->pLeft->x.pList!=0 ); nKey = pF->pFExpr->pLeft->x.pList->nExpr; if( ALWAYS(!pF->bOBUnique) ) nKey++; } iTop = sqlite3VdbeAddOp1(v, OP_Rewind, pF->iOBTab); VdbeCoverage(v); for(j=nArg-1; j>=0; j--){ sqlite3VdbeAddOp3(v, OP_Column, pF->iOBTab, nKey+j, regAgg+j); } if( pF->bUseSubtype ){ int regSubtype = sqlite3GetTempReg(pParse); int iBaseCol = nKey + nArg + (pF->bOBPayload==0 && pF->bOBUnique==0); for(j=nArg-1; j>=0; j--){ sqlite3VdbeAddOp3(v, OP_Column, pF->iOBTab, iBaseCol+j, regSubtype); sqlite3VdbeAddOp2(v, OP_SetSubtype, regSubtype, regAgg+j); } sqlite3ReleaseTempReg(pParse, regSubtype); } sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); sqlite3VdbeAddOp2(v, OP_Next, pF->iOBTab, iTop+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, iTop); sqlite3ReleaseTempRange(pParse, regAgg, nArg); } |
︙ | ︙ | |||
6770 6771 6772 6773 6774 6775 6776 6777 6778 6779 6780 6781 6782 6783 | int addrNext = 0; int regAgg; int regAggSz = 0; int regDistinct = 0; ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); assert( !IsWindowFunc(pF->pFExpr) ); pList = pF->pFExpr->x.pList; if( ExprHasProperty(pF->pFExpr, EP_WinFunc) ){ Expr *pFilter = pF->pFExpr->y.pWin->pFilter; if( pAggInfo->nAccumulator && (pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) && regAcc ){ | > | 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798 | int addrNext = 0; int regAgg; int regAggSz = 0; int regDistinct = 0; ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); assert( !IsWindowFunc(pF->pFExpr) ); assert( pF->pFunc!=0 ); pList = pF->pFExpr->x.pList; if( ExprHasProperty(pF->pFExpr, EP_WinFunc) ){ Expr *pFilter = pF->pFExpr->y.pWin->pFilter; if( pAggInfo->nAccumulator && (pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) && regAcc ){ |
︙ | ︙ | |||
6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 | assert( pOBList->nExpr>0 ); regAggSz = pOBList->nExpr; if( !pF->bOBUnique ){ regAggSz++; /* One register for OP_Sequence */ } if( pF->bOBPayload ){ regAggSz += nArg; } regAggSz++; /* One extra register to hold result of MakeRecord */ regAgg = sqlite3GetTempRange(pParse, regAggSz); regDistinct = regAgg; sqlite3ExprCodeExprList(pParse, pOBList, regAgg, 0, SQLITE_ECEL_DUP); jj = pOBList->nExpr; if( !pF->bOBUnique ){ sqlite3VdbeAddOp2(v, OP_Sequence, pF->iOBTab, regAgg+jj); jj++; } if( pF->bOBPayload ){ regDistinct = regAgg+jj; sqlite3ExprCodeExprList(pParse, pList, regDistinct, 0, SQLITE_ECEL_DUP); } }else if( pList ){ nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); regDistinct = regAgg; sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP); }else{ | > > > > > > > > > > > | 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844 6845 6846 6847 6848 6849 6850 6851 6852 6853 6854 6855 6856 6857 6858 6859 6860 6861 6862 6863 6864 6865 | assert( pOBList->nExpr>0 ); regAggSz = pOBList->nExpr; if( !pF->bOBUnique ){ regAggSz++; /* One register for OP_Sequence */ } if( pF->bOBPayload ){ regAggSz += nArg; } if( pF->bUseSubtype ){ regAggSz += nArg; } regAggSz++; /* One extra register to hold result of MakeRecord */ regAgg = sqlite3GetTempRange(pParse, regAggSz); regDistinct = regAgg; sqlite3ExprCodeExprList(pParse, pOBList, regAgg, 0, SQLITE_ECEL_DUP); jj = pOBList->nExpr; if( !pF->bOBUnique ){ sqlite3VdbeAddOp2(v, OP_Sequence, pF->iOBTab, regAgg+jj); jj++; } if( pF->bOBPayload ){ regDistinct = regAgg+jj; sqlite3ExprCodeExprList(pParse, pList, regDistinct, 0, SQLITE_ECEL_DUP); jj += nArg; } if( pF->bUseSubtype ){ int kk; int regBase = pF->bOBPayload ? regDistinct : regAgg; for(kk=0; kk<nArg; kk++, jj++){ sqlite3VdbeAddOp2(v, OP_GetSubtype, regBase+kk, regAgg+jj); } } }else if( pList ){ nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); regDistinct = regAgg; sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP); }else{ |
︙ | ︙ |
Changes to src/sqliteInt.h.
︙ | ︙ | |||
2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 | Expr *pFExpr; /* Expression encoding the function */ FuncDef *pFunc; /* The aggregate function implementation */ int iDistinct; /* Ephemeral table used to enforce DISTINCT */ int iDistAddr; /* Address of OP_OpenEphemeral */ int iOBTab; /* Ephemeral table to implement ORDER BY */ u8 bOBPayload; /* iOBTab has payload columns separate from key */ u8 bOBUnique; /* Enforce uniqueness on iOBTab keys */ } *aFunc; int nFunc; /* Number of entries in aFunc[] */ u32 selId; /* Select to which this AggInfo belongs */ #ifdef SQLITE_DEBUG Select *pSelect; /* SELECT statement that this AggInfo supports */ #endif }; | > | 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 | Expr *pFExpr; /* Expression encoding the function */ FuncDef *pFunc; /* The aggregate function implementation */ int iDistinct; /* Ephemeral table used to enforce DISTINCT */ int iDistAddr; /* Address of OP_OpenEphemeral */ int iOBTab; /* Ephemeral table to implement ORDER BY */ u8 bOBPayload; /* iOBTab has payload columns separate from key */ u8 bOBUnique; /* Enforce uniqueness on iOBTab keys */ u8 bUseSubtype; /* Transfer subtype info through sorter */ } *aFunc; int nFunc; /* Number of entries in aFunc[] */ u32 selId; /* Select to which this AggInfo belongs */ #ifdef SQLITE_DEBUG Select *pSelect; /* SELECT statement that this AggInfo supports */ #endif }; |
︙ | ︙ |
Changes to src/vdbe.c.
︙ | ︙ | |||
8669 8670 8671 8672 8673 8674 8675 8676 8677 8678 8679 8680 8681 8682 | ** Clear the subtype from register P1. */ case OP_ClrSubtype: { /* in1 */ pIn1 = &aMem[pOp->p1]; pIn1->flags &= ~MEM_Subtype; break; } /* Opcode: FilterAdd P1 * P3 P4 * ** Synopsis: filter(P1) += key(P3@P4) ** ** Compute a hash on the P4 registers starting with r[P3] and ** add that hash to the bloom filter contained in r[P1]. */ | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 8669 8670 8671 8672 8673 8674 8675 8676 8677 8678 8679 8680 8681 8682 8683 8684 8685 8686 8687 8688 8689 8690 8691 8692 8693 8694 8695 8696 8697 8698 8699 8700 8701 8702 8703 8704 8705 8706 8707 8708 8709 8710 8711 8712 8713 8714 8715 8716 8717 8718 | ** Clear the subtype from register P1. */ case OP_ClrSubtype: { /* in1 */ pIn1 = &aMem[pOp->p1]; pIn1->flags &= ~MEM_Subtype; break; } /* Opcode: GetSubtype P1 P2 * * * ** Synopsis: r[P2] = r[P1].subtype ** ** Extract the subtype value from register P1 and write that subtype ** into register P2. If P1 has no subtype, then P1 gets a NULL. */ case OP_GetSubtype: { /* in1 out2 */ pIn1 = &aMem[pOp->p1]; pOut = &aMem[pOp->p2]; if( pIn1->flags & MEM_Subtype ){ sqlite3VdbeMemSetInt64(pOut, pIn1->eSubtype); }else{ sqlite3VdbeMemSetNull(pOut); } break; } /* Opcode: SetSubtype P1 P2 * * * ** Synopsis: r[P2].subtype = r[P1] ** ** Set the subtype value of register P2 to the integer from register P1. ** If P1 is NULL, clear the subtype from p2. */ case OP_SetSubtype: { /* in1 out2 */ pIn1 = &aMem[pOp->p1]; pOut = &aMem[pOp->p2]; if( pIn1->flags & MEM_Null ){ pOut->flags &= ~MEM_Subtype; }else{ assert( pIn1->flags & MEM_Int ); pOut->flags |= MEM_Subtype; pOut->eSubtype = (u8)(pIn1->u.i & 0xff); } break; } /* Opcode: FilterAdd P1 * P3 P4 * ** Synopsis: filter(P1) += key(P3@P4) ** ** Compute a hash on the P4 registers starting with r[P3] and ** add that hash to the bloom filter contained in r[P1]. */ |
︙ | ︙ |
Changes to test/aggorderby.test.
︙ | ︙ | |||
113 114 115 116 117 118 119 120 121 122 | WITH c(x,y,z) AS (VALUES('a',4,5),('b',3,6),('b',2,7),('c',1,8)) SELECT group_concat(DISTINCT x ORDER BY y, z) FROM c; } {c,b,a} do_execsql_test aggorderby-8.2 { WITH c(x,y) AS (VALUES(1,1),(2,2),(3,3),(3,4),(3,5),(3,6)) SELECT sum(DISTINCT x ORDER BY y) FROM c; } 6 finish_test | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 | WITH c(x,y,z) AS (VALUES('a',4,5),('b',3,6),('b',2,7),('c',1,8)) SELECT group_concat(DISTINCT x ORDER BY y, z) FROM c; } {c,b,a} do_execsql_test aggorderby-8.2 { WITH c(x,y) AS (VALUES(1,1),(2,2),(3,3),(3,4),(3,5),(3,6)) SELECT sum(DISTINCT x ORDER BY y) FROM c; } 6 # Subtype information is transfered through the sorter for aggregates # that make use of subtype info. # do_execsql_test aggorderby-9.0 { WITH c(x,y) AS (VALUES ('{a:3}', 3), ('[1,1]', 1), ('[4,4]', 4), ('{x:2}', 2)) SELECT json_group_array(json(x) ORDER BY y) FROM c; } {{[[1,1],{"x":2},{"a":3},[4,4]]}} do_execsql_test aggorderby-9.1 { WITH c(x,y) AS (VALUES ('[4,4]', 4), ('{a:3}', 3), ('[4,4]', 4), ('[1,1]', 1), ('[4,4]', 4), ('{x:2}', 2)) SELECT json_group_array(DISTINCT json(x) ORDER BY y) FROM c; } {{[[1,1],{"x":2},{"a":3},[4,4]]}} do_execsql_test aggorderby-9.2 { WITH c(x,y) AS (VALUES ('{a:3}', 3), ('[1,1]', 1), ('[4,4]', 4), ('{x:2}', 2)) SELECT json_group_array(json(x) ORDER BY json(x)) FROM c; } {{[[1,1],[4,4],{"a":3},{"x":2}]}} do_execsql_test aggorderby-9.3 { WITH c(x,y) AS (VALUES ('[4,4]', 4), ('{a:3}', 3), ('[4,4]', 4), ('[1,1]', 1), ('[4,4]', 4), ('{x:2}', 2)) SELECT json_group_array(DISTINCT json(x) ORDER BY json(x)) FROM c; } {{[[1,1],[4,4],{"a":3},{"x":2}]}} finish_test |