SQLite

Check-in [d302a389]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Enhance aggregate order-by so that it transmits subtype information through the sorter. Fix for the deficiency reported by forum post 87347ad2fb5a8f76.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: d302a389460d0c15775a8b5f5afbac2c1d8a91bc282bc9b04c583ca04a8c09c6
User & Date: drh 2023-12-14 15:11:39
Context
2023-12-14
15:31
Improve the error message returned by an fts5 'rebuild' command on an external content table if there is a problem with the content table or view. (check-in: 0fbf4b8a user: dan tags: trunk)
15:11
Enhance aggregate order-by so that it transmits subtype information through the sorter. Fix for the deficiency reported by forum post 87347ad2fb5a8f76. (check-in: d302a389 user: drh tags: trunk)
13:58
Pass subtype information through the aggregate ORDER BY sorter for aggregate functions that use subtype information. (Closed-Leaf check-in: 3536f403 user: drh tags: agg-orderby-subtype)
2023-12-13
20:37
In CLI, fix .read inability to open 2GB+ files on WIN32. (check-in: 56c80a62 user: larrybr tags: trunk)
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/expr.c.

6817
6818
6819
6820
6821
6822
6823


6824
6825
6826
6827
6828
6829
6830
                               pExpr->x.pList->a[0].pExpr,0)==0
              ){
                pItem->bOBPayload = 0;
                pItem->bOBUnique = ExprHasProperty(pExpr, EP_Distinct);
              }else{
                pItem->bOBPayload = 1;
              }


            }else{
              pItem->iOBTab = -1;
            }
            if( ExprHasProperty(pExpr, EP_Distinct) && !pItem->bOBUnique ){
              pItem->iDistinct = pParse->nTab++;
            }else{
              pItem->iDistinct = -1;







>
>







6817
6818
6819
6820
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
                               pExpr->x.pList->a[0].pExpr,0)==0
              ){
                pItem->bOBPayload = 0;
                pItem->bOBUnique = ExprHasProperty(pExpr, EP_Distinct);
              }else{
                pItem->bOBPayload = 1;
              }
              pItem->bUseSubtype =
                    (pItem->pFunc->funcFlags & SQLITE_SUBTYPE)!=0;
            }else{
              pItem->iOBTab = -1;
            }
            if( ExprHasProperty(pExpr, EP_Distinct) && !pItem->bOBUnique ){
              pItem->iDistinct = pParse->nTab++;
            }else{
              pItem->iDistinct = -1;

Changes to src/select.c.

6655
6656
6657
6658
6659
6660
6661

6662
6663
6664
6665
6666
6667
6668
6669
6670



6671
6672
6673
6674
6675
6676
6677
    if( pFunc->iOBTab>=0 ){
      ExprList *pOBList;
      KeyInfo *pKeyInfo;
      int nExtra = 0;
      assert( pFunc->pFExpr->pLeft!=0 );
      assert( pFunc->pFExpr->pLeft->op==TK_ORDER );
      assert( ExprUseXList(pFunc->pFExpr->pLeft) );

      pOBList = pFunc->pFExpr->pLeft->x.pList;
      if( !pFunc->bOBUnique ){
        nExtra++;  /* One extra column for the OP_Sequence */
      }
      if( pFunc->bOBPayload ){
        /* extra columns for the function arguments */
        assert( ExprUseXList(pFunc->pFExpr) );
        nExtra += pFunc->pFExpr->x.pList->nExpr;
      }



      pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOBList, 0, nExtra);
      if( !pFunc->bOBUnique && pParse->nErr==0 ){
        pKeyInfo->nKeyField++;
      }
      sqlite3VdbeAddOp4(v, OP_OpenEphemeral,
            pFunc->iOBTab, pOBList->nExpr+nExtra, 0,
            (char*)pKeyInfo, P4_KEYINFO);







>









>
>
>







6655
6656
6657
6658
6659
6660
6661
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
6678
6679
6680
6681
    if( pFunc->iOBTab>=0 ){
      ExprList *pOBList;
      KeyInfo *pKeyInfo;
      int nExtra = 0;
      assert( pFunc->pFExpr->pLeft!=0 );
      assert( pFunc->pFExpr->pLeft->op==TK_ORDER );
      assert( ExprUseXList(pFunc->pFExpr->pLeft) );
      assert( pFunc->pFunc!=0 );
      pOBList = pFunc->pFExpr->pLeft->x.pList;
      if( !pFunc->bOBUnique ){
        nExtra++;  /* One extra column for the OP_Sequence */
      }
      if( pFunc->bOBPayload ){
        /* extra columns for the function arguments */
        assert( ExprUseXList(pFunc->pFExpr) );
        nExtra += pFunc->pFExpr->x.pList->nExpr;
      }
      if( pFunc->bUseSubtype ){
        nExtra += pFunc->pFExpr->x.pList->nExpr;
      }
      pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOBList, 0, nExtra);
      if( !pFunc->bOBUnique && pParse->nErr==0 ){
        pKeyInfo->nKeyField++;
      }
      sqlite3VdbeAddOp4(v, OP_OpenEphemeral,
            pFunc->iOBTab, pOBList->nExpr+nExtra, 0,
            (char*)pKeyInfo, P4_KEYINFO);
6690
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706

6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722









6723
6724
6725
6726
6727
6728
6729
  int i;
  struct AggInfo_func *pF;
  for(i=0, pF=pAggInfo->aFunc; i<pAggInfo->nFunc; i++, pF++){
    ExprList *pList;
    assert( ExprUseXList(pF->pFExpr) );
    pList = pF->pFExpr->x.pList;
    if( pF->iOBTab>=0 ){
      /* For an ORDER BY aggregate, calls to OP_AggStep where deferred and
      ** all content was stored in emphermal table pF->iOBTab.  Extract that
      ** content now (in ORDER BY order) and make all calls to OP_AggStep
      ** before doing the OP_AggFinal call. */
      int iTop;        /* Start of loop for extracting columns */
      int nArg;        /* Number of columns to extract */
      int nKey;        /* Key columns to be skipped */
      int regAgg;      /* Extract into this array */
      int j;           /* Loop counter */
      

      nArg = pList->nExpr;
      regAgg = sqlite3GetTempRange(pParse, nArg);

      if( pF->bOBPayload==0 ){
        nKey = 0;
      }else{
        assert( pF->pFExpr->pLeft!=0 );
        assert( ExprUseXList(pF->pFExpr->pLeft) );
        assert( pF->pFExpr->pLeft->x.pList!=0 );
        nKey = pF->pFExpr->pLeft->x.pList->nExpr;
        if( ALWAYS(!pF->bOBUnique) ) nKey++;
      }
      iTop = sqlite3VdbeAddOp1(v, OP_Rewind, pF->iOBTab); VdbeCoverage(v);
      for(j=nArg-1; j>=0; j--){
        sqlite3VdbeAddOp3(v, OP_Column, pF->iOBTab, nKey+j, regAgg+j);
      }









      sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i));
      sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF);
      sqlite3VdbeChangeP5(v, (u8)nArg);
      sqlite3VdbeAddOp2(v, OP_Next, pF->iOBTab, iTop+1); VdbeCoverage(v);
      sqlite3VdbeJumpHere(v, iTop);
      sqlite3ReleaseTempRange(pParse, regAgg, nArg);
    }







|
|
|






|
>
















>
>
>
>
>
>
>
>
>







6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706
6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722
6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742
6743
  int i;
  struct AggInfo_func *pF;
  for(i=0, pF=pAggInfo->aFunc; i<pAggInfo->nFunc; i++, pF++){
    ExprList *pList;
    assert( ExprUseXList(pF->pFExpr) );
    pList = pF->pFExpr->x.pList;
    if( pF->iOBTab>=0 ){
      /* For an ORDER BY aggregate, calls to OP_AggStep were deferred.  Inputs
      ** were stored in emphermal table pF->iOBTab.  Here, we extract those
      ** inputs (in ORDER BY order) and make all calls to OP_AggStep
      ** before doing the OP_AggFinal call. */
      int iTop;        /* Start of loop for extracting columns */
      int nArg;        /* Number of columns to extract */
      int nKey;        /* Key columns to be skipped */
      int regAgg;      /* Extract into this array */
      int j;           /* Loop counter */
     
      assert( pF->pFunc!=0 );
      nArg = pList->nExpr;
      regAgg = sqlite3GetTempRange(pParse, nArg);

      if( pF->bOBPayload==0 ){
        nKey = 0;
      }else{
        assert( pF->pFExpr->pLeft!=0 );
        assert( ExprUseXList(pF->pFExpr->pLeft) );
        assert( pF->pFExpr->pLeft->x.pList!=0 );
        nKey = pF->pFExpr->pLeft->x.pList->nExpr;
        if( ALWAYS(!pF->bOBUnique) ) nKey++;
      }
      iTop = sqlite3VdbeAddOp1(v, OP_Rewind, pF->iOBTab); VdbeCoverage(v);
      for(j=nArg-1; j>=0; j--){
        sqlite3VdbeAddOp3(v, OP_Column, pF->iOBTab, nKey+j, regAgg+j);
      }
      if( pF->bUseSubtype ){
        int regSubtype = sqlite3GetTempReg(pParse);
        int iBaseCol = nKey + nArg + (pF->bOBPayload==0 && pF->bOBUnique==0);
        for(j=nArg-1; j>=0; j--){
          sqlite3VdbeAddOp3(v, OP_Column, pF->iOBTab, iBaseCol+j, regSubtype);
          sqlite3VdbeAddOp2(v, OP_SetSubtype, regSubtype, regAgg+j);
        }
        sqlite3ReleaseTempReg(pParse, regSubtype);
      }
      sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i));
      sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF);
      sqlite3VdbeChangeP5(v, (u8)nArg);
      sqlite3VdbeAddOp2(v, OP_Next, pF->iOBTab, iTop+1); VdbeCoverage(v);
      sqlite3VdbeJumpHere(v, iTop);
      sqlite3ReleaseTempRange(pParse, regAgg, nArg);
    }
6770
6771
6772
6773
6774
6775
6776

6777
6778
6779
6780
6781
6782
6783
    int addrNext = 0;
    int regAgg;
    int regAggSz = 0;
    int regDistinct = 0;
    ExprList *pList;
    assert( ExprUseXList(pF->pFExpr) );
    assert( !IsWindowFunc(pF->pFExpr) );

    pList = pF->pFExpr->x.pList;
    if( ExprHasProperty(pF->pFExpr, EP_WinFunc) ){
      Expr *pFilter = pF->pFExpr->y.pWin->pFilter;
      if( pAggInfo->nAccumulator
       && (pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL)
       && regAcc
      ){







>







6784
6785
6786
6787
6788
6789
6790
6791
6792
6793
6794
6795
6796
6797
6798
    int addrNext = 0;
    int regAgg;
    int regAggSz = 0;
    int regDistinct = 0;
    ExprList *pList;
    assert( ExprUseXList(pF->pFExpr) );
    assert( !IsWindowFunc(pF->pFExpr) );
    assert( pF->pFunc!=0 );
    pList = pF->pFExpr->x.pList;
    if( ExprHasProperty(pF->pFExpr, EP_WinFunc) ){
      Expr *pFilter = pF->pFExpr->y.pWin->pFilter;
      if( pAggInfo->nAccumulator
       && (pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL)
       && regAcc
      ){
6813
6814
6815
6816
6817
6818
6819



6820
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832








6833
6834
6835
6836
6837
6838
6839
      assert( pOBList->nExpr>0 );
      regAggSz = pOBList->nExpr;
      if( !pF->bOBUnique ){
        regAggSz++;   /* One register for OP_Sequence */
      }
      if( pF->bOBPayload ){
        regAggSz += nArg;



      }
      regAggSz++;  /* One extra register to hold result of MakeRecord */
      regAgg = sqlite3GetTempRange(pParse, regAggSz);
      regDistinct = regAgg;
      sqlite3ExprCodeExprList(pParse, pOBList, regAgg, 0, SQLITE_ECEL_DUP);
      jj = pOBList->nExpr;
      if( !pF->bOBUnique ){
        sqlite3VdbeAddOp2(v, OP_Sequence, pF->iOBTab, regAgg+jj);
        jj++;
      }
      if( pF->bOBPayload ){
        regDistinct = regAgg+jj;
        sqlite3ExprCodeExprList(pParse, pList, regDistinct, 0, SQLITE_ECEL_DUP);








      }
    }else if( pList ){
      nArg = pList->nExpr;
      regAgg = sqlite3GetTempRange(pParse, nArg);
      regDistinct = regAgg;
      sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP);
    }else{







>
>
>













>
>
>
>
>
>
>
>







6828
6829
6830
6831
6832
6833
6834
6835
6836
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861
6862
6863
6864
6865
      assert( pOBList->nExpr>0 );
      regAggSz = pOBList->nExpr;
      if( !pF->bOBUnique ){
        regAggSz++;   /* One register for OP_Sequence */
      }
      if( pF->bOBPayload ){
        regAggSz += nArg;
      }
      if( pF->bUseSubtype ){
        regAggSz += nArg;
      }
      regAggSz++;  /* One extra register to hold result of MakeRecord */
      regAgg = sqlite3GetTempRange(pParse, regAggSz);
      regDistinct = regAgg;
      sqlite3ExprCodeExprList(pParse, pOBList, regAgg, 0, SQLITE_ECEL_DUP);
      jj = pOBList->nExpr;
      if( !pF->bOBUnique ){
        sqlite3VdbeAddOp2(v, OP_Sequence, pF->iOBTab, regAgg+jj);
        jj++;
      }
      if( pF->bOBPayload ){
        regDistinct = regAgg+jj;
        sqlite3ExprCodeExprList(pParse, pList, regDistinct, 0, SQLITE_ECEL_DUP);
        jj += nArg;
      }
      if( pF->bUseSubtype ){
        int kk;
        int regBase = pF->bOBPayload ? regDistinct : regAgg;
        for(kk=0; kk<nArg; kk++, jj++){
          sqlite3VdbeAddOp2(v, OP_GetSubtype, regBase+kk, regAgg+jj);
        }
      }
    }else if( pList ){
      nArg = pList->nExpr;
      regAgg = sqlite3GetTempRange(pParse, nArg);
      regDistinct = regAgg;
      sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP);
    }else{

Changes to src/sqliteInt.h.

2862
2863
2864
2865
2866
2867
2868

2869
2870
2871
2872
2873
2874
2875
    Expr *pFExpr;            /* Expression encoding the function */
    FuncDef *pFunc;          /* The aggregate function implementation */
    int iDistinct;           /* Ephemeral table used to enforce DISTINCT */
    int iDistAddr;           /* Address of OP_OpenEphemeral */
    int iOBTab;              /* Ephemeral table to implement ORDER BY */
    u8 bOBPayload;           /* iOBTab has payload columns separate from key */
    u8 bOBUnique;            /* Enforce uniqueness on iOBTab keys */

  } *aFunc;
  int nFunc;              /* Number of entries in aFunc[] */
  u32 selId;              /* Select to which this AggInfo belongs */
#ifdef SQLITE_DEBUG
  Select *pSelect;        /* SELECT statement that this AggInfo supports */
#endif
};







>







2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
    Expr *pFExpr;            /* Expression encoding the function */
    FuncDef *pFunc;          /* The aggregate function implementation */
    int iDistinct;           /* Ephemeral table used to enforce DISTINCT */
    int iDistAddr;           /* Address of OP_OpenEphemeral */
    int iOBTab;              /* Ephemeral table to implement ORDER BY */
    u8 bOBPayload;           /* iOBTab has payload columns separate from key */
    u8 bOBUnique;            /* Enforce uniqueness on iOBTab keys */
    u8 bUseSubtype;          /* Transfer subtype info through sorter */
  } *aFunc;
  int nFunc;              /* Number of entries in aFunc[] */
  u32 selId;              /* Select to which this AggInfo belongs */
#ifdef SQLITE_DEBUG
  Select *pSelect;        /* SELECT statement that this AggInfo supports */
#endif
};

Changes to src/vdbe.c.

8669
8670
8671
8672
8673
8674
8675




































8676
8677
8678
8679
8680
8681
8682
** Clear the subtype from register P1.
*/
case OP_ClrSubtype: {   /* in1 */
  pIn1 = &aMem[pOp->p1];
  pIn1->flags &= ~MEM_Subtype;
  break;
}





































/* Opcode: FilterAdd P1 * P3 P4 *
** Synopsis: filter(P1) += key(P3@P4)
**
** Compute a hash on the P4 registers starting with r[P3] and
** add that hash to the bloom filter contained in r[P1].
*/







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







8669
8670
8671
8672
8673
8674
8675
8676
8677
8678
8679
8680
8681
8682
8683
8684
8685
8686
8687
8688
8689
8690
8691
8692
8693
8694
8695
8696
8697
8698
8699
8700
8701
8702
8703
8704
8705
8706
8707
8708
8709
8710
8711
8712
8713
8714
8715
8716
8717
8718
** Clear the subtype from register P1.
*/
case OP_ClrSubtype: {   /* in1 */
  pIn1 = &aMem[pOp->p1];
  pIn1->flags &= ~MEM_Subtype;
  break;
}

/* Opcode: GetSubtype P1 P2 * * *
** Synopsis:  r[P2] = r[P1].subtype
**
** Extract the subtype value from register P1 and write that subtype
** into register P2.  If P1 has no subtype, then P1 gets a NULL.
*/
case OP_GetSubtype: {   /* in1 out2 */
  pIn1 = &aMem[pOp->p1];
  pOut = &aMem[pOp->p2];
  if( pIn1->flags & MEM_Subtype ){
    sqlite3VdbeMemSetInt64(pOut, pIn1->eSubtype);
  }else{
    sqlite3VdbeMemSetNull(pOut);
  }
  break;
}

/* Opcode: SetSubtype P1 P2 * * *
** Synopsis:  r[P2].subtype = r[P1]
**
** Set the subtype value of register P2 to the integer from register P1.
** If P1 is NULL, clear the subtype from p2.
*/
case OP_SetSubtype: {   /* in1 out2 */
  pIn1 = &aMem[pOp->p1];
  pOut = &aMem[pOp->p2];
  if( pIn1->flags & MEM_Null ){
    pOut->flags &= ~MEM_Subtype;
  }else{
    assert( pIn1->flags & MEM_Int );
    pOut->flags |= MEM_Subtype;
    pOut->eSubtype = (u8)(pIn1->u.i & 0xff);
  }
  break;
}

/* Opcode: FilterAdd P1 * P3 P4 *
** Synopsis: filter(P1) += key(P3@P4)
**
** Compute a hash on the P4 registers starting with r[P3] and
** add that hash to the bloom filter contained in r[P1].
*/

Changes to test/aggorderby.test.

113
114
115
116
117
118
119








































120
121
122
  WITH c(x,y,z) AS (VALUES('a',4,5),('b',3,6),('b',2,7),('c',1,8))
  SELECT group_concat(DISTINCT x ORDER BY y, z) FROM c;
} {c,b,a}
do_execsql_test aggorderby-8.2 {
  WITH c(x,y) AS (VALUES(1,1),(2,2),(3,3),(3,4),(3,5),(3,6))
  SELECT sum(DISTINCT x ORDER BY y) FROM c;
} 6










































finish_test







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
  WITH c(x,y,z) AS (VALUES('a',4,5),('b',3,6),('b',2,7),('c',1,8))
  SELECT group_concat(DISTINCT x ORDER BY y, z) FROM c;
} {c,b,a}
do_execsql_test aggorderby-8.2 {
  WITH c(x,y) AS (VALUES(1,1),(2,2),(3,3),(3,4),(3,5),(3,6))
  SELECT sum(DISTINCT x ORDER BY y) FROM c;
} 6

# Subtype information is transfered through the sorter for aggregates
# that make use of subtype info.
#
do_execsql_test aggorderby-9.0 {
  WITH c(x,y) AS (VALUES
    ('{a:3}', 3),
    ('[1,1]', 1),
    ('[4,4]', 4),
    ('{x:2}', 2))
  SELECT json_group_array(json(x) ORDER BY y) FROM c;
} {{[[1,1],{"x":2},{"a":3},[4,4]]}}
do_execsql_test aggorderby-9.1 {
  WITH c(x,y) AS (VALUES
    ('[4,4]', 4),
    ('{a:3}', 3),
    ('[4,4]', 4),
    ('[1,1]', 1),
    ('[4,4]', 4),
    ('{x:2}', 2))
  SELECT json_group_array(DISTINCT json(x) ORDER BY y) FROM c;
} {{[[1,1],{"x":2},{"a":3},[4,4]]}}
do_execsql_test aggorderby-9.2 {
  WITH c(x,y) AS (VALUES
    ('{a:3}', 3),
    ('[1,1]', 1),
    ('[4,4]', 4),
    ('{x:2}', 2))
  SELECT json_group_array(json(x) ORDER BY json(x)) FROM c;
} {{[[1,1],[4,4],{"a":3},{"x":2}]}}
do_execsql_test aggorderby-9.3 {
  WITH c(x,y) AS (VALUES
    ('[4,4]', 4),
    ('{a:3}', 3),
    ('[4,4]', 4),
    ('[1,1]', 1),
    ('[4,4]', 4),
    ('{x:2}', 2))
  SELECT json_group_array(DISTINCT json(x) ORDER BY json(x)) FROM c;
} {{[[1,1],[4,4],{"a":3},{"x":2}]}}


finish_test