SQLite

Changes On Branch OP_SorterColumns
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch OP_SorterColumns Excluding Merge-Ins

This is equivalent to a diff from 91384a7d to 134e65c0

2014-10-13
13:00
Remove the OPFLAG_CLEARCACHE flag from OP_Column. In its place, change the P3 parameter of OP_SorterData to be the index of the pseudo-table cursor whose record header cache is to be cleared. This gives a small size reduction and performance increase. (check-in: 20062f49 user: drh tags: trunk)
12:30
Use OP_SorterColumns in aggregate queries. Remove OPFLAG_CLEARCACHE. (Closed-Leaf check-in: 134e65c0 user: drh tags: OP_SorterColumns)
01:23
Add the OP_SorterColumns opcode - an experiment in using a special case opcode to decode the Sorter output rather than the generic OP_Column. This might be faster. And with further work, it could eventually eliminate the need for OP_OpenPseudo. (check-in: b9c695e8 user: drh tags: OP_SorterColumns)
2014-10-12
22:37
Remove the VdbeCursor.lastRowid cache of the current rowid, since maintaining the correct cache value uses more CPU cycles than just recomputing the rowid on the occasions when it is actually needed. Replace it with the VdbeCursor.aOffset field which used to be computed from VdbeCursor.aType when needed. Saves 100 bytes of code space and runs 0.2% faster. (check-in: 91384a7d user: drh tags: trunk)
2014-10-11
23:31
Performance optimization and very slight size reduction for OP_Column. (check-in: 869c30e4 user: drh tags: trunk)

Changes to src/select.c.

1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
  int nColumn,      /* Number of columns of data */
  SelectDest *pDest /* Write the sorted results here */
){
  Vdbe *v = pParse->pVdbe;                     /* The prepared statement */
  int addrBreak = sqlite3VdbeMakeLabel(v);     /* Jump here to exit loop */
  int addrContinue = sqlite3VdbeMakeLabel(v);  /* Jump here for next cycle */
  int addr;
  int addrOnce = 0;
  int iTab;
  ExprList *pOrderBy = pSort->pOrderBy;
  int eDest = pDest->eDest;
  int iParm = pDest->iSDParm;
  int regRow;
  int regRowid;
  int nKey;
  int iSortTab;                   /* Sorter cursor to read from */
  int nSortData;                  /* Trailing values to read from sorter */
  u8 p5;                          /* p5 parameter for 1st OP_Column */
  int i;
  int bSeq;                       /* True if sorter record includes seq. no. */
#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
  struct ExprList_item *aOutEx = p->pEList->a;
#endif

  if( pSort->labelBkOut ){
    sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut);
    sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak);







<







<

<

<







1167
1168
1169
1170
1171
1172
1173

1174
1175
1176
1177
1178
1179
1180

1181

1182

1183
1184
1185
1186
1187
1188
1189
  int nColumn,      /* Number of columns of data */
  SelectDest *pDest /* Write the sorted results here */
){
  Vdbe *v = pParse->pVdbe;                     /* The prepared statement */
  int addrBreak = sqlite3VdbeMakeLabel(v);     /* Jump here to exit loop */
  int addrContinue = sqlite3VdbeMakeLabel(v);  /* Jump here for next cycle */
  int addr;

  int iTab;
  ExprList *pOrderBy = pSort->pOrderBy;
  int eDest = pDest->eDest;
  int iParm = pDest->iSDParm;
  int regRow;
  int regRowid;
  int nKey;

  int nSortData;                  /* Trailing values to read from sorter */

  int i;

#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
  struct ExprList_item *aOutEx = p->pEList->a;
#endif

  if( pSort->labelBkOut ){
    sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut);
    sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak);
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231

1232
1233
1234
1235
1236
1237
1238
    regRowid = sqlite3GetTempReg(pParse);
    regRow = sqlite3GetTempReg(pParse);
    nSortData = 1;
  }
  nKey = pOrderBy->nExpr - pSort->nOBSat;
  if( pSort->sortFlags & SORTFLAG_UseSorter ){
    int regSortOut = ++pParse->nMem;
    iSortTab = pParse->nTab++;
    if( pSort->labelBkOut ){
      addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v);
    }
    sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData);
    if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
    addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak);
    VdbeCoverage(v);
    codeOffset(v, p->iOffset, addrContinue);
    sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut);
    p5 = OPFLAG_CLEARCACHE;
    bSeq = 0;
  }else{
    addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v);
    codeOffset(v, p->iOffset, addrContinue);
    iSortTab = iTab;
    p5 = 0;
    bSeq = 1;
  }
  for(i=0; i<nSortData; i++){
    sqlite3VdbeAddOp3(v, OP_Column, iSortTab, nKey+bSeq+i, regRow+i);
    if( i==0 ) sqlite3VdbeChangeP5(v, p5);
    VdbeComment((v, "%s", aOutEx[i].zName ? aOutEx[i].zName : aOutEx[i].zSpan));

  }
  switch( eDest ){
    case SRT_Table:
    case SRT_EphemTab: {
      testcase( eDest==SRT_Table );
      testcase( eDest==SRT_EphemTab );
      sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid);







<
<
<
<
<
<




|
<



<
<
<
<
|
|
<
|
>







1198
1199
1200
1201
1202
1203
1204






1205
1206
1207
1208
1209

1210
1211
1212




1213
1214

1215
1216
1217
1218
1219
1220
1221
1222
1223
    regRowid = sqlite3GetTempReg(pParse);
    regRow = sqlite3GetTempReg(pParse);
    nSortData = 1;
  }
  nKey = pOrderBy->nExpr - pSort->nOBSat;
  if( pSort->sortFlags & SORTFLAG_UseSorter ){
    int regSortOut = ++pParse->nMem;






    addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak);
    VdbeCoverage(v);
    codeOffset(v, p->iOffset, addrContinue);
    sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut);
    sqlite3VdbeAddOp4Int(v, OP_SorterColumns, nKey, nSortData, regRow, regSortOut);

  }else{
    addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v);
    codeOffset(v, p->iOffset, addrContinue);




    for(i=0; i<nSortData; i++){
      sqlite3VdbeAddOp3(v, OP_Column, iTab, nKey+i+1, regRow+i);

      VdbeComment((v, "%s", aOutEx[i].zName ? aOutEx[i].zName : aOutEx[i].zSpan));
    }
  }
  switch( eDest ){
    case SRT_Table:
    case SRT_EphemTab: {
      testcase( eDest==SRT_Table );
      testcase( eDest==SRT_EphemTab );
      sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid);
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165

5166
5167
5168
5169
5170
5171
5172
      ** Then compare the current GROUP BY terms against the GROUP BY terms
      ** from the previous row currently stored in a0, a1, a2...
      */
      addrTopOfLoop = sqlite3VdbeCurrentAddr(v);
      sqlite3ExprCacheClear(pParse);
      if( groupBySort ){
        sqlite3VdbeAddOp2(v, OP_SorterData, sAggInfo.sortingIdx, sortOut);
      }
      for(j=0; j<pGroupBy->nExpr; j++){
        if( groupBySort ){
          sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j);
          if( j==0 ) sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE);
        }else{

          sAggInfo.directMode = 1;
          sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j);
        }
      }
      sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr,
                          (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO);
      j1 = sqlite3VdbeCurrentAddr(v);







<
|
<
|
|
|
>







5138
5139
5140
5141
5142
5143
5144

5145

5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
      ** Then compare the current GROUP BY terms against the GROUP BY terms
      ** from the previous row currently stored in a0, a1, a2...
      */
      addrTopOfLoop = sqlite3VdbeCurrentAddr(v);
      sqlite3ExprCacheClear(pParse);
      if( groupBySort ){
        sqlite3VdbeAddOp2(v, OP_SorterData, sAggInfo.sortingIdx, sortOut);

        j = pGroupBy->nExpr;

        sqlite3VdbeAddOp4Int(v, OP_SorterColumns, 0, j, iBMem, sortOut);
        sqlite3VdbeAddOp1(v, OP_NullRow, sortPTab);
      }else{
        for(j=0; j<pGroupBy->nExpr; j++){
          sAggInfo.directMode = 1;
          sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j);
        }
      }
      sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr,
                          (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO);
      j1 = sqlite3VdbeCurrentAddr(v);

Changes to src/sqliteInt.h.

2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
*/
#define OPFLAG_NCHANGE       0x01    /* Set to update db->nChange */
#define OPFLAG_EPHEM         0x01    /* OP_Column: Ephemeral output is ok */
#define OPFLAG_LASTROWID     0x02    /* Set to update db->lastRowid */
#define OPFLAG_ISUPDATE      0x04    /* This OP_Insert is an sql UPDATE */
#define OPFLAG_APPEND        0x08    /* This is likely to be an append */
#define OPFLAG_USESEEKRESULT 0x10    /* Try to avoid a seek in BtreeInsert() */
#define OPFLAG_CLEARCACHE    0x20    /* Clear pseudo-table cache in OP_Column */
#define OPFLAG_LENGTHARG     0x40    /* OP_Column only used for length() */
#define OPFLAG_TYPEOFARG     0x80    /* OP_Column only used for typeof() */
#define OPFLAG_BULKCSR       0x01    /* OP_Open** used to open bulk cursor */
#define OPFLAG_P2ISREG       0x02    /* P2 to OP_Open** is a register number */
#define OPFLAG_PERMUTE       0x01    /* OP_Compare: use the permutation */

/*







<







2662
2663
2664
2665
2666
2667
2668

2669
2670
2671
2672
2673
2674
2675
*/
#define OPFLAG_NCHANGE       0x01    /* Set to update db->nChange */
#define OPFLAG_EPHEM         0x01    /* OP_Column: Ephemeral output is ok */
#define OPFLAG_LASTROWID     0x02    /* Set to update db->lastRowid */
#define OPFLAG_ISUPDATE      0x04    /* This OP_Insert is an sql UPDATE */
#define OPFLAG_APPEND        0x08    /* This is likely to be an append */
#define OPFLAG_USESEEKRESULT 0x10    /* Try to avoid a seek in BtreeInsert() */

#define OPFLAG_LENGTHARG     0x40    /* OP_Column only used for length() */
#define OPFLAG_TYPEOFARG     0x80    /* OP_Column only used for typeof() */
#define OPFLAG_BULKCSR       0x01    /* OP_Open** used to open bulk cursor */
#define OPFLAG_P2ISREG       0x02    /* P2 to OP_Open** is a register number */
#define OPFLAG_PERMUTE       0x01    /* OP_Compare: use the permutation */

/*

Changes to src/vdbe.c.

2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
**
** The value extracted is stored in register P3.
**
** If the column contains fewer than P2 fields, then extract a NULL.  Or,
** if the P4 argument is a P4_MEM use the value of the P4 argument as
** the result.
**
** If the OPFLAG_CLEARCACHE bit is set on P5 and P1 is a pseudo-table cursor,
** then the cache of the cursor is reset prior to extracting the column.
** The first OP_Column against a pseudo-table after the value of the content
** register has changed should have this bit set.
**
** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 when
** the result is guaranteed to only be used as the argument of a length()
** or typeof() function, respectively.  The loading of large blobs can be
** skipped for length() and all content loading can be skipped for typeof().
*/
case OP_Column: {
  i64 payloadSize64; /* Number of bytes in the record */







<
<
<
<
<







2235
2236
2237
2238
2239
2240
2241





2242
2243
2244
2245
2246
2247
2248
**
** The value extracted is stored in register P3.
**
** If the column contains fewer than P2 fields, then extract a NULL.  Or,
** if the P4 argument is a P4_MEM use the value of the P4 argument as
** the result.
**





** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 when
** the result is guaranteed to only be used as the argument of a length()
** or typeof() function, respectively.  The loading of large blobs can be
** skipped for length() and all content loading can be skipped for typeof().
*/
case OP_Column: {
  i64 payloadSize64; /* Number of bytes in the record */
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
  pCrsr = pC->pCursor;
  assert( pCrsr!=0 || pC->pseudoTableReg>0 ); /* pCrsr NULL on PseudoTables */
  assert( pCrsr!=0 || pC->nullRow );          /* pC->nullRow on PseudoTables */

  /* If the cursor cache is stale, bring it up-to-date */
  rc = sqlite3VdbeCursorMoveto(pC);
  if( rc ) goto abort_due_to_error;
  if( pC->cacheStatus!=p->cacheCtr || (pOp->p5&OPFLAG_CLEARCACHE)!=0 ){
    if( pC->nullRow ){
      if( pCrsr==0 ){
        assert( pC->pseudoTableReg>0 );
        pReg = &aMem[pC->pseudoTableReg];
        assert( pReg->flags & MEM_Blob );
        assert( memIsValid(pReg) );
        pC->payloadSize = pC->szRow = avail = pReg->n;







|







2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
  pCrsr = pC->pCursor;
  assert( pCrsr!=0 || pC->pseudoTableReg>0 ); /* pCrsr NULL on PseudoTables */
  assert( pCrsr!=0 || pC->nullRow );          /* pC->nullRow on PseudoTables */

  /* If the cursor cache is stale, bring it up-to-date */
  rc = sqlite3VdbeCursorMoveto(pC);
  if( rc ) goto abort_due_to_error;
  if( pC->cacheStatus!=p->cacheCtr ){
    if( pC->nullRow ){
      if( pCrsr==0 ){
        assert( pC->pseudoTableReg>0 );
        pReg = &aMem[pC->pseudoTableReg];
        assert( pReg->flags & MEM_Blob );
        assert( memIsValid(pReg) );
        pC->payloadSize = pC->szRow = avail = pReg->n;
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262

















































4263
4264
4265
4266
4267
4268
4269

/* Opcode: SorterData P1 P2 * * *
** Synopsis: r[P2]=data
**
** Write into register P2 the current sorter data for sorter cursor P1.
*/
case OP_SorterData: {
  VdbeCursor *pC;

  pOut = &aMem[pOp->p2];
  pC = p->apCsr[pOp->p1];
  assert( isSorter(pC) );
  rc = sqlite3VdbeSorterRowkey(pC, pOut);
  assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) );
  break;
}


















































/* Opcode: RowData P1 P2 * * *
** Synopsis: r[P2]=data
**
** Write into register P2 the complete row data for cursor P1.
** There is no interpretation of the data.  
** It is just copied onto the P2 register exactly as 







|








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313

/* Opcode: SorterData P1 P2 * * *
** Synopsis: r[P2]=data
**
** Write into register P2 the current sorter data for sorter cursor P1.
*/
case OP_SorterData: {
  VdbeCursor *pC;         /* Sorting cursor defined by P1 */

  pOut = &aMem[pOp->p2];
  pC = p->apCsr[pOp->p1];
  assert( isSorter(pC) );
  rc = sqlite3VdbeSorterRowkey(pC, pOut);
  assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) );
  break;
}

/* Opcode: SorterColumns P1 P2 P3 P4 *
** Synopsis: r[P3@P2]=decode(r[P4])
**
** The P4 register contains a record that has just come out of a sorter.
** Decode columns P1 through P1+P2-1 into registers P3..P3+P2-1.
**
** This opcode is much faster than multiple calls to Column since it 
** does not need to deal with corrupt record detection or default values
** or any of the other complications associated with a record read
** from disk.
*/
case OP_SorterColumns: {
  Mem *pDest;             /* Register P3 output register */
  Mem *pLast;             /* Register P3+P2-1 */
  u32 serial_type;        /* Serial type of a column value */
  u32 idx;                /* Index into the record header */
  u32 d;                  /* Index into the data of the record */
  int nSkip;              /* Number of initial columns to skip */
  const u8 *aKey;         /* Complete text of the record */

  assert( pOp->p4type==P4_INT32 );
  assert( pOp->p4.i>0 && pOp->p4.i<=(p->nMem - p->nCursor) );
  assert( pOp->p3>pOp->p4.i || pOp->p3+pOp->p2<=pOp->p4.i );
  assert( pOp->p1>=0 );
  assert( pOp->p2>0 );
  assert( aMem[pOp->p4.i].flags & MEM_Blob );
  aKey = (const u8*)aMem[pOp->p4.i].z;
  pDest = &aMem[pOp->p3];
  pLast = &pDest[pOp->p2-1];
  idx = getVarint32(aKey, d);
  nSkip = pOp->p1;
  while( nSkip-- ){
    assert( d<=aMem[pOp->p4.i].n );
    idx += getVarint32(&aKey[idx], serial_type);
    d += sqlite3VdbeSerialTypeLen(serial_type);
  }
  do{
    assert( d<=aMem[pOp->p4.i].n );
    idx += getVarint32(&aKey[idx], serial_type);
    if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest);
    d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pDest);
    pDest->enc = encoding;
    Deephemeralize(pDest);
    REGISTER_TRACE((int)(pDest-aMem), pDest);
    pDest++;
  }while( pDest<=pLast );
  break;
}

/* Opcode: RowData P1 P2 * * *
** Synopsis: r[P2]=data
**
** Write into register P2 the complete row data for cursor P1.
** There is no interpretation of the data.  
** It is just copied onto the P2 register exactly as