SQLite

Check-in [2accf32b]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Be more aggressive about reusing subqueries that appear on the RHS of IN operators that have been replicated due to the predicate push-down optimization.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | reuse-subqueries
Files: files | file ages | folders
SHA3-256: 2accf32b6e45a396503c29eecc14a103bcc7b4c313cde921b26b489704060177
User & Date: drh 2024-07-04 16:57:11
Context
2024-07-04
17:49
Update EXPLAIN output to include P4_SUBRTNSIG. (check-in: 61e56923 user: dan tags: reuse-subqueries)
16:57
Be more aggressive about reusing subqueries that appear on the RHS of IN operators that have been replicated due to the predicate push-down optimization. (check-in: 2accf32b user: drh tags: reuse-subqueries)
11:15
Add comment using the name "predicate push-down optimization" to what we have also called "WHERE-clause push down". No changes to code. (check-in: be77fe70 user: drh tags: trunk)
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/expr.c.

3416
3417
3418
3419
3420
3421
3422








































3423
3424
3425
3426
3427
3428
3429
  }else
#endif
  {
    sqlite3ErrorMsg(pParse, "row value misused");
  }
}









































#ifndef SQLITE_OMIT_SUBQUERY
/*
** Generate code that will construct an ephemeral table containing all terms
** in the RHS of an IN operator.  The IN operator can be in either of two
** forms:
**
**     x IN (4,5,11)              -- IN operator with list on right-hand side







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
  }else
#endif
  {
    sqlite3ErrorMsg(pParse, "row value misused");
  }
}

#ifndef SQLITE_OMIT_SUBQUERY
/*
** Scan all previously generated bytecode looking for an OP_BeginSubrtn
** that is compatible with pExpr.  If found, add the y.sub values
** to pExpr and return true.  If not found, return false.
*/
static int findCompatibleInRhsSubrtn(
  Parse *pParse,          /* Parsing context */
  Expr *pExpr,            /* IN operator with RHS that we want to reuse */
  SubrtnSig *pNewSig      /* Signature for the IN operator */
){
  VdbeOp *pOp, *pEnd;
  SubrtnSig *pSig;
  Vdbe *v;

  if( pNewSig==0 ) return 0;
  assert( pExpr->op==TK_IN );
  assert( !ExprUseYSub(pExpr) );
  assert( ExprUseXSelect(pExpr) );
  v = pParse->pVdbe;
  assert( v!=0 );
  pOp = sqlite3VdbeGetOp(v, 1);
  pEnd = sqlite3VdbeGetLastOp(v);
  for(; pOp<pEnd; pOp++){
    if( pOp->opcode!=OP_BeginSubrtn ) continue;
    if( pOp->p4type!=P4_SUBRTNSIG ) continue;
    pSig = pOp->p4.pSubrtnSig;
    assert( pSig!=0 );
    if( pNewSig->selId!=pSig->selId ) continue;
    if( strcmp(pNewSig->zAff,pSig->zAff)!=0 ) continue;
    pExpr->y.sub.iAddr = pSig->iAddr;
    pExpr->y.sub.regReturn = pSig->regReturn;
    pExpr->iTable = pSig->iTable;
    ExprSetProperty(pExpr, EP_Subrtn);
    return 1;
  }
  return 0;
}
#endif /* SQLITE_OMIT_SUBQUERY */

#ifndef SQLITE_OMIT_SUBQUERY
/*
** Generate code that will construct an ephemeral table containing all terms
** in the RHS of an IN operator.  The IN operator can be in either of two
** forms:
**
**     x IN (4,5,11)              -- IN operator with list on right-hand side
3465
3466
3467
3468
3469
3470
3471
3472
3473














3474



3475
3476


3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487




3488
3489
3490
3491
3492
3493
3494
3495
3496
3497





3498
3499
3500
3501
3502
3503
3504
3505
  **    *  The right-hand side is an expression list containing variables
  **    *  We are inside a trigger
  **
  ** If all of the above are false, then we can compute the RHS just once
  ** and reuse it many names.
  */
  if( !ExprHasProperty(pExpr, EP_VarSelect) && pParse->iSelfTab==0 ){
    /* Reuse of the RHS is allowed */
    /* If this routine has already been coded, but the previous code














    ** might not have been invoked yet, so invoke it now as a subroutine.



    */
    if( ExprHasProperty(pExpr, EP_Subrtn) ){


      addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
      if( ExprUseXSelect(pExpr) ){
        ExplainQueryPlan((pParse, 0, "REUSE LIST SUBQUERY %d",
              pExpr->x.pSelect->selId));
      }
      assert( ExprUseYSub(pExpr) );
      sqlite3VdbeAddOp2(v, OP_Gosub, pExpr->y.sub.regReturn,
                        pExpr->y.sub.iAddr);
      assert( iTab!=pExpr->iTable );
      sqlite3VdbeAddOp2(v, OP_OpenDup, iTab, pExpr->iTable);
      sqlite3VdbeJumpHere(v, addrOnce);




      return;
    }

    /* Begin coding the subroutine */
    assert( !ExprUseYWin(pExpr) );
    ExprSetProperty(pExpr, EP_Subrtn);
    assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) );
    pExpr->y.sub.regReturn = ++pParse->nMem;
    pExpr->y.sub.iAddr =
      sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pExpr->y.sub.regReturn) + 1;






    addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
  }

  /* Check to see if this is a vector IN operator */
  pLeft = pExpr->pLeft;
  nVal = sqlite3ExprVectorSize(pLeft);








|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>

|
>
>











>
>
>
>










>
>
>
>
>
|







3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
  **    *  The right-hand side is an expression list containing variables
  **    *  We are inside a trigger
  **
  ** If all of the above are false, then we can compute the RHS just once
  ** and reuse it many names.
  */
  if( !ExprHasProperty(pExpr, EP_VarSelect) && pParse->iSelfTab==0 ){
    /* Reuse of the RHS is allowed
    **
    ** Compute a signature for the RHS of the IN operator to facility
    ** finding and reusing prior instances of the same IN operator.
    */
    SubrtnSig *pSig;
    if( !ExprUseXSelect(pExpr) ){
      pSig = 0;
    }else{
      assert( pExpr->x.pSelect!=0 );
      pSig = sqlite3DbMallocRawNN(pParse->db, sizeof(pSig[0]));
      if( pSig ){
        pSig->selId = pExpr->x.pSelect->selId;
        pSig->zAff = exprINAffinity(pParse, pExpr);
      }
    }

    /* Check to see if there is a prior materialization of the RHS of
    ** this IN operator.  If there is, then make use of that prior
    ** materialization rather than recomputing it.
    */
    if( ExprHasProperty(pExpr, EP_Subrtn) 
     || findCompatibleInRhsSubrtn(pParse, pExpr, pSig)
    ){
      addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
      if( ExprUseXSelect(pExpr) ){
        ExplainQueryPlan((pParse, 0, "REUSE LIST SUBQUERY %d",
              pExpr->x.pSelect->selId));
      }
      assert( ExprUseYSub(pExpr) );
      sqlite3VdbeAddOp2(v, OP_Gosub, pExpr->y.sub.regReturn,
                        pExpr->y.sub.iAddr);
      assert( iTab!=pExpr->iTable );
      sqlite3VdbeAddOp2(v, OP_OpenDup, iTab, pExpr->iTable);
      sqlite3VdbeJumpHere(v, addrOnce);
      if( pSig ){
        sqlite3DbFree(pParse->db, pSig->zAff);
        sqlite3DbFree(pParse->db, pSig);
      }
      return;
    }

    /* Begin coding the subroutine */
    assert( !ExprUseYWin(pExpr) );
    ExprSetProperty(pExpr, EP_Subrtn);
    assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) );
    pExpr->y.sub.regReturn = ++pParse->nMem;
    pExpr->y.sub.iAddr =
      sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pExpr->y.sub.regReturn) + 1;
    if( pSig ){
      pSig->iAddr = pExpr->y.sub.iAddr;
      pSig->regReturn = pExpr->y.sub.regReturn;
      pSig->iTable = iTab;
      sqlite3VdbeChangeP4(v, -1, (const char*)pSig, P4_SUBRTNSIG);
    }
    addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
  }

  /* Check to see if this is a vector IN operator */
  pLeft = pExpr->pLeft;
  nVal = sqlite3ExprVectorSize(pLeft);

Changes to src/vdbe.h.

28
29
30
31
32
33
34













35
36
37
38
39
40
41

/*
** The names of the following types declared in vdbeInt.h are required
** for the VdbeOp definition.
*/
typedef struct sqlite3_value Mem;
typedef struct SubProgram SubProgram;














/*
** A single instruction of the virtual machine has an opcode
** and as many as three operands.  The instruction is recorded
** as an instance of the following structure:
*/
struct VdbeOp {







>
>
>
>
>
>
>
>
>
>
>
>
>







28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

/*
** The names of the following types declared in vdbeInt.h are required
** for the VdbeOp definition.
*/
typedef struct sqlite3_value Mem;
typedef struct SubProgram SubProgram;
typedef struct SubrtnSig SubrtnSig;

/*
** A signature for a reusable subroutine that materializes the RHS of
** an IN operator.
*/
struct SubrtnSig {
  int selId;          /* SELECT-id for the SELECT statement on the RHS */
  char *zAff;         /* Affinity of the overall IN expression */
  int iTable;         /* Ephemeral table generated by the subroutine */
  int iAddr;          /* Subroutine entry address */
  int regReturn;      /* Register used to hold return address */
};

/*
** A single instruction of the virtual machine has an opcode
** and as many as three operands.  The instruction is recorded
** as an instance of the following structure:
*/
struct VdbeOp {
56
57
58
59
60
61
62

63
64
65
66
67
68
69
    CollSeq *pColl;        /* Used when p4type is P4_COLLSEQ */
    Mem *pMem;             /* Used when p4type is P4_MEM */
    VTable *pVtab;         /* Used when p4type is P4_VTAB */
    KeyInfo *pKeyInfo;     /* Used when p4type is P4_KEYINFO */
    u32 *ai;               /* Used when p4type is P4_INTARRAY */
    SubProgram *pProgram;  /* Used when p4type is P4_SUBPROGRAM */
    Table *pTab;           /* Used when p4type is P4_TABLE */

#ifdef SQLITE_ENABLE_CURSOR_HINTS
    Expr *pExpr;           /* Used when p4type is P4_EXPR */
#endif
  } p4;
#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
  char *zComment;          /* Comment to improve readability */
#endif







>







69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
    CollSeq *pColl;        /* Used when p4type is P4_COLLSEQ */
    Mem *pMem;             /* Used when p4type is P4_MEM */
    VTable *pVtab;         /* Used when p4type is P4_VTAB */
    KeyInfo *pKeyInfo;     /* Used when p4type is P4_KEYINFO */
    u32 *ai;               /* Used when p4type is P4_INTARRAY */
    SubProgram *pProgram;  /* Used when p4type is P4_SUBPROGRAM */
    Table *pTab;           /* Used when p4type is P4_TABLE */
    SubrtnSig *pSubrtnSig; /* Used when p4type is P4_SUBRTNSIG */
#ifdef SQLITE_ENABLE_CURSOR_HINTS
    Expr *pExpr;           /* Used when p4type is P4_EXPR */
#endif
  } p4;
#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
  char *zComment;          /* Comment to improve readability */
#endif
123
124
125
126
127
128
129

130
131
132
133
134
135
136
#define P4_MEM        (-10) /* P4 is a pointer to a Mem*    structure */
#define P4_VTAB       (-11) /* P4 is a pointer to an sqlite3_vtab structure */
#define P4_REAL       (-12) /* P4 is a 64-bit floating point value */
#define P4_INT64      (-13) /* P4 is a 64-bit signed integer */
#define P4_INTARRAY   (-14) /* P4 is a vector of 32-bit integers */
#define P4_FUNCCTX    (-15) /* P4 is a pointer to an sqlite3_context object */
#define P4_TABLEREF   (-16) /* Like P4_TABLE, but reference counted */


/* Error message codes for OP_Halt */
#define P5_ConstraintNotNull 1
#define P5_ConstraintUnique  2
#define P5_ConstraintCheck   3
#define P5_ConstraintFK      4








>







137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
#define P4_MEM        (-10) /* P4 is a pointer to a Mem*    structure */
#define P4_VTAB       (-11) /* P4 is a pointer to an sqlite3_vtab structure */
#define P4_REAL       (-12) /* P4 is a 64-bit floating point value */
#define P4_INT64      (-13) /* P4 is a 64-bit signed integer */
#define P4_INTARRAY   (-14) /* P4 is a vector of 32-bit integers */
#define P4_FUNCCTX    (-15) /* P4 is a pointer to an sqlite3_context object */
#define P4_TABLEREF   (-16) /* Like P4_TABLE, but reference counted */
#define P4_SUBRTNSIG  (-17) /* P4 is a SubrtnSig pointer */

/* Error message codes for OP_Halt */
#define P5_ConstraintNotNull 1
#define P5_ConstraintUnique  2
#define P5_ConstraintCheck   3
#define P5_ConstraintFK      4

Changes to src/vdbeaux.c.

1409
1410
1411
1412
1413
1414
1415






1416
1417
1418
1419
1420
1421
1422
      if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4);
      break;
    }
    case P4_TABLEREF: {
      if( db->pnBytesFreed==0 ) sqlite3DeleteTable(db, (Table*)p4);
      break;
    }






  }
}

/*
** Free the space allocated for aOp and any p4 values allocated for the
** opcodes contained within. If aOp is not NULL it is assumed to contain
** nOp entries.







>
>
>
>
>
>







1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
      if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4);
      break;
    }
    case P4_TABLEREF: {
      if( db->pnBytesFreed==0 ) sqlite3DeleteTable(db, (Table*)p4);
      break;
    }
    case P4_SUBRTNSIG: {
      SubrtnSig *pSig = (SubrtnSig*)p4;
      sqlite3DbFree(db, pSig->zAff);
      sqlite3DbFree(db, pSig);
      break;
    }
  }
}

/*
** Free the space allocated for aOp and any p4 values allocated for the
** opcodes contained within. If aOp is not NULL it is assumed to contain
** nOp entries.
1987
1988
1989
1990
1991
1992
1993



1994
1995
1996
1997
1998
1999
2000
    case P4_SUBPROGRAM: {
      zP4 = "program";
      break;
    }
    case P4_TABLE: {
      zP4 = pOp->p4.pTab->zName;
      break;



    }
    default: {
      zP4 = pOp->p4.z;
    }
  }
  if( zP4 ) sqlite3_str_appendall(&x, zP4);
  if( (x.accError & SQLITE_NOMEM)!=0 ){







>
>
>







1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
    case P4_SUBPROGRAM: {
      zP4 = "program";
      break;
    }
    case P4_TABLE: {
      zP4 = pOp->p4.pTab->zName;
      break;
    }
    case P4_SUBRTNSIG: {
      break;
    }
    default: {
      zP4 = pOp->p4.z;
    }
  }
  if( zP4 ) sqlite3_str_appendall(&x, zP4);
  if( (x.accError & SQLITE_NOMEM)!=0 ){

Changes to test/pushdown.test.

275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
  |     |  `--LIST SUBQUERY xxxxxx
  |     |     |--MATERIALIZE k
  |     |     |  `--SCAN 3 CONSTANT ROWS
  |     |     |--SCAN k
  |     |     `--CREATE BLOOM FILTER
  |     `--UNION ALL
  |        |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?)
  |        `--LIST SUBQUERY xxxxxx
  |           |--SCAN k
  |           `--CREATE BLOOM FILTER
  |--SEARCH t0
  `--LIST SUBQUERY xxxxxx
     |--SCAN k
     `--CREATE BLOOM FILTER
}
# ^^^^--- The key feature above is that the SEARCH for each subquery
# uses all three fields of the index w, x, and y.  Prior to the push-down
# of "expr IN table", only the w term of the index would be used.  Similar
# for the following tests:
#
do_eqp_test 6.2 {







|
<
<

|
<
<







275
276
277
278
279
280
281
282


283
284


285
286
287
288
289
290
291
  |     |  `--LIST SUBQUERY xxxxxx
  |     |     |--MATERIALIZE k
  |     |     |  `--SCAN 3 CONSTANT ROWS
  |     |     |--SCAN k
  |     |     `--CREATE BLOOM FILTER
  |     `--UNION ALL
  |        |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?)
  |        `--REUSE LIST SUBQUERY xxxxxx


  |--SEARCH t0
  `--REUSE LIST SUBQUERY xxxxxx


}
# ^^^^--- The key feature above is that the SEARCH for each subquery
# uses all three fields of the index w, x, and y.  Prior to the push-down
# of "expr IN table", only the w term of the index would be used.  Similar
# for the following tests:
#
do_eqp_test 6.2 {
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
  |     |  `--LIST SUBQUERY xxxxxx
  |     |     |--CO-ROUTINE v1
  |     |     |  `--SCAN 3 CONSTANT ROWS
  |     |     |--SCAN v1
  |     |     `--CREATE BLOOM FILTER
  |     `--UNION ALL
  |        |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?)
  |        `--LIST SUBQUERY xxxxxx
  |           |--CO-ROUTINE v1
  |           |  `--SCAN 3 CONSTANT ROWS
  |           |--SCAN v1
  |           `--CREATE BLOOM FILTER
  |--SEARCH t0
  `--LIST SUBQUERY xxxxxx
     |--CO-ROUTINE v1
     |  `--SCAN 3 CONSTANT ROWS
     |--SCAN v1
     `--CREATE BLOOM FILTER
}
do_eqp_test 6.3 {
  SELECT max(z) FROM t0 WHERE w=123 AND x IN k1 AND y BETWEEN 44 AND 55;
} {
  QUERY PLAN
  |--CO-ROUTINE t0
  |  `--COMPOUND QUERY
  |     |--LEFT-MOST SUBQUERY
  |     |  |--SEARCH t01 USING INDEX t01x (w=? AND x=? AND y>? AND y<?)
  |     |  `--LIST SUBQUERY xxxxxx
  |     |     |--SCAN k1
  |     |     `--CREATE BLOOM FILTER
  |     `--UNION ALL
  |        |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?)
  |        `--LIST SUBQUERY xxxxxx
  |           |--SCAN k1
  |           `--CREATE BLOOM FILTER
  |--SEARCH t0
  `--LIST SUBQUERY xxxxxx
     |--SCAN k1
     `--CREATE BLOOM FILTER
}

finish_test







|
<
<
<
<

|
<
<
<
<














|
<
<

|
<
<



299
300
301
302
303
304
305
306




307
308




309
310
311
312
313
314
315
316
317
318
319
320
321
322
323


324
325


326
327
328
  |     |  `--LIST SUBQUERY xxxxxx
  |     |     |--CO-ROUTINE v1
  |     |     |  `--SCAN 3 CONSTANT ROWS
  |     |     |--SCAN v1
  |     |     `--CREATE BLOOM FILTER
  |     `--UNION ALL
  |        |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?)
  |        `--REUSE LIST SUBQUERY xxxxxx




  |--SEARCH t0
  `--REUSE LIST SUBQUERY xxxxxx




}
do_eqp_test 6.3 {
  SELECT max(z) FROM t0 WHERE w=123 AND x IN k1 AND y BETWEEN 44 AND 55;
} {
  QUERY PLAN
  |--CO-ROUTINE t0
  |  `--COMPOUND QUERY
  |     |--LEFT-MOST SUBQUERY
  |     |  |--SEARCH t01 USING INDEX t01x (w=? AND x=? AND y>? AND y<?)
  |     |  `--LIST SUBQUERY xxxxxx
  |     |     |--SCAN k1
  |     |     `--CREATE BLOOM FILTER
  |     `--UNION ALL
  |        |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?)
  |        `--REUSE LIST SUBQUERY xxxxxx


  |--SEARCH t0
  `--REUSE LIST SUBQUERY xxxxxx


}

finish_test