SQLite

Check-in [31e175fc]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix LATERAL subqueries so that they are able to reference other subqueries to their left that are implemented as co-routines. See Forum post dfe2cd37ca3a9a80.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | lateral-join
Files: files | file ages | folders
SHA3-256: 31e175fcd0ff823941d4f0f53f7bc6a65fb5b00de78cdb6a9d622cc96b6a6f15
User & Date: drh 2024-07-21 10:35:55
Context
2024-07-21
18:03
Assert added to ensure that LATERAL subqueries are materialized instead of implemented as a co-routine. (check-in: 6c14b94f user: drh tags: lateral-join)
10:35
Fix LATERAL subqueries so that they are able to reference other subqueries to their left that are implemented as co-routines. See Forum post dfe2cd37ca3a9a80. (check-in: 31e175fc user: drh tags: lateral-join)
2024-07-20
17:38
Fix rule (1c-ii) of the fromClauseTermCanBeCoroutine() decision so that it work with LATERAL. Add testcase() macros to verify bitmask conditions are all checked. (check-in: 23d83a41 user: drh tags: lateral-join)
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/resolve.c.

1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
          sNC.pNext = 0;
          pSubNC = pOuterNC;
        }
        sqlite3ResolveSelectNames(pParse, pItem->pSelect, pSubNC);
        p->pSrc->nSrc = nSrc;
        if( sNC.nRef>nRef2 ){
          pItem->fg.isCorrelated = 1;
          /* Add JT_LATERAL to the left-most term of the FROM clause as a
          ** marker that this FROM clause contains one or more LATERALs. */
          p->pSrc->a[0].fg.jointype |= JT_LATERAL;
        }
        pParse->zAuthContext = zSavedContext;
        if( pParse->nErr ) return WRC_Abort;
        assert( db->mallocFailed==0 );

        /* If the number of references to the outer context changed when
        ** expressions in the sub-select were resolved, the sub-select







<
<
<







1912
1913
1914
1915
1916
1917
1918



1919
1920
1921
1922
1923
1924
1925
          sNC.pNext = 0;
          pSubNC = pOuterNC;
        }
        sqlite3ResolveSelectNames(pParse, pItem->pSelect, pSubNC);
        p->pSrc->nSrc = nSrc;
        if( sNC.nRef>nRef2 ){
          pItem->fg.isCorrelated = 1;



        }
        pParse->zAuthContext = zSavedContext;
        if( pParse->nErr ) return WRC_Abort;
        assert( db->mallocFailed==0 );

        /* If the number of references to the outer context changed when
        ** expressions in the sub-select were resolved, the sub-select

Changes to src/select.c.

7252
7253
7254
7255
7256
7257
7258
7259
7260
7261
7262
7263
7264
7265
7266
7267
7268
7269
7270
7271
7272
7273
7274
7275
7276
7277
7278
7279
7280
7281
7282
7283
7284
**                    being used as the outer loop if the sqlite3WhereBegin()
**                    routine nominates it to that position.
**              (iii) The query is not a UPDATE ... FROM
**    (2)  The subquery is not a CTE that should be materialized because
**         (a) the AS MATERIALIZED keyword is used, or
**         (b) the CTE is used multiple times and does not have the
**             NOT MATERIALIZED keyword
**    (3)  The FROM clause does not contain a RIGHT JOIN nor a LATERAL JOIN.
**    (4)  The SQLITE_Coroutine optimization disable flag is not set
**    (5)  The subquery is not self-joined
*/
static int fromClauseTermCanBeCoroutine(
  Parse *pParse,          /* Parsing context */
  SrcList *pTabList,      /* FROM clause */
  int i,                  /* Which term of the FROM clause holds the subquery */
  int selFlags            /* Flags on the SELECT statement */
){
  SrcItem *pItem = &pTabList->a[i];
  if( pItem->fg.isCte ){
    const CteUse *pCteUse = pItem->u2.pCteUse;
    if( pCteUse->eM10d==M10d_Yes ) return 0;                          /* (2a) */
    if( pCteUse->nUse>=2 && pCteUse->eM10d!=M10d_No ) return 0;       /* (2b) */
  }
  testcase( pTabList->a[0].fg.jointype & JT_LTORJ );
  testcase( pTabList->a[0].fg.jointype & JT_LATERAL );
  if( pTabList->a[0].fg.jointype & (JT_LTORJ|JT_LATERAL) ) return 0;  /* (3)  */
  if( OptimizationDisabled(pParse->db, SQLITE_Coroutines) ) return 0; /* (4)  */
  if( isSelfJoinView(pTabList, pItem, i+1, pTabList->nSrc)!=0 ){
    return 0;                                                          /* (5) */
  }
  if( i==0 ){
    if( pTabList->nSrc==1 ) return 1;                             /* (1a) */
    if( pTabList->a[1].fg.jointype & JT_CROSS ) return 1;         /* (1b) */







|















|
<
<







7252
7253
7254
7255
7256
7257
7258
7259
7260
7261
7262
7263
7264
7265
7266
7267
7268
7269
7270
7271
7272
7273
7274
7275


7276
7277
7278
7279
7280
7281
7282
**                    being used as the outer loop if the sqlite3WhereBegin()
**                    routine nominates it to that position.
**              (iii) The query is not a UPDATE ... FROM
**    (2)  The subquery is not a CTE that should be materialized because
**         (a) the AS MATERIALIZED keyword is used, or
**         (b) the CTE is used multiple times and does not have the
**             NOT MATERIALIZED keyword
**    (3)  The FROM clause does not contain a RIGHT JOIN
**    (4)  The SQLITE_Coroutine optimization disable flag is not set
**    (5)  The subquery is not self-joined
*/
static int fromClauseTermCanBeCoroutine(
  Parse *pParse,          /* Parsing context */
  SrcList *pTabList,      /* FROM clause */
  int i,                  /* Which term of the FROM clause holds the subquery */
  int selFlags            /* Flags on the SELECT statement */
){
  SrcItem *pItem = &pTabList->a[i];
  if( pItem->fg.isCte ){
    const CteUse *pCteUse = pItem->u2.pCteUse;
    if( pCteUse->eM10d==M10d_Yes ) return 0;                          /* (2a) */
    if( pCteUse->nUse>=2 && pCteUse->eM10d!=M10d_No ) return 0;       /* (2b) */
  }
  if( pTabList->a[0].fg.jointype & JT_LTORJ ) return 0;               /* (3)  */


  if( OptimizationDisabled(pParse->db, SQLITE_Coroutines) ) return 0; /* (4)  */
  if( isSelfJoinView(pTabList, pItem, i+1, pTabList->nSrc)!=0 ){
    return 0;                                                          /* (5) */
  }
  if( i==0 ){
    if( pTabList->nSrc==1 ) return 1;                             /* (1a) */
    if( pTabList->a[1].fg.jointype & JT_CROSS ) return 1;         /* (1b) */
7812
7813
7814
7815
7816
7817
7818









7819
7820
7821
7822
7823
7824
7825
        VdbeNoopComment((v, "materialize %!S", pItem));
      }
      sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor);

      ExplainQueryPlan2(addrExplain, (pParse, 1, "MATERIALIZE %!S", pItem));
      sqlite3Select(pParse, pSub, &dest);
      pItem->pTab->nRowLogEst = pSub->nSelectRow;









      if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr);
      sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1);
      VdbeComment((v, "end %!S", pItem));
      sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1);
      sqlite3VdbeJumpHere(v, topAddr);
      sqlite3ClearTempRegCache(pParse);
      if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){







>
>
>
>
>
>
>
>
>







7810
7811
7812
7813
7814
7815
7816
7817
7818
7819
7820
7821
7822
7823
7824
7825
7826
7827
7828
7829
7830
7831
7832
        VdbeNoopComment((v, "materialize %!S", pItem));
      }
      sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor);

      ExplainQueryPlan2(addrExplain, (pParse, 1, "MATERIALIZE %!S", pItem));
      sqlite3Select(pParse, pSub, &dest);
      pItem->pTab->nRowLogEst = pSub->nSelectRow;
      if( pItem->fg.isLateral && pItem->fg.isCorrelated ){
        int kk;
        for(kk=0; kk<i; kk++){
          SrcItem *pX = &pTabList->a[kk];
          if( pX->fg.viaCoroutine==0 ) continue;
          sqlite3TranslateColumnToCopy(pParse, topAddr+1,
             pX->iCursor, pX->regResult, 0);
        }
      }
      if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr);
      sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1);
      VdbeComment((v, "end %!S", pItem));
      sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1);
      sqlite3VdbeJumpHere(v, topAddr);
      sqlite3ClearTempRegCache(pParse);
      if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){

Changes to src/sqliteInt.h.

5015
5016
5017
5018
5019
5020
5021







5022
5023
5024
5025
5026
5027
5028
void sqlite3CodeChangeCount(Vdbe*,int,const char*);
void sqlite3DeleteFrom(Parse*, SrcList*, Expr*, ExprList*, Expr*);
void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*,
                   Upsert*);
WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,
                             ExprList*,Select*,u16,int);
void sqlite3WhereEnd(WhereInfo*);







LogEst sqlite3WhereOutputRowCount(WhereInfo*);
int sqlite3WhereIsDistinct(WhereInfo*);
int sqlite3WhereIsOrdered(WhereInfo*);
int sqlite3WhereOrderByLimitOptLabel(WhereInfo*);
void sqlite3WhereMinMaxOptEarlyOut(Vdbe*,WhereInfo*);
int sqlite3WhereIsSorted(WhereInfo*);
int sqlite3WhereContinueLabel(WhereInfo*);







>
>
>
>
>
>
>







5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
void sqlite3CodeChangeCount(Vdbe*,int,const char*);
void sqlite3DeleteFrom(Parse*, SrcList*, Expr*, ExprList*, Expr*);
void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*,
                   Upsert*);
WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,
                             ExprList*,Select*,u16,int);
void sqlite3WhereEnd(WhereInfo*);
void sqlite3TranslateColumnToCopy(
  Parse *pParse,      /* Parsing context */
  int iStart,         /* Translate from this opcode to the end */
  int iTabCur,        /* OP_Column/OP_Rowid references to this table */
  int iRegister,      /* The first column is in this register */
  int iAutoidxCur     /* If non-zero, cursor of autoindex being generated */
);
LogEst sqlite3WhereOutputRowCount(WhereInfo*);
int sqlite3WhereIsDistinct(WhereInfo*);
int sqlite3WhereIsOrdered(WhereInfo*);
int sqlite3WhereOrderByLimitOptLabel(WhereInfo*);
void sqlite3WhereMinMaxOptEarlyOut(Vdbe*,WhereInfo*);
int sqlite3WhereIsSorted(WhereInfo*);
int sqlite3WhereContinueLabel(WhereInfo*);

Changes to src/where.c.

704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
** instead of via table lookup.
**
** If the iAutoidxCur is not zero, then any OP_Rowid instructions on
** cursor iTabCur are transformed into OP_Sequence opcode for the
** iAutoidxCur cursor, in order to generate unique rowids for the
** automatic index being generated.
*/
static void translateColumnToCopy(
  Parse *pParse,      /* Parsing context */
  int iStart,         /* Translate from this opcode to the end */
  int iTabCur,        /* OP_Column/OP_Rowid references to this table */
  int iRegister,      /* The first column is in this register */
  int iAutoidxCur     /* If non-zero, cursor of autoindex being generated */
){
  Vdbe *v = pParse->pVdbe;







|







704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
** instead of via table lookup.
**
** If the iAutoidxCur is not zero, then any OP_Rowid instructions on
** cursor iTabCur are transformed into OP_Sequence opcode for the
** iAutoidxCur cursor, in order to generate unique rowids for the
** automatic index being generated.
*/
void sqlite3TranslateColumnToCopy(
  Parse *pParse,      /* Parsing context */
  int iStart,         /* Translate from this opcode to the end */
  int iTabCur,        /* OP_Column/OP_Rowid references to this table */
  int iRegister,      /* The first column is in this register */
  int iAutoidxCur     /* If non-zero, cursor of autoindex being generated */
){
  Vdbe *v = pParse->pVdbe;
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
  sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord);
  sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
  if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
  if( pSrc->fg.viaCoroutine ){
    sqlite3VdbeChangeP2(v, addrCounter, regBase+n);
    testcase( pParse->db->mallocFailed );
    assert( pLevel->iIdxCur>0 );
    translateColumnToCopy(pParse, addrTop, pLevel->iTabCur,
                          pSrc->regResult, pLevel->iIdxCur);
    sqlite3VdbeGoto(v, addrTop);
    pSrc->fg.viaCoroutine = 0;
  }else{
    sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v);
    sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX);
  }
  sqlite3VdbeJumpHere(v, addrTop);







|
|







1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
  sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord);
  sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
  if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
  if( pSrc->fg.viaCoroutine ){
    sqlite3VdbeChangeP2(v, addrCounter, regBase+n);
    testcase( pParse->db->mallocFailed );
    assert( pLevel->iIdxCur>0 );
    sqlite3TranslateColumnToCopy(pParse, addrTop, pLevel->iTabCur,
                                 pSrc->regResult, pLevel->iIdxCur);
    sqlite3VdbeGoto(v, addrTop);
    pSrc->fg.viaCoroutine = 0;
  }else{
    sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v);
    sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX);
  }
  sqlite3VdbeJumpHere(v, addrTop);
7288
7289
7290
7291
7292
7293
7294
7295
7296
7297
7298
7299
7300
7301
7302
7303
    /* For a co-routine, change all OP_Column references to the table of
    ** the co-routine into OP_Copy of result contained in a register.
    ** OP_Rowid becomes OP_Null.
    */
    if( pTabItem->fg.viaCoroutine ){
      testcase( pParse->db->mallocFailed );
      assert( pTabItem->regResult>=0 );
      translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur,
                            pTabItem->regResult, 0);
      continue;
    }

    /* If this scan uses an index, make VDBE code substitutions to read data
    ** from the index instead of from the table where possible.  In some cases
    ** this optimization prevents the table from ever being read, which can
    ** yield a significant performance boost.







|
|







7288
7289
7290
7291
7292
7293
7294
7295
7296
7297
7298
7299
7300
7301
7302
7303
    /* For a co-routine, change all OP_Column references to the table of
    ** the co-routine into OP_Copy of result contained in a register.
    ** OP_Rowid becomes OP_Null.
    */
    if( pTabItem->fg.viaCoroutine ){
      testcase( pParse->db->mallocFailed );
      assert( pTabItem->regResult>=0 );
      sqlite3TranslateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur,
                                   pTabItem->regResult, 0);
      continue;
    }

    /* If this scan uses an index, make VDBE code substitutions to read data
    ** from the index instead of from the table where possible.  In some cases
    ** this optimization prevents the table from ever being read, which can
    ** yield a significant performance boost.

Changes to test/joinL.test.

160
161
162
163
164
165
166






167
168
  INSERT INTO lateral VALUES(1,2);
  SELECT * FROM lateral;
  ATTACH ':memory:' AS lateral;
  CREATE TABLE lateral.t2(x,y);
  INSERT INTO t2 VALUES(98,99);
  SELECT * FROM t2;
} {1 2 98 99}







finish_test







>
>
>
>
>
>


160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
  INSERT INTO lateral VALUES(1,2);
  SELECT * FROM lateral;
  ATTACH ':memory:' AS lateral;
  CREATE TABLE lateral.t2(x,y);
  INSERT INTO t2 VALUES(98,99);
  SELECT * FROM t2;
} {1 2 98 99}

# https://sqlite.org/forum/forumpost/dfe2cd37ca3a9a80
#
do_execsql_test 4.0 {
  SELECT * FROM (VALUES (1), (2)) JOIN LATERAL (select COUNT(*), column1);
} {1 1 1 2 1 2}

finish_test