SQLite

Check-in [f911f1c4]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Drop support for the view-scan optimization (check-in [609fbb94b8f01d67]) as it was causing multiple performance regressions. In its place, reduce the estimated row count for DISTINCT subsqueries by a factor of 8.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: f911f1c4977fbcae041243955cf2b98d8cc8baa337885a69be0f2b9bd2efa6f3
User & Date: drh 2023-09-15 19:51:18
Context
2023-09-15
20:28
Simplifications and performance optimizations for the RTree extension. (check-in: 04a333f5 user: drh tags: trunk)
20:04
Drop support for the view-scan optimization as it was causing multiple performance regressions. In its place, reduce the estimated row count for DISTINCT subsqueries by a factor of 8. (check-in: 796a65fa user: drh tags: branch-3.28)
19:51
Drop support for the view-scan optimization (check-in [609fbb94b8f01d67]) as it was causing multiple performance regressions. In its place, reduce the estimated row count for DISTINCT subsqueries by a factor of 8. (check-in: f911f1c4 user: drh tags: trunk)
19:27
Minor simplification to the DISTINCT output row count change. (Closed-Leaf check-in: 0738386d user: drh tags: rethink-viewscan)
10:24
Do not try to convert a double into an unsigned 64-bit integer, as that does not work on all platforms. A double can only be converted into a signed 64-bit integer. This is a fix for the problem reported in forum post 9f6db917e1c05d40. (check-in: ce339046 user: drh tags: trunk)
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/where.c.

3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
      ** better.
      */
#ifdef SQLITE_ENABLE_STAT4
      pNew->rRun = rSize + 16 - 2*((pTab->tabFlags & TF_HasStat4)!=0);
#else
      pNew->rRun = rSize + 16;
#endif
      if( IsView(pTab) || (pTab->tabFlags & TF_Ephemeral)!=0 ){
        pNew->wsFlags |= WHERE_VIEWSCAN;
      }
      ApplyCostMultiplier(pNew->rRun, pTab->costMult);
      whereLoopOutputAdjust(pWC, pNew, rSize);
      rc = whereLoopInsert(pBuilder, pNew);
      pNew->nOut = rSize;
      if( rc ) break;
    }else{
      Bitmask m;







<
<
<







3696
3697
3698
3699
3700
3701
3702



3703
3704
3705
3706
3707
3708
3709
      ** better.
      */
#ifdef SQLITE_ENABLE_STAT4
      pNew->rRun = rSize + 16 - 2*((pTab->tabFlags & TF_HasStat4)!=0);
#else
      pNew->rRun = rSize + 16;
#endif



      ApplyCostMultiplier(pNew->rRun, pTab->costMult);
      whereLoopOutputAdjust(pWC, pNew, rSize);
      rc = whereLoopInsert(pBuilder, pNew);
      pNew->nOut = rSize;
      if( rc ) break;
    }else{
      Bitmask m;
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
               aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy,
               rUnsorted, rCost));
        }else{
          rCost = rUnsorted;
          rUnsorted -= 2;  /* TUNING:  Slight bias in favor of no-sort plans */
        }

        /* TUNING:  A full-scan of a VIEW or subquery in the outer loop
        ** is not so bad. */
        if( iLoop==0 && (pWLoop->wsFlags & WHERE_VIEWSCAN)!=0 && nLoop>1 ){
          rCost += -10;
          nOut += -30;
          WHERETRACE(0x80,("VIEWSCAN cost reduction for %c\n",pWLoop->cId));
        }

        /* Check to see if pWLoop should be added to the set of
        ** mxChoice best-so-far paths.
        **
        ** First look for an existing path among best-so-far paths
        ** that covers the same set of loops and has the same isOrdered
        ** setting as the current path candidate.
        **







<
<
<
<
<
<
<
<







5114
5115
5116
5117
5118
5119
5120








5121
5122
5123
5124
5125
5126
5127
               aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy,
               rUnsorted, rCost));
        }else{
          rCost = rUnsorted;
          rUnsorted -= 2;  /* TUNING:  Slight bias in favor of no-sort plans */
        }









        /* Check to see if pWLoop should be added to the set of
        ** mxChoice best-so-far paths.
        **
        ** First look for an existing path among best-so-far paths
        ** that covers the same set of loops and has the same isOrdered
        ** setting as the current path candidate.
        **
6139
6140
6141
6142
6143
6144
6145










6146
6147
6148
6149
6150
6151
6152
 
    wherePathSolver(pWInfo, 0);
    if( db->mallocFailed ) goto whereBeginError;
    if( pWInfo->pOrderBy ){
       wherePathSolver(pWInfo, pWInfo->nRowOut+1);
       if( db->mallocFailed ) goto whereBeginError;
    }










  }
  assert( pWInfo->pTabList!=0 );
  if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){
    whereReverseScanOrder(pWInfo);
  }
  if( pParse->nErr ){
    goto whereBeginError;







>
>
>
>
>
>
>
>
>
>







6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
 
    wherePathSolver(pWInfo, 0);
    if( db->mallocFailed ) goto whereBeginError;
    if( pWInfo->pOrderBy ){
       wherePathSolver(pWInfo, pWInfo->nRowOut+1);
       if( db->mallocFailed ) goto whereBeginError;
    }

    /* TUNING:  Assume that a DISTINCT clause on a subquery reduces
    ** the output size by a factor of 8 (LogEst -30).
    */
    if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){
      WHERETRACE(0x0080,("nRowOut reduced from %d to %d due to DISTINCT\n",
                         pWInfo->nRowOut, pWInfo->nRowOut-30));
      pWInfo->nRowOut -= 30;
    }

  }
  assert( pWInfo->pTabList!=0 );
  if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){
    whereReverseScanOrder(pWInfo);
  }
  if( pParse->nErr ){
    goto whereBeginError;

Changes to src/whereInt.h.

629
630
631
632
633
634
635
636
637
638
639
#define WHERE_IN_EARLYOUT  0x00040000  /* Perhaps quit IN loops early */
#define WHERE_BIGNULL_SORT 0x00080000  /* Column nEq of index is BIGNULL */
#define WHERE_IN_SEEKSCAN  0x00100000  /* Seek-scan optimization for IN */
#define WHERE_TRANSCONS    0x00200000  /* Uses a transitive constraint */
#define WHERE_BLOOMFILTER  0x00400000  /* Consider using a Bloom-filter */
#define WHERE_SELFCULL     0x00800000  /* nOut reduced by extra WHERE terms */
#define WHERE_OMIT_OFFSET  0x01000000  /* Set offset counter to zero */
#define WHERE_VIEWSCAN     0x02000000  /* A full-scan of a VIEW or subquery */
#define WHERE_EXPRIDX      0x04000000  /* Uses an index-on-expressions */

#endif /* !defined(SQLITE_WHEREINT_H) */







|



629
630
631
632
633
634
635
636
637
638
639
#define WHERE_IN_EARLYOUT  0x00040000  /* Perhaps quit IN loops early */
#define WHERE_BIGNULL_SORT 0x00080000  /* Column nEq of index is BIGNULL */
#define WHERE_IN_SEEKSCAN  0x00100000  /* Seek-scan optimization for IN */
#define WHERE_TRANSCONS    0x00200000  /* Uses a transitive constraint */
#define WHERE_BLOOMFILTER  0x00400000  /* Consider using a Bloom-filter */
#define WHERE_SELFCULL     0x00800000  /* nOut reduced by extra WHERE terms */
#define WHERE_OMIT_OFFSET  0x01000000  /* Set offset counter to zero */
                      /*   0x02000000  -- available for reuse */
#define WHERE_EXPRIDX      0x04000000  /* Uses an index-on-expressions */

#endif /* !defined(SQLITE_WHEREINT_H) */

Changes to test/subquery.test.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# 2005 January 19
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing correlated subqueries
#
# $Id: subquery.test,v 1.17 2009/01/09 01:12:28 drh Exp $
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

ifcapable !subquery {
  finish_test
  return













<
<







1
2
3
4
5
6
7
8
9
10
11
12
13


14
15
16
17
18
19
20
# 2005 January 19
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing correlated subqueries
#



set testdir [file dirname $argv0]
source $testdir/tester.tcl

ifcapable !subquery {
  finish_test
  return
608
609
610
611
612
613
614
615









































616
do_execsql_test subquery-9.3 {
  INSERT INTO t1 VALUES(2);
  SELECT (SELECT DISTINCT x FROM t1 ORDER BY +x LIMIT 1 OFFSET 1) FROM t1;
} {2 2 2 2}
do_execsql_test subquery-9.4 {
  SELECT (SELECT DISTINCT x FROM t1 ORDER BY +x LIMIT 1 OFFSET 2) FROM t1;
} {{} {} {} {}}










































finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
do_execsql_test subquery-9.3 {
  INSERT INTO t1 VALUES(2);
  SELECT (SELECT DISTINCT x FROM t1 ORDER BY +x LIMIT 1 OFFSET 1) FROM t1;
} {2 2 2 2}
do_execsql_test subquery-9.4 {
  SELECT (SELECT DISTINCT x FROM t1 ORDER BY +x LIMIT 1 OFFSET 2) FROM t1;
} {{} {} {} {}}

# 2023-09-15
# Query planner performance regression reported by private email 
# on 2023-09-14, caused by VIEWSCAN optimization of check-in 609fbb94b8f01d67
# from 2022-09-01.
#
reset_db
do_execsql_test subquery-10.1 {
  CREATE TABLE t1(aa TEXT, bb INT, cc TEXT);
  CREATE INDEX x11 on t1(bb);
  CREATE INDEX x12 on t1(aa);
  CREATE TABLE t2(aa TEXT, xx INT);
  ANALYZE sqlite_master;
  INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES('t1', 'x11', '156789 28');
  INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES('t1', 'x12', '156789 1');
  ANALYZE sqlite_master;
}
do_eqp_test subquery-10.2 {
  WITH v1(aa,cc,bb) AS (SELECT aa, cc, bb FROM t1 WHERE bb=12345),
       v2(aa,mx)    AS (SELECT aa, max(xx) FROM t2 GROUP BY aa)
  SELECT * FROM v1 JOIN v2 ON v1.aa=v2.aa;
} {
  QUERY PLAN
  |--CO-ROUTINE v2
  |  |--SCAN t2
  |  `--USE TEMP B-TREE FOR GROUP BY
  |--SEARCH t1 USING INDEX x11 (bb=?)
  `--SEARCH v2 USING AUTOMATIC COVERING INDEX (aa=?)
}
# ^^^^^^^^^^^^^
# Prior to the fix the incorrect (slow) plan caused by the
# VIEWSCAN optimization was:
#
# QUERY PLAN
# |--CO-ROUTINE v2
# |  |--SCAN t2
# |  `--USE TEMP B-TREE FOR GROUP BY
# |--SCAN v2
# `--SEARCH t1 USING INDEX x12 (aa=?)
#


finish_test

Changes to test/with3.test.

128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
  QUERY PLAN
  |--CO-ROUTINE c
  |  |--SETUP
  |  |  |--SCAN CONSTANT ROW
  |  |  `--SCALAR SUBQUERY xxxxxx
  |  |     `--SCAN w2
  |  `--RECURSIVE STEP
  |     |--SCAN c
  |     `--SCAN w1
  |--SCAN c
  |--SEARCH w2 USING INTEGER PRIMARY KEY (rowid=?)
  `--SEARCH w1 USING INTEGER PRIMARY KEY (rowid=?)
}

do_execsql_test 4.0 {
  WITH t5(t5col1) AS (







|
|







128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
  QUERY PLAN
  |--CO-ROUTINE c
  |  |--SETUP
  |  |  |--SCAN CONSTANT ROW
  |  |  `--SCALAR SUBQUERY xxxxxx
  |  |     `--SCAN w2
  |  `--RECURSIVE STEP
  |     |--SCAN w1
  |     `--SCAN c
  |--SCAN c
  |--SEARCH w2 USING INTEGER PRIMARY KEY (rowid=?)
  `--SEARCH w1 USING INTEGER PRIMARY KEY (rowid=?)
}

do_execsql_test 4.0 {
  WITH t5(t5col1) AS (