SQLite

Check-in [44200596]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Do a better job of detecting when a WHERE clause term might be useful to an expression index. Fix for performance regression reported by forum thread e65800d8cb.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 44200596aa943963bc6ca98b5d4fd5b9235d1109d8dfc1a75eeae353b4239142
User & Date: drh 2023-02-10 21:53:33
References
2024-03-07
12:34
Do not allow the query planner to be tricked into thinking that an index on a constant expression might be useful for something. Problem reported on forum post ecdfc02339. This is a follow-up to the fixes at [44200596aa943963] and [2d2b91cc0f6fed8c]. (check-in: 720ce06d user: drh tags: trunk)
2023-02-13
18:42
Do not allow WHERE clause terms to match constant string index terms, which can happen if DQS_DDL is enabled. Follow-up to [44200596aa943963]. dbsqlfuzz 54c9db85ed4af7055f5fd0d50877875c82b11d46. (check-in: 2d2b91cc user: drh tags: trunk)
12:46
In the LIKE optimization, do not analyze the new virtual WHERE clause terms until both have been added, since they are expected to be consecutive and the analysis might add complementary terms. This fixes a problem caused by [44200596aa943963] and discovered by dbsqlfuzz and recorded as case 7e3b5983727d843b910b2d9ab556e4afcd777cfb. (check-in: d35de3ad user: drh tags: trunk)
Context
2023-02-11
21:11
Change a variable from 32 to 64-bits to avoid a harmless compiler warning in Xcode. Forum post 402d733c22. (check-in: 0216ce23 user: drh tags: trunk)
2023-02-10
21:53
Do a better job of detecting when a WHERE clause term might be useful to an expression index. Fix for performance regression reported by forum thread e65800d8cb. (check-in: 44200596 user: drh tags: trunk)
17:17
Fix a problem with the fts5 trigram tokenizer and LIKE or GLOB patterns for which contain runs of 2 or fewer non-wildcard characters that are 3 or more bytes when encoded as utf-8. (check-in: 00714b39 user: dan tags: trunk)
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/whereexpr.c.

970
971
972
973
974
975
976
977
978
979

980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997

998
999
1000
1001
1002
1003
1004
1005
1006


1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025



1026




1027
1028
1029
1030
1031
1032
1033
**
** If pExpr is a TK_COLUMN column reference, then this routine always returns
** true even if that particular column is not indexed, because the column
** might be added to an automatic index later.
*/
static SQLITE_NOINLINE int exprMightBeIndexed2(
  SrcList *pFrom,        /* The FROM clause */
  Bitmask mPrereq,       /* Bitmask of FROM clause terms referenced by pExpr */
  int *aiCurCol,         /* Write the referenced table cursor and column here */
  Expr *pExpr            /* An operand of a comparison operator */

){
  Index *pIdx;
  int i;
  int iCur;
  for(i=0; mPrereq>1; i++, mPrereq>>=1){}
  iCur = pFrom->a[i].iCursor;
  for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){
    if( pIdx->aColExpr==0 ) continue;
    for(i=0; i<pIdx->nKeyCol; i++){
      if( pIdx->aiColumn[i]!=XN_EXPR ) continue;
      assert( pIdx->bHasExpr );
      if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){
        aiCurCol[0] = iCur;
        aiCurCol[1] = XN_EXPR;
        return 1;
      }
    }
  }

  return 0;
}
static int exprMightBeIndexed(
  SrcList *pFrom,        /* The FROM clause */
  Bitmask mPrereq,       /* Bitmask of FROM clause terms referenced by pExpr */
  int *aiCurCol,         /* Write the referenced table cursor & column here */
  Expr *pExpr,           /* An operand of a comparison operator */
  int op                 /* The specific comparison operator */
){


  /* If this expression is a vector to the left or right of a 
  ** inequality constraint (>, <, >= or <=), perform the processing 
  ** on the first element of the vector.  */
  assert( TK_GT+1==TK_LE && TK_GT+2==TK_LT && TK_GT+3==TK_GE );
  assert( TK_IS<TK_GE && TK_ISNULL<TK_GE && TK_IN<TK_GE );
  assert( op<=TK_GE );
  if( pExpr->op==TK_VECTOR && (op>=TK_GT && ALWAYS(op<=TK_GE)) ){
    assert( ExprUseXList(pExpr) );
    pExpr = pExpr->x.pList->a[0].pExpr;

  }

  if( pExpr->op==TK_COLUMN ){
    aiCurCol[0] = pExpr->iTable;
    aiCurCol[1] = pExpr->iColumn;
    return 1;
  }
  if( mPrereq==0 ) return 0;                 /* No table references */
  if( (mPrereq&(mPrereq-1))!=0 ) return 0;   /* Refs more than one table */



  return exprMightBeIndexed2(pFrom,mPrereq,aiCurCol,pExpr);




}


/*
** The input to this routine is an WhereTerm structure with only the
** "pExpr" field filled in.  The job of this routine is to analyze the
** subexpression and populate all the other fields of the WhereTerm







<

|
>




|
|
|
|
|
|
|
|
|
|
|
|
|
|
>




<




>
>









<







|
|
>
>
>
|
>
>
>
>







970
971
972
973
974
975
976

977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002

1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017

1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
**
** If pExpr is a TK_COLUMN column reference, then this routine always returns
** true even if that particular column is not indexed, because the column
** might be added to an automatic index later.
*/
static SQLITE_NOINLINE int exprMightBeIndexed2(
  SrcList *pFrom,        /* The FROM clause */

  int *aiCurCol,         /* Write the referenced table cursor and column here */
  Expr *pExpr,           /* An operand of a comparison operator */
  int j                  /* Start looking with the j-th pFrom entry */
){
  Index *pIdx;
  int i;
  int iCur;
  do{
    iCur = pFrom->a[j].iCursor;
    for(pIdx=pFrom->a[j].pTab->pIndex; pIdx; pIdx=pIdx->pNext){
      if( pIdx->aColExpr==0 ) continue;
      for(i=0; i<pIdx->nKeyCol; i++){
        if( pIdx->aiColumn[i]!=XN_EXPR ) continue;
        assert( pIdx->bHasExpr );
        if( sqlite3ExprCompareSkip(pExpr,pIdx->aColExpr->a[i].pExpr,iCur)==0 ){
          aiCurCol[0] = iCur;
          aiCurCol[1] = XN_EXPR;
          return 1;
        }
      }
    }
  }while( ++j < pFrom->nSrc );
  return 0;
}
static int exprMightBeIndexed(
  SrcList *pFrom,        /* The FROM clause */

  int *aiCurCol,         /* Write the referenced table cursor & column here */
  Expr *pExpr,           /* An operand of a comparison operator */
  int op                 /* The specific comparison operator */
){
  int i;

  /* If this expression is a vector to the left or right of a 
  ** inequality constraint (>, <, >= or <=), perform the processing 
  ** on the first element of the vector.  */
  assert( TK_GT+1==TK_LE && TK_GT+2==TK_LT && TK_GT+3==TK_GE );
  assert( TK_IS<TK_GE && TK_ISNULL<TK_GE && TK_IN<TK_GE );
  assert( op<=TK_GE );
  if( pExpr->op==TK_VECTOR && (op>=TK_GT && ALWAYS(op<=TK_GE)) ){
    assert( ExprUseXList(pExpr) );
    pExpr = pExpr->x.pList->a[0].pExpr;

  }

  if( pExpr->op==TK_COLUMN ){
    aiCurCol[0] = pExpr->iTable;
    aiCurCol[1] = pExpr->iColumn;
    return 1;
  }

  for(i=0; i<pFrom->nSrc; i++){
    Index *pIdx;
    for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){
      if( pIdx->aColExpr ){
        return exprMightBeIndexed2(pFrom,aiCurCol,pExpr,i);
      }
    }
  }
  return 0;
}


/*
** The input to this routine is an WhereTerm structure with only the
** "pExpr" field filled in.  The job of this routine is to analyze the
** subexpression and populate all the other fields of the WhereTerm
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
    if( pTerm->u.x.iField>0 ){
      assert( op==TK_IN );
      assert( pLeft->op==TK_VECTOR );
      assert( ExprUseXList(pLeft) );
      pLeft = pLeft->x.pList->a[pTerm->u.x.iField-1].pExpr;
    }

    if( exprMightBeIndexed(pSrc, prereqLeft, aiCurCol, pLeft, op) ){
      pTerm->leftCursor = aiCurCol[0];
      assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
      pTerm->u.x.leftColumn = aiCurCol[1];
      pTerm->eOperator = operatorMask(op) & opMask;
    }
    if( op==TK_IS ) pTerm->wtFlags |= TERM_IS;
    if( pRight 
     && exprMightBeIndexed(pSrc, pTerm->prereqRight, aiCurCol, pRight, op)
     && !ExprHasProperty(pRight, EP_FixedCol)
    ){
      WhereTerm *pNew;
      Expr *pDup;
      u16 eExtraOp = 0;        /* Extra bits for pNew->eOperator */
      assert( pTerm->u.x.iField==0 );
      if( pTerm->leftCursor>=0 ){







|







|







1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
    if( pTerm->u.x.iField>0 ){
      assert( op==TK_IN );
      assert( pLeft->op==TK_VECTOR );
      assert( ExprUseXList(pLeft) );
      pLeft = pLeft->x.pList->a[pTerm->u.x.iField-1].pExpr;
    }

    if( exprMightBeIndexed(pSrc, aiCurCol, pLeft, op) ){
      pTerm->leftCursor = aiCurCol[0];
      assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
      pTerm->u.x.leftColumn = aiCurCol[1];
      pTerm->eOperator = operatorMask(op) & opMask;
    }
    if( op==TK_IS ) pTerm->wtFlags |= TERM_IS;
    if( pRight 
     && exprMightBeIndexed(pSrc, aiCurCol, pRight, op)
     && !ExprHasProperty(pRight, EP_FixedCol)
    ){
      WhereTerm *pNew;
      Expr *pDup;
      u16 eExtraOp = 0;        /* Extra bits for pNew->eOperator */
      assert( pTerm->u.x.iField==0 );
      if( pTerm->leftCursor>=0 ){

Changes to test/whereL.test.

185
186
187
188
189
190
191



















192
193
        FROM tableB
        WHERE RunYearMonth = 202004
    ) AS B
    ON A.ID = B.ID
    AND A.RunYearMonth = B.RunYearMonth;
} {4 202004 4 202004 5 202004 5 202004}





















finish_test







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
        FROM tableB
        WHERE RunYearMonth = 202004
    ) AS B
    ON A.ID = B.ID
    AND A.RunYearMonth = B.RunYearMonth;
} {4 202004 4 202004 5 202004 5 202004}

# 2023-02-10 https://sqlite.org/forum/forumpost/0a539c76db3b9e29
# The original constant propagation implementation caused a performance
# regression.  Because "abs(v)" was rewritten into "abs(1)" it no longer
# matches the indexed column and the index is not used.
# 
reset_db
do_execsql_test 700 {
  CREATE TABLE t1(v INTEGER);
  WITH RECURSIVE c(x) AS (VALUES(-10) UNION ALL SELECT x+1 FROM c WHERE x<10)
    INSERT INTO t1(v) SELECT x FROM c;
  CREATE INDEX idx ON t1( abs(v) );
  SELECT v FROM t1 WHERE abs(v)=1 and v=1;
} 1
do_eqp_test 710 {
  SELECT v FROM t1 WHERE abs(v)=1 and v=1;
} {
  QUERY PLAN
  `--SEARCH t1 USING INDEX idx (<expr>=?)
}

finish_test