000001 /*
000002 ** 2001 September 15
000003 **
000004 ** The author disclaims copyright to this source code. In place of
000005 ** a legal notice, here is a blessing:
000006 **
000007 ** May you do good and not evil.
000008 ** May you find forgiveness for yourself and forgive others.
000009 ** May you share freely, never taking more than you give.
000010 **
000011 *************************************************************************
000012 ** This module contains C code that generates VDBE code used to process
000013 ** the WHERE clause of SQL statements. This module is responsible for
000014 ** generating the code that loops through a table looking for applicable
000015 ** rows. Indices are selected and used to speed the search when doing
000016 ** so is applicable. Because this module is responsible for selecting
000017 ** indices, you might also think of this module as the "query optimizer".
000018 */
000019 #include "sqliteInt.h"
000020 #include "whereInt.h"
000021
000022 /*
000023 ** Extra information appended to the end of sqlite3_index_info but not
000024 ** visible to the xBestIndex function, at least not directly. The
000025 ** sqlite3_vtab_collation() interface knows how to reach it, however.
000026 **
000027 ** This object is not an API and can be changed from one release to the
000028 ** next. As long as allocateIndexInfo() and sqlite3_vtab_collation()
000029 ** agree on the structure, all will be well.
000030 */
000031 typedef struct HiddenIndexInfo HiddenIndexInfo;
000032 struct HiddenIndexInfo {
000033 WhereClause *pWC; /* The Where clause being analyzed */
000034 Parse *pParse; /* The parsing context */
000035 int eDistinct; /* Value to return from sqlite3_vtab_distinct() */
000036 u32 mIn; /* Mask of terms that are <col> IN (...) */
000037 u32 mHandleIn; /* Terms that vtab will handle as <col> IN (...) */
000038 sqlite3_value *aRhs[1]; /* RHS values for constraints. MUST BE LAST
000039 ** because extra space is allocated to hold up
000040 ** to nTerm such values */
000041 };
000042
000043 /* Forward declaration of methods */
000044 static int whereLoopResize(sqlite3*, WhereLoop*, int);
000045
000046 /*
000047 ** Return the estimated number of output rows from a WHERE clause
000048 */
000049 LogEst sqlite3WhereOutputRowCount(WhereInfo *pWInfo){
000050 return pWInfo->nRowOut;
000051 }
000052
000053 /*
000054 ** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this
000055 ** WHERE clause returns outputs for DISTINCT processing.
000056 */
000057 int sqlite3WhereIsDistinct(WhereInfo *pWInfo){
000058 return pWInfo->eDistinct;
000059 }
000060
000061 /*
000062 ** Return the number of ORDER BY terms that are satisfied by the
000063 ** WHERE clause. A return of 0 means that the output must be
000064 ** completely sorted. A return equal to the number of ORDER BY
000065 ** terms means that no sorting is needed at all. A return that
000066 ** is positive but less than the number of ORDER BY terms means that
000067 ** block sorting is required.
000068 */
000069 int sqlite3WhereIsOrdered(WhereInfo *pWInfo){
000070 return pWInfo->nOBSat<0 ? 0 : pWInfo->nOBSat;
000071 }
000072
000073 /*
000074 ** In the ORDER BY LIMIT optimization, if the inner-most loop is known
000075 ** to emit rows in increasing order, and if the last row emitted by the
000076 ** inner-most loop did not fit within the sorter, then we can skip all
000077 ** subsequent rows for the current iteration of the inner loop (because they
000078 ** will not fit in the sorter either) and continue with the second inner
000079 ** loop - the loop immediately outside the inner-most.
000080 **
000081 ** When a row does not fit in the sorter (because the sorter already
000082 ** holds LIMIT+OFFSET rows that are smaller), then a jump is made to the
000083 ** label returned by this function.
000084 **
000085 ** If the ORDER BY LIMIT optimization applies, the jump destination should
000086 ** be the continuation for the second-inner-most loop. If the ORDER BY
000087 ** LIMIT optimization does not apply, then the jump destination should
000088 ** be the continuation for the inner-most loop.
000089 **
000090 ** It is always safe for this routine to return the continuation of the
000091 ** inner-most loop, in the sense that a correct answer will result.
000092 ** Returning the continuation the second inner loop is an optimization
000093 ** that might make the code run a little faster, but should not change
000094 ** the final answer.
000095 */
000096 int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){
000097 WhereLevel *pInner;
000098 if( !pWInfo->bOrderedInnerLoop ){
000099 /* The ORDER BY LIMIT optimization does not apply. Jump to the
000100 ** continuation of the inner-most loop. */
000101 return pWInfo->iContinue;
000102 }
000103 pInner = &pWInfo->a[pWInfo->nLevel-1];
000104 assert( pInner->addrNxt!=0 );
000105 return pInner->pRJ ? pWInfo->iContinue : pInner->addrNxt;
000106 }
000107
000108 /*
000109 ** While generating code for the min/max optimization, after handling
000110 ** the aggregate-step call to min() or max(), check to see if any
000111 ** additional looping is required. If the output order is such that
000112 ** we are certain that the correct answer has already been found, then
000113 ** code an OP_Goto to by pass subsequent processing.
000114 **
000115 ** Any extra OP_Goto that is coded here is an optimization. The
000116 ** correct answer should be obtained regardless. This OP_Goto just
000117 ** makes the answer appear faster.
000118 */
000119 void sqlite3WhereMinMaxOptEarlyOut(Vdbe *v, WhereInfo *pWInfo){
000120 WhereLevel *pInner;
000121 int i;
000122 if( !pWInfo->bOrderedInnerLoop ) return;
000123 if( pWInfo->nOBSat==0 ) return;
000124 for(i=pWInfo->nLevel-1; i>=0; i--){
000125 pInner = &pWInfo->a[i];
000126 if( (pInner->pWLoop->wsFlags & WHERE_COLUMN_IN)!=0 ){
000127 sqlite3VdbeGoto(v, pInner->addrNxt);
000128 return;
000129 }
000130 }
000131 sqlite3VdbeGoto(v, pWInfo->iBreak);
000132 }
000133
000134 /*
000135 ** Return the VDBE address or label to jump to in order to continue
000136 ** immediately with the next row of a WHERE clause.
000137 */
000138 int sqlite3WhereContinueLabel(WhereInfo *pWInfo){
000139 assert( pWInfo->iContinue!=0 );
000140 return pWInfo->iContinue;
000141 }
000142
000143 /*
000144 ** Return the VDBE address or label to jump to in order to break
000145 ** out of a WHERE loop.
000146 */
000147 int sqlite3WhereBreakLabel(WhereInfo *pWInfo){
000148 return pWInfo->iBreak;
000149 }
000150
000151 /*
000152 ** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to
000153 ** operate directly on the rowids returned by a WHERE clause. Return
000154 ** ONEPASS_SINGLE (1) if the statement can operation directly because only
000155 ** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass
000156 ** optimization can be used on multiple
000157 **
000158 ** If the ONEPASS optimization is used (if this routine returns true)
000159 ** then also write the indices of open cursors used by ONEPASS
000160 ** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data
000161 ** table and iaCur[1] gets the cursor used by an auxiliary index.
000162 ** Either value may be -1, indicating that cursor is not used.
000163 ** Any cursors returned will have been opened for writing.
000164 **
000165 ** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is
000166 ** unable to use the ONEPASS optimization.
000167 */
000168 int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){
000169 memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2);
000170 #ifdef WHERETRACE_ENABLED
000171 if( sqlite3WhereTrace && pWInfo->eOnePass!=ONEPASS_OFF ){
000172 sqlite3DebugPrintf("%s cursors: %d %d\n",
000173 pWInfo->eOnePass==ONEPASS_SINGLE ? "ONEPASS_SINGLE" : "ONEPASS_MULTI",
000174 aiCur[0], aiCur[1]);
000175 }
000176 #endif
000177 return pWInfo->eOnePass;
000178 }
000179
000180 /*
000181 ** Return TRUE if the WHERE loop uses the OP_DeferredSeek opcode to move
000182 ** the data cursor to the row selected by the index cursor.
000183 */
000184 int sqlite3WhereUsesDeferredSeek(WhereInfo *pWInfo){
000185 return pWInfo->bDeferredSeek;
000186 }
000187
000188 /*
000189 ** Move the content of pSrc into pDest
000190 */
000191 static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){
000192 pDest->n = pSrc->n;
000193 memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0]));
000194 }
000195
000196 /*
000197 ** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet.
000198 **
000199 ** The new entry might overwrite an existing entry, or it might be
000200 ** appended, or it might be discarded. Do whatever is the right thing
000201 ** so that pSet keeps the N_OR_COST best entries seen so far.
000202 */
000203 static int whereOrInsert(
000204 WhereOrSet *pSet, /* The WhereOrSet to be updated */
000205 Bitmask prereq, /* Prerequisites of the new entry */
000206 LogEst rRun, /* Run-cost of the new entry */
000207 LogEst nOut /* Number of outputs for the new entry */
000208 ){
000209 u16 i;
000210 WhereOrCost *p;
000211 for(i=pSet->n, p=pSet->a; i>0; i--, p++){
000212 if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){
000213 goto whereOrInsert_done;
000214 }
000215 if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){
000216 return 0;
000217 }
000218 }
000219 if( pSet->n<N_OR_COST ){
000220 p = &pSet->a[pSet->n++];
000221 p->nOut = nOut;
000222 }else{
000223 p = pSet->a;
000224 for(i=1; i<pSet->n; i++){
000225 if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i;
000226 }
000227 if( p->rRun<=rRun ) return 0;
000228 }
000229 whereOrInsert_done:
000230 p->prereq = prereq;
000231 p->rRun = rRun;
000232 if( p->nOut>nOut ) p->nOut = nOut;
000233 return 1;
000234 }
000235
000236 /*
000237 ** Return the bitmask for the given cursor number. Return 0 if
000238 ** iCursor is not in the set.
000239 */
000240 Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){
000241 int i;
000242 assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 );
000243 assert( pMaskSet->n>0 || pMaskSet->ix[0]<0 );
000244 assert( iCursor>=-1 );
000245 if( pMaskSet->ix[0]==iCursor ){
000246 return 1;
000247 }
000248 for(i=1; i<pMaskSet->n; i++){
000249 if( pMaskSet->ix[i]==iCursor ){
000250 return MASKBIT(i);
000251 }
000252 }
000253 return 0;
000254 }
000255
000256 /* Allocate memory that is automatically freed when pWInfo is freed.
000257 */
000258 void *sqlite3WhereMalloc(WhereInfo *pWInfo, u64 nByte){
000259 WhereMemBlock *pBlock;
000260 pBlock = sqlite3DbMallocRawNN(pWInfo->pParse->db, nByte+sizeof(*pBlock));
000261 if( pBlock ){
000262 pBlock->pNext = pWInfo->pMemToFree;
000263 pBlock->sz = nByte;
000264 pWInfo->pMemToFree = pBlock;
000265 pBlock++;
000266 }
000267 return (void*)pBlock;
000268 }
000269 void *sqlite3WhereRealloc(WhereInfo *pWInfo, void *pOld, u64 nByte){
000270 void *pNew = sqlite3WhereMalloc(pWInfo, nByte);
000271 if( pNew && pOld ){
000272 WhereMemBlock *pOldBlk = (WhereMemBlock*)pOld;
000273 pOldBlk--;
000274 assert( pOldBlk->sz<nByte );
000275 memcpy(pNew, pOld, pOldBlk->sz);
000276 }
000277 return pNew;
000278 }
000279
000280 /*
000281 ** Create a new mask for cursor iCursor.
000282 **
000283 ** There is one cursor per table in the FROM clause. The number of
000284 ** tables in the FROM clause is limited by a test early in the
000285 ** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[]
000286 ** array will never overflow.
000287 */
000288 static void createMask(WhereMaskSet *pMaskSet, int iCursor){
000289 assert( pMaskSet->n < ArraySize(pMaskSet->ix) );
000290 pMaskSet->ix[pMaskSet->n++] = iCursor;
000291 }
000292
000293 /*
000294 ** If the right-hand branch of the expression is a TK_COLUMN, then return
000295 ** a pointer to the right-hand branch. Otherwise, return NULL.
000296 */
000297 static Expr *whereRightSubexprIsColumn(Expr *p){
000298 p = sqlite3ExprSkipCollateAndLikely(p->pRight);
000299 if( ALWAYS(p!=0) && p->op==TK_COLUMN && !ExprHasProperty(p, EP_FixedCol) ){
000300 return p;
000301 }
000302 return 0;
000303 }
000304
000305 /*
000306 ** Advance to the next WhereTerm that matches according to the criteria
000307 ** established when the pScan object was initialized by whereScanInit().
000308 ** Return NULL if there are no more matching WhereTerms.
000309 */
000310 static WhereTerm *whereScanNext(WhereScan *pScan){
000311 int iCur; /* The cursor on the LHS of the term */
000312 i16 iColumn; /* The column on the LHS of the term. -1 for IPK */
000313 Expr *pX; /* An expression being tested */
000314 WhereClause *pWC; /* Shorthand for pScan->pWC */
000315 WhereTerm *pTerm; /* The term being tested */
000316 int k = pScan->k; /* Where to start scanning */
000317
000318 assert( pScan->iEquiv<=pScan->nEquiv );
000319 pWC = pScan->pWC;
000320 while(1){
000321 iColumn = pScan->aiColumn[pScan->iEquiv-1];
000322 iCur = pScan->aiCur[pScan->iEquiv-1];
000323 assert( pWC!=0 );
000324 assert( iCur>=0 );
000325 do{
000326 for(pTerm=pWC->a+k; k<pWC->nTerm; k++, pTerm++){
000327 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 || pTerm->leftCursor<0 );
000328 if( pTerm->leftCursor==iCur
000329 && pTerm->u.x.leftColumn==iColumn
000330 && (iColumn!=XN_EXPR
000331 || sqlite3ExprCompareSkip(pTerm->pExpr->pLeft,
000332 pScan->pIdxExpr,iCur)==0)
000333 && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_OuterON))
000334 ){
000335 if( (pTerm->eOperator & WO_EQUIV)!=0
000336 && pScan->nEquiv<ArraySize(pScan->aiCur)
000337 && (pX = whereRightSubexprIsColumn(pTerm->pExpr))!=0
000338 ){
000339 int j;
000340 for(j=0; j<pScan->nEquiv; j++){
000341 if( pScan->aiCur[j]==pX->iTable
000342 && pScan->aiColumn[j]==pX->iColumn ){
000343 break;
000344 }
000345 }
000346 if( j==pScan->nEquiv ){
000347 pScan->aiCur[j] = pX->iTable;
000348 pScan->aiColumn[j] = pX->iColumn;
000349 pScan->nEquiv++;
000350 }
000351 }
000352 if( (pTerm->eOperator & pScan->opMask)!=0 ){
000353 /* Verify the affinity and collating sequence match */
000354 if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){
000355 CollSeq *pColl;
000356 Parse *pParse = pWC->pWInfo->pParse;
000357 pX = pTerm->pExpr;
000358 if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){
000359 continue;
000360 }
000361 assert(pX->pLeft);
000362 pColl = sqlite3ExprCompareCollSeq(pParse, pX);
000363 if( pColl==0 ) pColl = pParse->db->pDfltColl;
000364 if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){
000365 continue;
000366 }
000367 }
000368 if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0
000369 && (pX = pTerm->pExpr->pRight, ALWAYS(pX!=0))
000370 && pX->op==TK_COLUMN
000371 && pX->iTable==pScan->aiCur[0]
000372 && pX->iColumn==pScan->aiColumn[0]
000373 ){
000374 testcase( pTerm->eOperator & WO_IS );
000375 continue;
000376 }
000377 pScan->pWC = pWC;
000378 pScan->k = k+1;
000379 #ifdef WHERETRACE_ENABLED
000380 if( sqlite3WhereTrace & 0x20000 ){
000381 int ii;
000382 sqlite3DebugPrintf("SCAN-TERM %p: nEquiv=%d",
000383 pTerm, pScan->nEquiv);
000384 for(ii=0; ii<pScan->nEquiv; ii++){
000385 sqlite3DebugPrintf(" {%d:%d}",
000386 pScan->aiCur[ii], pScan->aiColumn[ii]);
000387 }
000388 sqlite3DebugPrintf("\n");
000389 }
000390 #endif
000391 return pTerm;
000392 }
000393 }
000394 }
000395 pWC = pWC->pOuter;
000396 k = 0;
000397 }while( pWC!=0 );
000398 if( pScan->iEquiv>=pScan->nEquiv ) break;
000399 pWC = pScan->pOrigWC;
000400 k = 0;
000401 pScan->iEquiv++;
000402 }
000403 return 0;
000404 }
000405
000406 /*
000407 ** This is whereScanInit() for the case of an index on an expression.
000408 ** It is factored out into a separate tail-recursion subroutine so that
000409 ** the normal whereScanInit() routine, which is a high-runner, does not
000410 ** need to push registers onto the stack as part of its prologue.
000411 */
000412 static SQLITE_NOINLINE WhereTerm *whereScanInitIndexExpr(WhereScan *pScan){
000413 pScan->idxaff = sqlite3ExprAffinity(pScan->pIdxExpr);
000414 return whereScanNext(pScan);
000415 }
000416
000417 /*
000418 ** Initialize a WHERE clause scanner object. Return a pointer to the
000419 ** first match. Return NULL if there are no matches.
000420 **
000421 ** The scanner will be searching the WHERE clause pWC. It will look
000422 ** for terms of the form "X <op> <expr>" where X is column iColumn of table
000423 ** iCur. Or if pIdx!=0 then X is column iColumn of index pIdx. pIdx
000424 ** must be one of the indexes of table iCur.
000425 **
000426 ** The <op> must be one of the operators described by opMask.
000427 **
000428 ** If the search is for X and the WHERE clause contains terms of the
000429 ** form X=Y then this routine might also return terms of the form
000430 ** "Y <op> <expr>". The number of levels of transitivity is limited,
000431 ** but is enough to handle most commonly occurring SQL statements.
000432 **
000433 ** If X is not the INTEGER PRIMARY KEY then X must be compatible with
000434 ** index pIdx.
000435 */
000436 static WhereTerm *whereScanInit(
000437 WhereScan *pScan, /* The WhereScan object being initialized */
000438 WhereClause *pWC, /* The WHERE clause to be scanned */
000439 int iCur, /* Cursor to scan for */
000440 int iColumn, /* Column to scan for */
000441 u32 opMask, /* Operator(s) to scan for */
000442 Index *pIdx /* Must be compatible with this index */
000443 ){
000444 pScan->pOrigWC = pWC;
000445 pScan->pWC = pWC;
000446 pScan->pIdxExpr = 0;
000447 pScan->idxaff = 0;
000448 pScan->zCollName = 0;
000449 pScan->opMask = opMask;
000450 pScan->k = 0;
000451 pScan->aiCur[0] = iCur;
000452 pScan->nEquiv = 1;
000453 pScan->iEquiv = 1;
000454 if( pIdx ){
000455 int j = iColumn;
000456 iColumn = pIdx->aiColumn[j];
000457 if( iColumn==pIdx->pTable->iPKey ){
000458 iColumn = XN_ROWID;
000459 }else if( iColumn>=0 ){
000460 pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity;
000461 pScan->zCollName = pIdx->azColl[j];
000462 }else if( iColumn==XN_EXPR ){
000463 pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr;
000464 pScan->zCollName = pIdx->azColl[j];
000465 pScan->aiColumn[0] = XN_EXPR;
000466 return whereScanInitIndexExpr(pScan);
000467 }
000468 }else if( iColumn==XN_EXPR ){
000469 return 0;
000470 }
000471 pScan->aiColumn[0] = iColumn;
000472 return whereScanNext(pScan);
000473 }
000474
000475 /*
000476 ** Search for a term in the WHERE clause that is of the form "X <op> <expr>"
000477 ** where X is a reference to the iColumn of table iCur or of index pIdx
000478 ** if pIdx!=0 and <op> is one of the WO_xx operator codes specified by
000479 ** the op parameter. Return a pointer to the term. Return 0 if not found.
000480 **
000481 ** If pIdx!=0 then it must be one of the indexes of table iCur.
000482 ** Search for terms matching the iColumn-th column of pIdx
000483 ** rather than the iColumn-th column of table iCur.
000484 **
000485 ** The term returned might by Y=<expr> if there is another constraint in
000486 ** the WHERE clause that specifies that X=Y. Any such constraints will be
000487 ** identified by the WO_EQUIV bit in the pTerm->eOperator field. The
000488 ** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11
000489 ** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10
000490 ** other equivalent values. Hence a search for X will return <expr> if X=A1
000491 ** and A1=A2 and A2=A3 and ... and A9=A10 and A10=<expr>.
000492 **
000493 ** If there are multiple terms in the WHERE clause of the form "X <op> <expr>"
000494 ** then try for the one with no dependencies on <expr> - in other words where
000495 ** <expr> is a constant expression of some kind. Only return entries of
000496 ** the form "X <op> Y" where Y is a column in another table if no terms of
000497 ** the form "X <op> <const-expr>" exist. If no terms with a constant RHS
000498 ** exist, try to return a term that does not use WO_EQUIV.
000499 */
000500 WhereTerm *sqlite3WhereFindTerm(
000501 WhereClause *pWC, /* The WHERE clause to be searched */
000502 int iCur, /* Cursor number of LHS */
000503 int iColumn, /* Column number of LHS */
000504 Bitmask notReady, /* RHS must not overlap with this mask */
000505 u32 op, /* Mask of WO_xx values describing operator */
000506 Index *pIdx /* Must be compatible with this index, if not NULL */
000507 ){
000508 WhereTerm *pResult = 0;
000509 WhereTerm *p;
000510 WhereScan scan;
000511
000512 p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx);
000513 op &= WO_EQ|WO_IS;
000514 while( p ){
000515 if( (p->prereqRight & notReady)==0 ){
000516 if( p->prereqRight==0 && (p->eOperator&op)!=0 ){
000517 testcase( p->eOperator & WO_IS );
000518 return p;
000519 }
000520 if( pResult==0 ) pResult = p;
000521 }
000522 p = whereScanNext(&scan);
000523 }
000524 return pResult;
000525 }
000526
000527 /*
000528 ** This function searches pList for an entry that matches the iCol-th column
000529 ** of index pIdx.
000530 **
000531 ** If such an expression is found, its index in pList->a[] is returned. If
000532 ** no expression is found, -1 is returned.
000533 */
000534 static int findIndexCol(
000535 Parse *pParse, /* Parse context */
000536 ExprList *pList, /* Expression list to search */
000537 int iBase, /* Cursor for table associated with pIdx */
000538 Index *pIdx, /* Index to match column of */
000539 int iCol /* Column of index to match */
000540 ){
000541 int i;
000542 const char *zColl = pIdx->azColl[iCol];
000543
000544 for(i=0; i<pList->nExpr; i++){
000545 Expr *p = sqlite3ExprSkipCollateAndLikely(pList->a[i].pExpr);
000546 if( ALWAYS(p!=0)
000547 && (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN)
000548 && p->iColumn==pIdx->aiColumn[iCol]
000549 && p->iTable==iBase
000550 ){
000551 CollSeq *pColl = sqlite3ExprNNCollSeq(pParse, pList->a[i].pExpr);
000552 if( 0==sqlite3StrICmp(pColl->zName, zColl) ){
000553 return i;
000554 }
000555 }
000556 }
000557
000558 return -1;
000559 }
000560
000561 /*
000562 ** Return TRUE if the iCol-th column of index pIdx is NOT NULL
000563 */
000564 static int indexColumnNotNull(Index *pIdx, int iCol){
000565 int j;
000566 assert( pIdx!=0 );
000567 assert( iCol>=0 && iCol<pIdx->nColumn );
000568 j = pIdx->aiColumn[iCol];
000569 if( j>=0 ){
000570 return pIdx->pTable->aCol[j].notNull;
000571 }else if( j==(-1) ){
000572 return 1;
000573 }else{
000574 assert( j==(-2) );
000575 return 0; /* Assume an indexed expression can always yield a NULL */
000576
000577 }
000578 }
000579
000580 /*
000581 ** Return true if the DISTINCT expression-list passed as the third argument
000582 ** is redundant.
000583 **
000584 ** A DISTINCT list is redundant if any subset of the columns in the
000585 ** DISTINCT list are collectively unique and individually non-null.
000586 */
000587 static int isDistinctRedundant(
000588 Parse *pParse, /* Parsing context */
000589 SrcList *pTabList, /* The FROM clause */
000590 WhereClause *pWC, /* The WHERE clause */
000591 ExprList *pDistinct /* The result set that needs to be DISTINCT */
000592 ){
000593 Table *pTab;
000594 Index *pIdx;
000595 int i;
000596 int iBase;
000597
000598 /* If there is more than one table or sub-select in the FROM clause of
000599 ** this query, then it will not be possible to show that the DISTINCT
000600 ** clause is redundant. */
000601 if( pTabList->nSrc!=1 ) return 0;
000602 iBase = pTabList->a[0].iCursor;
000603 pTab = pTabList->a[0].pTab;
000604
000605 /* If any of the expressions is an IPK column on table iBase, then return
000606 ** true. Note: The (p->iTable==iBase) part of this test may be false if the
000607 ** current SELECT is a correlated sub-query.
000608 */
000609 for(i=0; i<pDistinct->nExpr; i++){
000610 Expr *p = sqlite3ExprSkipCollateAndLikely(pDistinct->a[i].pExpr);
000611 if( NEVER(p==0) ) continue;
000612 if( p->op!=TK_COLUMN && p->op!=TK_AGG_COLUMN ) continue;
000613 if( p->iTable==iBase && p->iColumn<0 ) return 1;
000614 }
000615
000616 /* Loop through all indices on the table, checking each to see if it makes
000617 ** the DISTINCT qualifier redundant. It does so if:
000618 **
000619 ** 1. The index is itself UNIQUE, and
000620 **
000621 ** 2. All of the columns in the index are either part of the pDistinct
000622 ** list, or else the WHERE clause contains a term of the form "col=X",
000623 ** where X is a constant value. The collation sequences of the
000624 ** comparison and select-list expressions must match those of the index.
000625 **
000626 ** 3. All of those index columns for which the WHERE clause does not
000627 ** contain a "col=X" term are subject to a NOT NULL constraint.
000628 */
000629 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
000630 if( !IsUniqueIndex(pIdx) ) continue;
000631 if( pIdx->pPartIdxWhere ) continue;
000632 for(i=0; i<pIdx->nKeyCol; i++){
000633 if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){
000634 if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break;
000635 if( indexColumnNotNull(pIdx, i)==0 ) break;
000636 }
000637 }
000638 if( i==pIdx->nKeyCol ){
000639 /* This index implies that the DISTINCT qualifier is redundant. */
000640 return 1;
000641 }
000642 }
000643
000644 return 0;
000645 }
000646
000647
000648 /*
000649 ** Estimate the logarithm of the input value to base 2.
000650 */
000651 static LogEst estLog(LogEst N){
000652 return N<=10 ? 0 : sqlite3LogEst(N) - 33;
000653 }
000654
000655 /*
000656 ** Convert OP_Column opcodes to OP_Copy in previously generated code.
000657 **
000658 ** This routine runs over generated VDBE code and translates OP_Column
000659 ** opcodes into OP_Copy when the table is being accessed via co-routine
000660 ** instead of via table lookup.
000661 **
000662 ** If the iAutoidxCur is not zero, then any OP_Rowid instructions on
000663 ** cursor iTabCur are transformed into OP_Sequence opcode for the
000664 ** iAutoidxCur cursor, in order to generate unique rowids for the
000665 ** automatic index being generated.
000666 */
000667 static void translateColumnToCopy(
000668 Parse *pParse, /* Parsing context */
000669 int iStart, /* Translate from this opcode to the end */
000670 int iTabCur, /* OP_Column/OP_Rowid references to this table */
000671 int iRegister, /* The first column is in this register */
000672 int iAutoidxCur /* If non-zero, cursor of autoindex being generated */
000673 ){
000674 Vdbe *v = pParse->pVdbe;
000675 VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart);
000676 int iEnd = sqlite3VdbeCurrentAddr(v);
000677 if( pParse->db->mallocFailed ) return;
000678 for(; iStart<iEnd; iStart++, pOp++){
000679 if( pOp->p1!=iTabCur ) continue;
000680 if( pOp->opcode==OP_Column ){
000681 pOp->opcode = OP_Copy;
000682 pOp->p1 = pOp->p2 + iRegister;
000683 pOp->p2 = pOp->p3;
000684 pOp->p3 = 0;
000685 pOp->p5 = 2; /* Cause the MEM_Subtype flag to be cleared */
000686 }else if( pOp->opcode==OP_Rowid ){
000687 pOp->opcode = OP_Sequence;
000688 pOp->p1 = iAutoidxCur;
000689 #ifdef SQLITE_ALLOW_ROWID_IN_VIEW
000690 if( iAutoidxCur==0 ){
000691 pOp->opcode = OP_Null;
000692 pOp->p3 = 0;
000693 }
000694 #endif
000695 }
000696 }
000697 }
000698
000699 /*
000700 ** Two routines for printing the content of an sqlite3_index_info
000701 ** structure. Used for testing and debugging only. If neither
000702 ** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines
000703 ** are no-ops.
000704 */
000705 #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED)
000706 static void whereTraceIndexInfoInputs(sqlite3_index_info *p){
000707 int i;
000708 if( (sqlite3WhereTrace & 0x10)==0 ) return;
000709 for(i=0; i<p->nConstraint; i++){
000710 sqlite3DebugPrintf(
000711 " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n",
000712 i,
000713 p->aConstraint[i].iColumn,
000714 p->aConstraint[i].iTermOffset,
000715 p->aConstraint[i].op,
000716 p->aConstraint[i].usable,
000717 sqlite3_vtab_collation(p,i));
000718 }
000719 for(i=0; i<p->nOrderBy; i++){
000720 sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n",
000721 i,
000722 p->aOrderBy[i].iColumn,
000723 p->aOrderBy[i].desc);
000724 }
000725 }
000726 static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){
000727 int i;
000728 if( (sqlite3WhereTrace & 0x10)==0 ) return;
000729 for(i=0; i<p->nConstraint; i++){
000730 sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n",
000731 i,
000732 p->aConstraintUsage[i].argvIndex,
000733 p->aConstraintUsage[i].omit);
000734 }
000735 sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum);
000736 sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr);
000737 sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed);
000738 sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost);
000739 sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows);
000740 }
000741 #else
000742 #define whereTraceIndexInfoInputs(A)
000743 #define whereTraceIndexInfoOutputs(A)
000744 #endif
000745
000746 /*
000747 ** We know that pSrc is an operand of an outer join. Return true if
000748 ** pTerm is a constraint that is compatible with that join.
000749 **
000750 ** pTerm must be EP_OuterON if pSrc is the right operand of an
000751 ** outer join. pTerm can be either EP_OuterON or EP_InnerON if pSrc
000752 ** is the left operand of a RIGHT join.
000753 **
000754 ** See https://sqlite.org/forum/forumpost/206d99a16dd9212f
000755 ** for an example of a WHERE clause constraints that may not be used on
000756 ** the right table of a RIGHT JOIN because the constraint implies a
000757 ** not-NULL condition on the left table of the RIGHT JOIN.
000758 */
000759 static int constraintCompatibleWithOuterJoin(
000760 const WhereTerm *pTerm, /* WHERE clause term to check */
000761 const SrcItem *pSrc /* Table we are trying to access */
000762 ){
000763 assert( (pSrc->fg.jointype&(JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ); /* By caller */
000764 testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT );
000765 testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ );
000766 testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) )
000767 testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) );
000768 if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON)
000769 || pTerm->pExpr->w.iJoin != pSrc->iCursor
000770 ){
000771 return 0;
000772 }
000773 if( (pSrc->fg.jointype & (JT_LEFT|JT_RIGHT))!=0
000774 && ExprHasProperty(pTerm->pExpr, EP_InnerON)
000775 ){
000776 return 0;
000777 }
000778 return 1;
000779 }
000780
000781
000782
000783 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
000784 /*
000785 ** Return TRUE if the WHERE clause term pTerm is of a form where it
000786 ** could be used with an index to access pSrc, assuming an appropriate
000787 ** index existed.
000788 */
000789 static int termCanDriveIndex(
000790 const WhereTerm *pTerm, /* WHERE clause term to check */
000791 const SrcItem *pSrc, /* Table we are trying to access */
000792 const Bitmask notReady /* Tables in outer loops of the join */
000793 ){
000794 char aff;
000795 if( pTerm->leftCursor!=pSrc->iCursor ) return 0;
000796 if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0;
000797 assert( (pSrc->fg.jointype & JT_RIGHT)==0 );
000798 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
000799 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
000800 ){
000801 return 0; /* See https://sqlite.org/forum/forumpost/51e6959f61 */
000802 }
000803 if( (pTerm->prereqRight & notReady)!=0 ) return 0;
000804 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
000805 if( pTerm->u.x.leftColumn<0 ) return 0;
000806 aff = pSrc->pTab->aCol[pTerm->u.x.leftColumn].affinity;
000807 if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0;
000808 testcase( pTerm->pExpr->op==TK_IS );
000809 return 1;
000810 }
000811 #endif
000812
000813
000814 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
000815
000816 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
000817 /*
000818 ** Argument pIdx represents an automatic index that the current statement
000819 ** will create and populate. Add an OP_Explain with text of the form:
000820 **
000821 ** CREATE AUTOMATIC INDEX ON <table>(<cols>) [WHERE <expr>]
000822 **
000823 ** This is only required if sqlite3_stmt_scanstatus() is enabled, to
000824 ** associate an SQLITE_SCANSTAT_NCYCLE and SQLITE_SCANSTAT_NLOOP
000825 ** values with. In order to avoid breaking legacy code and test cases,
000826 ** the OP_Explain is not added if this is an EXPLAIN QUERY PLAN command.
000827 */
000828 static void explainAutomaticIndex(
000829 Parse *pParse,
000830 Index *pIdx, /* Automatic index to explain */
000831 int bPartial, /* True if pIdx is a partial index */
000832 int *pAddrExplain /* OUT: Address of OP_Explain */
000833 ){
000834 if( IS_STMT_SCANSTATUS(pParse->db) && pParse->explain!=2 ){
000835 Table *pTab = pIdx->pTable;
000836 const char *zSep = "";
000837 char *zText = 0;
000838 int ii = 0;
000839 sqlite3_str *pStr = sqlite3_str_new(pParse->db);
000840 sqlite3_str_appendf(pStr,"CREATE AUTOMATIC INDEX ON %s(", pTab->zName);
000841 assert( pIdx->nColumn>1 );
000842 assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID );
000843 for(ii=0; ii<(pIdx->nColumn-1); ii++){
000844 const char *zName = 0;
000845 int iCol = pIdx->aiColumn[ii];
000846
000847 zName = pTab->aCol[iCol].zCnName;
000848 sqlite3_str_appendf(pStr, "%s%s", zSep, zName);
000849 zSep = ", ";
000850 }
000851 zText = sqlite3_str_finish(pStr);
000852 if( zText==0 ){
000853 sqlite3OomFault(pParse->db);
000854 }else{
000855 *pAddrExplain = sqlite3VdbeExplain(
000856 pParse, 0, "%s)%s", zText, (bPartial ? " WHERE <expr>" : "")
000857 );
000858 sqlite3_free(zText);
000859 }
000860 }
000861 }
000862 #else
000863 # define explainAutomaticIndex(a,b,c,d)
000864 #endif
000865
000866 /*
000867 ** Generate code to construct the Index object for an automatic index
000868 ** and to set up the WhereLevel object pLevel so that the code generator
000869 ** makes use of the automatic index.
000870 */
000871 static SQLITE_NOINLINE void constructAutomaticIndex(
000872 Parse *pParse, /* The parsing context */
000873 WhereClause *pWC, /* The WHERE clause */
000874 const Bitmask notReady, /* Mask of cursors that are not available */
000875 WhereLevel *pLevel /* Write new index here */
000876 ){
000877 int nKeyCol; /* Number of columns in the constructed index */
000878 WhereTerm *pTerm; /* A single term of the WHERE clause */
000879 WhereTerm *pWCEnd; /* End of pWC->a[] */
000880 Index *pIdx; /* Object describing the transient index */
000881 Vdbe *v; /* Prepared statement under construction */
000882 int addrInit; /* Address of the initialization bypass jump */
000883 Table *pTable; /* The table being indexed */
000884 int addrTop; /* Top of the index fill loop */
000885 int regRecord; /* Register holding an index record */
000886 int n; /* Column counter */
000887 int i; /* Loop counter */
000888 int mxBitCol; /* Maximum column in pSrc->colUsed */
000889 CollSeq *pColl; /* Collating sequence to on a column */
000890 WhereLoop *pLoop; /* The Loop object */
000891 char *zNotUsed; /* Extra space on the end of pIdx */
000892 Bitmask idxCols; /* Bitmap of columns used for indexing */
000893 Bitmask extraCols; /* Bitmap of additional columns */
000894 u8 sentWarning = 0; /* True if a warning has been issued */
000895 u8 useBloomFilter = 0; /* True to also add a Bloom filter */
000896 Expr *pPartial = 0; /* Partial Index Expression */
000897 int iContinue = 0; /* Jump here to skip excluded rows */
000898 SrcList *pTabList; /* The complete FROM clause */
000899 SrcItem *pSrc; /* The FROM clause term to get the next index */
000900 int addrCounter = 0; /* Address where integer counter is initialized */
000901 int regBase; /* Array of registers where record is assembled */
000902 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
000903 int addrExp = 0; /* Address of OP_Explain */
000904 #endif
000905
000906 /* Generate code to skip over the creation and initialization of the
000907 ** transient index on 2nd and subsequent iterations of the loop. */
000908 v = pParse->pVdbe;
000909 assert( v!=0 );
000910 addrInit = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
000911
000912 /* Count the number of columns that will be added to the index
000913 ** and used to match WHERE clause constraints */
000914 nKeyCol = 0;
000915 pTabList = pWC->pWInfo->pTabList;
000916 pSrc = &pTabList->a[pLevel->iFrom];
000917 pTable = pSrc->pTab;
000918 pWCEnd = &pWC->a[pWC->nTerm];
000919 pLoop = pLevel->pWLoop;
000920 idxCols = 0;
000921 for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
000922 Expr *pExpr = pTerm->pExpr;
000923 /* Make the automatic index a partial index if there are terms in the
000924 ** WHERE clause (or the ON clause of a LEFT join) that constrain which
000925 ** rows of the target table (pSrc) that can be used. */
000926 if( (pTerm->wtFlags & TERM_VIRTUAL)==0
000927 && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom)
000928 ){
000929 pPartial = sqlite3ExprAnd(pParse, pPartial,
000930 sqlite3ExprDup(pParse->db, pExpr, 0));
000931 }
000932 if( termCanDriveIndex(pTerm, pSrc, notReady) ){
000933 int iCol;
000934 Bitmask cMask;
000935 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
000936 iCol = pTerm->u.x.leftColumn;
000937 cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
000938 testcase( iCol==BMS );
000939 testcase( iCol==BMS-1 );
000940 if( !sentWarning ){
000941 sqlite3_log(SQLITE_WARNING_AUTOINDEX,
000942 "automatic index on %s(%s)", pTable->zName,
000943 pTable->aCol[iCol].zCnName);
000944 sentWarning = 1;
000945 }
000946 if( (idxCols & cMask)==0 ){
000947 if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){
000948 goto end_auto_index_create;
000949 }
000950 pLoop->aLTerm[nKeyCol++] = pTerm;
000951 idxCols |= cMask;
000952 }
000953 }
000954 }
000955 assert( nKeyCol>0 || pParse->db->mallocFailed );
000956 pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol;
000957 pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED
000958 | WHERE_AUTO_INDEX;
000959
000960 /* Count the number of additional columns needed to create a
000961 ** covering index. A "covering index" is an index that contains all
000962 ** columns that are needed by the query. With a covering index, the
000963 ** original table never needs to be accessed. Automatic indices must
000964 ** be a covering index because the index will not be updated if the
000965 ** original table changes and the index and table cannot both be used
000966 ** if they go out of sync.
000967 */
000968 if( IsView(pTable) ){
000969 extraCols = ALLBITS;
000970 }else{
000971 extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1));
000972 }
000973 mxBitCol = MIN(BMS-1,pTable->nCol);
000974 testcase( pTable->nCol==BMS-1 );
000975 testcase( pTable->nCol==BMS-2 );
000976 for(i=0; i<mxBitCol; i++){
000977 if( extraCols & MASKBIT(i) ) nKeyCol++;
000978 }
000979 if( pSrc->colUsed & MASKBIT(BMS-1) ){
000980 nKeyCol += pTable->nCol - BMS + 1;
000981 }
000982
000983 /* Construct the Index object to describe this index */
000984 pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed);
000985 if( pIdx==0 ) goto end_auto_index_create;
000986 pLoop->u.btree.pIndex = pIdx;
000987 pIdx->zName = "auto-index";
000988 pIdx->pTable = pTable;
000989 n = 0;
000990 idxCols = 0;
000991 for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
000992 if( termCanDriveIndex(pTerm, pSrc, notReady) ){
000993 int iCol;
000994 Bitmask cMask;
000995 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
000996 iCol = pTerm->u.x.leftColumn;
000997 cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
000998 testcase( iCol==BMS-1 );
000999 testcase( iCol==BMS );
001000 if( (idxCols & cMask)==0 ){
001001 Expr *pX = pTerm->pExpr;
001002 idxCols |= cMask;
001003 pIdx->aiColumn[n] = pTerm->u.x.leftColumn;
001004 pColl = sqlite3ExprCompareCollSeq(pParse, pX);
001005 assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */
001006 pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY;
001007 n++;
001008 if( ALWAYS(pX->pLeft!=0)
001009 && sqlite3ExprAffinity(pX->pLeft)!=SQLITE_AFF_TEXT
001010 ){
001011 /* TUNING: only use a Bloom filter on an automatic index
001012 ** if one or more key columns has the ability to hold numeric
001013 ** values, since strings all have the same hash in the Bloom
001014 ** filter implementation and hence a Bloom filter on a text column
001015 ** is not usually helpful. */
001016 useBloomFilter = 1;
001017 }
001018 }
001019 }
001020 }
001021 assert( (u32)n==pLoop->u.btree.nEq );
001022
001023 /* Add additional columns needed to make the automatic index into
001024 ** a covering index */
001025 for(i=0; i<mxBitCol; i++){
001026 if( extraCols & MASKBIT(i) ){
001027 pIdx->aiColumn[n] = i;
001028 pIdx->azColl[n] = sqlite3StrBINARY;
001029 n++;
001030 }
001031 }
001032 if( pSrc->colUsed & MASKBIT(BMS-1) ){
001033 for(i=BMS-1; i<pTable->nCol; i++){
001034 pIdx->aiColumn[n] = i;
001035 pIdx->azColl[n] = sqlite3StrBINARY;
001036 n++;
001037 }
001038 }
001039 assert( n==nKeyCol );
001040 pIdx->aiColumn[n] = XN_ROWID;
001041 pIdx->azColl[n] = sqlite3StrBINARY;
001042
001043 /* Create the automatic index */
001044 explainAutomaticIndex(pParse, pIdx, pPartial!=0, &addrExp);
001045 assert( pLevel->iIdxCur>=0 );
001046 pLevel->iIdxCur = pParse->nTab++;
001047 sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1);
001048 sqlite3VdbeSetP4KeyInfo(pParse, pIdx);
001049 VdbeComment((v, "for %s", pTable->zName));
001050 if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) && useBloomFilter ){
001051 sqlite3WhereExplainBloomFilter(pParse, pWC->pWInfo, pLevel);
001052 pLevel->regFilter = ++pParse->nMem;
001053 sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter);
001054 }
001055
001056 /* Fill the automatic index with content */
001057 assert( pSrc == &pWC->pWInfo->pTabList->a[pLevel->iFrom] );
001058 if( pSrc->fg.viaCoroutine ){
001059 int regYield = pSrc->regReturn;
001060 addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0);
001061 sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSrc->addrFillSub);
001062 addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield);
001063 VdbeCoverage(v);
001064 VdbeComment((v, "next row of %s", pSrc->pTab->zName));
001065 }else{
001066 addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v);
001067 }
001068 if( pPartial ){
001069 iContinue = sqlite3VdbeMakeLabel(pParse);
001070 sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL);
001071 pLoop->wsFlags |= WHERE_PARTIALIDX;
001072 }
001073 regRecord = sqlite3GetTempReg(pParse);
001074 regBase = sqlite3GenerateIndexKey(
001075 pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0
001076 );
001077 if( pLevel->regFilter ){
001078 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0,
001079 regBase, pLoop->u.btree.nEq);
001080 }
001081 sqlite3VdbeScanStatusCounters(v, addrExp, addrExp, sqlite3VdbeCurrentAddr(v));
001082 sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord);
001083 sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
001084 if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
001085 if( pSrc->fg.viaCoroutine ){
001086 sqlite3VdbeChangeP2(v, addrCounter, regBase+n);
001087 testcase( pParse->db->mallocFailed );
001088 assert( pLevel->iIdxCur>0 );
001089 translateColumnToCopy(pParse, addrTop, pLevel->iTabCur,
001090 pSrc->regResult, pLevel->iIdxCur);
001091 sqlite3VdbeGoto(v, addrTop);
001092 pSrc->fg.viaCoroutine = 0;
001093 }else{
001094 sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v);
001095 sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX);
001096 }
001097 sqlite3VdbeJumpHere(v, addrTop);
001098 sqlite3ReleaseTempReg(pParse, regRecord);
001099
001100 /* Jump here when skipping the initialization */
001101 sqlite3VdbeJumpHere(v, addrInit);
001102 sqlite3VdbeScanStatusRange(v, addrExp, addrExp, -1);
001103
001104 end_auto_index_create:
001105 sqlite3ExprDelete(pParse->db, pPartial);
001106 }
001107 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
001108
001109 /*
001110 ** Generate bytecode that will initialize a Bloom filter that is appropriate
001111 ** for pLevel.
001112 **
001113 ** If there are inner loops within pLevel that have the WHERE_BLOOMFILTER
001114 ** flag set, initialize a Bloomfilter for them as well. Except don't do
001115 ** this recursive initialization if the SQLITE_BloomPulldown optimization has
001116 ** been turned off.
001117 **
001118 ** When the Bloom filter is initialized, the WHERE_BLOOMFILTER flag is cleared
001119 ** from the loop, but the regFilter value is set to a register that implements
001120 ** the Bloom filter. When regFilter is positive, the
001121 ** sqlite3WhereCodeOneLoopStart() will generate code to test the Bloom filter
001122 ** and skip the subsequence B-Tree seek if the Bloom filter indicates that
001123 ** no matching rows exist.
001124 **
001125 ** This routine may only be called if it has previously been determined that
001126 ** the loop would benefit from a Bloom filter, and the WHERE_BLOOMFILTER bit
001127 ** is set.
001128 */
001129 static SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
001130 WhereInfo *pWInfo, /* The WHERE clause */
001131 int iLevel, /* Index in pWInfo->a[] that is pLevel */
001132 WhereLevel *pLevel, /* Make a Bloom filter for this FROM term */
001133 Bitmask notReady /* Loops that are not ready */
001134 ){
001135 int addrOnce; /* Address of opening OP_Once */
001136 int addrTop; /* Address of OP_Rewind */
001137 int addrCont; /* Jump here to skip a row */
001138 const WhereTerm *pTerm; /* For looping over WHERE clause terms */
001139 const WhereTerm *pWCEnd; /* Last WHERE clause term */
001140 Parse *pParse = pWInfo->pParse; /* Parsing context */
001141 Vdbe *v = pParse->pVdbe; /* VDBE under construction */
001142 WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */
001143 int iCur; /* Cursor for table getting the filter */
001144 IndexedExpr *saved_pIdxEpr; /* saved copy of Parse.pIdxEpr */
001145
001146 saved_pIdxEpr = pParse->pIdxEpr;
001147 pParse->pIdxEpr = 0;
001148
001149 assert( pLoop!=0 );
001150 assert( v!=0 );
001151 assert( pLoop->wsFlags & WHERE_BLOOMFILTER );
001152
001153 addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
001154 do{
001155 const SrcList *pTabList;
001156 const SrcItem *pItem;
001157 const Table *pTab;
001158 u64 sz;
001159 int iSrc;
001160 sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel);
001161 addrCont = sqlite3VdbeMakeLabel(pParse);
001162 iCur = pLevel->iTabCur;
001163 pLevel->regFilter = ++pParse->nMem;
001164
001165 /* The Bloom filter is a Blob held in a register. Initialize it
001166 ** to zero-filled blob of at least 80K bits, but maybe more if the
001167 ** estimated size of the table is larger. We could actually
001168 ** measure the size of the table at run-time using OP_Count with
001169 ** P3==1 and use that value to initialize the blob. But that makes
001170 ** testing complicated. By basing the blob size on the value in the
001171 ** sqlite_stat1 table, testing is much easier.
001172 */
001173 pTabList = pWInfo->pTabList;
001174 iSrc = pLevel->iFrom;
001175 pItem = &pTabList->a[iSrc];
001176 assert( pItem!=0 );
001177 pTab = pItem->pTab;
001178 assert( pTab!=0 );
001179 sz = sqlite3LogEstToInt(pTab->nRowLogEst);
001180 if( sz<10000 ){
001181 sz = 10000;
001182 }else if( sz>10000000 ){
001183 sz = 10000000;
001184 }
001185 sqlite3VdbeAddOp2(v, OP_Blob, (int)sz, pLevel->regFilter);
001186
001187 addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
001188 pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm];
001189 for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
001190 Expr *pExpr = pTerm->pExpr;
001191 if( (pTerm->wtFlags & TERM_VIRTUAL)==0
001192 && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc)
001193 ){
001194 sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL);
001195 }
001196 }
001197 if( pLoop->wsFlags & WHERE_IPK ){
001198 int r1 = sqlite3GetTempReg(pParse);
001199 sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1);
001200 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, 1);
001201 sqlite3ReleaseTempReg(pParse, r1);
001202 }else{
001203 Index *pIdx = pLoop->u.btree.pIndex;
001204 int n = pLoop->u.btree.nEq;
001205 int r1 = sqlite3GetTempRange(pParse, n);
001206 int jj;
001207 for(jj=0; jj<n; jj++){
001208 assert( pIdx->pTable==pItem->pTab );
001209 sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iCur, jj, r1+jj);
001210 }
001211 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n);
001212 sqlite3ReleaseTempRange(pParse, r1, n);
001213 }
001214 sqlite3VdbeResolveLabel(v, addrCont);
001215 sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1);
001216 VdbeCoverage(v);
001217 sqlite3VdbeJumpHere(v, addrTop);
001218 pLoop->wsFlags &= ~WHERE_BLOOMFILTER;
001219 if( OptimizationDisabled(pParse->db, SQLITE_BloomPulldown) ) break;
001220 while( ++iLevel < pWInfo->nLevel ){
001221 const SrcItem *pTabItem;
001222 pLevel = &pWInfo->a[iLevel];
001223 pTabItem = &pWInfo->pTabList->a[pLevel->iFrom];
001224 if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ) ) continue;
001225 pLoop = pLevel->pWLoop;
001226 if( NEVER(pLoop==0) ) continue;
001227 if( pLoop->prereq & notReady ) continue;
001228 if( (pLoop->wsFlags & (WHERE_BLOOMFILTER|WHERE_COLUMN_IN))
001229 ==WHERE_BLOOMFILTER
001230 ){
001231 /* This is a candidate for bloom-filter pull-down (early evaluation).
001232 ** The test that WHERE_COLUMN_IN is omitted is important, as we are
001233 ** not able to do early evaluation of bloom filters that make use of
001234 ** the IN operator */
001235 break;
001236 }
001237 }
001238 }while( iLevel < pWInfo->nLevel );
001239 sqlite3VdbeJumpHere(v, addrOnce);
001240 pParse->pIdxEpr = saved_pIdxEpr;
001241 }
001242
001243
001244 #ifndef SQLITE_OMIT_VIRTUALTABLE
001245 /*
001246 ** Allocate and populate an sqlite3_index_info structure. It is the
001247 ** responsibility of the caller to eventually release the structure
001248 ** by passing the pointer returned by this function to freeIndexInfo().
001249 */
001250 static sqlite3_index_info *allocateIndexInfo(
001251 WhereInfo *pWInfo, /* The WHERE clause */
001252 WhereClause *pWC, /* The WHERE clause being analyzed */
001253 Bitmask mUnusable, /* Ignore terms with these prereqs */
001254 SrcItem *pSrc, /* The FROM clause term that is the vtab */
001255 u16 *pmNoOmit /* Mask of terms not to omit */
001256 ){
001257 int i, j;
001258 int nTerm;
001259 Parse *pParse = pWInfo->pParse;
001260 struct sqlite3_index_constraint *pIdxCons;
001261 struct sqlite3_index_orderby *pIdxOrderBy;
001262 struct sqlite3_index_constraint_usage *pUsage;
001263 struct HiddenIndexInfo *pHidden;
001264 WhereTerm *pTerm;
001265 int nOrderBy;
001266 sqlite3_index_info *pIdxInfo;
001267 u16 mNoOmit = 0;
001268 const Table *pTab;
001269 int eDistinct = 0;
001270 ExprList *pOrderBy = pWInfo->pOrderBy;
001271
001272 assert( pSrc!=0 );
001273 pTab = pSrc->pTab;
001274 assert( pTab!=0 );
001275 assert( IsVirtual(pTab) );
001276
001277 /* Find all WHERE clause constraints referring to this virtual table.
001278 ** Mark each term with the TERM_OK flag. Set nTerm to the number of
001279 ** terms found.
001280 */
001281 for(i=nTerm=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
001282 pTerm->wtFlags &= ~TERM_OK;
001283 if( pTerm->leftCursor != pSrc->iCursor ) continue;
001284 if( pTerm->prereqRight & mUnusable ) continue;
001285 assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) );
001286 testcase( pTerm->eOperator & WO_IN );
001287 testcase( pTerm->eOperator & WO_ISNULL );
001288 testcase( pTerm->eOperator & WO_IS );
001289 testcase( pTerm->eOperator & WO_ALL );
001290 if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue;
001291 if( pTerm->wtFlags & TERM_VNULL ) continue;
001292
001293 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
001294 assert( pTerm->u.x.leftColumn>=XN_ROWID );
001295 assert( pTerm->u.x.leftColumn<pTab->nCol );
001296 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
001297 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
001298 ){
001299 continue;
001300 }
001301 nTerm++;
001302 pTerm->wtFlags |= TERM_OK;
001303 }
001304
001305 /* If the ORDER BY clause contains only columns in the current
001306 ** virtual table then allocate space for the aOrderBy part of
001307 ** the sqlite3_index_info structure.
001308 */
001309 nOrderBy = 0;
001310 if( pOrderBy ){
001311 int n = pOrderBy->nExpr;
001312 for(i=0; i<n; i++){
001313 Expr *pExpr = pOrderBy->a[i].pExpr;
001314 Expr *pE2;
001315
001316 /* Skip over constant terms in the ORDER BY clause */
001317 if( sqlite3ExprIsConstant(pExpr) ){
001318 continue;
001319 }
001320
001321 /* Virtual tables are unable to deal with NULLS FIRST */
001322 if( pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL ) break;
001323
001324 /* First case - a direct column references without a COLLATE operator */
001325 if( pExpr->op==TK_COLUMN && pExpr->iTable==pSrc->iCursor ){
001326 assert( pExpr->iColumn>=XN_ROWID && pExpr->iColumn<pTab->nCol );
001327 continue;
001328 }
001329
001330 /* 2nd case - a column reference with a COLLATE operator. Only match
001331 ** of the COLLATE operator matches the collation of the column. */
001332 if( pExpr->op==TK_COLLATE
001333 && (pE2 = pExpr->pLeft)->op==TK_COLUMN
001334 && pE2->iTable==pSrc->iCursor
001335 ){
001336 const char *zColl; /* The collating sequence name */
001337 assert( !ExprHasProperty(pExpr, EP_IntValue) );
001338 assert( pExpr->u.zToken!=0 );
001339 assert( pE2->iColumn>=XN_ROWID && pE2->iColumn<pTab->nCol );
001340 pExpr->iColumn = pE2->iColumn;
001341 if( pE2->iColumn<0 ) continue; /* Collseq does not matter for rowid */
001342 zColl = sqlite3ColumnColl(&pTab->aCol[pE2->iColumn]);
001343 if( zColl==0 ) zColl = sqlite3StrBINARY;
001344 if( sqlite3_stricmp(pExpr->u.zToken, zColl)==0 ) continue;
001345 }
001346
001347 /* No matches cause a break out of the loop */
001348 break;
001349 }
001350 if( i==n ){
001351 nOrderBy = n;
001352 if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) ){
001353 eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0);
001354 }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){
001355 eDistinct = 1;
001356 }
001357 }
001358 }
001359
001360 /* Allocate the sqlite3_index_info structure
001361 */
001362 pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo)
001363 + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm
001364 + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden)
001365 + sizeof(sqlite3_value*)*nTerm );
001366 if( pIdxInfo==0 ){
001367 sqlite3ErrorMsg(pParse, "out of memory");
001368 return 0;
001369 }
001370 pHidden = (struct HiddenIndexInfo*)&pIdxInfo[1];
001371 pIdxCons = (struct sqlite3_index_constraint*)&pHidden->aRhs[nTerm];
001372 pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm];
001373 pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy];
001374 pIdxInfo->aConstraint = pIdxCons;
001375 pIdxInfo->aOrderBy = pIdxOrderBy;
001376 pIdxInfo->aConstraintUsage = pUsage;
001377 pHidden->pWC = pWC;
001378 pHidden->pParse = pParse;
001379 pHidden->eDistinct = eDistinct;
001380 pHidden->mIn = 0;
001381 for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
001382 u16 op;
001383 if( (pTerm->wtFlags & TERM_OK)==0 ) continue;
001384 pIdxCons[j].iColumn = pTerm->u.x.leftColumn;
001385 pIdxCons[j].iTermOffset = i;
001386 op = pTerm->eOperator & WO_ALL;
001387 if( op==WO_IN ){
001388 if( (pTerm->wtFlags & TERM_SLICE)==0 ){
001389 pHidden->mIn |= SMASKBIT32(j);
001390 }
001391 op = WO_EQ;
001392 }
001393 if( op==WO_AUX ){
001394 pIdxCons[j].op = pTerm->eMatchOp;
001395 }else if( op & (WO_ISNULL|WO_IS) ){
001396 if( op==WO_ISNULL ){
001397 pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL;
001398 }else{
001399 pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS;
001400 }
001401 }else{
001402 pIdxCons[j].op = (u8)op;
001403 /* The direct assignment in the previous line is possible only because
001404 ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The
001405 ** following asserts verify this fact. */
001406 assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ );
001407 assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT );
001408 assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE );
001409 assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT );
001410 assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE );
001411 assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) );
001412
001413 if( op & (WO_LT|WO_LE|WO_GT|WO_GE)
001414 && sqlite3ExprIsVector(pTerm->pExpr->pRight)
001415 ){
001416 testcase( j!=i );
001417 if( j<16 ) mNoOmit |= (1 << j);
001418 if( op==WO_LT ) pIdxCons[j].op = WO_LE;
001419 if( op==WO_GT ) pIdxCons[j].op = WO_GE;
001420 }
001421 }
001422
001423 j++;
001424 }
001425 assert( j==nTerm );
001426 pIdxInfo->nConstraint = j;
001427 for(i=j=0; i<nOrderBy; i++){
001428 Expr *pExpr = pOrderBy->a[i].pExpr;
001429 if( sqlite3ExprIsConstant(pExpr) ) continue;
001430 assert( pExpr->op==TK_COLUMN
001431 || (pExpr->op==TK_COLLATE && pExpr->pLeft->op==TK_COLUMN
001432 && pExpr->iColumn==pExpr->pLeft->iColumn) );
001433 pIdxOrderBy[j].iColumn = pExpr->iColumn;
001434 pIdxOrderBy[j].desc = pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC;
001435 j++;
001436 }
001437 pIdxInfo->nOrderBy = j;
001438
001439 *pmNoOmit = mNoOmit;
001440 return pIdxInfo;
001441 }
001442
001443 /*
001444 ** Free an sqlite3_index_info structure allocated by allocateIndexInfo()
001445 ** and possibly modified by xBestIndex methods.
001446 */
001447 static void freeIndexInfo(sqlite3 *db, sqlite3_index_info *pIdxInfo){
001448 HiddenIndexInfo *pHidden;
001449 int i;
001450 assert( pIdxInfo!=0 );
001451 pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
001452 assert( pHidden->pParse!=0 );
001453 assert( pHidden->pParse->db==db );
001454 for(i=0; i<pIdxInfo->nConstraint; i++){
001455 sqlite3ValueFree(pHidden->aRhs[i]); /* IMP: R-14553-25174 */
001456 pHidden->aRhs[i] = 0;
001457 }
001458 sqlite3DbFree(db, pIdxInfo);
001459 }
001460
001461 /*
001462 ** The table object reference passed as the second argument to this function
001463 ** must represent a virtual table. This function invokes the xBestIndex()
001464 ** method of the virtual table with the sqlite3_index_info object that
001465 ** comes in as the 3rd argument to this function.
001466 **
001467 ** If an error occurs, pParse is populated with an error message and an
001468 ** appropriate error code is returned. A return of SQLITE_CONSTRAINT from
001469 ** xBestIndex is not considered an error. SQLITE_CONSTRAINT indicates that
001470 ** the current configuration of "unusable" flags in sqlite3_index_info can
001471 ** not result in a valid plan.
001472 **
001473 ** Whether or not an error is returned, it is the responsibility of the
001474 ** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates
001475 ** that this is required.
001476 */
001477 static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){
001478 sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab;
001479 int rc;
001480
001481 whereTraceIndexInfoInputs(p);
001482 pParse->db->nSchemaLock++;
001483 rc = pVtab->pModule->xBestIndex(pVtab, p);
001484 pParse->db->nSchemaLock--;
001485 whereTraceIndexInfoOutputs(p);
001486
001487 if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){
001488 if( rc==SQLITE_NOMEM ){
001489 sqlite3OomFault(pParse->db);
001490 }else if( !pVtab->zErrMsg ){
001491 sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc));
001492 }else{
001493 sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg);
001494 }
001495 }
001496 if( pTab->u.vtab.p->bAllSchemas ){
001497 sqlite3VtabUsesAllSchemas(pParse);
001498 }
001499 sqlite3_free(pVtab->zErrMsg);
001500 pVtab->zErrMsg = 0;
001501 return rc;
001502 }
001503 #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */
001504
001505 #ifdef SQLITE_ENABLE_STAT4
001506 /*
001507 ** Estimate the location of a particular key among all keys in an
001508 ** index. Store the results in aStat as follows:
001509 **
001510 ** aStat[0] Est. number of rows less than pRec
001511 ** aStat[1] Est. number of rows equal to pRec
001512 **
001513 ** Return the index of the sample that is the smallest sample that
001514 ** is greater than or equal to pRec. Note that this index is not an index
001515 ** into the aSample[] array - it is an index into a virtual set of samples
001516 ** based on the contents of aSample[] and the number of fields in record
001517 ** pRec.
001518 */
001519 static int whereKeyStats(
001520 Parse *pParse, /* Database connection */
001521 Index *pIdx, /* Index to consider domain of */
001522 UnpackedRecord *pRec, /* Vector of values to consider */
001523 int roundUp, /* Round up if true. Round down if false */
001524 tRowcnt *aStat /* OUT: stats written here */
001525 ){
001526 IndexSample *aSample = pIdx->aSample;
001527 int iCol; /* Index of required stats in anEq[] etc. */
001528 int i; /* Index of first sample >= pRec */
001529 int iSample; /* Smallest sample larger than or equal to pRec */
001530 int iMin = 0; /* Smallest sample not yet tested */
001531 int iTest; /* Next sample to test */
001532 int res; /* Result of comparison operation */
001533 int nField; /* Number of fields in pRec */
001534 tRowcnt iLower = 0; /* anLt[] + anEq[] of largest sample pRec is > */
001535
001536 #ifndef SQLITE_DEBUG
001537 UNUSED_PARAMETER( pParse );
001538 #endif
001539 assert( pRec!=0 );
001540 assert( pIdx->nSample>0 );
001541 assert( pRec->nField>0 );
001542
001543
001544 /* Do a binary search to find the first sample greater than or equal
001545 ** to pRec. If pRec contains a single field, the set of samples to search
001546 ** is simply the aSample[] array. If the samples in aSample[] contain more
001547 ** than one fields, all fields following the first are ignored.
001548 **
001549 ** If pRec contains N fields, where N is more than one, then as well as the
001550 ** samples in aSample[] (truncated to N fields), the search also has to
001551 ** consider prefixes of those samples. For example, if the set of samples
001552 ** in aSample is:
001553 **
001554 ** aSample[0] = (a, 5)
001555 ** aSample[1] = (a, 10)
001556 ** aSample[2] = (b, 5)
001557 ** aSample[3] = (c, 100)
001558 ** aSample[4] = (c, 105)
001559 **
001560 ** Then the search space should ideally be the samples above and the
001561 ** unique prefixes [a], [b] and [c]. But since that is hard to organize,
001562 ** the code actually searches this set:
001563 **
001564 ** 0: (a)
001565 ** 1: (a, 5)
001566 ** 2: (a, 10)
001567 ** 3: (a, 10)
001568 ** 4: (b)
001569 ** 5: (b, 5)
001570 ** 6: (c)
001571 ** 7: (c, 100)
001572 ** 8: (c, 105)
001573 ** 9: (c, 105)
001574 **
001575 ** For each sample in the aSample[] array, N samples are present in the
001576 ** effective sample array. In the above, samples 0 and 1 are based on
001577 ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc.
001578 **
001579 ** Often, sample i of each block of N effective samples has (i+1) fields.
001580 ** Except, each sample may be extended to ensure that it is greater than or
001581 ** equal to the previous sample in the array. For example, in the above,
001582 ** sample 2 is the first sample of a block of N samples, so at first it
001583 ** appears that it should be 1 field in size. However, that would make it
001584 ** smaller than sample 1, so the binary search would not work. As a result,
001585 ** it is extended to two fields. The duplicates that this creates do not
001586 ** cause any problems.
001587 */
001588 if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
001589 nField = pIdx->nKeyCol;
001590 }else{
001591 nField = pIdx->nColumn;
001592 }
001593 nField = MIN(pRec->nField, nField);
001594 iCol = 0;
001595 iSample = pIdx->nSample * nField;
001596 do{
001597 int iSamp; /* Index in aSample[] of test sample */
001598 int n; /* Number of fields in test sample */
001599
001600 iTest = (iMin+iSample)/2;
001601 iSamp = iTest / nField;
001602 if( iSamp>0 ){
001603 /* The proposed effective sample is a prefix of sample aSample[iSamp].
001604 ** Specifically, the shortest prefix of at least (1 + iTest%nField)
001605 ** fields that is greater than the previous effective sample. */
001606 for(n=(iTest % nField) + 1; n<nField; n++){
001607 if( aSample[iSamp-1].anLt[n-1]!=aSample[iSamp].anLt[n-1] ) break;
001608 }
001609 }else{
001610 n = iTest + 1;
001611 }
001612
001613 pRec->nField = n;
001614 res = sqlite3VdbeRecordCompare(aSample[iSamp].n, aSample[iSamp].p, pRec);
001615 if( res<0 ){
001616 iLower = aSample[iSamp].anLt[n-1] + aSample[iSamp].anEq[n-1];
001617 iMin = iTest+1;
001618 }else if( res==0 && n<nField ){
001619 iLower = aSample[iSamp].anLt[n-1];
001620 iMin = iTest+1;
001621 res = -1;
001622 }else{
001623 iSample = iTest;
001624 iCol = n-1;
001625 }
001626 }while( res && iMin<iSample );
001627 i = iSample / nField;
001628
001629 #ifdef SQLITE_DEBUG
001630 /* The following assert statements check that the binary search code
001631 ** above found the right answer. This block serves no purpose other
001632 ** than to invoke the asserts. */
001633 if( pParse->db->mallocFailed==0 ){
001634 if( res==0 ){
001635 /* If (res==0) is true, then pRec must be equal to sample i. */
001636 assert( i<pIdx->nSample );
001637 assert( iCol==nField-1 );
001638 pRec->nField = nField;
001639 assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)
001640 || pParse->db->mallocFailed
001641 );
001642 }else{
001643 /* Unless i==pIdx->nSample, indicating that pRec is larger than
001644 ** all samples in the aSample[] array, pRec must be smaller than the
001645 ** (iCol+1) field prefix of sample i. */
001646 assert( i<=pIdx->nSample && i>=0 );
001647 pRec->nField = iCol+1;
001648 assert( i==pIdx->nSample
001649 || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0
001650 || pParse->db->mallocFailed );
001651
001652 /* if i==0 and iCol==0, then record pRec is smaller than all samples
001653 ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must
001654 ** be greater than or equal to the (iCol) field prefix of sample i.
001655 ** If (i>0), then pRec must also be greater than sample (i-1). */
001656 if( iCol>0 ){
001657 pRec->nField = iCol;
001658 assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0
001659 || pParse->db->mallocFailed || CORRUPT_DB );
001660 }
001661 if( i>0 ){
001662 pRec->nField = nField;
001663 assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0
001664 || pParse->db->mallocFailed || CORRUPT_DB );
001665 }
001666 }
001667 }
001668 #endif /* ifdef SQLITE_DEBUG */
001669
001670 if( res==0 ){
001671 /* Record pRec is equal to sample i */
001672 assert( iCol==nField-1 );
001673 aStat[0] = aSample[i].anLt[iCol];
001674 aStat[1] = aSample[i].anEq[iCol];
001675 }else{
001676 /* At this point, the (iCol+1) field prefix of aSample[i] is the first
001677 ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec
001678 ** is larger than all samples in the array. */
001679 tRowcnt iUpper, iGap;
001680 if( i>=pIdx->nSample ){
001681 iUpper = pIdx->nRowEst0;
001682 }else{
001683 iUpper = aSample[i].anLt[iCol];
001684 }
001685
001686 if( iLower>=iUpper ){
001687 iGap = 0;
001688 }else{
001689 iGap = iUpper - iLower;
001690 }
001691 if( roundUp ){
001692 iGap = (iGap*2)/3;
001693 }else{
001694 iGap = iGap/3;
001695 }
001696 aStat[0] = iLower + iGap;
001697 aStat[1] = pIdx->aAvgEq[nField-1];
001698 }
001699
001700 /* Restore the pRec->nField value before returning. */
001701 pRec->nField = nField;
001702 return i;
001703 }
001704 #endif /* SQLITE_ENABLE_STAT4 */
001705
001706 /*
001707 ** If it is not NULL, pTerm is a term that provides an upper or lower
001708 ** bound on a range scan. Without considering pTerm, it is estimated
001709 ** that the scan will visit nNew rows. This function returns the number
001710 ** estimated to be visited after taking pTerm into account.
001711 **
001712 ** If the user explicitly specified a likelihood() value for this term,
001713 ** then the return value is the likelihood multiplied by the number of
001714 ** input rows. Otherwise, this function assumes that an "IS NOT NULL" term
001715 ** has a likelihood of 0.50, and any other term a likelihood of 0.25.
001716 */
001717 static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){
001718 LogEst nRet = nNew;
001719 if( pTerm ){
001720 if( pTerm->truthProb<=0 ){
001721 nRet += pTerm->truthProb;
001722 }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){
001723 nRet -= 20; assert( 20==sqlite3LogEst(4) );
001724 }
001725 }
001726 return nRet;
001727 }
001728
001729
001730 #ifdef SQLITE_ENABLE_STAT4
001731 /*
001732 ** Return the affinity for a single column of an index.
001733 */
001734 char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){
001735 assert( iCol>=0 && iCol<pIdx->nColumn );
001736 if( !pIdx->zColAff ){
001737 if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB;
001738 }
001739 assert( pIdx->zColAff[iCol]!=0 );
001740 return pIdx->zColAff[iCol];
001741 }
001742 #endif
001743
001744
001745 #ifdef SQLITE_ENABLE_STAT4
001746 /*
001747 ** This function is called to estimate the number of rows visited by a
001748 ** range-scan on a skip-scan index. For example:
001749 **
001750 ** CREATE INDEX i1 ON t1(a, b, c);
001751 ** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?;
001752 **
001753 ** Value pLoop->nOut is currently set to the estimated number of rows
001754 ** visited for scanning (a=? AND b=?). This function reduces that estimate
001755 ** by some factor to account for the (c BETWEEN ? AND ?) expression based
001756 ** on the stat4 data for the index. this scan will be performed multiple
001757 ** times (once for each (a,b) combination that matches a=?) is dealt with
001758 ** by the caller.
001759 **
001760 ** It does this by scanning through all stat4 samples, comparing values
001761 ** extracted from pLower and pUpper with the corresponding column in each
001762 ** sample. If L and U are the number of samples found to be less than or
001763 ** equal to the values extracted from pLower and pUpper respectively, and
001764 ** N is the total number of samples, the pLoop->nOut value is adjusted
001765 ** as follows:
001766 **
001767 ** nOut = nOut * ( min(U - L, 1) / N )
001768 **
001769 ** If pLower is NULL, or a value cannot be extracted from the term, L is
001770 ** set to zero. If pUpper is NULL, or a value cannot be extracted from it,
001771 ** U is set to N.
001772 **
001773 ** Normally, this function sets *pbDone to 1 before returning. However,
001774 ** if no value can be extracted from either pLower or pUpper (and so the
001775 ** estimate of the number of rows delivered remains unchanged), *pbDone
001776 ** is left as is.
001777 **
001778 ** If an error occurs, an SQLite error code is returned. Otherwise,
001779 ** SQLITE_OK.
001780 */
001781 static int whereRangeSkipScanEst(
001782 Parse *pParse, /* Parsing & code generating context */
001783 WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */
001784 WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */
001785 WhereLoop *pLoop, /* Update the .nOut value of this loop */
001786 int *pbDone /* Set to true if at least one expr. value extracted */
001787 ){
001788 Index *p = pLoop->u.btree.pIndex;
001789 int nEq = pLoop->u.btree.nEq;
001790 sqlite3 *db = pParse->db;
001791 int nLower = -1;
001792 int nUpper = p->nSample+1;
001793 int rc = SQLITE_OK;
001794 u8 aff = sqlite3IndexColumnAffinity(db, p, nEq);
001795 CollSeq *pColl;
001796
001797 sqlite3_value *p1 = 0; /* Value extracted from pLower */
001798 sqlite3_value *p2 = 0; /* Value extracted from pUpper */
001799 sqlite3_value *pVal = 0; /* Value extracted from record */
001800
001801 pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]);
001802 if( pLower ){
001803 rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1);
001804 nLower = 0;
001805 }
001806 if( pUpper && rc==SQLITE_OK ){
001807 rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2);
001808 nUpper = p2 ? 0 : p->nSample;
001809 }
001810
001811 if( p1 || p2 ){
001812 int i;
001813 int nDiff;
001814 for(i=0; rc==SQLITE_OK && i<p->nSample; i++){
001815 rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal);
001816 if( rc==SQLITE_OK && p1 ){
001817 int res = sqlite3MemCompare(p1, pVal, pColl);
001818 if( res>=0 ) nLower++;
001819 }
001820 if( rc==SQLITE_OK && p2 ){
001821 int res = sqlite3MemCompare(p2, pVal, pColl);
001822 if( res>=0 ) nUpper++;
001823 }
001824 }
001825 nDiff = (nUpper - nLower);
001826 if( nDiff<=0 ) nDiff = 1;
001827
001828 /* If there is both an upper and lower bound specified, and the
001829 ** comparisons indicate that they are close together, use the fallback
001830 ** method (assume that the scan visits 1/64 of the rows) for estimating
001831 ** the number of rows visited. Otherwise, estimate the number of rows
001832 ** using the method described in the header comment for this function. */
001833 if( nDiff!=1 || pUpper==0 || pLower==0 ){
001834 int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff));
001835 pLoop->nOut -= nAdjust;
001836 *pbDone = 1;
001837 WHERETRACE(0x20, ("range skip-scan regions: %u..%u adjust=%d est=%d\n",
001838 nLower, nUpper, nAdjust*-1, pLoop->nOut));
001839 }
001840
001841 }else{
001842 assert( *pbDone==0 );
001843 }
001844
001845 sqlite3ValueFree(p1);
001846 sqlite3ValueFree(p2);
001847 sqlite3ValueFree(pVal);
001848
001849 return rc;
001850 }
001851 #endif /* SQLITE_ENABLE_STAT4 */
001852
001853 /*
001854 ** This function is used to estimate the number of rows that will be visited
001855 ** by scanning an index for a range of values. The range may have an upper
001856 ** bound, a lower bound, or both. The WHERE clause terms that set the upper
001857 ** and lower bounds are represented by pLower and pUpper respectively. For
001858 ** example, assuming that index p is on t1(a):
001859 **
001860 ** ... FROM t1 WHERE a > ? AND a < ? ...
001861 ** |_____| |_____|
001862 ** | |
001863 ** pLower pUpper
001864 **
001865 ** If either of the upper or lower bound is not present, then NULL is passed in
001866 ** place of the corresponding WhereTerm.
001867 **
001868 ** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index
001869 ** column subject to the range constraint. Or, equivalently, the number of
001870 ** equality constraints optimized by the proposed index scan. For example,
001871 ** assuming index p is on t1(a, b), and the SQL query is:
001872 **
001873 ** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ...
001874 **
001875 ** then nEq is set to 1 (as the range restricted column, b, is the second
001876 ** left-most column of the index). Or, if the query is:
001877 **
001878 ** ... FROM t1 WHERE a > ? AND a < ? ...
001879 **
001880 ** then nEq is set to 0.
001881 **
001882 ** When this function is called, *pnOut is set to the sqlite3LogEst() of the
001883 ** number of rows that the index scan is expected to visit without
001884 ** considering the range constraints. If nEq is 0, then *pnOut is the number of
001885 ** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced)
001886 ** to account for the range constraints pLower and pUpper.
001887 **
001888 ** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be
001889 ** used, a single range inequality reduces the search space by a factor of 4.
001890 ** and a pair of constraints (x>? AND x<?) reduces the expected number of
001891 ** rows visited by a factor of 64.
001892 */
001893 static int whereRangeScanEst(
001894 Parse *pParse, /* Parsing & code generating context */
001895 WhereLoopBuilder *pBuilder,
001896 WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */
001897 WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */
001898 WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */
001899 ){
001900 int rc = SQLITE_OK;
001901 int nOut = pLoop->nOut;
001902 LogEst nNew;
001903
001904 #ifdef SQLITE_ENABLE_STAT4
001905 Index *p = pLoop->u.btree.pIndex;
001906 int nEq = pLoop->u.btree.nEq;
001907
001908 if( p->nSample>0 && ALWAYS(nEq<p->nSampleCol)
001909 && OptimizationEnabled(pParse->db, SQLITE_Stat4)
001910 ){
001911 if( nEq==pBuilder->nRecValid ){
001912 UnpackedRecord *pRec = pBuilder->pRec;
001913 tRowcnt a[2];
001914 int nBtm = pLoop->u.btree.nBtm;
001915 int nTop = pLoop->u.btree.nTop;
001916
001917 /* Variable iLower will be set to the estimate of the number of rows in
001918 ** the index that are less than the lower bound of the range query. The
001919 ** lower bound being the concatenation of $P and $L, where $P is the
001920 ** key-prefix formed by the nEq values matched against the nEq left-most
001921 ** columns of the index, and $L is the value in pLower.
001922 **
001923 ** Or, if pLower is NULL or $L cannot be extracted from it (because it
001924 ** is not a simple variable or literal value), the lower bound of the
001925 ** range is $P. Due to a quirk in the way whereKeyStats() works, even
001926 ** if $L is available, whereKeyStats() is called for both ($P) and
001927 ** ($P:$L) and the larger of the two returned values is used.
001928 **
001929 ** Similarly, iUpper is to be set to the estimate of the number of rows
001930 ** less than the upper bound of the range query. Where the upper bound
001931 ** is either ($P) or ($P:$U). Again, even if $U is available, both values
001932 ** of iUpper are requested of whereKeyStats() and the smaller used.
001933 **
001934 ** The number of rows between the two bounds is then just iUpper-iLower.
001935 */
001936 tRowcnt iLower; /* Rows less than the lower bound */
001937 tRowcnt iUpper; /* Rows less than the upper bound */
001938 int iLwrIdx = -2; /* aSample[] for the lower bound */
001939 int iUprIdx = -1; /* aSample[] for the upper bound */
001940
001941 if( pRec ){
001942 testcase( pRec->nField!=pBuilder->nRecValid );
001943 pRec->nField = pBuilder->nRecValid;
001944 }
001945 /* Determine iLower and iUpper using ($P) only. */
001946 if( nEq==0 ){
001947 iLower = 0;
001948 iUpper = p->nRowEst0;
001949 }else{
001950 /* Note: this call could be optimized away - since the same values must
001951 ** have been requested when testing key $P in whereEqualScanEst(). */
001952 whereKeyStats(pParse, p, pRec, 0, a);
001953 iLower = a[0];
001954 iUpper = a[0] + a[1];
001955 }
001956
001957 assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 );
001958 assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 );
001959 assert( p->aSortOrder!=0 );
001960 if( p->aSortOrder[nEq] ){
001961 /* The roles of pLower and pUpper are swapped for a DESC index */
001962 SWAP(WhereTerm*, pLower, pUpper);
001963 SWAP(int, nBtm, nTop);
001964 }
001965
001966 /* If possible, improve on the iLower estimate using ($P:$L). */
001967 if( pLower ){
001968 int n; /* Values extracted from pExpr */
001969 Expr *pExpr = pLower->pExpr->pRight;
001970 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nBtm, nEq, &n);
001971 if( rc==SQLITE_OK && n ){
001972 tRowcnt iNew;
001973 u16 mask = WO_GT|WO_LE;
001974 if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
001975 iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a);
001976 iNew = a[0] + ((pLower->eOperator & mask) ? a[1] : 0);
001977 if( iNew>iLower ) iLower = iNew;
001978 nOut--;
001979 pLower = 0;
001980 }
001981 }
001982
001983 /* If possible, improve on the iUpper estimate using ($P:$U). */
001984 if( pUpper ){
001985 int n; /* Values extracted from pExpr */
001986 Expr *pExpr = pUpper->pExpr->pRight;
001987 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nTop, nEq, &n);
001988 if( rc==SQLITE_OK && n ){
001989 tRowcnt iNew;
001990 u16 mask = WO_GT|WO_LE;
001991 if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
001992 iUprIdx = whereKeyStats(pParse, p, pRec, 1, a);
001993 iNew = a[0] + ((pUpper->eOperator & mask) ? a[1] : 0);
001994 if( iNew<iUpper ) iUpper = iNew;
001995 nOut--;
001996 pUpper = 0;
001997 }
001998 }
001999
002000 pBuilder->pRec = pRec;
002001 if( rc==SQLITE_OK ){
002002 if( iUpper>iLower ){
002003 nNew = sqlite3LogEst(iUpper - iLower);
002004 /* TUNING: If both iUpper and iLower are derived from the same
002005 ** sample, then assume they are 4x more selective. This brings
002006 ** the estimated selectivity more in line with what it would be
002007 ** if estimated without the use of STAT4 tables. */
002008 if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) );
002009 }else{
002010 nNew = 10; assert( 10==sqlite3LogEst(2) );
002011 }
002012 if( nNew<nOut ){
002013 nOut = nNew;
002014 }
002015 WHERETRACE(0x20, ("STAT4 range scan: %u..%u est=%d\n",
002016 (u32)iLower, (u32)iUpper, nOut));
002017 }
002018 }else{
002019 int bDone = 0;
002020 rc = whereRangeSkipScanEst(pParse, pLower, pUpper, pLoop, &bDone);
002021 if( bDone ) return rc;
002022 }
002023 }
002024 #else
002025 UNUSED_PARAMETER(pParse);
002026 UNUSED_PARAMETER(pBuilder);
002027 assert( pLower || pUpper );
002028 #endif
002029 assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 || pParse->nErr>0 );
002030 nNew = whereRangeAdjust(pLower, nOut);
002031 nNew = whereRangeAdjust(pUpper, nNew);
002032
002033 /* TUNING: If there is both an upper and lower limit and neither limit
002034 ** has an application-defined likelihood(), assume the range is
002035 ** reduced by an additional 75%. This means that, by default, an open-ended
002036 ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the
002037 ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to
002038 ** match 1/64 of the index. */
002039 if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){
002040 nNew -= 20;
002041 }
002042
002043 nOut -= (pLower!=0) + (pUpper!=0);
002044 if( nNew<10 ) nNew = 10;
002045 if( nNew<nOut ) nOut = nNew;
002046 #if defined(WHERETRACE_ENABLED)
002047 if( pLoop->nOut>nOut ){
002048 WHERETRACE(0x20,("Range scan lowers nOut from %d to %d\n",
002049 pLoop->nOut, nOut));
002050 }
002051 #endif
002052 pLoop->nOut = (LogEst)nOut;
002053 return rc;
002054 }
002055
002056 #ifdef SQLITE_ENABLE_STAT4
002057 /*
002058 ** Estimate the number of rows that will be returned based on
002059 ** an equality constraint x=VALUE and where that VALUE occurs in
002060 ** the histogram data. This only works when x is the left-most
002061 ** column of an index and sqlite_stat4 histogram data is available
002062 ** for that index. When pExpr==NULL that means the constraint is
002063 ** "x IS NULL" instead of "x=VALUE".
002064 **
002065 ** Write the estimated row count into *pnRow and return SQLITE_OK.
002066 ** If unable to make an estimate, leave *pnRow unchanged and return
002067 ** non-zero.
002068 **
002069 ** This routine can fail if it is unable to load a collating sequence
002070 ** required for string comparison, or if unable to allocate memory
002071 ** for a UTF conversion required for comparison. The error is stored
002072 ** in the pParse structure.
002073 */
002074 static int whereEqualScanEst(
002075 Parse *pParse, /* Parsing & code generating context */
002076 WhereLoopBuilder *pBuilder,
002077 Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */
002078 tRowcnt *pnRow /* Write the revised row estimate here */
002079 ){
002080 Index *p = pBuilder->pNew->u.btree.pIndex;
002081 int nEq = pBuilder->pNew->u.btree.nEq;
002082 UnpackedRecord *pRec = pBuilder->pRec;
002083 int rc; /* Subfunction return code */
002084 tRowcnt a[2]; /* Statistics */
002085 int bOk;
002086
002087 assert( nEq>=1 );
002088 assert( nEq<=p->nColumn );
002089 assert( p->aSample!=0 );
002090 assert( p->nSample>0 );
002091 assert( pBuilder->nRecValid<nEq );
002092
002093 /* If values are not available for all fields of the index to the left
002094 ** of this one, no estimate can be made. Return SQLITE_NOTFOUND. */
002095 if( pBuilder->nRecValid<(nEq-1) ){
002096 return SQLITE_NOTFOUND;
002097 }
002098
002099 /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue()
002100 ** below would return the same value. */
002101 if( nEq>=p->nColumn ){
002102 *pnRow = 1;
002103 return SQLITE_OK;
002104 }
002105
002106 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, 1, nEq-1, &bOk);
002107 pBuilder->pRec = pRec;
002108 if( rc!=SQLITE_OK ) return rc;
002109 if( bOk==0 ) return SQLITE_NOTFOUND;
002110 pBuilder->nRecValid = nEq;
002111
002112 whereKeyStats(pParse, p, pRec, 0, a);
002113 WHERETRACE(0x20,("equality scan regions %s(%d): %d\n",
002114 p->zName, nEq-1, (int)a[1]));
002115 *pnRow = a[1];
002116
002117 return rc;
002118 }
002119 #endif /* SQLITE_ENABLE_STAT4 */
002120
002121 #ifdef SQLITE_ENABLE_STAT4
002122 /*
002123 ** Estimate the number of rows that will be returned based on
002124 ** an IN constraint where the right-hand side of the IN operator
002125 ** is a list of values. Example:
002126 **
002127 ** WHERE x IN (1,2,3,4)
002128 **
002129 ** Write the estimated row count into *pnRow and return SQLITE_OK.
002130 ** If unable to make an estimate, leave *pnRow unchanged and return
002131 ** non-zero.
002132 **
002133 ** This routine can fail if it is unable to load a collating sequence
002134 ** required for string comparison, or if unable to allocate memory
002135 ** for a UTF conversion required for comparison. The error is stored
002136 ** in the pParse structure.
002137 */
002138 static int whereInScanEst(
002139 Parse *pParse, /* Parsing & code generating context */
002140 WhereLoopBuilder *pBuilder,
002141 ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */
002142 tRowcnt *pnRow /* Write the revised row estimate here */
002143 ){
002144 Index *p = pBuilder->pNew->u.btree.pIndex;
002145 i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]);
002146 int nRecValid = pBuilder->nRecValid;
002147 int rc = SQLITE_OK; /* Subfunction return code */
002148 tRowcnt nEst; /* Number of rows for a single term */
002149 tRowcnt nRowEst = 0; /* New estimate of the number of rows */
002150 int i; /* Loop counter */
002151
002152 assert( p->aSample!=0 );
002153 for(i=0; rc==SQLITE_OK && i<pList->nExpr; i++){
002154 nEst = nRow0;
002155 rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst);
002156 nRowEst += nEst;
002157 pBuilder->nRecValid = nRecValid;
002158 }
002159
002160 if( rc==SQLITE_OK ){
002161 if( nRowEst > (tRowcnt)nRow0 ) nRowEst = nRow0;
002162 *pnRow = nRowEst;
002163 WHERETRACE(0x20,("IN row estimate: est=%d\n", nRowEst));
002164 }
002165 assert( pBuilder->nRecValid==nRecValid );
002166 return rc;
002167 }
002168 #endif /* SQLITE_ENABLE_STAT4 */
002169
002170
002171 #ifdef WHERETRACE_ENABLED
002172 /*
002173 ** Print the content of a WhereTerm object
002174 */
002175 void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){
002176 if( pTerm==0 ){
002177 sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm);
002178 }else{
002179 char zType[8];
002180 char zLeft[50];
002181 memcpy(zType, "....", 5);
002182 if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V';
002183 if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E';
002184 if( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) zType[2] = 'L';
002185 if( pTerm->wtFlags & TERM_CODED ) zType[3] = 'C';
002186 if( pTerm->eOperator & WO_SINGLE ){
002187 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
002188 sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}",
002189 pTerm->leftCursor, pTerm->u.x.leftColumn);
002190 }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){
002191 sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%llx",
002192 pTerm->u.pOrInfo->indexable);
002193 }else{
002194 sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor);
002195 }
002196 sqlite3DebugPrintf(
002197 "TERM-%-3d %p %s %-12s op=%03x wtFlags=%04x",
002198 iTerm, pTerm, zType, zLeft, pTerm->eOperator, pTerm->wtFlags);
002199 /* The 0x10000 .wheretrace flag causes extra information to be
002200 ** shown about each Term */
002201 if( sqlite3WhereTrace & 0x10000 ){
002202 sqlite3DebugPrintf(" prob=%-3d prereq=%llx,%llx",
002203 pTerm->truthProb, (u64)pTerm->prereqAll, (u64)pTerm->prereqRight);
002204 }
002205 if( (pTerm->eOperator & (WO_OR|WO_AND))==0 && pTerm->u.x.iField ){
002206 sqlite3DebugPrintf(" iField=%d", pTerm->u.x.iField);
002207 }
002208 if( pTerm->iParent>=0 ){
002209 sqlite3DebugPrintf(" iParent=%d", pTerm->iParent);
002210 }
002211 sqlite3DebugPrintf("\n");
002212 sqlite3TreeViewExpr(0, pTerm->pExpr, 0);
002213 }
002214 }
002215 #endif
002216
002217 #ifdef WHERETRACE_ENABLED
002218 /*
002219 ** Show the complete content of a WhereClause
002220 */
002221 void sqlite3WhereClausePrint(WhereClause *pWC){
002222 int i;
002223 for(i=0; i<pWC->nTerm; i++){
002224 sqlite3WhereTermPrint(&pWC->a[i], i);
002225 }
002226 }
002227 #endif
002228
002229 #ifdef WHERETRACE_ENABLED
002230 /*
002231 ** Print a WhereLoop object for debugging purposes
002232 */
002233 void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){
002234 WhereInfo *pWInfo = pWC->pWInfo;
002235 int nb = 1+(pWInfo->pTabList->nSrc+3)/4;
002236 SrcItem *pItem = pWInfo->pTabList->a + p->iTab;
002237 Table *pTab = pItem->pTab;
002238 Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1;
002239 sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId,
002240 p->iTab, nb, p->maskSelf, nb, p->prereq & mAll);
002241 sqlite3DebugPrintf(" %12s",
002242 pItem->zAlias ? pItem->zAlias : pTab->zName);
002243 if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
002244 const char *zName;
002245 if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){
002246 if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){
002247 int i = sqlite3Strlen30(zName) - 1;
002248 while( zName[i]!='_' ) i--;
002249 zName += i;
002250 }
002251 sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq);
002252 }else{
002253 sqlite3DebugPrintf("%20s","");
002254 }
002255 }else{
002256 char *z;
002257 if( p->u.vtab.idxStr ){
002258 z = sqlite3_mprintf("(%d,\"%s\",%#x)",
002259 p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask);
002260 }else{
002261 z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask);
002262 }
002263 sqlite3DebugPrintf(" %-19s", z);
002264 sqlite3_free(z);
002265 }
002266 if( p->wsFlags & WHERE_SKIPSCAN ){
002267 sqlite3DebugPrintf(" f %06x %d-%d", p->wsFlags, p->nLTerm,p->nSkip);
002268 }else{
002269 sqlite3DebugPrintf(" f %06x N %d", p->wsFlags, p->nLTerm);
002270 }
002271 sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut);
002272 if( p->nLTerm && (sqlite3WhereTrace & 0x4000)!=0 ){
002273 int i;
002274 for(i=0; i<p->nLTerm; i++){
002275 sqlite3WhereTermPrint(p->aLTerm[i], i);
002276 }
002277 }
002278 }
002279 #endif
002280
002281 /*
002282 ** Convert bulk memory into a valid WhereLoop that can be passed
002283 ** to whereLoopClear harmlessly.
002284 */
002285 static void whereLoopInit(WhereLoop *p){
002286 p->aLTerm = p->aLTermSpace;
002287 p->nLTerm = 0;
002288 p->nLSlot = ArraySize(p->aLTermSpace);
002289 p->wsFlags = 0;
002290 }
002291
002292 /*
002293 ** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact.
002294 */
002295 static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){
002296 if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){
002297 if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){
002298 sqlite3_free(p->u.vtab.idxStr);
002299 p->u.vtab.needFree = 0;
002300 p->u.vtab.idxStr = 0;
002301 }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){
002302 sqlite3DbFree(db, p->u.btree.pIndex->zColAff);
002303 sqlite3DbFreeNN(db, p->u.btree.pIndex);
002304 p->u.btree.pIndex = 0;
002305 }
002306 }
002307 }
002308
002309 /*
002310 ** Deallocate internal memory used by a WhereLoop object. Leave the
002311 ** object in an initialized state, as if it had been newly allocated.
002312 */
002313 static void whereLoopClear(sqlite3 *db, WhereLoop *p){
002314 if( p->aLTerm!=p->aLTermSpace ){
002315 sqlite3DbFreeNN(db, p->aLTerm);
002316 p->aLTerm = p->aLTermSpace;
002317 p->nLSlot = ArraySize(p->aLTermSpace);
002318 }
002319 whereLoopClearUnion(db, p);
002320 p->nLTerm = 0;
002321 p->wsFlags = 0;
002322 }
002323
002324 /*
002325 ** Increase the memory allocation for pLoop->aLTerm[] to be at least n.
002326 */
002327 static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){
002328 WhereTerm **paNew;
002329 if( p->nLSlot>=n ) return SQLITE_OK;
002330 n = (n+7)&~7;
002331 paNew = sqlite3DbMallocRawNN(db, sizeof(p->aLTerm[0])*n);
002332 if( paNew==0 ) return SQLITE_NOMEM_BKPT;
002333 memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot);
002334 if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFreeNN(db, p->aLTerm);
002335 p->aLTerm = paNew;
002336 p->nLSlot = n;
002337 return SQLITE_OK;
002338 }
002339
002340 /*
002341 ** Transfer content from the second pLoop into the first.
002342 */
002343 static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){
002344 whereLoopClearUnion(db, pTo);
002345 if( pFrom->nLTerm > pTo->nLSlot
002346 && whereLoopResize(db, pTo, pFrom->nLTerm)
002347 ){
002348 memset(pTo, 0, WHERE_LOOP_XFER_SZ);
002349 return SQLITE_NOMEM_BKPT;
002350 }
002351 memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ);
002352 memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0]));
002353 if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){
002354 pFrom->u.vtab.needFree = 0;
002355 }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){
002356 pFrom->u.btree.pIndex = 0;
002357 }
002358 return SQLITE_OK;
002359 }
002360
002361 /*
002362 ** Delete a WhereLoop object
002363 */
002364 static void whereLoopDelete(sqlite3 *db, WhereLoop *p){
002365 assert( db!=0 );
002366 whereLoopClear(db, p);
002367 sqlite3DbNNFreeNN(db, p);
002368 }
002369
002370 /*
002371 ** Free a WhereInfo structure
002372 */
002373 static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
002374 assert( pWInfo!=0 );
002375 assert( db!=0 );
002376 sqlite3WhereClauseClear(&pWInfo->sWC);
002377 while( pWInfo->pLoops ){
002378 WhereLoop *p = pWInfo->pLoops;
002379 pWInfo->pLoops = p->pNextLoop;
002380 whereLoopDelete(db, p);
002381 }
002382 while( pWInfo->pMemToFree ){
002383 WhereMemBlock *pNext = pWInfo->pMemToFree->pNext;
002384 sqlite3DbNNFreeNN(db, pWInfo->pMemToFree);
002385 pWInfo->pMemToFree = pNext;
002386 }
002387 sqlite3DbNNFreeNN(db, pWInfo);
002388 }
002389
002390 /*
002391 ** Return TRUE if all of the following are true:
002392 **
002393 ** (1) X has the same or lower cost, or returns the same or fewer rows,
002394 ** than Y.
002395 ** (2) X uses fewer WHERE clause terms than Y
002396 ** (3) Every WHERE clause term used by X is also used by Y
002397 ** (4) X skips at least as many columns as Y
002398 ** (5) If X is a covering index, than Y is too
002399 **
002400 ** Conditions (2) and (3) mean that X is a "proper subset" of Y.
002401 ** If X is a proper subset of Y then Y is a better choice and ought
002402 ** to have a lower cost. This routine returns TRUE when that cost
002403 ** relationship is inverted and needs to be adjusted. Constraint (4)
002404 ** was added because if X uses skip-scan less than Y it still might
002405 ** deserve a lower cost even if it is a proper subset of Y. Constraint (5)
002406 ** was added because a covering index probably deserves to have a lower cost
002407 ** than a non-covering index even if it is a proper subset.
002408 */
002409 static int whereLoopCheaperProperSubset(
002410 const WhereLoop *pX, /* First WhereLoop to compare */
002411 const WhereLoop *pY /* Compare against this WhereLoop */
002412 ){
002413 int i, j;
002414 if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){
002415 return 0; /* X is not a subset of Y */
002416 }
002417 if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0;
002418 if( pY->nSkip > pX->nSkip ) return 0;
002419 for(i=pX->nLTerm-1; i>=0; i--){
002420 if( pX->aLTerm[i]==0 ) continue;
002421 for(j=pY->nLTerm-1; j>=0; j--){
002422 if( pY->aLTerm[j]==pX->aLTerm[i] ) break;
002423 }
002424 if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */
002425 }
002426 if( (pX->wsFlags&WHERE_IDX_ONLY)!=0
002427 && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){
002428 return 0; /* Constraint (5) */
002429 }
002430 return 1; /* All conditions meet */
002431 }
002432
002433 /*
002434 ** Try to adjust the cost and number of output rows of WhereLoop pTemplate
002435 ** upwards or downwards so that:
002436 **
002437 ** (1) pTemplate costs less than any other WhereLoops that are a proper
002438 ** subset of pTemplate
002439 **
002440 ** (2) pTemplate costs more than any other WhereLoops for which pTemplate
002441 ** is a proper subset.
002442 **
002443 ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer
002444 ** WHERE clause terms than Y and that every WHERE clause term used by X is
002445 ** also used by Y.
002446 */
002447 static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){
002448 if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return;
002449 for(; p; p=p->pNextLoop){
002450 if( p->iTab!=pTemplate->iTab ) continue;
002451 if( (p->wsFlags & WHERE_INDEXED)==0 ) continue;
002452 if( whereLoopCheaperProperSubset(p, pTemplate) ){
002453 /* Adjust pTemplate cost downward so that it is cheaper than its
002454 ** subset p. */
002455 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
002456 pTemplate->rRun, pTemplate->nOut,
002457 MIN(p->rRun, pTemplate->rRun),
002458 MIN(p->nOut - 1, pTemplate->nOut)));
002459 pTemplate->rRun = MIN(p->rRun, pTemplate->rRun);
002460 pTemplate->nOut = MIN(p->nOut - 1, pTemplate->nOut);
002461 }else if( whereLoopCheaperProperSubset(pTemplate, p) ){
002462 /* Adjust pTemplate cost upward so that it is costlier than p since
002463 ** pTemplate is a proper subset of p */
002464 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
002465 pTemplate->rRun, pTemplate->nOut,
002466 MAX(p->rRun, pTemplate->rRun),
002467 MAX(p->nOut + 1, pTemplate->nOut)));
002468 pTemplate->rRun = MAX(p->rRun, pTemplate->rRun);
002469 pTemplate->nOut = MAX(p->nOut + 1, pTemplate->nOut);
002470 }
002471 }
002472 }
002473
002474 /*
002475 ** Search the list of WhereLoops in *ppPrev looking for one that can be
002476 ** replaced by pTemplate.
002477 **
002478 ** Return NULL if pTemplate does not belong on the WhereLoop list.
002479 ** In other words if pTemplate ought to be dropped from further consideration.
002480 **
002481 ** If pX is a WhereLoop that pTemplate can replace, then return the
002482 ** link that points to pX.
002483 **
002484 ** If pTemplate cannot replace any existing element of the list but needs
002485 ** to be added to the list as a new entry, then return a pointer to the
002486 ** tail of the list.
002487 */
002488 static WhereLoop **whereLoopFindLesser(
002489 WhereLoop **ppPrev,
002490 const WhereLoop *pTemplate
002491 ){
002492 WhereLoop *p;
002493 for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){
002494 if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){
002495 /* If either the iTab or iSortIdx values for two WhereLoop are different
002496 ** then those WhereLoops need to be considered separately. Neither is
002497 ** a candidate to replace the other. */
002498 continue;
002499 }
002500 /* In the current implementation, the rSetup value is either zero
002501 ** or the cost of building an automatic index (NlogN) and the NlogN
002502 ** is the same for compatible WhereLoops. */
002503 assert( p->rSetup==0 || pTemplate->rSetup==0
002504 || p->rSetup==pTemplate->rSetup );
002505
002506 /* whereLoopAddBtree() always generates and inserts the automatic index
002507 ** case first. Hence compatible candidate WhereLoops never have a larger
002508 ** rSetup. Call this SETUP-INVARIANT */
002509 assert( p->rSetup>=pTemplate->rSetup );
002510
002511 /* Any loop using an application-defined index (or PRIMARY KEY or
002512 ** UNIQUE constraint) with one or more == constraints is better
002513 ** than an automatic index. Unless it is a skip-scan. */
002514 if( (p->wsFlags & WHERE_AUTO_INDEX)!=0
002515 && (pTemplate->nSkip)==0
002516 && (pTemplate->wsFlags & WHERE_INDEXED)!=0
002517 && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0
002518 && (p->prereq & pTemplate->prereq)==pTemplate->prereq
002519 ){
002520 break;
002521 }
002522
002523 /* If existing WhereLoop p is better than pTemplate, pTemplate can be
002524 ** discarded. WhereLoop p is better if:
002525 ** (1) p has no more dependencies than pTemplate, and
002526 ** (2) p has an equal or lower cost than pTemplate
002527 */
002528 if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */
002529 && p->rSetup<=pTemplate->rSetup /* (2a) */
002530 && p->rRun<=pTemplate->rRun /* (2b) */
002531 && p->nOut<=pTemplate->nOut /* (2c) */
002532 ){
002533 return 0; /* Discard pTemplate */
002534 }
002535
002536 /* If pTemplate is always better than p, then cause p to be overwritten
002537 ** with pTemplate. pTemplate is better than p if:
002538 ** (1) pTemplate has no more dependencies than p, and
002539 ** (2) pTemplate has an equal or lower cost than p.
002540 */
002541 if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */
002542 && p->rRun>=pTemplate->rRun /* (2a) */
002543 && p->nOut>=pTemplate->nOut /* (2b) */
002544 ){
002545 assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */
002546 break; /* Cause p to be overwritten by pTemplate */
002547 }
002548 }
002549 return ppPrev;
002550 }
002551
002552 /*
002553 ** Insert or replace a WhereLoop entry using the template supplied.
002554 **
002555 ** An existing WhereLoop entry might be overwritten if the new template
002556 ** is better and has fewer dependencies. Or the template will be ignored
002557 ** and no insert will occur if an existing WhereLoop is faster and has
002558 ** fewer dependencies than the template. Otherwise a new WhereLoop is
002559 ** added based on the template.
002560 **
002561 ** If pBuilder->pOrSet is not NULL then we care about only the
002562 ** prerequisites and rRun and nOut costs of the N best loops. That
002563 ** information is gathered in the pBuilder->pOrSet object. This special
002564 ** processing mode is used only for OR clause processing.
002565 **
002566 ** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we
002567 ** still might overwrite similar loops with the new template if the
002568 ** new template is better. Loops may be overwritten if the following
002569 ** conditions are met:
002570 **
002571 ** (1) They have the same iTab.
002572 ** (2) They have the same iSortIdx.
002573 ** (3) The template has same or fewer dependencies than the current loop
002574 ** (4) The template has the same or lower cost than the current loop
002575 */
002576 static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
002577 WhereLoop **ppPrev, *p;
002578 WhereInfo *pWInfo = pBuilder->pWInfo;
002579 sqlite3 *db = pWInfo->pParse->db;
002580 int rc;
002581
002582 /* Stop the search once we hit the query planner search limit */
002583 if( pBuilder->iPlanLimit==0 ){
002584 WHERETRACE(0xffffffff,("=== query planner search limit reached ===\n"));
002585 if( pBuilder->pOrSet ) pBuilder->pOrSet->n = 0;
002586 return SQLITE_DONE;
002587 }
002588 pBuilder->iPlanLimit--;
002589
002590 whereLoopAdjustCost(pWInfo->pLoops, pTemplate);
002591
002592 /* If pBuilder->pOrSet is defined, then only keep track of the costs
002593 ** and prereqs.
002594 */
002595 if( pBuilder->pOrSet!=0 ){
002596 if( pTemplate->nLTerm ){
002597 #if WHERETRACE_ENABLED
002598 u16 n = pBuilder->pOrSet->n;
002599 int x =
002600 #endif
002601 whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun,
002602 pTemplate->nOut);
002603 #if WHERETRACE_ENABLED /* 0x8 */
002604 if( sqlite3WhereTrace & 0x8 ){
002605 sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n);
002606 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
002607 }
002608 #endif
002609 }
002610 return SQLITE_OK;
002611 }
002612
002613 /* Look for an existing WhereLoop to replace with pTemplate
002614 */
002615 ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate);
002616
002617 if( ppPrev==0 ){
002618 /* There already exists a WhereLoop on the list that is better
002619 ** than pTemplate, so just ignore pTemplate */
002620 #if WHERETRACE_ENABLED /* 0x8 */
002621 if( sqlite3WhereTrace & 0x8 ){
002622 sqlite3DebugPrintf(" skip: ");
002623 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
002624 }
002625 #endif
002626 return SQLITE_OK;
002627 }else{
002628 p = *ppPrev;
002629 }
002630
002631 /* If we reach this point it means that either p[] should be overwritten
002632 ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new
002633 ** WhereLoop and insert it.
002634 */
002635 #if WHERETRACE_ENABLED /* 0x8 */
002636 if( sqlite3WhereTrace & 0x8 ){
002637 if( p!=0 ){
002638 sqlite3DebugPrintf("replace: ");
002639 sqlite3WhereLoopPrint(p, pBuilder->pWC);
002640 sqlite3DebugPrintf(" with: ");
002641 }else{
002642 sqlite3DebugPrintf(" add: ");
002643 }
002644 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
002645 }
002646 #endif
002647 if( p==0 ){
002648 /* Allocate a new WhereLoop to add to the end of the list */
002649 *ppPrev = p = sqlite3DbMallocRawNN(db, sizeof(WhereLoop));
002650 if( p==0 ) return SQLITE_NOMEM_BKPT;
002651 whereLoopInit(p);
002652 p->pNextLoop = 0;
002653 }else{
002654 /* We will be overwriting WhereLoop p[]. But before we do, first
002655 ** go through the rest of the list and delete any other entries besides
002656 ** p[] that are also supplanted by pTemplate */
002657 WhereLoop **ppTail = &p->pNextLoop;
002658 WhereLoop *pToDel;
002659 while( *ppTail ){
002660 ppTail = whereLoopFindLesser(ppTail, pTemplate);
002661 if( ppTail==0 ) break;
002662 pToDel = *ppTail;
002663 if( pToDel==0 ) break;
002664 *ppTail = pToDel->pNextLoop;
002665 #if WHERETRACE_ENABLED /* 0x8 */
002666 if( sqlite3WhereTrace & 0x8 ){
002667 sqlite3DebugPrintf(" delete: ");
002668 sqlite3WhereLoopPrint(pToDel, pBuilder->pWC);
002669 }
002670 #endif
002671 whereLoopDelete(db, pToDel);
002672 }
002673 }
002674 rc = whereLoopXfer(db, p, pTemplate);
002675 if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
002676 Index *pIndex = p->u.btree.pIndex;
002677 if( pIndex && pIndex->idxType==SQLITE_IDXTYPE_IPK ){
002678 p->u.btree.pIndex = 0;
002679 }
002680 }
002681 return rc;
002682 }
002683
002684 /*
002685 ** Adjust the WhereLoop.nOut value downward to account for terms of the
002686 ** WHERE clause that reference the loop but which are not used by an
002687 ** index.
002688 *
002689 ** For every WHERE clause term that is not used by the index
002690 ** and which has a truth probability assigned by one of the likelihood(),
002691 ** likely(), or unlikely() SQL functions, reduce the estimated number
002692 ** of output rows by the probability specified.
002693 **
002694 ** TUNING: For every WHERE clause term that is not used by the index
002695 ** and which does not have an assigned truth probability, heuristics
002696 ** described below are used to try to estimate the truth probability.
002697 ** TODO --> Perhaps this is something that could be improved by better
002698 ** table statistics.
002699 **
002700 ** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75%
002701 ** value corresponds to -1 in LogEst notation, so this means decrement
002702 ** the WhereLoop.nOut field for every such WHERE clause term.
002703 **
002704 ** Heuristic 2: If there exists one or more WHERE clause terms of the
002705 ** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the
002706 ** final output row estimate is no greater than 1/4 of the total number
002707 ** of rows in the table. In other words, assume that x==EXPR will filter
002708 ** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the
002709 ** "x" column is boolean or else -1 or 0 or 1 is a common default value
002710 ** on the "x" column and so in that case only cap the output row estimate
002711 ** at 1/2 instead of 1/4.
002712 */
002713 static void whereLoopOutputAdjust(
002714 WhereClause *pWC, /* The WHERE clause */
002715 WhereLoop *pLoop, /* The loop to adjust downward */
002716 LogEst nRow /* Number of rows in the entire table */
002717 ){
002718 WhereTerm *pTerm, *pX;
002719 Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf);
002720 int i, j;
002721 LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */
002722
002723 assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 );
002724 for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){
002725 assert( pTerm!=0 );
002726 if( (pTerm->prereqAll & notAllowed)!=0 ) continue;
002727 if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue;
002728 if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) continue;
002729 for(j=pLoop->nLTerm-1; j>=0; j--){
002730 pX = pLoop->aLTerm[j];
002731 if( pX==0 ) continue;
002732 if( pX==pTerm ) break;
002733 if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break;
002734 }
002735 if( j<0 ){
002736 sqlite3ProgressCheck(pWC->pWInfo->pParse);
002737 if( pLoop->maskSelf==pTerm->prereqAll ){
002738 /* If there are extra terms in the WHERE clause not used by an index
002739 ** that depend only on the table being scanned, and that will tend to
002740 ** cause many rows to be omitted, then mark that table as
002741 ** "self-culling".
002742 **
002743 ** 2022-03-24: Self-culling only applies if either the extra terms
002744 ** are straight comparison operators that are non-true with NULL
002745 ** operand, or if the loop is not an OUTER JOIN.
002746 */
002747 if( (pTerm->eOperator & 0x3f)!=0
002748 || (pWC->pWInfo->pTabList->a[pLoop->iTab].fg.jointype
002749 & (JT_LEFT|JT_LTORJ))==0
002750 ){
002751 pLoop->wsFlags |= WHERE_SELFCULL;
002752 }
002753 }
002754 if( pTerm->truthProb<=0 ){
002755 /* If a truth probability is specified using the likelihood() hints,
002756 ** then use the probability provided by the application. */
002757 pLoop->nOut += pTerm->truthProb;
002758 }else{
002759 /* In the absence of explicit truth probabilities, use heuristics to
002760 ** guess a reasonable truth probability. */
002761 pLoop->nOut--;
002762 if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0
002763 && (pTerm->wtFlags & TERM_HIGHTRUTH)==0 /* tag-20200224-1 */
002764 ){
002765 Expr *pRight = pTerm->pExpr->pRight;
002766 int k = 0;
002767 testcase( pTerm->pExpr->op==TK_IS );
002768 if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){
002769 k = 10;
002770 }else{
002771 k = 20;
002772 }
002773 if( iReduce<k ){
002774 pTerm->wtFlags |= TERM_HEURTRUTH;
002775 iReduce = k;
002776 }
002777 }
002778 }
002779 }
002780 }
002781 if( pLoop->nOut > nRow-iReduce ){
002782 pLoop->nOut = nRow - iReduce;
002783 }
002784 }
002785
002786 /*
002787 ** Term pTerm is a vector range comparison operation. The first comparison
002788 ** in the vector can be optimized using column nEq of the index. This
002789 ** function returns the total number of vector elements that can be used
002790 ** as part of the range comparison.
002791 **
002792 ** For example, if the query is:
002793 **
002794 ** WHERE a = ? AND (b, c, d) > (?, ?, ?)
002795 **
002796 ** and the index:
002797 **
002798 ** CREATE INDEX ... ON (a, b, c, d, e)
002799 **
002800 ** then this function would be invoked with nEq=1. The value returned in
002801 ** this case is 3.
002802 */
002803 static int whereRangeVectorLen(
002804 Parse *pParse, /* Parsing context */
002805 int iCur, /* Cursor open on pIdx */
002806 Index *pIdx, /* The index to be used for a inequality constraint */
002807 int nEq, /* Number of prior equality constraints on same index */
002808 WhereTerm *pTerm /* The vector inequality constraint */
002809 ){
002810 int nCmp = sqlite3ExprVectorSize(pTerm->pExpr->pLeft);
002811 int i;
002812
002813 nCmp = MIN(nCmp, (pIdx->nColumn - nEq));
002814 for(i=1; i<nCmp; i++){
002815 /* Test if comparison i of pTerm is compatible with column (i+nEq)
002816 ** of the index. If not, exit the loop. */
002817 char aff; /* Comparison affinity */
002818 char idxaff = 0; /* Indexed columns affinity */
002819 CollSeq *pColl; /* Comparison collation sequence */
002820 Expr *pLhs, *pRhs;
002821
002822 assert( ExprUseXList(pTerm->pExpr->pLeft) );
002823 pLhs = pTerm->pExpr->pLeft->x.pList->a[i].pExpr;
002824 pRhs = pTerm->pExpr->pRight;
002825 if( ExprUseXSelect(pRhs) ){
002826 pRhs = pRhs->x.pSelect->pEList->a[i].pExpr;
002827 }else{
002828 pRhs = pRhs->x.pList->a[i].pExpr;
002829 }
002830
002831 /* Check that the LHS of the comparison is a column reference to
002832 ** the right column of the right source table. And that the sort
002833 ** order of the index column is the same as the sort order of the
002834 ** leftmost index column. */
002835 if( pLhs->op!=TK_COLUMN
002836 || pLhs->iTable!=iCur
002837 || pLhs->iColumn!=pIdx->aiColumn[i+nEq]
002838 || pIdx->aSortOrder[i+nEq]!=pIdx->aSortOrder[nEq]
002839 ){
002840 break;
002841 }
002842
002843 testcase( pLhs->iColumn==XN_ROWID );
002844 aff = sqlite3CompareAffinity(pRhs, sqlite3ExprAffinity(pLhs));
002845 idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn);
002846 if( aff!=idxaff ) break;
002847
002848 pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs);
002849 if( pColl==0 ) break;
002850 if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break;
002851 }
002852 return i;
002853 }
002854
002855 /*
002856 ** Adjust the cost C by the costMult factor T. This only occurs if
002857 ** compiled with -DSQLITE_ENABLE_COSTMULT
002858 */
002859 #ifdef SQLITE_ENABLE_COSTMULT
002860 # define ApplyCostMultiplier(C,T) C += T
002861 #else
002862 # define ApplyCostMultiplier(C,T)
002863 #endif
002864
002865 /*
002866 ** We have so far matched pBuilder->pNew->u.btree.nEq terms of the
002867 ** index pIndex. Try to match one more.
002868 **
002869 ** When this function is called, pBuilder->pNew->nOut contains the
002870 ** number of rows expected to be visited by filtering using the nEq
002871 ** terms only. If it is modified, this value is restored before this
002872 ** function returns.
002873 **
002874 ** If pProbe->idxType==SQLITE_IDXTYPE_IPK, that means pIndex is
002875 ** a fake index used for the INTEGER PRIMARY KEY.
002876 */
002877 static int whereLoopAddBtreeIndex(
002878 WhereLoopBuilder *pBuilder, /* The WhereLoop factory */
002879 SrcItem *pSrc, /* FROM clause term being analyzed */
002880 Index *pProbe, /* An index on pSrc */
002881 LogEst nInMul /* log(Number of iterations due to IN) */
002882 ){
002883 WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyze context */
002884 Parse *pParse = pWInfo->pParse; /* Parsing context */
002885 sqlite3 *db = pParse->db; /* Database connection malloc context */
002886 WhereLoop *pNew; /* Template WhereLoop under construction */
002887 WhereTerm *pTerm; /* A WhereTerm under consideration */
002888 int opMask; /* Valid operators for constraints */
002889 WhereScan scan; /* Iterator for WHERE terms */
002890 Bitmask saved_prereq; /* Original value of pNew->prereq */
002891 u16 saved_nLTerm; /* Original value of pNew->nLTerm */
002892 u16 saved_nEq; /* Original value of pNew->u.btree.nEq */
002893 u16 saved_nBtm; /* Original value of pNew->u.btree.nBtm */
002894 u16 saved_nTop; /* Original value of pNew->u.btree.nTop */
002895 u16 saved_nSkip; /* Original value of pNew->nSkip */
002896 u32 saved_wsFlags; /* Original value of pNew->wsFlags */
002897 LogEst saved_nOut; /* Original value of pNew->nOut */
002898 int rc = SQLITE_OK; /* Return code */
002899 LogEst rSize; /* Number of rows in the table */
002900 LogEst rLogSize; /* Logarithm of table size */
002901 WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */
002902
002903 pNew = pBuilder->pNew;
002904 assert( db->mallocFailed==0 || pParse->nErr>0 );
002905 if( pParse->nErr ){
002906 return pParse->rc;
002907 }
002908 WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d, rRun=%d\n",
002909 pProbe->pTable->zName,pProbe->zName,
002910 pNew->u.btree.nEq, pNew->nSkip, pNew->rRun));
002911
002912 assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 );
002913 assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 );
002914 if( pNew->wsFlags & WHERE_BTM_LIMIT ){
002915 opMask = WO_LT|WO_LE;
002916 }else{
002917 assert( pNew->u.btree.nBtm==0 );
002918 opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS;
002919 }
002920 if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
002921
002922 assert( pNew->u.btree.nEq<pProbe->nColumn );
002923 assert( pNew->u.btree.nEq<pProbe->nKeyCol
002924 || pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY );
002925
002926 saved_nEq = pNew->u.btree.nEq;
002927 saved_nBtm = pNew->u.btree.nBtm;
002928 saved_nTop = pNew->u.btree.nTop;
002929 saved_nSkip = pNew->nSkip;
002930 saved_nLTerm = pNew->nLTerm;
002931 saved_wsFlags = pNew->wsFlags;
002932 saved_prereq = pNew->prereq;
002933 saved_nOut = pNew->nOut;
002934 pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq,
002935 opMask, pProbe);
002936 pNew->rSetup = 0;
002937 rSize = pProbe->aiRowLogEst[0];
002938 rLogSize = estLog(rSize);
002939 for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){
002940 u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */
002941 LogEst rCostIdx;
002942 LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */
002943 int nIn = 0;
002944 #ifdef SQLITE_ENABLE_STAT4
002945 int nRecValid = pBuilder->nRecValid;
002946 #endif
002947 if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0)
002948 && indexColumnNotNull(pProbe, saved_nEq)
002949 ){
002950 continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */
002951 }
002952 if( pTerm->prereqRight & pNew->maskSelf ) continue;
002953
002954 /* Do not allow the upper bound of a LIKE optimization range constraint
002955 ** to mix with a lower range bound from some other source */
002956 if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue;
002957
002958 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
002959 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
002960 ){
002961 continue;
002962 }
002963 if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){
002964 pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE;
002965 }else{
002966 pBuilder->bldFlags1 |= SQLITE_BLDF1_INDEXED;
002967 }
002968 pNew->wsFlags = saved_wsFlags;
002969 pNew->u.btree.nEq = saved_nEq;
002970 pNew->u.btree.nBtm = saved_nBtm;
002971 pNew->u.btree.nTop = saved_nTop;
002972 pNew->nLTerm = saved_nLTerm;
002973 if( pNew->nLTerm>=pNew->nLSlot
002974 && whereLoopResize(db, pNew, pNew->nLTerm+1)
002975 ){
002976 break; /* OOM while trying to enlarge the pNew->aLTerm array */
002977 }
002978 pNew->aLTerm[pNew->nLTerm++] = pTerm;
002979 pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf;
002980
002981 assert( nInMul==0
002982 || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0
002983 || (pNew->wsFlags & WHERE_COLUMN_IN)!=0
002984 || (pNew->wsFlags & WHERE_SKIPSCAN)!=0
002985 );
002986
002987 if( eOp & WO_IN ){
002988 Expr *pExpr = pTerm->pExpr;
002989 if( ExprUseXSelect(pExpr) ){
002990 /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */
002991 int i;
002992 nIn = 46; assert( 46==sqlite3LogEst(25) );
002993
002994 /* The expression may actually be of the form (x, y) IN (SELECT...).
002995 ** In this case there is a separate term for each of (x) and (y).
002996 ** However, the nIn multiplier should only be applied once, not once
002997 ** for each such term. The following loop checks that pTerm is the
002998 ** first such term in use, and sets nIn back to 0 if it is not. */
002999 for(i=0; i<pNew->nLTerm-1; i++){
003000 if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0;
003001 }
003002 }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){
003003 /* "x IN (value, value, ...)" */
003004 nIn = sqlite3LogEst(pExpr->x.pList->nExpr);
003005 }
003006 if( pProbe->hasStat1 && rLogSize>=10 ){
003007 LogEst M, logK, x;
003008 /* Let:
003009 ** N = the total number of rows in the table
003010 ** K = the number of entries on the RHS of the IN operator
003011 ** M = the number of rows in the table that match terms to the
003012 ** to the left in the same index. If the IN operator is on
003013 ** the left-most index column, M==N.
003014 **
003015 ** Given the definitions above, it is better to omit the IN operator
003016 ** from the index lookup and instead do a scan of the M elements,
003017 ** testing each scanned row against the IN operator separately, if:
003018 **
003019 ** M*log(K) < K*log(N)
003020 **
003021 ** Our estimates for M, K, and N might be inaccurate, so we build in
003022 ** a safety margin of 2 (LogEst: 10) that favors using the IN operator
003023 ** with the index, as using an index has better worst-case behavior.
003024 ** If we do not have real sqlite_stat1 data, always prefer to use
003025 ** the index. Do not bother with this optimization on very small
003026 ** tables (less than 2 rows) as it is pointless in that case.
003027 */
003028 M = pProbe->aiRowLogEst[saved_nEq];
003029 logK = estLog(nIn);
003030 /* TUNING v----- 10 to bias toward indexed IN */
003031 x = M + logK + 10 - (nIn + rLogSize);
003032 if( x>=0 ){
003033 WHERETRACE(0x40,
003034 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d) "
003035 "prefers indexed lookup\n",
003036 saved_nEq, M, logK, nIn, rLogSize, x));
003037 }else if( nInMul<2 && OptimizationEnabled(db, SQLITE_SeekScan) ){
003038 WHERETRACE(0x40,
003039 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
003040 " nInMul=%d) prefers skip-scan\n",
003041 saved_nEq, M, logK, nIn, rLogSize, x, nInMul));
003042 pNew->wsFlags |= WHERE_IN_SEEKSCAN;
003043 }else{
003044 WHERETRACE(0x40,
003045 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
003046 " nInMul=%d) prefers normal scan\n",
003047 saved_nEq, M, logK, nIn, rLogSize, x, nInMul));
003048 continue;
003049 }
003050 }
003051 pNew->wsFlags |= WHERE_COLUMN_IN;
003052 }else if( eOp & (WO_EQ|WO_IS) ){
003053 int iCol = pProbe->aiColumn[saved_nEq];
003054 pNew->wsFlags |= WHERE_COLUMN_EQ;
003055 assert( saved_nEq==pNew->u.btree.nEq );
003056 if( iCol==XN_ROWID
003057 || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1)
003058 ){
003059 if( iCol==XN_ROWID || pProbe->uniqNotNull
003060 || (pProbe->nKeyCol==1 && pProbe->onError && eOp==WO_EQ)
003061 ){
003062 pNew->wsFlags |= WHERE_ONEROW;
003063 }else{
003064 pNew->wsFlags |= WHERE_UNQ_WANTED;
003065 }
003066 }
003067 if( scan.iEquiv>1 ) pNew->wsFlags |= WHERE_TRANSCONS;
003068 }else if( eOp & WO_ISNULL ){
003069 pNew->wsFlags |= WHERE_COLUMN_NULL;
003070 }else{
003071 int nVecLen = whereRangeVectorLen(
003072 pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm
003073 );
003074 if( eOp & (WO_GT|WO_GE) ){
003075 testcase( eOp & WO_GT );
003076 testcase( eOp & WO_GE );
003077 pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT;
003078 pNew->u.btree.nBtm = nVecLen;
003079 pBtm = pTerm;
003080 pTop = 0;
003081 if( pTerm->wtFlags & TERM_LIKEOPT ){
003082 /* Range constraints that come from the LIKE optimization are
003083 ** always used in pairs. */
003084 pTop = &pTerm[1];
003085 assert( (pTop-(pTerm->pWC->a))<pTerm->pWC->nTerm );
003086 assert( pTop->wtFlags & TERM_LIKEOPT );
003087 assert( pTop->eOperator==WO_LT );
003088 if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
003089 pNew->aLTerm[pNew->nLTerm++] = pTop;
003090 pNew->wsFlags |= WHERE_TOP_LIMIT;
003091 pNew->u.btree.nTop = 1;
003092 }
003093 }else{
003094 assert( eOp & (WO_LT|WO_LE) );
003095 testcase( eOp & WO_LT );
003096 testcase( eOp & WO_LE );
003097 pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT;
003098 pNew->u.btree.nTop = nVecLen;
003099 pTop = pTerm;
003100 pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ?
003101 pNew->aLTerm[pNew->nLTerm-2] : 0;
003102 }
003103 }
003104
003105 /* At this point pNew->nOut is set to the number of rows expected to
003106 ** be visited by the index scan before considering term pTerm, or the
003107 ** values of nIn and nInMul. In other words, assuming that all
003108 ** "x IN(...)" terms are replaced with "x = ?". This block updates
003109 ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */
003110 assert( pNew->nOut==saved_nOut );
003111 if( pNew->wsFlags & WHERE_COLUMN_RANGE ){
003112 /* Adjust nOut using stat4 data. Or, if there is no stat4
003113 ** data, using some other estimate. */
003114 whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew);
003115 }else{
003116 int nEq = ++pNew->u.btree.nEq;
003117 assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) );
003118
003119 assert( pNew->nOut==saved_nOut );
003120 if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){
003121 assert( (eOp & WO_IN) || nIn==0 );
003122 testcase( eOp & WO_IN );
003123 pNew->nOut += pTerm->truthProb;
003124 pNew->nOut -= nIn;
003125 }else{
003126 #ifdef SQLITE_ENABLE_STAT4
003127 tRowcnt nOut = 0;
003128 if( nInMul==0
003129 && pProbe->nSample
003130 && ALWAYS(pNew->u.btree.nEq<=pProbe->nSampleCol)
003131 && ((eOp & WO_IN)==0 || ExprUseXList(pTerm->pExpr))
003132 && OptimizationEnabled(db, SQLITE_Stat4)
003133 ){
003134 Expr *pExpr = pTerm->pExpr;
003135 if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){
003136 testcase( eOp & WO_EQ );
003137 testcase( eOp & WO_IS );
003138 testcase( eOp & WO_ISNULL );
003139 rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut);
003140 }else{
003141 rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut);
003142 }
003143 if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK;
003144 if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */
003145 if( nOut ){
003146 pNew->nOut = sqlite3LogEst(nOut);
003147 if( nEq==1
003148 /* TUNING: Mark terms as "low selectivity" if they seem likely
003149 ** to be true for half or more of the rows in the table.
003150 ** See tag-202002240-1 */
003151 && pNew->nOut+10 > pProbe->aiRowLogEst[0]
003152 ){
003153 #if WHERETRACE_ENABLED /* 0x01 */
003154 if( sqlite3WhereTrace & 0x20 ){
003155 sqlite3DebugPrintf(
003156 "STAT4 determines term has low selectivity:\n");
003157 sqlite3WhereTermPrint(pTerm, 999);
003158 }
003159 #endif
003160 pTerm->wtFlags |= TERM_HIGHTRUTH;
003161 if( pTerm->wtFlags & TERM_HEURTRUTH ){
003162 /* If the term has previously been used with an assumption of
003163 ** higher selectivity, then set the flag to rerun the
003164 ** loop computations. */
003165 pBuilder->bldFlags2 |= SQLITE_BLDF2_2NDPASS;
003166 }
003167 }
003168 if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut;
003169 pNew->nOut -= nIn;
003170 }
003171 }
003172 if( nOut==0 )
003173 #endif
003174 {
003175 pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]);
003176 if( eOp & WO_ISNULL ){
003177 /* TUNING: If there is no likelihood() value, assume that a
003178 ** "col IS NULL" expression matches twice as many rows
003179 ** as (col=?). */
003180 pNew->nOut += 10;
003181 }
003182 }
003183 }
003184 }
003185
003186 /* Set rCostIdx to the cost of visiting selected rows in index. Add
003187 ** it to pNew->rRun, which is currently set to the cost of the index
003188 ** seek only. Then, if this is a non-covering index, add the cost of
003189 ** visiting the rows in the main table. */
003190 assert( pSrc->pTab->szTabRow>0 );
003191 if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){
003192 /* The pProbe->szIdxRow is low for an IPK table since the interior
003193 ** pages are small. Thus szIdxRow gives a good estimate of seek cost.
003194 ** But the leaf pages are full-size, so pProbe->szIdxRow would badly
003195 ** under-estimate the scanning cost. */
003196 rCostIdx = pNew->nOut + 16;
003197 }else{
003198 rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow;
003199 }
003200 pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx);
003201 if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){
003202 pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16);
003203 }
003204 ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult);
003205
003206 nOutUnadjusted = pNew->nOut;
003207 pNew->rRun += nInMul + nIn;
003208 pNew->nOut += nInMul + nIn;
003209 whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize);
003210 rc = whereLoopInsert(pBuilder, pNew);
003211
003212 if( pNew->wsFlags & WHERE_COLUMN_RANGE ){
003213 pNew->nOut = saved_nOut;
003214 }else{
003215 pNew->nOut = nOutUnadjusted;
003216 }
003217
003218 if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0
003219 && pNew->u.btree.nEq<pProbe->nColumn
003220 && (pNew->u.btree.nEq<pProbe->nKeyCol ||
003221 pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY)
003222 ){
003223 if( pNew->u.btree.nEq>3 ){
003224 sqlite3ProgressCheck(pParse);
003225 }
003226 whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn);
003227 }
003228 pNew->nOut = saved_nOut;
003229 #ifdef SQLITE_ENABLE_STAT4
003230 pBuilder->nRecValid = nRecValid;
003231 #endif
003232 }
003233 pNew->prereq = saved_prereq;
003234 pNew->u.btree.nEq = saved_nEq;
003235 pNew->u.btree.nBtm = saved_nBtm;
003236 pNew->u.btree.nTop = saved_nTop;
003237 pNew->nSkip = saved_nSkip;
003238 pNew->wsFlags = saved_wsFlags;
003239 pNew->nOut = saved_nOut;
003240 pNew->nLTerm = saved_nLTerm;
003241
003242 /* Consider using a skip-scan if there are no WHERE clause constraints
003243 ** available for the left-most terms of the index, and if the average
003244 ** number of repeats in the left-most terms is at least 18.
003245 **
003246 ** The magic number 18 is selected on the basis that scanning 17 rows
003247 ** is almost always quicker than an index seek (even though if the index
003248 ** contains fewer than 2^17 rows we assume otherwise in other parts of
003249 ** the code). And, even if it is not, it should not be too much slower.
003250 ** On the other hand, the extra seeks could end up being significantly
003251 ** more expensive. */
003252 assert( 42==sqlite3LogEst(18) );
003253 if( saved_nEq==saved_nSkip
003254 && saved_nEq+1<pProbe->nKeyCol
003255 && saved_nEq==pNew->nLTerm
003256 && pProbe->noSkipScan==0
003257 && pProbe->hasStat1!=0
003258 && OptimizationEnabled(db, SQLITE_SkipScan)
003259 && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */
003260 && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK
003261 ){
003262 LogEst nIter;
003263 pNew->u.btree.nEq++;
003264 pNew->nSkip++;
003265 pNew->aLTerm[pNew->nLTerm++] = 0;
003266 pNew->wsFlags |= WHERE_SKIPSCAN;
003267 nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1];
003268 pNew->nOut -= nIter;
003269 /* TUNING: Because uncertainties in the estimates for skip-scan queries,
003270 ** add a 1.375 fudge factor to make skip-scan slightly less likely. */
003271 nIter += 5;
003272 whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul);
003273 pNew->nOut = saved_nOut;
003274 pNew->u.btree.nEq = saved_nEq;
003275 pNew->nSkip = saved_nSkip;
003276 pNew->wsFlags = saved_wsFlags;
003277 }
003278
003279 WHERETRACE(0x800, ("END %s.addBtreeIdx(%s), nEq=%d, rc=%d\n",
003280 pProbe->pTable->zName, pProbe->zName, saved_nEq, rc));
003281 return rc;
003282 }
003283
003284 /*
003285 ** Return True if it is possible that pIndex might be useful in
003286 ** implementing the ORDER BY clause in pBuilder.
003287 **
003288 ** Return False if pBuilder does not contain an ORDER BY clause or
003289 ** if there is no way for pIndex to be useful in implementing that
003290 ** ORDER BY clause.
003291 */
003292 static int indexMightHelpWithOrderBy(
003293 WhereLoopBuilder *pBuilder,
003294 Index *pIndex,
003295 int iCursor
003296 ){
003297 ExprList *pOB;
003298 ExprList *aColExpr;
003299 int ii, jj;
003300
003301 if( pIndex->bUnordered ) return 0;
003302 if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0;
003303 for(ii=0; ii<pOB->nExpr; ii++){
003304 Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr);
003305 if( NEVER(pExpr==0) ) continue;
003306 if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){
003307 if( pExpr->iColumn<0 ) return 1;
003308 for(jj=0; jj<pIndex->nKeyCol; jj++){
003309 if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1;
003310 }
003311 }else if( (aColExpr = pIndex->aColExpr)!=0 ){
003312 for(jj=0; jj<pIndex->nKeyCol; jj++){
003313 if( pIndex->aiColumn[jj]!=XN_EXPR ) continue;
003314 if( sqlite3ExprCompareSkip(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){
003315 return 1;
003316 }
003317 }
003318 }
003319 }
003320 return 0;
003321 }
003322
003323 /* Check to see if a partial index with pPartIndexWhere can be used
003324 ** in the current query. Return true if it can be and false if not.
003325 */
003326 static int whereUsablePartialIndex(
003327 int iTab, /* The table for which we want an index */
003328 u8 jointype, /* The JT_* flags on the join */
003329 WhereClause *pWC, /* The WHERE clause of the query */
003330 Expr *pWhere /* The WHERE clause from the partial index */
003331 ){
003332 int i;
003333 WhereTerm *pTerm;
003334 Parse *pParse;
003335
003336 if( jointype & JT_LTORJ ) return 0;
003337 pParse = pWC->pWInfo->pParse;
003338 while( pWhere->op==TK_AND ){
003339 if( !whereUsablePartialIndex(iTab,jointype,pWC,pWhere->pLeft) ) return 0;
003340 pWhere = pWhere->pRight;
003341 }
003342 if( pParse->db->flags & SQLITE_EnableQPSG ) pParse = 0;
003343 for(i=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
003344 Expr *pExpr;
003345 pExpr = pTerm->pExpr;
003346 if( (!ExprHasProperty(pExpr, EP_OuterON) || pExpr->w.iJoin==iTab)
003347 && ((jointype & JT_OUTER)==0 || ExprHasProperty(pExpr, EP_OuterON))
003348 && sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, iTab)
003349 && (pTerm->wtFlags & TERM_VNULL)==0
003350 ){
003351 return 1;
003352 }
003353 }
003354 return 0;
003355 }
003356
003357 /*
003358 ** pIdx is an index containing expressions. Check it see if any of the
003359 ** expressions in the index match the pExpr expression.
003360 */
003361 static int exprIsCoveredByIndex(
003362 const Expr *pExpr,
003363 const Index *pIdx,
003364 int iTabCur
003365 ){
003366 int i;
003367 for(i=0; i<pIdx->nColumn; i++){
003368 if( pIdx->aiColumn[i]==XN_EXPR
003369 && sqlite3ExprCompare(0, pExpr, pIdx->aColExpr->a[i].pExpr, iTabCur)==0
003370 ){
003371 return 1;
003372 }
003373 }
003374 return 0;
003375 }
003376
003377 /*
003378 ** Structure passed to the whereIsCoveringIndex Walker callback.
003379 */
003380 typedef struct CoveringIndexCheck CoveringIndexCheck;
003381 struct CoveringIndexCheck {
003382 Index *pIdx; /* The index */
003383 int iTabCur; /* Cursor number for the corresponding table */
003384 u8 bExpr; /* Uses an indexed expression */
003385 u8 bUnidx; /* Uses an unindexed column not within an indexed expr */
003386 };
003387
003388 /*
003389 ** Information passed in is pWalk->u.pCovIdxCk. Call it pCk.
003390 **
003391 ** If the Expr node references the table with cursor pCk->iTabCur, then
003392 ** make sure that column is covered by the index pCk->pIdx. We know that
003393 ** all columns less than 63 (really BMS-1) are covered, so we don't need
003394 ** to check them. But we do need to check any column at 63 or greater.
003395 **
003396 ** If the index does not cover the column, then set pWalk->eCode to
003397 ** non-zero and return WRC_Abort to stop the search.
003398 **
003399 ** If this node does not disprove that the index can be a covering index,
003400 ** then just return WRC_Continue, to continue the search.
003401 **
003402 ** If pCk->pIdx contains indexed expressions and one of those expressions
003403 ** matches pExpr, then prune the search.
003404 */
003405 static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){
003406 int i; /* Loop counter */
003407 const Index *pIdx; /* The index of interest */
003408 const i16 *aiColumn; /* Columns contained in the index */
003409 u16 nColumn; /* Number of columns in the index */
003410 CoveringIndexCheck *pCk; /* Info about this search */
003411
003412 pCk = pWalk->u.pCovIdxCk;
003413 pIdx = pCk->pIdx;
003414 if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){
003415 /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/
003416 if( pExpr->iTable!=pCk->iTabCur ) return WRC_Continue;
003417 pIdx = pWalk->u.pCovIdxCk->pIdx;
003418 aiColumn = pIdx->aiColumn;
003419 nColumn = pIdx->nColumn;
003420 for(i=0; i<nColumn; i++){
003421 if( aiColumn[i]==pExpr->iColumn ) return WRC_Continue;
003422 }
003423 pCk->bUnidx = 1;
003424 return WRC_Abort;
003425 }else if( pIdx->bHasExpr
003426 && exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){
003427 pCk->bExpr = 1;
003428 return WRC_Prune;
003429 }
003430 return WRC_Continue;
003431 }
003432
003433
003434 /*
003435 ** pIdx is an index that covers all of the low-number columns used by
003436 ** pWInfo->pSelect (columns from 0 through 62) or an index that has
003437 ** expressions terms. Hence, we cannot determine whether or not it is
003438 ** a covering index by using the colUsed bitmasks. We have to do a search
003439 ** to see if the index is covering. This routine does that search.
003440 **
003441 ** The return value is one of these:
003442 **
003443 ** 0 The index is definitely not a covering index
003444 **
003445 ** WHERE_IDX_ONLY The index is definitely a covering index
003446 **
003447 ** WHERE_EXPRIDX The index is likely a covering index, but it is
003448 ** difficult to determine precisely because of the
003449 ** expressions that are indexed. Score it as a
003450 ** covering index, but still keep the main table open
003451 ** just in case we need it.
003452 **
003453 ** This routine is an optimization. It is always safe to return zero.
003454 ** But returning one of the other two values when zero should have been
003455 ** returned can lead to incorrect bytecode and assertion faults.
003456 */
003457 static SQLITE_NOINLINE u32 whereIsCoveringIndex(
003458 WhereInfo *pWInfo, /* The WHERE clause context */
003459 Index *pIdx, /* Index that is being tested */
003460 int iTabCur /* Cursor for the table being indexed */
003461 ){
003462 int i, rc;
003463 struct CoveringIndexCheck ck;
003464 Walker w;
003465 if( pWInfo->pSelect==0 ){
003466 /* We don't have access to the full query, so we cannot check to see
003467 ** if pIdx is covering. Assume it is not. */
003468 return 0;
003469 }
003470 if( pIdx->bHasExpr==0 ){
003471 for(i=0; i<pIdx->nColumn; i++){
003472 if( pIdx->aiColumn[i]>=BMS-1 ) break;
003473 }
003474 if( i>=pIdx->nColumn ){
003475 /* pIdx does not index any columns greater than 62, but we know from
003476 ** colMask that columns greater than 62 are used, so this is not a
003477 ** covering index */
003478 return 0;
003479 }
003480 }
003481 ck.pIdx = pIdx;
003482 ck.iTabCur = iTabCur;
003483 ck.bExpr = 0;
003484 ck.bUnidx = 0;
003485 memset(&w, 0, sizeof(w));
003486 w.xExprCallback = whereIsCoveringIndexWalkCallback;
003487 w.xSelectCallback = sqlite3SelectWalkNoop;
003488 w.u.pCovIdxCk = &ck;
003489 sqlite3WalkSelect(&w, pWInfo->pSelect);
003490 if( ck.bUnidx ){
003491 rc = 0;
003492 }else if( ck.bExpr ){
003493 rc = WHERE_EXPRIDX;
003494 }else{
003495 rc = WHERE_IDX_ONLY;
003496 }
003497 return rc;
003498 }
003499
003500 /*
003501 ** Add all WhereLoop objects for a single table of the join where the table
003502 ** is identified by pBuilder->pNew->iTab. That table is guaranteed to be
003503 ** a b-tree table, not a virtual table.
003504 **
003505 ** The costs (WhereLoop.rRun) of the b-tree loops added by this function
003506 ** are calculated as follows:
003507 **
003508 ** For a full scan, assuming the table (or index) contains nRow rows:
003509 **
003510 ** cost = nRow * 3.0 // full-table scan
003511 ** cost = nRow * K // scan of covering index
003512 ** cost = nRow * (K+3.0) // scan of non-covering index
003513 **
003514 ** where K is a value between 1.1 and 3.0 set based on the relative
003515 ** estimated average size of the index and table records.
003516 **
003517 ** For an index scan, where nVisit is the number of index rows visited
003518 ** by the scan, and nSeek is the number of seek operations required on
003519 ** the index b-tree:
003520 **
003521 ** cost = nSeek * (log(nRow) + K * nVisit) // covering index
003522 ** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index
003523 **
003524 ** Normally, nSeek is 1. nSeek values greater than 1 come about if the
003525 ** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when
003526 ** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans.
003527 **
003528 ** The estimated values (nRow, nVisit, nSeek) often contain a large amount
003529 ** of uncertainty. For this reason, scoring is designed to pick plans that
003530 ** "do the least harm" if the estimates are inaccurate. For example, a
003531 ** log(nRow) factor is omitted from a non-covering index scan in order to
003532 ** bias the scoring in favor of using an index, since the worst-case
003533 ** performance of using an index is far better than the worst-case performance
003534 ** of a full table scan.
003535 */
003536 static int whereLoopAddBtree(
003537 WhereLoopBuilder *pBuilder, /* WHERE clause information */
003538 Bitmask mPrereq /* Extra prerequisites for using this table */
003539 ){
003540 WhereInfo *pWInfo; /* WHERE analysis context */
003541 Index *pProbe; /* An index we are evaluating */
003542 Index sPk; /* A fake index object for the primary key */
003543 LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */
003544 i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */
003545 SrcList *pTabList; /* The FROM clause */
003546 SrcItem *pSrc; /* The FROM clause btree term to add */
003547 WhereLoop *pNew; /* Template WhereLoop object */
003548 int rc = SQLITE_OK; /* Return code */
003549 int iSortIdx = 1; /* Index number */
003550 int b; /* A boolean value */
003551 LogEst rSize; /* number of rows in the table */
003552 WhereClause *pWC; /* The parsed WHERE clause */
003553 Table *pTab; /* Table being queried */
003554
003555 pNew = pBuilder->pNew;
003556 pWInfo = pBuilder->pWInfo;
003557 pTabList = pWInfo->pTabList;
003558 pSrc = pTabList->a + pNew->iTab;
003559 pTab = pSrc->pTab;
003560 pWC = pBuilder->pWC;
003561 assert( !IsVirtual(pSrc->pTab) );
003562
003563 if( pSrc->fg.isIndexedBy ){
003564 assert( pSrc->fg.isCte==0 );
003565 /* An INDEXED BY clause specifies a particular index to use */
003566 pProbe = pSrc->u2.pIBIndex;
003567 }else if( !HasRowid(pTab) ){
003568 pProbe = pTab->pIndex;
003569 }else{
003570 /* There is no INDEXED BY clause. Create a fake Index object in local
003571 ** variable sPk to represent the rowid primary key index. Make this
003572 ** fake index the first in a chain of Index objects with all of the real
003573 ** indices to follow */
003574 Index *pFirst; /* First of real indices on the table */
003575 memset(&sPk, 0, sizeof(Index));
003576 sPk.nKeyCol = 1;
003577 sPk.nColumn = 1;
003578 sPk.aiColumn = &aiColumnPk;
003579 sPk.aiRowLogEst = aiRowEstPk;
003580 sPk.onError = OE_Replace;
003581 sPk.pTable = pTab;
003582 sPk.szIdxRow = 3; /* TUNING: Interior rows of IPK table are very small */
003583 sPk.idxType = SQLITE_IDXTYPE_IPK;
003584 aiRowEstPk[0] = pTab->nRowLogEst;
003585 aiRowEstPk[1] = 0;
003586 pFirst = pSrc->pTab->pIndex;
003587 if( pSrc->fg.notIndexed==0 ){
003588 /* The real indices of the table are only considered if the
003589 ** NOT INDEXED qualifier is omitted from the FROM clause */
003590 sPk.pNext = pFirst;
003591 }
003592 pProbe = &sPk;
003593 }
003594 rSize = pTab->nRowLogEst;
003595
003596 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
003597 /* Automatic indexes */
003598 if( !pBuilder->pOrSet /* Not part of an OR optimization */
003599 && (pWInfo->wctrlFlags & (WHERE_RIGHT_JOIN|WHERE_OR_SUBCLAUSE))==0
003600 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0
003601 && !pSrc->fg.isIndexedBy /* Has no INDEXED BY clause */
003602 && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */
003603 && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */
003604 && !pSrc->fg.isCorrelated /* Not a correlated subquery */
003605 && !pSrc->fg.isRecursive /* Not a recursive common table expression. */
003606 && (pSrc->fg.jointype & JT_RIGHT)==0 /* Not the right tab of a RIGHT JOIN */
003607 ){
003608 /* Generate auto-index WhereLoops */
003609 LogEst rLogSize; /* Logarithm of the number of rows in the table */
003610 WhereTerm *pTerm;
003611 WhereTerm *pWCEnd = pWC->a + pWC->nTerm;
003612 rLogSize = estLog(rSize);
003613 for(pTerm=pWC->a; rc==SQLITE_OK && pTerm<pWCEnd; pTerm++){
003614 if( pTerm->prereqRight & pNew->maskSelf ) continue;
003615 if( termCanDriveIndex(pTerm, pSrc, 0) ){
003616 pNew->u.btree.nEq = 1;
003617 pNew->nSkip = 0;
003618 pNew->u.btree.pIndex = 0;
003619 pNew->nLTerm = 1;
003620 pNew->aLTerm[0] = pTerm;
003621 /* TUNING: One-time cost for computing the automatic index is
003622 ** estimated to be X*N*log2(N) where N is the number of rows in
003623 ** the table being indexed and where X is 7 (LogEst=28) for normal
003624 ** tables or 0.5 (LogEst=-10) for views and subqueries. The value
003625 ** of X is smaller for views and subqueries so that the query planner
003626 ** will be more aggressive about generating automatic indexes for
003627 ** those objects, since there is no opportunity to add schema
003628 ** indexes on subqueries and views. */
003629 pNew->rSetup = rLogSize + rSize;
003630 if( !IsView(pTab) && (pTab->tabFlags & TF_Ephemeral)==0 ){
003631 pNew->rSetup += 28;
003632 }else{
003633 pNew->rSetup -= 25; /* Greatly reduced setup cost for auto indexes
003634 ** on ephemeral materializations of views */
003635 }
003636 ApplyCostMultiplier(pNew->rSetup, pTab->costMult);
003637 if( pNew->rSetup<0 ) pNew->rSetup = 0;
003638 /* TUNING: Each index lookup yields 20 rows in the table. This
003639 ** is more than the usual guess of 10 rows, since we have no way
003640 ** of knowing how selective the index will ultimately be. It would
003641 ** not be unreasonable to make this value much larger. */
003642 pNew->nOut = 43; assert( 43==sqlite3LogEst(20) );
003643 pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut);
003644 pNew->wsFlags = WHERE_AUTO_INDEX;
003645 pNew->prereq = mPrereq | pTerm->prereqRight;
003646 rc = whereLoopInsert(pBuilder, pNew);
003647 }
003648 }
003649 }
003650 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
003651
003652 /* Loop over all indices. If there was an INDEXED BY clause, then only
003653 ** consider index pProbe. */
003654 for(; rc==SQLITE_OK && pProbe;
003655 pProbe=(pSrc->fg.isIndexedBy ? 0 : pProbe->pNext), iSortIdx++
003656 ){
003657 if( pProbe->pPartIdxWhere!=0
003658 && !whereUsablePartialIndex(pSrc->iCursor, pSrc->fg.jointype, pWC,
003659 pProbe->pPartIdxWhere)
003660 ){
003661 testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */
003662 continue; /* Partial index inappropriate for this query */
003663 }
003664 if( pProbe->bNoQuery ) continue;
003665 rSize = pProbe->aiRowLogEst[0];
003666 pNew->u.btree.nEq = 0;
003667 pNew->u.btree.nBtm = 0;
003668 pNew->u.btree.nTop = 0;
003669 pNew->nSkip = 0;
003670 pNew->nLTerm = 0;
003671 pNew->iSortIdx = 0;
003672 pNew->rSetup = 0;
003673 pNew->prereq = mPrereq;
003674 pNew->nOut = rSize;
003675 pNew->u.btree.pIndex = pProbe;
003676 b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor);
003677
003678 /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */
003679 assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 );
003680 if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){
003681 /* Integer primary key index */
003682 pNew->wsFlags = WHERE_IPK;
003683
003684 /* Full table scan */
003685 pNew->iSortIdx = b ? iSortIdx : 0;
003686 /* TUNING: Cost of full table scan is 3.0*N. The 3.0 factor is an
003687 ** extra cost designed to discourage the use of full table scans,
003688 ** since index lookups have better worst-case performance if our
003689 ** stat guesses are wrong. Reduce the 3.0 penalty slightly
003690 ** (to 2.75) if we have valid STAT4 information for the table.
003691 ** At 2.75, a full table scan is preferred over using an index on
003692 ** a column with just two distinct values where each value has about
003693 ** an equal number of appearances. Without STAT4 data, we still want
003694 ** to use an index in that case, since the constraint might be for
003695 ** the scarcer of the two values, and in that case an index lookup is
003696 ** better.
003697 */
003698 #ifdef SQLITE_ENABLE_STAT4
003699 pNew->rRun = rSize + 16 - 2*((pTab->tabFlags & TF_HasStat4)!=0);
003700 #else
003701 pNew->rRun = rSize + 16;
003702 #endif
003703 if( IsView(pTab) || (pTab->tabFlags & TF_Ephemeral)!=0 ){
003704 pNew->wsFlags |= WHERE_VIEWSCAN;
003705 }
003706 ApplyCostMultiplier(pNew->rRun, pTab->costMult);
003707 whereLoopOutputAdjust(pWC, pNew, rSize);
003708 rc = whereLoopInsert(pBuilder, pNew);
003709 pNew->nOut = rSize;
003710 if( rc ) break;
003711 }else{
003712 Bitmask m;
003713 if( pProbe->isCovering ){
003714 m = 0;
003715 pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
003716 }else{
003717 m = pSrc->colUsed & pProbe->colNotIdxed;
003718 pNew->wsFlags = WHERE_INDEXED;
003719 if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){
003720 u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor);
003721 if( isCov==0 ){
003722 WHERETRACE(0x200,
003723 ("-> %s is not a covering index"
003724 " according to whereIsCoveringIndex()\n", pProbe->zName));
003725 assert( m!=0 );
003726 }else{
003727 m = 0;
003728 pNew->wsFlags |= isCov;
003729 if( isCov & WHERE_IDX_ONLY ){
003730 WHERETRACE(0x200,
003731 ("-> %s is a covering expression index"
003732 " according to whereIsCoveringIndex()\n", pProbe->zName));
003733 }else{
003734 assert( isCov==WHERE_EXPRIDX );
003735 WHERETRACE(0x200,
003736 ("-> %s might be a covering expression index"
003737 " according to whereIsCoveringIndex()\n", pProbe->zName));
003738 }
003739 }
003740 }else if( m==0 ){
003741 WHERETRACE(0x200,
003742 ("-> %s a covering index according to bitmasks\n",
003743 pProbe->zName, m==0 ? "is" : "is not"));
003744 pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
003745 }
003746 }
003747
003748 /* Full scan via index */
003749 if( b
003750 || !HasRowid(pTab)
003751 || pProbe->pPartIdxWhere!=0
003752 || pSrc->fg.isIndexedBy
003753 || ( m==0
003754 && pProbe->bUnordered==0
003755 && (pProbe->szIdxRow<pTab->szTabRow)
003756 && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0
003757 && sqlite3GlobalConfig.bUseCis
003758 && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan)
003759 )
003760 ){
003761 pNew->iSortIdx = b ? iSortIdx : 0;
003762
003763 /* The cost of visiting the index rows is N*K, where K is
003764 ** between 1.1 and 3.0, depending on the relative sizes of the
003765 ** index and table rows. */
003766 pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow;
003767 if( m!=0 ){
003768 /* If this is a non-covering index scan, add in the cost of
003769 ** doing table lookups. The cost will be 3x the number of
003770 ** lookups. Take into account WHERE clause terms that can be
003771 ** satisfied using just the index, and that do not require a
003772 ** table lookup. */
003773 LogEst nLookup = rSize + 16; /* Base cost: N*3 */
003774 int ii;
003775 int iCur = pSrc->iCursor;
003776 WhereClause *pWC2 = &pWInfo->sWC;
003777 for(ii=0; ii<pWC2->nTerm; ii++){
003778 WhereTerm *pTerm = &pWC2->a[ii];
003779 if( !sqlite3ExprCoveredByIndex(pTerm->pExpr, iCur, pProbe) ){
003780 break;
003781 }
003782 /* pTerm can be evaluated using just the index. So reduce
003783 ** the expected number of table lookups accordingly */
003784 if( pTerm->truthProb<=0 ){
003785 nLookup += pTerm->truthProb;
003786 }else{
003787 nLookup--;
003788 if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19;
003789 }
003790 }
003791
003792 pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup);
003793 }
003794 ApplyCostMultiplier(pNew->rRun, pTab->costMult);
003795 whereLoopOutputAdjust(pWC, pNew, rSize);
003796 if( (pSrc->fg.jointype & JT_RIGHT)!=0 && pProbe->aColExpr ){
003797 /* Do not do an SCAN of a index-on-expression in a RIGHT JOIN
003798 ** because the cursor used to access the index might not be
003799 ** positioned to the correct row during the right-join no-match
003800 ** loop. */
003801 }else{
003802 rc = whereLoopInsert(pBuilder, pNew);
003803 }
003804 pNew->nOut = rSize;
003805 if( rc ) break;
003806 }
003807 }
003808
003809 pBuilder->bldFlags1 = 0;
003810 rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0);
003811 if( pBuilder->bldFlags1==SQLITE_BLDF1_INDEXED ){
003812 /* If a non-unique index is used, or if a prefix of the key for
003813 ** unique index is used (making the index functionally non-unique)
003814 ** then the sqlite_stat1 data becomes important for scoring the
003815 ** plan */
003816 pTab->tabFlags |= TF_StatsUsed;
003817 }
003818 #ifdef SQLITE_ENABLE_STAT4
003819 sqlite3Stat4ProbeFree(pBuilder->pRec);
003820 pBuilder->nRecValid = 0;
003821 pBuilder->pRec = 0;
003822 #endif
003823 }
003824 return rc;
003825 }
003826
003827 #ifndef SQLITE_OMIT_VIRTUALTABLE
003828
003829 /*
003830 ** Return true if pTerm is a virtual table LIMIT or OFFSET term.
003831 */
003832 static int isLimitTerm(WhereTerm *pTerm){
003833 assert( pTerm->eOperator==WO_AUX || pTerm->eMatchOp==0 );
003834 return pTerm->eMatchOp>=SQLITE_INDEX_CONSTRAINT_LIMIT
003835 && pTerm->eMatchOp<=SQLITE_INDEX_CONSTRAINT_OFFSET;
003836 }
003837
003838 /*
003839 ** Argument pIdxInfo is already populated with all constraints that may
003840 ** be used by the virtual table identified by pBuilder->pNew->iTab. This
003841 ** function marks a subset of those constraints usable, invokes the
003842 ** xBestIndex method and adds the returned plan to pBuilder.
003843 **
003844 ** A constraint is marked usable if:
003845 **
003846 ** * Argument mUsable indicates that its prerequisites are available, and
003847 **
003848 ** * It is not one of the operators specified in the mExclude mask passed
003849 ** as the fourth argument (which in practice is either WO_IN or 0).
003850 **
003851 ** Argument mPrereq is a mask of tables that must be scanned before the
003852 ** virtual table in question. These are added to the plans prerequisites
003853 ** before it is added to pBuilder.
003854 **
003855 ** Output parameter *pbIn is set to true if the plan added to pBuilder
003856 ** uses one or more WO_IN terms, or false otherwise.
003857 */
003858 static int whereLoopAddVirtualOne(
003859 WhereLoopBuilder *pBuilder,
003860 Bitmask mPrereq, /* Mask of tables that must be used. */
003861 Bitmask mUsable, /* Mask of usable tables */
003862 u16 mExclude, /* Exclude terms using these operators */
003863 sqlite3_index_info *pIdxInfo, /* Populated object for xBestIndex */
003864 u16 mNoOmit, /* Do not omit these constraints */
003865 int *pbIn, /* OUT: True if plan uses an IN(...) op */
003866 int *pbRetryLimit /* OUT: Retry without LIMIT/OFFSET */
003867 ){
003868 WhereClause *pWC = pBuilder->pWC;
003869 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
003870 struct sqlite3_index_constraint *pIdxCons;
003871 struct sqlite3_index_constraint_usage *pUsage = pIdxInfo->aConstraintUsage;
003872 int i;
003873 int mxTerm;
003874 int rc = SQLITE_OK;
003875 WhereLoop *pNew = pBuilder->pNew;
003876 Parse *pParse = pBuilder->pWInfo->pParse;
003877 SrcItem *pSrc = &pBuilder->pWInfo->pTabList->a[pNew->iTab];
003878 int nConstraint = pIdxInfo->nConstraint;
003879
003880 assert( (mUsable & mPrereq)==mPrereq );
003881 *pbIn = 0;
003882 pNew->prereq = mPrereq;
003883
003884 /* Set the usable flag on the subset of constraints identified by
003885 ** arguments mUsable and mExclude. */
003886 pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
003887 for(i=0; i<nConstraint; i++, pIdxCons++){
003888 WhereTerm *pTerm = &pWC->a[pIdxCons->iTermOffset];
003889 pIdxCons->usable = 0;
003890 if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight
003891 && (pTerm->eOperator & mExclude)==0
003892 && (pbRetryLimit || !isLimitTerm(pTerm))
003893 ){
003894 pIdxCons->usable = 1;
003895 }
003896 }
003897
003898 /* Initialize the output fields of the sqlite3_index_info structure */
003899 memset(pUsage, 0, sizeof(pUsage[0])*nConstraint);
003900 assert( pIdxInfo->needToFreeIdxStr==0 );
003901 pIdxInfo->idxStr = 0;
003902 pIdxInfo->idxNum = 0;
003903 pIdxInfo->orderByConsumed = 0;
003904 pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2;
003905 pIdxInfo->estimatedRows = 25;
003906 pIdxInfo->idxFlags = 0;
003907 pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed;
003908 pHidden->mHandleIn = 0;
003909
003910 /* Invoke the virtual table xBestIndex() method */
003911 rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo);
003912 if( rc ){
003913 if( rc==SQLITE_CONSTRAINT ){
003914 /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means
003915 ** that the particular combination of parameters provided is unusable.
003916 ** Make no entries in the loop table.
003917 */
003918 WHERETRACE(0xffffffff, (" ^^^^--- non-viable plan rejected!\n"));
003919 return SQLITE_OK;
003920 }
003921 return rc;
003922 }
003923
003924 mxTerm = -1;
003925 assert( pNew->nLSlot>=nConstraint );
003926 memset(pNew->aLTerm, 0, sizeof(pNew->aLTerm[0])*nConstraint );
003927 memset(&pNew->u.vtab, 0, sizeof(pNew->u.vtab));
003928 pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
003929 for(i=0; i<nConstraint; i++, pIdxCons++){
003930 int iTerm;
003931 if( (iTerm = pUsage[i].argvIndex - 1)>=0 ){
003932 WhereTerm *pTerm;
003933 int j = pIdxCons->iTermOffset;
003934 if( iTerm>=nConstraint
003935 || j<0
003936 || j>=pWC->nTerm
003937 || pNew->aLTerm[iTerm]!=0
003938 || pIdxCons->usable==0
003939 ){
003940 sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName);
003941 testcase( pIdxInfo->needToFreeIdxStr );
003942 return SQLITE_ERROR;
003943 }
003944 testcase( iTerm==nConstraint-1 );
003945 testcase( j==0 );
003946 testcase( j==pWC->nTerm-1 );
003947 pTerm = &pWC->a[j];
003948 pNew->prereq |= pTerm->prereqRight;
003949 assert( iTerm<pNew->nLSlot );
003950 pNew->aLTerm[iTerm] = pTerm;
003951 if( iTerm>mxTerm ) mxTerm = iTerm;
003952 testcase( iTerm==15 );
003953 testcase( iTerm==16 );
003954 if( pUsage[i].omit ){
003955 if( i<16 && ((1<<i)&mNoOmit)==0 ){
003956 testcase( i!=iTerm );
003957 pNew->u.vtab.omitMask |= 1<<iTerm;
003958 }else{
003959 testcase( i!=iTerm );
003960 }
003961 if( pTerm->eMatchOp==SQLITE_INDEX_CONSTRAINT_OFFSET ){
003962 pNew->u.vtab.bOmitOffset = 1;
003963 }
003964 }
003965 if( SMASKBIT32(i) & pHidden->mHandleIn ){
003966 pNew->u.vtab.mHandleIn |= MASKBIT32(iTerm);
003967 }else if( (pTerm->eOperator & WO_IN)!=0 ){
003968 /* A virtual table that is constrained by an IN clause may not
003969 ** consume the ORDER BY clause because (1) the order of IN terms
003970 ** is not necessarily related to the order of output terms and
003971 ** (2) Multiple outputs from a single IN value will not merge
003972 ** together. */
003973 pIdxInfo->orderByConsumed = 0;
003974 pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE;
003975 *pbIn = 1; assert( (mExclude & WO_IN)==0 );
003976 }
003977
003978 assert( pbRetryLimit || !isLimitTerm(pTerm) );
003979 if( isLimitTerm(pTerm) && *pbIn ){
003980 /* If there is an IN(...) term handled as an == (separate call to
003981 ** xFilter for each value on the RHS of the IN) and a LIMIT or
003982 ** OFFSET term handled as well, the plan is unusable. Set output
003983 ** variable *pbRetryLimit to true to tell the caller to retry with
003984 ** LIMIT and OFFSET disabled. */
003985 if( pIdxInfo->needToFreeIdxStr ){
003986 sqlite3_free(pIdxInfo->idxStr);
003987 pIdxInfo->idxStr = 0;
003988 pIdxInfo->needToFreeIdxStr = 0;
003989 }
003990 *pbRetryLimit = 1;
003991 return SQLITE_OK;
003992 }
003993 }
003994 }
003995
003996 pNew->nLTerm = mxTerm+1;
003997 for(i=0; i<=mxTerm; i++){
003998 if( pNew->aLTerm[i]==0 ){
003999 /* The non-zero argvIdx values must be contiguous. Raise an
004000 ** error if they are not */
004001 sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName);
004002 testcase( pIdxInfo->needToFreeIdxStr );
004003 return SQLITE_ERROR;
004004 }
004005 }
004006 assert( pNew->nLTerm<=pNew->nLSlot );
004007 pNew->u.vtab.idxNum = pIdxInfo->idxNum;
004008 pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr;
004009 pIdxInfo->needToFreeIdxStr = 0;
004010 pNew->u.vtab.idxStr = pIdxInfo->idxStr;
004011 pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ?
004012 pIdxInfo->nOrderBy : 0);
004013 pNew->rSetup = 0;
004014 pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost);
004015 pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows);
004016
004017 /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated
004018 ** that the scan will visit at most one row. Clear it otherwise. */
004019 if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){
004020 pNew->wsFlags |= WHERE_ONEROW;
004021 }else{
004022 pNew->wsFlags &= ~WHERE_ONEROW;
004023 }
004024 rc = whereLoopInsert(pBuilder, pNew);
004025 if( pNew->u.vtab.needFree ){
004026 sqlite3_free(pNew->u.vtab.idxStr);
004027 pNew->u.vtab.needFree = 0;
004028 }
004029 WHERETRACE(0xffffffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n",
004030 *pbIn, (sqlite3_uint64)mPrereq,
004031 (sqlite3_uint64)(pNew->prereq & ~mPrereq)));
004032
004033 return rc;
004034 }
004035
004036 /*
004037 ** Return the collating sequence for a constraint passed into xBestIndex.
004038 **
004039 ** pIdxInfo must be an sqlite3_index_info structure passed into xBestIndex.
004040 ** This routine depends on there being a HiddenIndexInfo structure immediately
004041 ** following the sqlite3_index_info structure.
004042 **
004043 ** Return a pointer to the collation name:
004044 **
004045 ** 1. If there is an explicit COLLATE operator on the constraint, return it.
004046 **
004047 ** 2. Else, if the column has an alternative collation, return that.
004048 **
004049 ** 3. Otherwise, return "BINARY".
004050 */
004051 const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int iCons){
004052 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004053 const char *zRet = 0;
004054 if( iCons>=0 && iCons<pIdxInfo->nConstraint ){
004055 CollSeq *pC = 0;
004056 int iTerm = pIdxInfo->aConstraint[iCons].iTermOffset;
004057 Expr *pX = pHidden->pWC->a[iTerm].pExpr;
004058 if( pX->pLeft ){
004059 pC = sqlite3ExprCompareCollSeq(pHidden->pParse, pX);
004060 }
004061 zRet = (pC ? pC->zName : sqlite3StrBINARY);
004062 }
004063 return zRet;
004064 }
004065
004066 /*
004067 ** Return true if constraint iCons is really an IN(...) constraint, or
004068 ** false otherwise. If iCons is an IN(...) constraint, set (if bHandle!=0)
004069 ** or clear (if bHandle==0) the flag to handle it using an iterator.
004070 */
004071 int sqlite3_vtab_in(sqlite3_index_info *pIdxInfo, int iCons, int bHandle){
004072 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004073 u32 m = SMASKBIT32(iCons);
004074 if( m & pHidden->mIn ){
004075 if( bHandle==0 ){
004076 pHidden->mHandleIn &= ~m;
004077 }else if( bHandle>0 ){
004078 pHidden->mHandleIn |= m;
004079 }
004080 return 1;
004081 }
004082 return 0;
004083 }
004084
004085 /*
004086 ** This interface is callable from within the xBestIndex callback only.
004087 **
004088 ** If possible, set (*ppVal) to point to an object containing the value
004089 ** on the right-hand-side of constraint iCons.
004090 */
004091 int sqlite3_vtab_rhs_value(
004092 sqlite3_index_info *pIdxInfo, /* Copy of first argument to xBestIndex */
004093 int iCons, /* Constraint for which RHS is wanted */
004094 sqlite3_value **ppVal /* Write value extracted here */
004095 ){
004096 HiddenIndexInfo *pH = (HiddenIndexInfo*)&pIdxInfo[1];
004097 sqlite3_value *pVal = 0;
004098 int rc = SQLITE_OK;
004099 if( iCons<0 || iCons>=pIdxInfo->nConstraint ){
004100 rc = SQLITE_MISUSE; /* EV: R-30545-25046 */
004101 }else{
004102 if( pH->aRhs[iCons]==0 ){
004103 WhereTerm *pTerm = &pH->pWC->a[pIdxInfo->aConstraint[iCons].iTermOffset];
004104 rc = sqlite3ValueFromExpr(
004105 pH->pParse->db, pTerm->pExpr->pRight, ENC(pH->pParse->db),
004106 SQLITE_AFF_BLOB, &pH->aRhs[iCons]
004107 );
004108 testcase( rc!=SQLITE_OK );
004109 }
004110 pVal = pH->aRhs[iCons];
004111 }
004112 *ppVal = pVal;
004113
004114 if( rc==SQLITE_OK && pVal==0 ){ /* IMP: R-19933-32160 */
004115 rc = SQLITE_NOTFOUND; /* IMP: R-36424-56542 */
004116 }
004117
004118 return rc;
004119 }
004120
004121 /*
004122 ** Return true if ORDER BY clause may be handled as DISTINCT.
004123 */
004124 int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){
004125 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004126 assert( pHidden->eDistinct>=0 && pHidden->eDistinct<=3 );
004127 return pHidden->eDistinct;
004128 }
004129
004130 /*
004131 ** Cause the prepared statement that is associated with a call to
004132 ** xBestIndex to potentially use all schemas. If the statement being
004133 ** prepared is read-only, then just start read transactions on all
004134 ** schemas. But if this is a write operation, start writes on all
004135 ** schemas.
004136 **
004137 ** This is used by the (built-in) sqlite_dbpage virtual table.
004138 */
004139 void sqlite3VtabUsesAllSchemas(Parse *pParse){
004140 int nDb = pParse->db->nDb;
004141 int i;
004142 for(i=0; i<nDb; i++){
004143 sqlite3CodeVerifySchema(pParse, i);
004144 }
004145 if( DbMaskNonZero(pParse->writeMask) ){
004146 for(i=0; i<nDb; i++){
004147 sqlite3BeginWriteOperation(pParse, 0, i);
004148 }
004149 }
004150 }
004151
004152 /*
004153 ** Add all WhereLoop objects for a table of the join identified by
004154 ** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table.
004155 **
004156 ** If there are no LEFT or CROSS JOIN joins in the query, both mPrereq and
004157 ** mUnusable are set to 0. Otherwise, mPrereq is a mask of all FROM clause
004158 ** entries that occur before the virtual table in the FROM clause and are
004159 ** separated from it by at least one LEFT or CROSS JOIN. Similarly, the
004160 ** mUnusable mask contains all FROM clause entries that occur after the
004161 ** virtual table and are separated from it by at least one LEFT or
004162 ** CROSS JOIN.
004163 **
004164 ** For example, if the query were:
004165 **
004166 ** ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6;
004167 **
004168 ** then mPrereq corresponds to (t1, t2) and mUnusable to (t5, t6).
004169 **
004170 ** All the tables in mPrereq must be scanned before the current virtual
004171 ** table. So any terms for which all prerequisites are satisfied by
004172 ** mPrereq may be specified as "usable" in all calls to xBestIndex.
004173 ** Conversely, all tables in mUnusable must be scanned after the current
004174 ** virtual table, so any terms for which the prerequisites overlap with
004175 ** mUnusable should always be configured as "not-usable" for xBestIndex.
004176 */
004177 static int whereLoopAddVirtual(
004178 WhereLoopBuilder *pBuilder, /* WHERE clause information */
004179 Bitmask mPrereq, /* Tables that must be scanned before this one */
004180 Bitmask mUnusable /* Tables that must be scanned after this one */
004181 ){
004182 int rc = SQLITE_OK; /* Return code */
004183 WhereInfo *pWInfo; /* WHERE analysis context */
004184 Parse *pParse; /* The parsing context */
004185 WhereClause *pWC; /* The WHERE clause */
004186 SrcItem *pSrc; /* The FROM clause term to search */
004187 sqlite3_index_info *p; /* Object to pass to xBestIndex() */
004188 int nConstraint; /* Number of constraints in p */
004189 int bIn; /* True if plan uses IN(...) operator */
004190 WhereLoop *pNew;
004191 Bitmask mBest; /* Tables used by best possible plan */
004192 u16 mNoOmit;
004193 int bRetry = 0; /* True to retry with LIMIT/OFFSET disabled */
004194
004195 assert( (mPrereq & mUnusable)==0 );
004196 pWInfo = pBuilder->pWInfo;
004197 pParse = pWInfo->pParse;
004198 pWC = pBuilder->pWC;
004199 pNew = pBuilder->pNew;
004200 pSrc = &pWInfo->pTabList->a[pNew->iTab];
004201 assert( IsVirtual(pSrc->pTab) );
004202 p = allocateIndexInfo(pWInfo, pWC, mUnusable, pSrc, &mNoOmit);
004203 if( p==0 ) return SQLITE_NOMEM_BKPT;
004204 pNew->rSetup = 0;
004205 pNew->wsFlags = WHERE_VIRTUALTABLE;
004206 pNew->nLTerm = 0;
004207 pNew->u.vtab.needFree = 0;
004208 nConstraint = p->nConstraint;
004209 if( whereLoopResize(pParse->db, pNew, nConstraint) ){
004210 freeIndexInfo(pParse->db, p);
004211 return SQLITE_NOMEM_BKPT;
004212 }
004213
004214 /* First call xBestIndex() with all constraints usable. */
004215 WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pTab->zName));
004216 WHERETRACE(0x800, (" VirtualOne: all usable\n"));
004217 rc = whereLoopAddVirtualOne(
004218 pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry
004219 );
004220 if( bRetry ){
004221 assert( rc==SQLITE_OK );
004222 rc = whereLoopAddVirtualOne(
004223 pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, 0
004224 );
004225 }
004226
004227 /* If the call to xBestIndex() with all terms enabled produced a plan
004228 ** that does not require any source tables (IOW: a plan with mBest==0)
004229 ** and does not use an IN(...) operator, then there is no point in making
004230 ** any further calls to xBestIndex() since they will all return the same
004231 ** result (if the xBestIndex() implementation is sane). */
004232 if( rc==SQLITE_OK && ((mBest = (pNew->prereq & ~mPrereq))!=0 || bIn) ){
004233 int seenZero = 0; /* True if a plan with no prereqs seen */
004234 int seenZeroNoIN = 0; /* Plan with no prereqs and no IN(...) seen */
004235 Bitmask mPrev = 0;
004236 Bitmask mBestNoIn = 0;
004237
004238 /* If the plan produced by the earlier call uses an IN(...) term, call
004239 ** xBestIndex again, this time with IN(...) terms disabled. */
004240 if( bIn ){
004241 WHERETRACE(0x800, (" VirtualOne: all usable w/o IN\n"));
004242 rc = whereLoopAddVirtualOne(
004243 pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn, 0);
004244 assert( bIn==0 );
004245 mBestNoIn = pNew->prereq & ~mPrereq;
004246 if( mBestNoIn==0 ){
004247 seenZero = 1;
004248 seenZeroNoIN = 1;
004249 }
004250 }
004251
004252 /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq)
004253 ** in the set of terms that apply to the current virtual table. */
004254 while( rc==SQLITE_OK ){
004255 int i;
004256 Bitmask mNext = ALLBITS;
004257 assert( mNext>0 );
004258 for(i=0; i<nConstraint; i++){
004259 Bitmask mThis = (
004260 pWC->a[p->aConstraint[i].iTermOffset].prereqRight & ~mPrereq
004261 );
004262 if( mThis>mPrev && mThis<mNext ) mNext = mThis;
004263 }
004264 mPrev = mNext;
004265 if( mNext==ALLBITS ) break;
004266 if( mNext==mBest || mNext==mBestNoIn ) continue;
004267 WHERETRACE(0x800, (" VirtualOne: mPrev=%04llx mNext=%04llx\n",
004268 (sqlite3_uint64)mPrev, (sqlite3_uint64)mNext));
004269 rc = whereLoopAddVirtualOne(
004270 pBuilder, mPrereq, mNext|mPrereq, 0, p, mNoOmit, &bIn, 0);
004271 if( pNew->prereq==mPrereq ){
004272 seenZero = 1;
004273 if( bIn==0 ) seenZeroNoIN = 1;
004274 }
004275 }
004276
004277 /* If the calls to xBestIndex() in the above loop did not find a plan
004278 ** that requires no source tables at all (i.e. one guaranteed to be
004279 ** usable), make a call here with all source tables disabled */
004280 if( rc==SQLITE_OK && seenZero==0 ){
004281 WHERETRACE(0x800, (" VirtualOne: all disabled\n"));
004282 rc = whereLoopAddVirtualOne(
004283 pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn, 0);
004284 if( bIn==0 ) seenZeroNoIN = 1;
004285 }
004286
004287 /* If the calls to xBestIndex() have so far failed to find a plan
004288 ** that requires no source tables at all and does not use an IN(...)
004289 ** operator, make a final call to obtain one here. */
004290 if( rc==SQLITE_OK && seenZeroNoIN==0 ){
004291 WHERETRACE(0x800, (" VirtualOne: all disabled and w/o IN\n"));
004292 rc = whereLoopAddVirtualOne(
004293 pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn, 0);
004294 }
004295 }
004296
004297 if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr);
004298 freeIndexInfo(pParse->db, p);
004299 WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pTab->zName, rc));
004300 return rc;
004301 }
004302 #endif /* SQLITE_OMIT_VIRTUALTABLE */
004303
004304 /*
004305 ** Add WhereLoop entries to handle OR terms. This works for either
004306 ** btrees or virtual tables.
004307 */
004308 static int whereLoopAddOr(
004309 WhereLoopBuilder *pBuilder,
004310 Bitmask mPrereq,
004311 Bitmask mUnusable
004312 ){
004313 WhereInfo *pWInfo = pBuilder->pWInfo;
004314 WhereClause *pWC;
004315 WhereLoop *pNew;
004316 WhereTerm *pTerm, *pWCEnd;
004317 int rc = SQLITE_OK;
004318 int iCur;
004319 WhereClause tempWC;
004320 WhereLoopBuilder sSubBuild;
004321 WhereOrSet sSum, sCur;
004322 SrcItem *pItem;
004323
004324 pWC = pBuilder->pWC;
004325 pWCEnd = pWC->a + pWC->nTerm;
004326 pNew = pBuilder->pNew;
004327 memset(&sSum, 0, sizeof(sSum));
004328 pItem = pWInfo->pTabList->a + pNew->iTab;
004329 iCur = pItem->iCursor;
004330
004331 /* The multi-index OR optimization does not work for RIGHT and FULL JOIN */
004332 if( pItem->fg.jointype & JT_RIGHT ) return SQLITE_OK;
004333
004334 for(pTerm=pWC->a; pTerm<pWCEnd && rc==SQLITE_OK; pTerm++){
004335 if( (pTerm->eOperator & WO_OR)!=0
004336 && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0
004337 ){
004338 WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc;
004339 WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm];
004340 WhereTerm *pOrTerm;
004341 int once = 1;
004342 int i, j;
004343
004344 sSubBuild = *pBuilder;
004345 sSubBuild.pOrSet = &sCur;
004346
004347 WHERETRACE(0x400, ("Begin processing OR-clause %p\n", pTerm));
004348 for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){
004349 if( (pOrTerm->eOperator & WO_AND)!=0 ){
004350 sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc;
004351 }else if( pOrTerm->leftCursor==iCur ){
004352 tempWC.pWInfo = pWC->pWInfo;
004353 tempWC.pOuter = pWC;
004354 tempWC.op = TK_AND;
004355 tempWC.nTerm = 1;
004356 tempWC.nBase = 1;
004357 tempWC.a = pOrTerm;
004358 sSubBuild.pWC = &tempWC;
004359 }else{
004360 continue;
004361 }
004362 sCur.n = 0;
004363 #ifdef WHERETRACE_ENABLED
004364 WHERETRACE(0x400, ("OR-term %d of %p has %d subterms:\n",
004365 (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm));
004366 if( sqlite3WhereTrace & 0x20000 ){
004367 sqlite3WhereClausePrint(sSubBuild.pWC);
004368 }
004369 #endif
004370 #ifndef SQLITE_OMIT_VIRTUALTABLE
004371 if( IsVirtual(pItem->pTab) ){
004372 rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable);
004373 }else
004374 #endif
004375 {
004376 rc = whereLoopAddBtree(&sSubBuild, mPrereq);
004377 }
004378 if( rc==SQLITE_OK ){
004379 rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable);
004380 }
004381 testcase( rc==SQLITE_NOMEM && sCur.n>0 );
004382 testcase( rc==SQLITE_DONE );
004383 if( sCur.n==0 ){
004384 sSum.n = 0;
004385 break;
004386 }else if( once ){
004387 whereOrMove(&sSum, &sCur);
004388 once = 0;
004389 }else{
004390 WhereOrSet sPrev;
004391 whereOrMove(&sPrev, &sSum);
004392 sSum.n = 0;
004393 for(i=0; i<sPrev.n; i++){
004394 for(j=0; j<sCur.n; j++){
004395 whereOrInsert(&sSum, sPrev.a[i].prereq | sCur.a[j].prereq,
004396 sqlite3LogEstAdd(sPrev.a[i].rRun, sCur.a[j].rRun),
004397 sqlite3LogEstAdd(sPrev.a[i].nOut, sCur.a[j].nOut));
004398 }
004399 }
004400 }
004401 }
004402 pNew->nLTerm = 1;
004403 pNew->aLTerm[0] = pTerm;
004404 pNew->wsFlags = WHERE_MULTI_OR;
004405 pNew->rSetup = 0;
004406 pNew->iSortIdx = 0;
004407 memset(&pNew->u, 0, sizeof(pNew->u));
004408 for(i=0; rc==SQLITE_OK && i<sSum.n; i++){
004409 /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs
004410 ** of all sub-scans required by the OR-scan. However, due to rounding
004411 ** errors, it may be that the cost of the OR-scan is equal to its
004412 ** most expensive sub-scan. Add the smallest possible penalty
004413 ** (equivalent to multiplying the cost by 1.07) to ensure that
004414 ** this does not happen. Otherwise, for WHERE clauses such as the
004415 ** following where there is an index on "y":
004416 **
004417 ** WHERE likelihood(x=?, 0.99) OR y=?
004418 **
004419 ** the planner may elect to "OR" together a full-table scan and an
004420 ** index lookup. And other similarly odd results. */
004421 pNew->rRun = sSum.a[i].rRun + 1;
004422 pNew->nOut = sSum.a[i].nOut;
004423 pNew->prereq = sSum.a[i].prereq;
004424 rc = whereLoopInsert(pBuilder, pNew);
004425 }
004426 WHERETRACE(0x400, ("End processing OR-clause %p\n", pTerm));
004427 }
004428 }
004429 return rc;
004430 }
004431
004432 /*
004433 ** Add all WhereLoop objects for all tables
004434 */
004435 static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
004436 WhereInfo *pWInfo = pBuilder->pWInfo;
004437 Bitmask mPrereq = 0;
004438 Bitmask mPrior = 0;
004439 int iTab;
004440 SrcList *pTabList = pWInfo->pTabList;
004441 SrcItem *pItem;
004442 SrcItem *pEnd = &pTabList->a[pWInfo->nLevel];
004443 sqlite3 *db = pWInfo->pParse->db;
004444 int rc = SQLITE_OK;
004445 int bFirstPastRJ = 0;
004446 int hasRightJoin = 0;
004447 WhereLoop *pNew;
004448
004449
004450 /* Loop over the tables in the join, from left to right */
004451 pNew = pBuilder->pNew;
004452
004453 /* Verify that pNew has already been initialized */
004454 assert( pNew->nLTerm==0 );
004455 assert( pNew->wsFlags==0 );
004456 assert( pNew->nLSlot>=ArraySize(pNew->aLTermSpace) );
004457 assert( pNew->aLTerm!=0 );
004458
004459 pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT;
004460 for(iTab=0, pItem=pTabList->a; pItem<pEnd; iTab++, pItem++){
004461 Bitmask mUnusable = 0;
004462 pNew->iTab = iTab;
004463 pBuilder->iPlanLimit += SQLITE_QUERY_PLANNER_LIMIT_INCR;
004464 pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor);
004465 if( bFirstPastRJ
004466 || (pItem->fg.jointype & (JT_OUTER|JT_CROSS|JT_LTORJ))!=0
004467 ){
004468 /* Add prerequisites to prevent reordering of FROM clause terms
004469 ** across CROSS joins and outer joins. The bFirstPastRJ boolean
004470 ** prevents the right operand of a RIGHT JOIN from being swapped with
004471 ** other elements even further to the right.
004472 **
004473 ** The JT_LTORJ case and the hasRightJoin flag work together to
004474 ** prevent FROM-clause terms from moving from the right side of
004475 ** a LEFT JOIN over to the left side of that join if the LEFT JOIN
004476 ** is itself on the left side of a RIGHT JOIN.
004477 */
004478 if( pItem->fg.jointype & JT_LTORJ ) hasRightJoin = 1;
004479 mPrereq |= mPrior;
004480 bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0;
004481 }else if( !hasRightJoin ){
004482 mPrereq = 0;
004483 }
004484 #ifndef SQLITE_OMIT_VIRTUALTABLE
004485 if( IsVirtual(pItem->pTab) ){
004486 SrcItem *p;
004487 for(p=&pItem[1]; p<pEnd; p++){
004488 if( mUnusable || (p->fg.jointype & (JT_OUTER|JT_CROSS)) ){
004489 mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor);
004490 }
004491 }
004492 rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable);
004493 }else
004494 #endif /* SQLITE_OMIT_VIRTUALTABLE */
004495 {
004496 rc = whereLoopAddBtree(pBuilder, mPrereq);
004497 }
004498 if( rc==SQLITE_OK && pBuilder->pWC->hasOr ){
004499 rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable);
004500 }
004501 mPrior |= pNew->maskSelf;
004502 if( rc || db->mallocFailed ){
004503 if( rc==SQLITE_DONE ){
004504 /* We hit the query planner search limit set by iPlanLimit */
004505 sqlite3_log(SQLITE_WARNING, "abbreviated query algorithm search");
004506 rc = SQLITE_OK;
004507 }else{
004508 break;
004509 }
004510 }
004511 }
004512
004513 whereLoopClear(db, pNew);
004514 return rc;
004515 }
004516
004517 /*
004518 ** Examine a WherePath (with the addition of the extra WhereLoop of the 6th
004519 ** parameters) to see if it outputs rows in the requested ORDER BY
004520 ** (or GROUP BY) without requiring a separate sort operation. Return N:
004521 **
004522 ** N>0: N terms of the ORDER BY clause are satisfied
004523 ** N==0: No terms of the ORDER BY clause are satisfied
004524 ** N<0: Unknown yet how many terms of ORDER BY might be satisfied.
004525 **
004526 ** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as
004527 ** strict. With GROUP BY and DISTINCT the only requirement is that
004528 ** equivalent rows appear immediately adjacent to one another. GROUP BY
004529 ** and DISTINCT do not require rows to appear in any particular order as long
004530 ** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT
004531 ** the pOrderBy terms can be matched in any order. With ORDER BY, the
004532 ** pOrderBy terms must be matched in strict left-to-right order.
004533 */
004534 static i8 wherePathSatisfiesOrderBy(
004535 WhereInfo *pWInfo, /* The WHERE clause */
004536 ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */
004537 WherePath *pPath, /* The WherePath to check */
004538 u16 wctrlFlags, /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */
004539 u16 nLoop, /* Number of entries in pPath->aLoop[] */
004540 WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */
004541 Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */
004542 ){
004543 u8 revSet; /* True if rev is known */
004544 u8 rev; /* Composite sort order */
004545 u8 revIdx; /* Index sort order */
004546 u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */
004547 u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */
004548 u8 isMatch; /* iColumn matches a term of the ORDER BY clause */
004549 u16 eqOpMask; /* Allowed equality operators */
004550 u16 nKeyCol; /* Number of key columns in pIndex */
004551 u16 nColumn; /* Total number of ordered columns in the index */
004552 u16 nOrderBy; /* Number terms in the ORDER BY clause */
004553 int iLoop; /* Index of WhereLoop in pPath being processed */
004554 int i, j; /* Loop counters */
004555 int iCur; /* Cursor number for current WhereLoop */
004556 int iColumn; /* A column number within table iCur */
004557 WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */
004558 WhereTerm *pTerm; /* A single term of the WHERE clause */
004559 Expr *pOBExpr; /* An expression from the ORDER BY clause */
004560 CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */
004561 Index *pIndex; /* The index associated with pLoop */
004562 sqlite3 *db = pWInfo->pParse->db; /* Database connection */
004563 Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */
004564 Bitmask obDone; /* Mask of all ORDER BY terms */
004565 Bitmask orderDistinctMask; /* Mask of all well-ordered loops */
004566 Bitmask ready; /* Mask of inner loops */
004567
004568 /*
004569 ** We say the WhereLoop is "one-row" if it generates no more than one
004570 ** row of output. A WhereLoop is one-row if all of the following are true:
004571 ** (a) All index columns match with WHERE_COLUMN_EQ.
004572 ** (b) The index is unique
004573 ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row.
004574 ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags.
004575 **
004576 ** We say the WhereLoop is "order-distinct" if the set of columns from
004577 ** that WhereLoop that are in the ORDER BY clause are different for every
004578 ** row of the WhereLoop. Every one-row WhereLoop is automatically
004579 ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause
004580 ** is not order-distinct. To be order-distinct is not quite the same as being
004581 ** UNIQUE since a UNIQUE column or index can have multiple rows that
004582 ** are NULL and NULL values are equivalent for the purpose of order-distinct.
004583 ** To be order-distinct, the columns must be UNIQUE and NOT NULL.
004584 **
004585 ** The rowid for a table is always UNIQUE and NOT NULL so whenever the
004586 ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is
004587 ** automatically order-distinct.
004588 */
004589
004590 assert( pOrderBy!=0 );
004591 if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0;
004592
004593 nOrderBy = pOrderBy->nExpr;
004594 testcase( nOrderBy==BMS-1 );
004595 if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */
004596 isOrderDistinct = 1;
004597 obDone = MASKBIT(nOrderBy)-1;
004598 orderDistinctMask = 0;
004599 ready = 0;
004600 eqOpMask = WO_EQ | WO_IS | WO_ISNULL;
004601 if( wctrlFlags & (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MAX|WHERE_ORDERBY_MIN) ){
004602 eqOpMask |= WO_IN;
004603 }
004604 for(iLoop=0; isOrderDistinct && obSat<obDone && iLoop<=nLoop; iLoop++){
004605 if( iLoop>0 ) ready |= pLoop->maskSelf;
004606 if( iLoop<nLoop ){
004607 pLoop = pPath->aLoop[iLoop];
004608 if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue;
004609 }else{
004610 pLoop = pLast;
004611 }
004612 if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){
004613 if( pLoop->u.vtab.isOrdered
004614 && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY)
004615 ){
004616 obSat = obDone;
004617 }
004618 break;
004619 }else if( wctrlFlags & WHERE_DISTINCTBY ){
004620 pLoop->u.btree.nDistinctCol = 0;
004621 }
004622 iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor;
004623
004624 /* Mark off any ORDER BY term X that is a column in the table of
004625 ** the current loop for which there is term in the WHERE
004626 ** clause of the form X IS NULL or X=? that reference only outer
004627 ** loops.
004628 */
004629 for(i=0; i<nOrderBy; i++){
004630 if( MASKBIT(i) & obSat ) continue;
004631 pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr);
004632 if( NEVER(pOBExpr==0) ) continue;
004633 if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue;
004634 if( pOBExpr->iTable!=iCur ) continue;
004635 pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn,
004636 ~ready, eqOpMask, 0);
004637 if( pTerm==0 ) continue;
004638 if( pTerm->eOperator==WO_IN ){
004639 /* IN terms are only valid for sorting in the ORDER BY LIMIT
004640 ** optimization, and then only if they are actually used
004641 ** by the query plan */
004642 assert( wctrlFlags &
004643 (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) );
004644 for(j=0; j<pLoop->nLTerm && pTerm!=pLoop->aLTerm[j]; j++){}
004645 if( j>=pLoop->nLTerm ) continue;
004646 }
004647 if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){
004648 Parse *pParse = pWInfo->pParse;
004649 CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pOrderBy->a[i].pExpr);
004650 CollSeq *pColl2 = sqlite3ExprCompareCollSeq(pParse, pTerm->pExpr);
004651 assert( pColl1 );
004652 if( pColl2==0 || sqlite3StrICmp(pColl1->zName, pColl2->zName) ){
004653 continue;
004654 }
004655 testcase( pTerm->pExpr->op==TK_IS );
004656 }
004657 obSat |= MASKBIT(i);
004658 }
004659
004660 if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){
004661 if( pLoop->wsFlags & WHERE_IPK ){
004662 pIndex = 0;
004663 nKeyCol = 0;
004664 nColumn = 1;
004665 }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){
004666 return 0;
004667 }else{
004668 nKeyCol = pIndex->nKeyCol;
004669 nColumn = pIndex->nColumn;
004670 assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) );
004671 assert( pIndex->aiColumn[nColumn-1]==XN_ROWID
004672 || !HasRowid(pIndex->pTable));
004673 /* All relevant terms of the index must also be non-NULL in order
004674 ** for isOrderDistinct to be true. So the isOrderDistint value
004675 ** computed here might be a false positive. Corrections will be
004676 ** made at tag-20210426-1 below */
004677 isOrderDistinct = IsUniqueIndex(pIndex)
004678 && (pLoop->wsFlags & WHERE_SKIPSCAN)==0;
004679 }
004680
004681 /* Loop through all columns of the index and deal with the ones
004682 ** that are not constrained by == or IN.
004683 */
004684 rev = revSet = 0;
004685 distinctColumns = 0;
004686 for(j=0; j<nColumn; j++){
004687 u8 bOnce = 1; /* True to run the ORDER BY search loop */
004688
004689 assert( j>=pLoop->u.btree.nEq
004690 || (pLoop->aLTerm[j]==0)==(j<pLoop->nSkip)
004691 );
004692 if( j<pLoop->u.btree.nEq && j>=pLoop->nSkip ){
004693 u16 eOp = pLoop->aLTerm[j]->eOperator;
004694
004695 /* Skip over == and IS and ISNULL terms. (Also skip IN terms when
004696 ** doing WHERE_ORDERBY_LIMIT processing). Except, IS and ISNULL
004697 ** terms imply that the index is not UNIQUE NOT NULL in which case
004698 ** the loop need to be marked as not order-distinct because it can
004699 ** have repeated NULL rows.
004700 **
004701 ** If the current term is a column of an ((?,?) IN (SELECT...))
004702 ** expression for which the SELECT returns more than one column,
004703 ** check that it is the only column used by this loop. Otherwise,
004704 ** if it is one of two or more, none of the columns can be
004705 ** considered to match an ORDER BY term.
004706 */
004707 if( (eOp & eqOpMask)!=0 ){
004708 if( eOp & (WO_ISNULL|WO_IS) ){
004709 testcase( eOp & WO_ISNULL );
004710 testcase( eOp & WO_IS );
004711 testcase( isOrderDistinct );
004712 isOrderDistinct = 0;
004713 }
004714 continue;
004715 }else if( ALWAYS(eOp & WO_IN) ){
004716 /* ALWAYS() justification: eOp is an equality operator due to the
004717 ** j<pLoop->u.btree.nEq constraint above. Any equality other
004718 ** than WO_IN is captured by the previous "if". So this one
004719 ** always has to be WO_IN. */
004720 Expr *pX = pLoop->aLTerm[j]->pExpr;
004721 for(i=j+1; i<pLoop->u.btree.nEq; i++){
004722 if( pLoop->aLTerm[i]->pExpr==pX ){
004723 assert( (pLoop->aLTerm[i]->eOperator & WO_IN) );
004724 bOnce = 0;
004725 break;
004726 }
004727 }
004728 }
004729 }
004730
004731 /* Get the column number in the table (iColumn) and sort order
004732 ** (revIdx) for the j-th column of the index.
004733 */
004734 if( pIndex ){
004735 iColumn = pIndex->aiColumn[j];
004736 revIdx = pIndex->aSortOrder[j] & KEYINFO_ORDER_DESC;
004737 if( iColumn==pIndex->pTable->iPKey ) iColumn = XN_ROWID;
004738 }else{
004739 iColumn = XN_ROWID;
004740 revIdx = 0;
004741 }
004742
004743 /* An unconstrained column that might be NULL means that this
004744 ** WhereLoop is not well-ordered. tag-20210426-1
004745 */
004746 if( isOrderDistinct ){
004747 if( iColumn>=0
004748 && j>=pLoop->u.btree.nEq
004749 && pIndex->pTable->aCol[iColumn].notNull==0
004750 ){
004751 isOrderDistinct = 0;
004752 }
004753 if( iColumn==XN_EXPR ){
004754 isOrderDistinct = 0;
004755 }
004756 }
004757
004758 /* Find the ORDER BY term that corresponds to the j-th column
004759 ** of the index and mark that ORDER BY term off
004760 */
004761 isMatch = 0;
004762 for(i=0; bOnce && i<nOrderBy; i++){
004763 if( MASKBIT(i) & obSat ) continue;
004764 pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr);
004765 testcase( wctrlFlags & WHERE_GROUPBY );
004766 testcase( wctrlFlags & WHERE_DISTINCTBY );
004767 if( NEVER(pOBExpr==0) ) continue;
004768 if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0;
004769 if( iColumn>=XN_ROWID ){
004770 if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue;
004771 if( pOBExpr->iTable!=iCur ) continue;
004772 if( pOBExpr->iColumn!=iColumn ) continue;
004773 }else{
004774 Expr *pIxExpr = pIndex->aColExpr->a[j].pExpr;
004775 if( sqlite3ExprCompareSkip(pOBExpr, pIxExpr, iCur) ){
004776 continue;
004777 }
004778 }
004779 if( iColumn!=XN_ROWID ){
004780 pColl = sqlite3ExprNNCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
004781 if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue;
004782 }
004783 if( wctrlFlags & WHERE_DISTINCTBY ){
004784 pLoop->u.btree.nDistinctCol = j+1;
004785 }
004786 isMatch = 1;
004787 break;
004788 }
004789 if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){
004790 /* Make sure the sort order is compatible in an ORDER BY clause.
004791 ** Sort order is irrelevant for a GROUP BY clause. */
004792 if( revSet ){
004793 if( (rev ^ revIdx)
004794 != (pOrderBy->a[i].fg.sortFlags&KEYINFO_ORDER_DESC)
004795 ){
004796 isMatch = 0;
004797 }
004798 }else{
004799 rev = revIdx ^ (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC);
004800 if( rev ) *pRevMask |= MASKBIT(iLoop);
004801 revSet = 1;
004802 }
004803 }
004804 if( isMatch && (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL) ){
004805 if( j==pLoop->u.btree.nEq ){
004806 pLoop->wsFlags |= WHERE_BIGNULL_SORT;
004807 }else{
004808 isMatch = 0;
004809 }
004810 }
004811 if( isMatch ){
004812 if( iColumn==XN_ROWID ){
004813 testcase( distinctColumns==0 );
004814 distinctColumns = 1;
004815 }
004816 obSat |= MASKBIT(i);
004817 }else{
004818 /* No match found */
004819 if( j==0 || j<nKeyCol ){
004820 testcase( isOrderDistinct!=0 );
004821 isOrderDistinct = 0;
004822 }
004823 break;
004824 }
004825 } /* end Loop over all index columns */
004826 if( distinctColumns ){
004827 testcase( isOrderDistinct==0 );
004828 isOrderDistinct = 1;
004829 }
004830 } /* end-if not one-row */
004831
004832 /* Mark off any other ORDER BY terms that reference pLoop */
004833 if( isOrderDistinct ){
004834 orderDistinctMask |= pLoop->maskSelf;
004835 for(i=0; i<nOrderBy; i++){
004836 Expr *p;
004837 Bitmask mTerm;
004838 if( MASKBIT(i) & obSat ) continue;
004839 p = pOrderBy->a[i].pExpr;
004840 mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p);
004841 if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue;
004842 if( (mTerm&~orderDistinctMask)==0 ){
004843 obSat |= MASKBIT(i);
004844 }
004845 }
004846 }
004847 } /* End the loop over all WhereLoops from outer-most down to inner-most */
004848 if( obSat==obDone ) return (i8)nOrderBy;
004849 if( !isOrderDistinct ){
004850 for(i=nOrderBy-1; i>0; i--){
004851 Bitmask m = ALWAYS(i<BMS) ? MASKBIT(i) - 1 : 0;
004852 if( (obSat&m)==m ) return i;
004853 }
004854 return 0;
004855 }
004856 return -1;
004857 }
004858
004859
004860 /*
004861 ** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(),
004862 ** the planner assumes that the specified pOrderBy list is actually a GROUP
004863 ** BY clause - and so any order that groups rows as required satisfies the
004864 ** request.
004865 **
004866 ** Normally, in this case it is not possible for the caller to determine
004867 ** whether or not the rows are really being delivered in sorted order, or
004868 ** just in some other order that provides the required grouping. However,
004869 ** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then
004870 ** this function may be called on the returned WhereInfo object. It returns
004871 ** true if the rows really will be sorted in the specified order, or false
004872 ** otherwise.
004873 **
004874 ** For example, assuming:
004875 **
004876 ** CREATE INDEX i1 ON t1(x, Y);
004877 **
004878 ** then
004879 **
004880 ** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1
004881 ** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0
004882 */
004883 int sqlite3WhereIsSorted(WhereInfo *pWInfo){
004884 assert( pWInfo->wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY) );
004885 assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP );
004886 return pWInfo->sorted;
004887 }
004888
004889 #ifdef WHERETRACE_ENABLED
004890 /* For debugging use only: */
004891 static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){
004892 static char zName[65];
004893 int i;
004894 for(i=0; i<nLoop; i++){ zName[i] = pPath->aLoop[i]->cId; }
004895 if( pLast ) zName[i++] = pLast->cId;
004896 zName[i] = 0;
004897 return zName;
004898 }
004899 #endif
004900
004901 /*
004902 ** Return the cost of sorting nRow rows, assuming that the keys have
004903 ** nOrderby columns and that the first nSorted columns are already in
004904 ** order.
004905 */
004906 static LogEst whereSortingCost(
004907 WhereInfo *pWInfo, /* Query planning context */
004908 LogEst nRow, /* Estimated number of rows to sort */
004909 int nOrderBy, /* Number of ORDER BY clause terms */
004910 int nSorted /* Number of initial ORDER BY terms naturally in order */
004911 ){
004912 /* Estimated cost of a full external sort, where N is
004913 ** the number of rows to sort is:
004914 **
004915 ** cost = (K * N * log(N)).
004916 **
004917 ** Or, if the order-by clause has X terms but only the last Y
004918 ** terms are out of order, then block-sorting will reduce the
004919 ** sorting cost to:
004920 **
004921 ** cost = (K * N * log(N)) * (Y/X)
004922 **
004923 ** The constant K is at least 2.0 but will be larger if there are a
004924 ** large number of columns to be sorted, as the sorting time is
004925 ** proportional to the amount of content to be sorted. The algorithm
004926 ** does not currently distinguish between fat columns (BLOBs and TEXTs)
004927 ** and skinny columns (INTs). It just uses the number of columns as
004928 ** an approximation for the row width.
004929 **
004930 ** And extra factor of 2.0 or 3.0 is added to the sorting cost if the sort
004931 ** is built using OP_IdxInsert and OP_Sort rather than with OP_SorterInsert.
004932 */
004933 LogEst rSortCost, nCol;
004934 assert( pWInfo->pSelect!=0 );
004935 assert( pWInfo->pSelect->pEList!=0 );
004936 /* TUNING: sorting cost proportional to the number of output columns: */
004937 nCol = sqlite3LogEst((pWInfo->pSelect->pEList->nExpr+59)/30);
004938 rSortCost = nRow + nCol;
004939 if( nSorted>0 ){
004940 /* Scale the result by (Y/X) */
004941 rSortCost += sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66;
004942 }
004943
004944 /* Multiple by log(M) where M is the number of output rows.
004945 ** Use the LIMIT for M if it is smaller. Or if this sort is for
004946 ** a DISTINCT operator, M will be the number of distinct output
004947 ** rows, so fudge it downwards a bit.
004948 */
004949 if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 ){
004950 rSortCost += 10; /* TUNING: Extra 2.0x if using LIMIT */
004951 if( nSorted!=0 ){
004952 rSortCost += 6; /* TUNING: Extra 1.5x if also using partial sort */
004953 }
004954 if( pWInfo->iLimit<nRow ){
004955 nRow = pWInfo->iLimit;
004956 }
004957 }else if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT) ){
004958 /* TUNING: In the sort for a DISTINCT operator, assume that the DISTINCT
004959 ** reduces the number of output rows by a factor of 2 */
004960 if( nRow>10 ){ nRow -= 10; assert( 10==sqlite3LogEst(2) ); }
004961 }
004962 rSortCost += estLog(nRow);
004963 return rSortCost;
004964 }
004965
004966 /*
004967 ** Given the list of WhereLoop objects at pWInfo->pLoops, this routine
004968 ** attempts to find the lowest cost path that visits each WhereLoop
004969 ** once. This path is then loaded into the pWInfo->a[].pWLoop fields.
004970 **
004971 ** Assume that the total number of output rows that will need to be sorted
004972 ** will be nRowEst (in the 10*log2 representation). Or, ignore sorting
004973 ** costs if nRowEst==0.
004974 **
004975 ** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation
004976 ** error occurs.
004977 */
004978 static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
004979 int mxChoice; /* Maximum number of simultaneous paths tracked */
004980 int nLoop; /* Number of terms in the join */
004981 Parse *pParse; /* Parsing context */
004982 int iLoop; /* Loop counter over the terms of the join */
004983 int ii, jj; /* Loop counters */
004984 int mxI = 0; /* Index of next entry to replace */
004985 int nOrderBy; /* Number of ORDER BY clause terms */
004986 LogEst mxCost = 0; /* Maximum cost of a set of paths */
004987 LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */
004988 int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */
004989 WherePath *aFrom; /* All nFrom paths at the previous level */
004990 WherePath *aTo; /* The nTo best paths at the current level */
004991 WherePath *pFrom; /* An element of aFrom[] that we are working on */
004992 WherePath *pTo; /* An element of aTo[] that we are working on */
004993 WhereLoop *pWLoop; /* One of the WhereLoop objects */
004994 WhereLoop **pX; /* Used to divy up the pSpace memory */
004995 LogEst *aSortCost = 0; /* Sorting and partial sorting costs */
004996 char *pSpace; /* Temporary memory used by this routine */
004997 int nSpace; /* Bytes of space allocated at pSpace */
004998
004999 pParse = pWInfo->pParse;
005000 nLoop = pWInfo->nLevel;
005001 /* TUNING: For simple queries, only the best path is tracked.
005002 ** For 2-way joins, the 5 best paths are followed.
005003 ** For joins of 3 or more tables, track the 10 best paths */
005004 mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10);
005005 assert( nLoop<=pWInfo->pTabList->nSrc );
005006 WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d, nQueryLoop=%d)\n",
005007 nRowEst, pParse->nQueryLoop));
005008
005009 /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this
005010 ** case the purpose of this call is to estimate the number of rows returned
005011 ** by the overall query. Once this estimate has been obtained, the caller
005012 ** will invoke this function a second time, passing the estimate as the
005013 ** nRowEst parameter. */
005014 if( pWInfo->pOrderBy==0 || nRowEst==0 ){
005015 nOrderBy = 0;
005016 }else{
005017 nOrderBy = pWInfo->pOrderBy->nExpr;
005018 }
005019
005020 /* Allocate and initialize space for aTo, aFrom and aSortCost[] */
005021 nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2;
005022 nSpace += sizeof(LogEst) * nOrderBy;
005023 pSpace = sqlite3StackAllocRawNN(pParse->db, nSpace);
005024 if( pSpace==0 ) return SQLITE_NOMEM_BKPT;
005025 aTo = (WherePath*)pSpace;
005026 aFrom = aTo+mxChoice;
005027 memset(aFrom, 0, sizeof(aFrom[0]));
005028 pX = (WhereLoop**)(aFrom+mxChoice);
005029 for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){
005030 pFrom->aLoop = pX;
005031 }
005032 if( nOrderBy ){
005033 /* If there is an ORDER BY clause and it is not being ignored, set up
005034 ** space for the aSortCost[] array. Each element of the aSortCost array
005035 ** is either zero - meaning it has not yet been initialized - or the
005036 ** cost of sorting nRowEst rows of data where the first X terms of
005037 ** the ORDER BY clause are already in order, where X is the array
005038 ** index. */
005039 aSortCost = (LogEst*)pX;
005040 memset(aSortCost, 0, sizeof(LogEst) * nOrderBy);
005041 }
005042 assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] );
005043 assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX );
005044
005045 /* Seed the search with a single WherePath containing zero WhereLoops.
005046 **
005047 ** TUNING: Do not let the number of iterations go above 28. If the cost
005048 ** of computing an automatic index is not paid back within the first 28
005049 ** rows, then do not use the automatic index. */
005050 aFrom[0].nRow = MIN(pParse->nQueryLoop, 48); assert( 48==sqlite3LogEst(28) );
005051 nFrom = 1;
005052 assert( aFrom[0].isOrdered==0 );
005053 if( nOrderBy ){
005054 /* If nLoop is zero, then there are no FROM terms in the query. Since
005055 ** in this case the query may return a maximum of one row, the results
005056 ** are already in the requested order. Set isOrdered to nOrderBy to
005057 ** indicate this. Or, if nLoop is greater than zero, set isOrdered to
005058 ** -1, indicating that the result set may or may not be ordered,
005059 ** depending on the loops added to the current plan. */
005060 aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy;
005061 }
005062
005063 /* Compute successively longer WherePaths using the previous generation
005064 ** of WherePaths as the basis for the next. Keep track of the mxChoice
005065 ** best paths at each generation */
005066 for(iLoop=0; iLoop<nLoop; iLoop++){
005067 nTo = 0;
005068 for(ii=0, pFrom=aFrom; ii<nFrom; ii++, pFrom++){
005069 for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
005070 LogEst nOut; /* Rows visited by (pFrom+pWLoop) */
005071 LogEst rCost; /* Cost of path (pFrom+pWLoop) */
005072 LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */
005073 i8 isOrdered; /* isOrdered for (pFrom+pWLoop) */
005074 Bitmask maskNew; /* Mask of src visited by (..) */
005075 Bitmask revMask; /* Mask of rev-order loops for (..) */
005076
005077 if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue;
005078 if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue;
005079 if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<3 ){
005080 /* Do not use an automatic index if the this loop is expected
005081 ** to run less than 1.25 times. It is tempting to also exclude
005082 ** automatic index usage on an outer loop, but sometimes an automatic
005083 ** index is useful in the outer loop of a correlated subquery. */
005084 assert( 10==sqlite3LogEst(2) );
005085 continue;
005086 }
005087
005088 /* At this point, pWLoop is a candidate to be the next loop.
005089 ** Compute its cost */
005090 rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow);
005091 rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted);
005092 nOut = pFrom->nRow + pWLoop->nOut;
005093 maskNew = pFrom->maskLoop | pWLoop->maskSelf;
005094 isOrdered = pFrom->isOrdered;
005095 if( isOrdered<0 ){
005096 revMask = 0;
005097 isOrdered = wherePathSatisfiesOrderBy(pWInfo,
005098 pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags,
005099 iLoop, pWLoop, &revMask);
005100 }else{
005101 revMask = pFrom->revLoop;
005102 }
005103 if( isOrdered>=0 && isOrdered<nOrderBy ){
005104 if( aSortCost[isOrdered]==0 ){
005105 aSortCost[isOrdered] = whereSortingCost(
005106 pWInfo, nRowEst, nOrderBy, isOrdered
005107 );
005108 }
005109 /* TUNING: Add a small extra penalty (3) to sorting as an
005110 ** extra encouragement to the query planner to select a plan
005111 ** where the rows emerge in the correct order without any sorting
005112 ** required. */
005113 rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 3;
005114
005115 WHERETRACE(0x002,
005116 ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n",
005117 aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy,
005118 rUnsorted, rCost));
005119 }else{
005120 rCost = rUnsorted;
005121 rUnsorted -= 2; /* TUNING: Slight bias in favor of no-sort plans */
005122 }
005123
005124 /* TUNING: A full-scan of a VIEW or subquery in the outer loop
005125 ** is not so bad. */
005126 if( iLoop==0 && (pWLoop->wsFlags & WHERE_VIEWSCAN)!=0 && nLoop>1 ){
005127 rCost += -10;
005128 nOut += -30;
005129 WHERETRACE(0x80,("VIEWSCAN cost reduction for %c\n",pWLoop->cId));
005130 }
005131
005132 /* Check to see if pWLoop should be added to the set of
005133 ** mxChoice best-so-far paths.
005134 **
005135 ** First look for an existing path among best-so-far paths
005136 ** that covers the same set of loops and has the same isOrdered
005137 ** setting as the current path candidate.
005138 **
005139 ** The term "((pTo->isOrdered^isOrdered)&0x80)==0" is equivalent
005140 ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range
005141 ** of legal values for isOrdered, -1..64.
005142 */
005143 for(jj=0, pTo=aTo; jj<nTo; jj++, pTo++){
005144 if( pTo->maskLoop==maskNew
005145 && ((pTo->isOrdered^isOrdered)&0x80)==0
005146 ){
005147 testcase( jj==nTo-1 );
005148 break;
005149 }
005150 }
005151 if( jj>=nTo ){
005152 /* None of the existing best-so-far paths match the candidate. */
005153 if( nTo>=mxChoice
005154 && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted))
005155 ){
005156 /* The current candidate is no better than any of the mxChoice
005157 ** paths currently in the best-so-far buffer. So discard
005158 ** this candidate as not viable. */
005159 #ifdef WHERETRACE_ENABLED /* 0x4 */
005160 if( sqlite3WhereTrace&0x4 ){
005161 sqlite3DebugPrintf("Skip %s cost=%-3d,%3d,%3d order=%c\n",
005162 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
005163 isOrdered>=0 ? isOrdered+'0' : '?');
005164 }
005165 #endif
005166 continue;
005167 }
005168 /* If we reach this points it means that the new candidate path
005169 ** needs to be added to the set of best-so-far paths. */
005170 if( nTo<mxChoice ){
005171 /* Increase the size of the aTo set by one */
005172 jj = nTo++;
005173 }else{
005174 /* New path replaces the prior worst to keep count below mxChoice */
005175 jj = mxI;
005176 }
005177 pTo = &aTo[jj];
005178 #ifdef WHERETRACE_ENABLED /* 0x4 */
005179 if( sqlite3WhereTrace&0x4 ){
005180 sqlite3DebugPrintf("New %s cost=%-3d,%3d,%3d order=%c\n",
005181 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
005182 isOrdered>=0 ? isOrdered+'0' : '?');
005183 }
005184 #endif
005185 }else{
005186 /* Control reaches here if best-so-far path pTo=aTo[jj] covers the
005187 ** same set of loops and has the same isOrdered setting as the
005188 ** candidate path. Check to see if the candidate should replace
005189 ** pTo or if the candidate should be skipped.
005190 **
005191 ** The conditional is an expanded vector comparison equivalent to:
005192 ** (pTo->rCost,pTo->nRow,pTo->rUnsorted) <= (rCost,nOut,rUnsorted)
005193 */
005194 if( pTo->rCost<rCost
005195 || (pTo->rCost==rCost
005196 && (pTo->nRow<nOut
005197 || (pTo->nRow==nOut && pTo->rUnsorted<=rUnsorted)
005198 )
005199 )
005200 ){
005201 #ifdef WHERETRACE_ENABLED /* 0x4 */
005202 if( sqlite3WhereTrace&0x4 ){
005203 sqlite3DebugPrintf(
005204 "Skip %s cost=%-3d,%3d,%3d order=%c",
005205 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
005206 isOrdered>=0 ? isOrdered+'0' : '?');
005207 sqlite3DebugPrintf(" vs %s cost=%-3d,%3d,%3d order=%c\n",
005208 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
005209 pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
005210 }
005211 #endif
005212 /* Discard the candidate path from further consideration */
005213 testcase( pTo->rCost==rCost );
005214 continue;
005215 }
005216 testcase( pTo->rCost==rCost+1 );
005217 /* Control reaches here if the candidate path is better than the
005218 ** pTo path. Replace pTo with the candidate. */
005219 #ifdef WHERETRACE_ENABLED /* 0x4 */
005220 if( sqlite3WhereTrace&0x4 ){
005221 sqlite3DebugPrintf(
005222 "Update %s cost=%-3d,%3d,%3d order=%c",
005223 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
005224 isOrdered>=0 ? isOrdered+'0' : '?');
005225 sqlite3DebugPrintf(" was %s cost=%-3d,%3d,%3d order=%c\n",
005226 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
005227 pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
005228 }
005229 #endif
005230 }
005231 /* pWLoop is a winner. Add it to the set of best so far */
005232 pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf;
005233 pTo->revLoop = revMask;
005234 pTo->nRow = nOut;
005235 pTo->rCost = rCost;
005236 pTo->rUnsorted = rUnsorted;
005237 pTo->isOrdered = isOrdered;
005238 memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop);
005239 pTo->aLoop[iLoop] = pWLoop;
005240 if( nTo>=mxChoice ){
005241 mxI = 0;
005242 mxCost = aTo[0].rCost;
005243 mxUnsorted = aTo[0].nRow;
005244 for(jj=1, pTo=&aTo[1]; jj<mxChoice; jj++, pTo++){
005245 if( pTo->rCost>mxCost
005246 || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted)
005247 ){
005248 mxCost = pTo->rCost;
005249 mxUnsorted = pTo->rUnsorted;
005250 mxI = jj;
005251 }
005252 }
005253 }
005254 }
005255 }
005256
005257 #ifdef WHERETRACE_ENABLED /* >=2 */
005258 if( sqlite3WhereTrace & 0x02 ){
005259 sqlite3DebugPrintf("---- after round %d ----\n", iLoop);
005260 for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){
005261 sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c",
005262 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
005263 pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?');
005264 if( pTo->isOrdered>0 ){
005265 sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop);
005266 }else{
005267 sqlite3DebugPrintf("\n");
005268 }
005269 }
005270 }
005271 #endif
005272
005273 /* Swap the roles of aFrom and aTo for the next generation */
005274 pFrom = aTo;
005275 aTo = aFrom;
005276 aFrom = pFrom;
005277 nFrom = nTo;
005278 }
005279
005280 if( nFrom==0 ){
005281 sqlite3ErrorMsg(pParse, "no query solution");
005282 sqlite3StackFreeNN(pParse->db, pSpace);
005283 return SQLITE_ERROR;
005284 }
005285
005286 /* Find the lowest cost path. pFrom will be left pointing to that path */
005287 pFrom = aFrom;
005288 for(ii=1; ii<nFrom; ii++){
005289 if( pFrom->rCost>aFrom[ii].rCost ) pFrom = &aFrom[ii];
005290 }
005291 assert( pWInfo->nLevel==nLoop );
005292 /* Load the lowest cost path into pWInfo */
005293 for(iLoop=0; iLoop<nLoop; iLoop++){
005294 WhereLevel *pLevel = pWInfo->a + iLoop;
005295 pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop];
005296 pLevel->iFrom = pWLoop->iTab;
005297 pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor;
005298 }
005299 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0
005300 && (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0
005301 && pWInfo->eDistinct==WHERE_DISTINCT_NOOP
005302 && nRowEst
005303 ){
005304 Bitmask notUsed;
005305 int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom,
005306 WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used);
005307 if( rc==pWInfo->pResultSet->nExpr ){
005308 pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
005309 }
005310 }
005311 pWInfo->bOrderedInnerLoop = 0;
005312 if( pWInfo->pOrderBy ){
005313 pWInfo->nOBSat = pFrom->isOrdered;
005314 if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){
005315 if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){
005316 pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
005317 }
005318 if( pWInfo->pSelect->pOrderBy
005319 && pWInfo->nOBSat > pWInfo->pSelect->pOrderBy->nExpr ){
005320 pWInfo->nOBSat = pWInfo->pSelect->pOrderBy->nExpr;
005321 }
005322 }else{
005323 pWInfo->revMask = pFrom->revLoop;
005324 if( pWInfo->nOBSat<=0 ){
005325 pWInfo->nOBSat = 0;
005326 if( nLoop>0 ){
005327 u32 wsFlags = pFrom->aLoop[nLoop-1]->wsFlags;
005328 if( (wsFlags & WHERE_ONEROW)==0
005329 && (wsFlags&(WHERE_IPK|WHERE_COLUMN_IN))!=(WHERE_IPK|WHERE_COLUMN_IN)
005330 ){
005331 Bitmask m = 0;
005332 int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom,
005333 WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m);
005334 testcase( wsFlags & WHERE_IPK );
005335 testcase( wsFlags & WHERE_COLUMN_IN );
005336 if( rc==pWInfo->pOrderBy->nExpr ){
005337 pWInfo->bOrderedInnerLoop = 1;
005338 pWInfo->revMask = m;
005339 }
005340 }
005341 }
005342 }else if( nLoop
005343 && pWInfo->nOBSat==1
005344 && (pWInfo->wctrlFlags & (WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX))!=0
005345 ){
005346 pWInfo->bOrderedInnerLoop = 1;
005347 }
005348 }
005349 if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)
005350 && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0
005351 ){
005352 Bitmask revMask = 0;
005353 int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy,
005354 pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask
005355 );
005356 assert( pWInfo->sorted==0 );
005357 if( nOrder==pWInfo->pOrderBy->nExpr ){
005358 pWInfo->sorted = 1;
005359 pWInfo->revMask = revMask;
005360 }
005361 }
005362 }
005363
005364
005365 pWInfo->nRowOut = pFrom->nRow;
005366
005367 /* Free temporary memory and return success */
005368 sqlite3StackFreeNN(pParse->db, pSpace);
005369 return SQLITE_OK;
005370 }
005371
005372 /*
005373 ** Most queries use only a single table (they are not joins) and have
005374 ** simple == constraints against indexed fields. This routine attempts
005375 ** to plan those simple cases using much less ceremony than the
005376 ** general-purpose query planner, and thereby yield faster sqlite3_prepare()
005377 ** times for the common case.
005378 **
005379 ** Return non-zero on success, if this query can be handled by this
005380 ** no-frills query planner. Return zero if this query needs the
005381 ** general-purpose query planner.
005382 */
005383 static int whereShortCut(WhereLoopBuilder *pBuilder){
005384 WhereInfo *pWInfo;
005385 SrcItem *pItem;
005386 WhereClause *pWC;
005387 WhereTerm *pTerm;
005388 WhereLoop *pLoop;
005389 int iCur;
005390 int j;
005391 Table *pTab;
005392 Index *pIdx;
005393 WhereScan scan;
005394
005395 pWInfo = pBuilder->pWInfo;
005396 if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0;
005397 assert( pWInfo->pTabList->nSrc>=1 );
005398 pItem = pWInfo->pTabList->a;
005399 pTab = pItem->pTab;
005400 if( IsVirtual(pTab) ) return 0;
005401 if( pItem->fg.isIndexedBy || pItem->fg.notIndexed ){
005402 testcase( pItem->fg.isIndexedBy );
005403 testcase( pItem->fg.notIndexed );
005404 return 0;
005405 }
005406 iCur = pItem->iCursor;
005407 pWC = &pWInfo->sWC;
005408 pLoop = pBuilder->pNew;
005409 pLoop->wsFlags = 0;
005410 pLoop->nSkip = 0;
005411 pTerm = whereScanInit(&scan, pWC, iCur, -1, WO_EQ|WO_IS, 0);
005412 while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan);
005413 if( pTerm ){
005414 testcase( pTerm->eOperator & WO_IS );
005415 pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW;
005416 pLoop->aLTerm[0] = pTerm;
005417 pLoop->nLTerm = 1;
005418 pLoop->u.btree.nEq = 1;
005419 /* TUNING: Cost of a rowid lookup is 10 */
005420 pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */
005421 }else{
005422 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
005423 int opMask;
005424 assert( pLoop->aLTermSpace==pLoop->aLTerm );
005425 if( !IsUniqueIndex(pIdx)
005426 || pIdx->pPartIdxWhere!=0
005427 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace)
005428 ) continue;
005429 opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ;
005430 for(j=0; j<pIdx->nKeyCol; j++){
005431 pTerm = whereScanInit(&scan, pWC, iCur, j, opMask, pIdx);
005432 while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan);
005433 if( pTerm==0 ) break;
005434 testcase( pTerm->eOperator & WO_IS );
005435 pLoop->aLTerm[j] = pTerm;
005436 }
005437 if( j!=pIdx->nKeyCol ) continue;
005438 pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED;
005439 if( pIdx->isCovering || (pItem->colUsed & pIdx->colNotIdxed)==0 ){
005440 pLoop->wsFlags |= WHERE_IDX_ONLY;
005441 }
005442 pLoop->nLTerm = j;
005443 pLoop->u.btree.nEq = j;
005444 pLoop->u.btree.pIndex = pIdx;
005445 /* TUNING: Cost of a unique index lookup is 15 */
005446 pLoop->rRun = 39; /* 39==sqlite3LogEst(15) */
005447 break;
005448 }
005449 }
005450 if( pLoop->wsFlags ){
005451 pLoop->nOut = (LogEst)1;
005452 pWInfo->a[0].pWLoop = pLoop;
005453 assert( pWInfo->sMaskSet.n==1 && iCur==pWInfo->sMaskSet.ix[0] );
005454 pLoop->maskSelf = 1; /* sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); */
005455 pWInfo->a[0].iTabCur = iCur;
005456 pWInfo->nRowOut = 1;
005457 if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr;
005458 if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){
005459 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
005460 }
005461 if( scan.iEquiv>1 ) pLoop->wsFlags |= WHERE_TRANSCONS;
005462 #ifdef SQLITE_DEBUG
005463 pLoop->cId = '0';
005464 #endif
005465 #ifdef WHERETRACE_ENABLED
005466 if( sqlite3WhereTrace & 0x02 ){
005467 sqlite3DebugPrintf("whereShortCut() used to compute solution\n");
005468 }
005469 #endif
005470 return 1;
005471 }
005472 return 0;
005473 }
005474
005475 /*
005476 ** Helper function for exprIsDeterministic().
005477 */
005478 static int exprNodeIsDeterministic(Walker *pWalker, Expr *pExpr){
005479 if( pExpr->op==TK_FUNCTION && ExprHasProperty(pExpr, EP_ConstFunc)==0 ){
005480 pWalker->eCode = 0;
005481 return WRC_Abort;
005482 }
005483 return WRC_Continue;
005484 }
005485
005486 /*
005487 ** Return true if the expression contains no non-deterministic SQL
005488 ** functions. Do not consider non-deterministic SQL functions that are
005489 ** part of sub-select statements.
005490 */
005491 static int exprIsDeterministic(Expr *p){
005492 Walker w;
005493 memset(&w, 0, sizeof(w));
005494 w.eCode = 1;
005495 w.xExprCallback = exprNodeIsDeterministic;
005496 w.xSelectCallback = sqlite3SelectWalkFail;
005497 sqlite3WalkExpr(&w, p);
005498 return w.eCode;
005499 }
005500
005501
005502 #ifdef WHERETRACE_ENABLED
005503 /*
005504 ** Display all WhereLoops in pWInfo
005505 */
005506 static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){
005507 if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */
005508 WhereLoop *p;
005509 int i;
005510 static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz"
005511 "ABCDEFGHIJKLMNOPQRSTUVWYXZ";
005512 for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){
005513 p->cId = zLabel[i%(sizeof(zLabel)-1)];
005514 sqlite3WhereLoopPrint(p, pWC);
005515 }
005516 }
005517 }
005518 # define WHERETRACE_ALL_LOOPS(W,C) showAllWhereLoops(W,C)
005519 #else
005520 # define WHERETRACE_ALL_LOOPS(W,C)
005521 #endif
005522
005523 /* Attempt to omit tables from a join that do not affect the result.
005524 ** For a table to not affect the result, the following must be true:
005525 **
005526 ** 1) The query must not be an aggregate.
005527 ** 2) The table must be the RHS of a LEFT JOIN.
005528 ** 3) Either the query must be DISTINCT, or else the ON or USING clause
005529 ** must contain a constraint that limits the scan of the table to
005530 ** at most a single row.
005531 ** 4) The table must not be referenced by any part of the query apart
005532 ** from its own USING or ON clause.
005533 ** 5) The table must not have an inner-join ON or USING clause if there is
005534 ** a RIGHT JOIN anywhere in the query. Otherwise the ON/USING clause
005535 ** might move from the right side to the left side of the RIGHT JOIN.
005536 ** Note: Due to (2), this condition can only arise if the table is
005537 ** the right-most table of a subquery that was flattened into the
005538 ** main query and that subquery was the right-hand operand of an
005539 ** inner join that held an ON or USING clause.
005540 **
005541 ** For example, given:
005542 **
005543 ** CREATE TABLE t1(ipk INTEGER PRIMARY KEY, v1);
005544 ** CREATE TABLE t2(ipk INTEGER PRIMARY KEY, v2);
005545 ** CREATE TABLE t3(ipk INTEGER PRIMARY KEY, v3);
005546 **
005547 ** then table t2 can be omitted from the following:
005548 **
005549 ** SELECT v1, v3 FROM t1
005550 ** LEFT JOIN t2 ON (t1.ipk=t2.ipk)
005551 ** LEFT JOIN t3 ON (t1.ipk=t3.ipk)
005552 **
005553 ** or from:
005554 **
005555 ** SELECT DISTINCT v1, v3 FROM t1
005556 ** LEFT JOIN t2
005557 ** LEFT JOIN t3 ON (t1.ipk=t3.ipk)
005558 */
005559 static SQLITE_NOINLINE Bitmask whereOmitNoopJoin(
005560 WhereInfo *pWInfo,
005561 Bitmask notReady
005562 ){
005563 int i;
005564 Bitmask tabUsed;
005565 int hasRightJoin;
005566
005567 /* Preconditions checked by the caller */
005568 assert( pWInfo->nLevel>=2 );
005569 assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_OmitNoopJoin) );
005570
005571 /* These two preconditions checked by the caller combine to guarantee
005572 ** condition (1) of the header comment */
005573 assert( pWInfo->pResultSet!=0 );
005574 assert( 0==(pWInfo->wctrlFlags & WHERE_AGG_DISTINCT) );
005575
005576 tabUsed = sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pResultSet);
005577 if( pWInfo->pOrderBy ){
005578 tabUsed |= sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pOrderBy);
005579 }
005580 hasRightJoin = (pWInfo->pTabList->a[0].fg.jointype & JT_LTORJ)!=0;
005581 for(i=pWInfo->nLevel-1; i>=1; i--){
005582 WhereTerm *pTerm, *pEnd;
005583 SrcItem *pItem;
005584 WhereLoop *pLoop;
005585 pLoop = pWInfo->a[i].pWLoop;
005586 pItem = &pWInfo->pTabList->a[pLoop->iTab];
005587 if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ) continue;
005588 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)==0
005589 && (pLoop->wsFlags & WHERE_ONEROW)==0
005590 ){
005591 continue;
005592 }
005593 if( (tabUsed & pLoop->maskSelf)!=0 ) continue;
005594 pEnd = pWInfo->sWC.a + pWInfo->sWC.nTerm;
005595 for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){
005596 if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){
005597 if( !ExprHasProperty(pTerm->pExpr, EP_OuterON)
005598 || pTerm->pExpr->w.iJoin!=pItem->iCursor
005599 ){
005600 break;
005601 }
005602 }
005603 if( hasRightJoin
005604 && ExprHasProperty(pTerm->pExpr, EP_InnerON)
005605 && pTerm->pExpr->w.iJoin==pItem->iCursor
005606 ){
005607 break; /* restriction (5) */
005608 }
005609 }
005610 if( pTerm<pEnd ) continue;
005611 WHERETRACE(0xffffffff, ("-> drop loop %c not used\n", pLoop->cId));
005612 notReady &= ~pLoop->maskSelf;
005613 for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){
005614 if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){
005615 pTerm->wtFlags |= TERM_CODED;
005616 }
005617 }
005618 if( i!=pWInfo->nLevel-1 ){
005619 int nByte = (pWInfo->nLevel-1-i) * sizeof(WhereLevel);
005620 memmove(&pWInfo->a[i], &pWInfo->a[i+1], nByte);
005621 }
005622 pWInfo->nLevel--;
005623 assert( pWInfo->nLevel>0 );
005624 }
005625 return notReady;
005626 }
005627
005628 /*
005629 ** Check to see if there are any SEARCH loops that might benefit from
005630 ** using a Bloom filter. Consider a Bloom filter if:
005631 **
005632 ** (1) The SEARCH happens more than N times where N is the number
005633 ** of rows in the table that is being considered for the Bloom
005634 ** filter.
005635 ** (2) Some searches are expected to find zero rows. (This is determined
005636 ** by the WHERE_SELFCULL flag on the term.)
005637 ** (3) Bloom-filter processing is not disabled. (Checked by the
005638 ** caller.)
005639 ** (4) The size of the table being searched is known by ANALYZE.
005640 **
005641 ** This block of code merely checks to see if a Bloom filter would be
005642 ** appropriate, and if so sets the WHERE_BLOOMFILTER flag on the
005643 ** WhereLoop. The implementation of the Bloom filter comes further
005644 ** down where the code for each WhereLoop is generated.
005645 */
005646 static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
005647 const WhereInfo *pWInfo
005648 ){
005649 int i;
005650 LogEst nSearch = 0;
005651
005652 assert( pWInfo->nLevel>=2 );
005653 assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_BloomFilter) );
005654 for(i=0; i<pWInfo->nLevel; i++){
005655 WhereLoop *pLoop = pWInfo->a[i].pWLoop;
005656 const unsigned int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ);
005657 SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab];
005658 Table *pTab = pItem->pTab;
005659 if( (pTab->tabFlags & TF_HasStat1)==0 ) break;
005660 pTab->tabFlags |= TF_StatsUsed;
005661 if( i>=1
005662 && (pLoop->wsFlags & reqFlags)==reqFlags
005663 /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */
005664 && ALWAYS((pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0)
005665 ){
005666 if( nSearch > pTab->nRowLogEst ){
005667 testcase( pItem->fg.jointype & JT_LEFT );
005668 pLoop->wsFlags |= WHERE_BLOOMFILTER;
005669 pLoop->wsFlags &= ~WHERE_IDX_ONLY;
005670 WHERETRACE(0xffffffff, (
005671 "-> use Bloom-filter on loop %c because there are ~%.1e "
005672 "lookups into %s which has only ~%.1e rows\n",
005673 pLoop->cId, (double)sqlite3LogEstToInt(nSearch), pTab->zName,
005674 (double)sqlite3LogEstToInt(pTab->nRowLogEst)));
005675 }
005676 }
005677 nSearch += pLoop->nOut;
005678 }
005679 }
005680
005681 /*
005682 ** This is an sqlite3ParserAddCleanup() callback that is invoked to
005683 ** free the Parse->pIdxEpr list when the Parse object is destroyed.
005684 */
005685 static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){
005686 Parse *pParse = (Parse*)pObject;
005687 while( pParse->pIdxEpr!=0 ){
005688 IndexedExpr *p = pParse->pIdxEpr;
005689 pParse->pIdxEpr = p->pIENext;
005690 sqlite3ExprDelete(db, p->pExpr);
005691 sqlite3DbFreeNN(db, p);
005692 }
005693 }
005694
005695 /*
005696 ** The index pIdx is used by a query and contains one or more expressions.
005697 ** In other words pIdx is an index on an expression. iIdxCur is the cursor
005698 ** number for the index and iDataCur is the cursor number for the corresponding
005699 ** table.
005700 **
005701 ** This routine adds IndexedExpr entries to the Parse->pIdxEpr field for
005702 ** each of the expressions in the index so that the expression code generator
005703 ** will know to replace occurrences of the indexed expression with
005704 ** references to the corresponding column of the index.
005705 */
005706 static SQLITE_NOINLINE void whereAddIndexedExpr(
005707 Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxEpr */
005708 Index *pIdx, /* The index-on-expression that contains the expressions */
005709 int iIdxCur, /* Cursor number for pIdx */
005710 SrcItem *pTabItem /* The FROM clause entry for the table */
005711 ){
005712 int i;
005713 IndexedExpr *p;
005714 Table *pTab;
005715 assert( pIdx->bHasExpr );
005716 pTab = pIdx->pTable;
005717 for(i=0; i<pIdx->nColumn; i++){
005718 Expr *pExpr;
005719 int j = pIdx->aiColumn[i];
005720 int bMaybeNullRow;
005721 if( j==XN_EXPR ){
005722 pExpr = pIdx->aColExpr->a[i].pExpr;
005723 testcase( pTabItem->fg.jointype & JT_LEFT );
005724 testcase( pTabItem->fg.jointype & JT_RIGHT );
005725 testcase( pTabItem->fg.jointype & JT_LTORJ );
005726 bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0;
005727 }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){
005728 pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]);
005729 bMaybeNullRow = 0;
005730 }else{
005731 continue;
005732 }
005733 if( sqlite3ExprIsConstant(pExpr) ) continue;
005734 p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr));
005735 if( p==0 ) break;
005736 p->pIENext = pParse->pIdxEpr;
005737 #ifdef WHERETRACE_ENABLED
005738 if( sqlite3WhereTrace & 0x200 ){
005739 sqlite3DebugPrintf("New pParse->pIdxEpr term {%d,%d}\n", iIdxCur, i);
005740 if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(pExpr);
005741 }
005742 #endif
005743 p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0);
005744 p->iDataCur = pTabItem->iCursor;
005745 p->iIdxCur = iIdxCur;
005746 p->iIdxCol = i;
005747 p->bMaybeNullRow = bMaybeNullRow;
005748 if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){
005749 p->aff = pIdx->zColAff[i];
005750 }
005751 #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
005752 p->zIdxName = pIdx->zName;
005753 #endif
005754 pParse->pIdxEpr = p;
005755 if( p->pIENext==0 ){
005756 sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pParse);
005757 }
005758 }
005759 }
005760
005761 /*
005762 ** Set the reverse-scan order mask to one for all tables in the query
005763 ** with the exception of MATERIALIZED common table expressions that have
005764 ** their own internal ORDER BY clauses.
005765 **
005766 ** This implements the PRAGMA reverse_unordered_selects=ON setting.
005767 ** (Also SQLITE_DBCONFIG_REVERSE_SCANORDER).
005768 */
005769 static SQLITE_NOINLINE void whereReverseScanOrder(WhereInfo *pWInfo){
005770 int ii;
005771 for(ii=0; ii<pWInfo->pTabList->nSrc; ii++){
005772 SrcItem *pItem = &pWInfo->pTabList->a[ii];
005773 if( !pItem->fg.isCte
005774 || pItem->u2.pCteUse->eM10d!=M10d_Yes
005775 || NEVER(pItem->pSelect==0)
005776 || pItem->pSelect->pOrderBy==0
005777 ){
005778 pWInfo->revMask |= MASKBIT(ii);
005779 }
005780 }
005781 }
005782
005783 /*
005784 ** Generate the beginning of the loop used for WHERE clause processing.
005785 ** The return value is a pointer to an opaque structure that contains
005786 ** information needed to terminate the loop. Later, the calling routine
005787 ** should invoke sqlite3WhereEnd() with the return value of this function
005788 ** in order to complete the WHERE clause processing.
005789 **
005790 ** If an error occurs, this routine returns NULL.
005791 **
005792 ** The basic idea is to do a nested loop, one loop for each table in
005793 ** the FROM clause of a select. (INSERT and UPDATE statements are the
005794 ** same as a SELECT with only a single table in the FROM clause.) For
005795 ** example, if the SQL is this:
005796 **
005797 ** SELECT * FROM t1, t2, t3 WHERE ...;
005798 **
005799 ** Then the code generated is conceptually like the following:
005800 **
005801 ** foreach row1 in t1 do \ Code generated
005802 ** foreach row2 in t2 do |-- by sqlite3WhereBegin()
005803 ** foreach row3 in t3 do /
005804 ** ...
005805 ** end \ Code generated
005806 ** end |-- by sqlite3WhereEnd()
005807 ** end /
005808 **
005809 ** Note that the loops might not be nested in the order in which they
005810 ** appear in the FROM clause if a different order is better able to make
005811 ** use of indices. Note also that when the IN operator appears in
005812 ** the WHERE clause, it might result in additional nested loops for
005813 ** scanning through all values on the right-hand side of the IN.
005814 **
005815 ** There are Btree cursors associated with each table. t1 uses cursor
005816 ** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor.
005817 ** And so forth. This routine generates code to open those VDBE cursors
005818 ** and sqlite3WhereEnd() generates the code to close them.
005819 **
005820 ** The code that sqlite3WhereBegin() generates leaves the cursors named
005821 ** in pTabList pointing at their appropriate entries. The [...] code
005822 ** can use OP_Column and OP_Rowid opcodes on these cursors to extract
005823 ** data from the various tables of the loop.
005824 **
005825 ** If the WHERE clause is empty, the foreach loops must each scan their
005826 ** entire tables. Thus a three-way join is an O(N^3) operation. But if
005827 ** the tables have indices and there are terms in the WHERE clause that
005828 ** refer to those indices, a complete table scan can be avoided and the
005829 ** code will run much faster. Most of the work of this routine is checking
005830 ** to see if there are indices that can be used to speed up the loop.
005831 **
005832 ** Terms of the WHERE clause are also used to limit which rows actually
005833 ** make it to the "..." in the middle of the loop. After each "foreach",
005834 ** terms of the WHERE clause that use only terms in that loop and outer
005835 ** loops are evaluated and if false a jump is made around all subsequent
005836 ** inner loops (or around the "..." if the test occurs within the inner-
005837 ** most loop)
005838 **
005839 ** OUTER JOINS
005840 **
005841 ** An outer join of tables t1 and t2 is conceptually coded as follows:
005842 **
005843 ** foreach row1 in t1 do
005844 ** flag = 0
005845 ** foreach row2 in t2 do
005846 ** start:
005847 ** ...
005848 ** flag = 1
005849 ** end
005850 ** if flag==0 then
005851 ** move the row2 cursor to a null row
005852 ** goto start
005853 ** fi
005854 ** end
005855 **
005856 ** ORDER BY CLAUSE PROCESSING
005857 **
005858 ** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause
005859 ** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement
005860 ** if there is one. If there is no ORDER BY clause or if this routine
005861 ** is called from an UPDATE or DELETE statement, then pOrderBy is NULL.
005862 **
005863 ** The iIdxCur parameter is the cursor number of an index. If
005864 ** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index
005865 ** to use for OR clause processing. The WHERE clause should use this
005866 ** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is
005867 ** the first cursor in an array of cursors for all indices. iIdxCur should
005868 ** be used to compute the appropriate cursor depending on which index is
005869 ** used.
005870 */
005871 WhereInfo *sqlite3WhereBegin(
005872 Parse *pParse, /* The parser context */
005873 SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */
005874 Expr *pWhere, /* The WHERE clause */
005875 ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */
005876 ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */
005877 Select *pSelect, /* The entire SELECT statement */
005878 u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */
005879 int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number
005880 ** If WHERE_USE_LIMIT, then the limit amount */
005881 ){
005882 int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */
005883 int nTabList; /* Number of elements in pTabList */
005884 WhereInfo *pWInfo; /* Will become the return value of this function */
005885 Vdbe *v = pParse->pVdbe; /* The virtual database engine */
005886 Bitmask notReady; /* Cursors that are not yet positioned */
005887 WhereLoopBuilder sWLB; /* The WhereLoop builder */
005888 WhereMaskSet *pMaskSet; /* The expression mask set */
005889 WhereLevel *pLevel; /* A single level in pWInfo->a[] */
005890 WhereLoop *pLoop; /* Pointer to a single WhereLoop object */
005891 int ii; /* Loop counter */
005892 sqlite3 *db; /* Database connection */
005893 int rc; /* Return code */
005894 u8 bFordelete = 0; /* OPFLAG_FORDELETE or zero, as appropriate */
005895
005896 assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || (
005897 (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0
005898 && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
005899 ));
005900
005901 /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */
005902 assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
005903 || (wctrlFlags & WHERE_USE_LIMIT)==0 );
005904
005905 /* Variable initialization */
005906 db = pParse->db;
005907 memset(&sWLB, 0, sizeof(sWLB));
005908
005909 /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */
005910 testcase( pOrderBy && pOrderBy->nExpr==BMS-1 );
005911 if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0;
005912
005913 /* The number of tables in the FROM clause is limited by the number of
005914 ** bits in a Bitmask
005915 */
005916 testcase( pTabList->nSrc==BMS );
005917 if( pTabList->nSrc>BMS ){
005918 sqlite3ErrorMsg(pParse, "at most %d tables in a join", BMS);
005919 return 0;
005920 }
005921
005922 /* This function normally generates a nested loop for all tables in
005923 ** pTabList. But if the WHERE_OR_SUBCLAUSE flag is set, then we should
005924 ** only generate code for the first table in pTabList and assume that
005925 ** any cursors associated with subsequent tables are uninitialized.
005926 */
005927 nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc;
005928
005929 /* Allocate and initialize the WhereInfo structure that will become the
005930 ** return value. A single allocation is used to store the WhereInfo
005931 ** struct, the contents of WhereInfo.a[], the WhereClause structure
005932 ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte
005933 ** field (type Bitmask) it must be aligned on an 8-byte boundary on
005934 ** some architectures. Hence the ROUND8() below.
005935 */
005936 nByteWInfo = ROUND8P(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel));
005937 pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop));
005938 if( db->mallocFailed ){
005939 sqlite3DbFree(db, pWInfo);
005940 pWInfo = 0;
005941 goto whereBeginError;
005942 }
005943 pWInfo->pParse = pParse;
005944 pWInfo->pTabList = pTabList;
005945 pWInfo->pOrderBy = pOrderBy;
005946 #if WHERETRACE_ENABLED
005947 pWInfo->pWhere = pWhere;
005948 #endif
005949 pWInfo->pResultSet = pResultSet;
005950 pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1;
005951 pWInfo->nLevel = nTabList;
005952 pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(pParse);
005953 pWInfo->wctrlFlags = wctrlFlags;
005954 pWInfo->iLimit = iAuxArg;
005955 pWInfo->savedNQueryLoop = pParse->nQueryLoop;
005956 pWInfo->pSelect = pSelect;
005957 memset(&pWInfo->nOBSat, 0,
005958 offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat));
005959 memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel));
005960 assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */
005961 pMaskSet = &pWInfo->sMaskSet;
005962 pMaskSet->n = 0;
005963 pMaskSet->ix[0] = -99; /* Initialize ix[0] to a value that can never be
005964 ** a valid cursor number, to avoid an initial
005965 ** test for pMaskSet->n==0 in sqlite3WhereGetMask() */
005966 sWLB.pWInfo = pWInfo;
005967 sWLB.pWC = &pWInfo->sWC;
005968 sWLB.pNew = (WhereLoop*)(((char*)pWInfo)+nByteWInfo);
005969 assert( EIGHT_BYTE_ALIGNMENT(sWLB.pNew) );
005970 whereLoopInit(sWLB.pNew);
005971 #ifdef SQLITE_DEBUG
005972 sWLB.pNew->cId = '*';
005973 #endif
005974
005975 /* Split the WHERE clause into separate subexpressions where each
005976 ** subexpression is separated by an AND operator.
005977 */
005978 sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo);
005979 sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND);
005980
005981 /* Special case: No FROM clause
005982 */
005983 if( nTabList==0 ){
005984 if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr;
005985 if( (wctrlFlags & WHERE_WANT_DISTINCT)!=0
005986 && OptimizationEnabled(db, SQLITE_DistinctOpt)
005987 ){
005988 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
005989 }
005990 ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW"));
005991 }else{
005992 /* Assign a bit from the bitmask to every term in the FROM clause.
005993 **
005994 ** The N-th term of the FROM clause is assigned a bitmask of 1<<N.
005995 **
005996 ** The rule of the previous sentence ensures that if X is the bitmask for
005997 ** a table T, then X-1 is the bitmask for all other tables to the left of T.
005998 ** Knowing the bitmask for all tables to the left of a left join is
005999 ** important. Ticket #3015.
006000 **
006001 ** Note that bitmasks are created for all pTabList->nSrc tables in
006002 ** pTabList, not just the first nTabList tables. nTabList is normally
006003 ** equal to pTabList->nSrc but might be shortened to 1 if the
006004 ** WHERE_OR_SUBCLAUSE flag is set.
006005 */
006006 ii = 0;
006007 do{
006008 createMask(pMaskSet, pTabList->a[ii].iCursor);
006009 sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC);
006010 }while( (++ii)<pTabList->nSrc );
006011 #ifdef SQLITE_DEBUG
006012 {
006013 Bitmask mx = 0;
006014 for(ii=0; ii<pTabList->nSrc; ii++){
006015 Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor);
006016 assert( m>=mx );
006017 mx = m;
006018 }
006019 }
006020 #endif
006021 }
006022
006023 /* Analyze all of the subexpressions. */
006024 sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC);
006025 if( pSelect && pSelect->pLimit ){
006026 sqlite3WhereAddLimit(&pWInfo->sWC, pSelect);
006027 }
006028 if( pParse->nErr ) goto whereBeginError;
006029
006030 /* The False-WHERE-Term-Bypass optimization:
006031 **
006032 ** If there are WHERE terms that are false, then no rows will be output,
006033 ** so skip over all of the code generated here.
006034 **
006035 ** Conditions:
006036 **
006037 ** (1) The WHERE term must not refer to any tables in the join.
006038 ** (2) The term must not come from an ON clause on the
006039 ** right-hand side of a LEFT or FULL JOIN.
006040 ** (3) The term must not come from an ON clause, or there must be
006041 ** no RIGHT or FULL OUTER joins in pTabList.
006042 ** (4) If the expression contains non-deterministic functions
006043 ** that are not within a sub-select. This is not required
006044 ** for correctness but rather to preserves SQLite's legacy
006045 ** behaviour in the following two cases:
006046 **
006047 ** WHERE random()>0; -- eval random() once per row
006048 ** WHERE (SELECT random())>0; -- eval random() just once overall
006049 **
006050 ** Note that the Where term need not be a constant in order for this
006051 ** optimization to apply, though it does need to be constant relative to
006052 ** the current subquery (condition 1). The term might include variables
006053 ** from outer queries so that the value of the term changes from one
006054 ** invocation of the current subquery to the next.
006055 */
006056 for(ii=0; ii<sWLB.pWC->nBase; ii++){
006057 WhereTerm *pT = &sWLB.pWC->a[ii]; /* A term of the WHERE clause */
006058 Expr *pX; /* The expression of pT */
006059 if( pT->wtFlags & TERM_VIRTUAL ) continue;
006060 pX = pT->pExpr;
006061 assert( pX!=0 );
006062 assert( pT->prereqAll!=0 || !ExprHasProperty(pX, EP_OuterON) );
006063 if( pT->prereqAll==0 /* Conditions (1) and (2) */
006064 && (nTabList==0 || exprIsDeterministic(pX)) /* Condition (4) */
006065 && !(ExprHasProperty(pX, EP_InnerON) /* Condition (3) */
006066 && (pTabList->a[0].fg.jointype & JT_LTORJ)!=0 )
006067 ){
006068 sqlite3ExprIfFalse(pParse, pX, pWInfo->iBreak, SQLITE_JUMPIFNULL);
006069 pT->wtFlags |= TERM_CODED;
006070 }
006071 }
006072
006073 if( wctrlFlags & WHERE_WANT_DISTINCT ){
006074 if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){
006075 /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via
006076 ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */
006077 wctrlFlags &= ~WHERE_WANT_DISTINCT;
006078 pWInfo->wctrlFlags &= ~WHERE_WANT_DISTINCT;
006079 }else if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){
006080 /* The DISTINCT marking is pointless. Ignore it. */
006081 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
006082 }else if( pOrderBy==0 ){
006083 /* Try to ORDER BY the result set to make distinct processing easier */
006084 pWInfo->wctrlFlags |= WHERE_DISTINCTBY;
006085 pWInfo->pOrderBy = pResultSet;
006086 }
006087 }
006088
006089 /* Construct the WhereLoop objects */
006090 #if defined(WHERETRACE_ENABLED)
006091 if( sqlite3WhereTrace & 0xffffffff ){
006092 sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags);
006093 if( wctrlFlags & WHERE_USE_LIMIT ){
006094 sqlite3DebugPrintf(", limit: %d", iAuxArg);
006095 }
006096 sqlite3DebugPrintf(")\n");
006097 if( sqlite3WhereTrace & 0x8000 ){
006098 Select sSelect;
006099 memset(&sSelect, 0, sizeof(sSelect));
006100 sSelect.selFlags = SF_WhereBegin;
006101 sSelect.pSrc = pTabList;
006102 sSelect.pWhere = pWhere;
006103 sSelect.pOrderBy = pOrderBy;
006104 sSelect.pEList = pResultSet;
006105 sqlite3TreeViewSelect(0, &sSelect, 0);
006106 }
006107 if( sqlite3WhereTrace & 0x4000 ){ /* Display all WHERE clause terms */
006108 sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n");
006109 sqlite3WhereClausePrint(sWLB.pWC);
006110 }
006111 }
006112 #endif
006113
006114 if( nTabList!=1 || whereShortCut(&sWLB)==0 ){
006115 rc = whereLoopAddAll(&sWLB);
006116 if( rc ) goto whereBeginError;
006117
006118 #ifdef SQLITE_ENABLE_STAT4
006119 /* If one or more WhereTerm.truthProb values were used in estimating
006120 ** loop parameters, but then those truthProb values were subsequently
006121 ** changed based on STAT4 information while computing subsequent loops,
006122 ** then we need to rerun the whole loop building process so that all
006123 ** loops will be built using the revised truthProb values. */
006124 if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){
006125 WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
006126 WHERETRACE(0xffffffff,
006127 ("**** Redo all loop computations due to"
006128 " TERM_HIGHTRUTH changes ****\n"));
006129 while( pWInfo->pLoops ){
006130 WhereLoop *p = pWInfo->pLoops;
006131 pWInfo->pLoops = p->pNextLoop;
006132 whereLoopDelete(db, p);
006133 }
006134 rc = whereLoopAddAll(&sWLB);
006135 if( rc ) goto whereBeginError;
006136 }
006137 #endif
006138 WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
006139
006140 wherePathSolver(pWInfo, 0);
006141 if( db->mallocFailed ) goto whereBeginError;
006142 if( pWInfo->pOrderBy ){
006143 wherePathSolver(pWInfo, pWInfo->nRowOut+1);
006144 if( db->mallocFailed ) goto whereBeginError;
006145 }
006146 }
006147 assert( pWInfo->pTabList!=0 );
006148 if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){
006149 whereReverseScanOrder(pWInfo);
006150 }
006151 if( pParse->nErr ){
006152 goto whereBeginError;
006153 }
006154 assert( db->mallocFailed==0 );
006155 #ifdef WHERETRACE_ENABLED
006156 if( sqlite3WhereTrace ){
006157 sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut);
006158 if( pWInfo->nOBSat>0 ){
006159 sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask);
006160 }
006161 switch( pWInfo->eDistinct ){
006162 case WHERE_DISTINCT_UNIQUE: {
006163 sqlite3DebugPrintf(" DISTINCT=unique");
006164 break;
006165 }
006166 case WHERE_DISTINCT_ORDERED: {
006167 sqlite3DebugPrintf(" DISTINCT=ordered");
006168 break;
006169 }
006170 case WHERE_DISTINCT_UNORDERED: {
006171 sqlite3DebugPrintf(" DISTINCT=unordered");
006172 break;
006173 }
006174 }
006175 sqlite3DebugPrintf("\n");
006176 for(ii=0; ii<pWInfo->nLevel; ii++){
006177 sqlite3WhereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC);
006178 }
006179 }
006180 #endif
006181
006182 /* Attempt to omit tables from a join that do not affect the result.
006183 ** See the comment on whereOmitNoopJoin() for further information.
006184 **
006185 ** This query optimization is factored out into a separate "no-inline"
006186 ** procedure to keep the sqlite3WhereBegin() procedure from becoming
006187 ** too large. If sqlite3WhereBegin() becomes too large, that prevents
006188 ** some C-compiler optimizers from in-lining the
006189 ** sqlite3WhereCodeOneLoopStart() procedure, and it is important to
006190 ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons.
006191 */
006192 notReady = ~(Bitmask)0;
006193 if( pWInfo->nLevel>=2
006194 && pResultSet!=0 /* these two combine to guarantee */
006195 && 0==(wctrlFlags & WHERE_AGG_DISTINCT) /* condition (1) above */
006196 && OptimizationEnabled(db, SQLITE_OmitNoopJoin)
006197 ){
006198 notReady = whereOmitNoopJoin(pWInfo, notReady);
006199 nTabList = pWInfo->nLevel;
006200 assert( nTabList>0 );
006201 }
006202
006203 /* Check to see if there are any SEARCH loops that might benefit from
006204 ** using a Bloom filter.
006205 */
006206 if( pWInfo->nLevel>=2
006207 && OptimizationEnabled(db, SQLITE_BloomFilter)
006208 ){
006209 whereCheckIfBloomFilterIsUseful(pWInfo);
006210 }
006211
006212 #if defined(WHERETRACE_ENABLED)
006213 if( sqlite3WhereTrace & 0x4000 ){ /* Display all terms of the WHERE clause */
006214 sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n");
006215 sqlite3WhereClausePrint(sWLB.pWC);
006216 }
006217 WHERETRACE(0xffffffff,("*** Optimizer Finished ***\n"));
006218 #endif
006219 pWInfo->pParse->nQueryLoop += pWInfo->nRowOut;
006220
006221 /* If the caller is an UPDATE or DELETE statement that is requesting
006222 ** to use a one-pass algorithm, determine if this is appropriate.
006223 **
006224 ** A one-pass approach can be used if the caller has requested one
006225 ** and either (a) the scan visits at most one row or (b) each
006226 ** of the following are true:
006227 **
006228 ** * the caller has indicated that a one-pass approach can be used
006229 ** with multiple rows (by setting WHERE_ONEPASS_MULTIROW), and
006230 ** * the table is not a virtual table, and
006231 ** * either the scan does not use the OR optimization or the caller
006232 ** is a DELETE operation (WHERE_DUPLICATES_OK is only specified
006233 ** for DELETE).
006234 **
006235 ** The last qualification is because an UPDATE statement uses
006236 ** WhereInfo.aiCurOnePass[1] to determine whether or not it really can
006237 ** use a one-pass approach, and this is not set accurately for scans
006238 ** that use the OR optimization.
006239 */
006240 assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 );
006241 if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){
006242 int wsFlags = pWInfo->a[0].pWLoop->wsFlags;
006243 int bOnerow = (wsFlags & WHERE_ONEROW)!=0;
006244 assert( !(wsFlags & WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pTab) );
006245 if( bOnerow || (
006246 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW)
006247 && !IsVirtual(pTabList->a[0].pTab)
006248 && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK))
006249 && OptimizationEnabled(db, SQLITE_OnePass)
006250 )){
006251 pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI;
006252 if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){
006253 if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){
006254 bFordelete = OPFLAG_FORDELETE;
006255 }
006256 pWInfo->a[0].pWLoop->wsFlags = (wsFlags & ~WHERE_IDX_ONLY);
006257 }
006258 }
006259 }
006260
006261 /* Open all tables in the pTabList and any indices selected for
006262 ** searching those tables.
006263 */
006264 for(ii=0, pLevel=pWInfo->a; ii<nTabList; ii++, pLevel++){
006265 Table *pTab; /* Table to open */
006266 int iDb; /* Index of database containing table/index */
006267 SrcItem *pTabItem;
006268
006269 pTabItem = &pTabList->a[pLevel->iFrom];
006270 pTab = pTabItem->pTab;
006271 iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
006272 pLoop = pLevel->pWLoop;
006273 if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){
006274 /* Do nothing */
006275 }else
006276 #ifndef SQLITE_OMIT_VIRTUALTABLE
006277 if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){
006278 const char *pVTab = (const char *)sqlite3GetVTable(db, pTab);
006279 int iCur = pTabItem->iCursor;
006280 sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB);
006281 }else if( IsVirtual(pTab) ){
006282 /* noop */
006283 }else
006284 #endif
006285 if( ((pLoop->wsFlags & WHERE_IDX_ONLY)==0
006286 && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0)
006287 || (pTabItem->fg.jointype & (JT_LTORJ|JT_RIGHT))!=0
006288 ){
006289 int op = OP_OpenRead;
006290 if( pWInfo->eOnePass!=ONEPASS_OFF ){
006291 op = OP_OpenWrite;
006292 pWInfo->aiCurOnePass[0] = pTabItem->iCursor;
006293 };
006294 sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op);
006295 assert( pTabItem->iCursor==pLevel->iTabCur );
006296 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 );
006297 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS );
006298 if( pWInfo->eOnePass==ONEPASS_OFF
006299 && pTab->nCol<BMS
006300 && (pTab->tabFlags & (TF_HasGenerated|TF_WithoutRowid))==0
006301 && (pLoop->wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))==0
006302 ){
006303 /* If we know that only a prefix of the record will be used,
006304 ** it is advantageous to reduce the "column count" field in
006305 ** the P4 operand of the OP_OpenRead/Write opcode. */
006306 Bitmask b = pTabItem->colUsed;
006307 int n = 0;
006308 for(; b; b=b>>1, n++){}
006309 sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32);
006310 assert( n<=pTab->nCol );
006311 }
006312 #ifdef SQLITE_ENABLE_CURSOR_HINTS
006313 if( pLoop->u.btree.pIndex!=0 && (pTab->tabFlags & TF_WithoutRowid)==0 ){
006314 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete);
006315 }else
006316 #endif
006317 {
006318 sqlite3VdbeChangeP5(v, bFordelete);
006319 }
006320 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
006321 sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0,
006322 (const u8*)&pTabItem->colUsed, P4_INT64);
006323 #endif
006324 }else{
006325 sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
006326 }
006327 if( pLoop->wsFlags & WHERE_INDEXED ){
006328 Index *pIx = pLoop->u.btree.pIndex;
006329 int iIndexCur;
006330 int op = OP_OpenRead;
006331 /* iAuxArg is always set to a positive value if ONEPASS is possible */
006332 assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 );
006333 if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx)
006334 && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0
006335 ){
006336 /* This is one term of an OR-optimization using the PRIMARY KEY of a
006337 ** WITHOUT ROWID table. No need for a separate index */
006338 iIndexCur = pLevel->iTabCur;
006339 op = 0;
006340 }else if( pWInfo->eOnePass!=ONEPASS_OFF ){
006341 Index *pJ = pTabItem->pTab->pIndex;
006342 iIndexCur = iAuxArg;
006343 assert( wctrlFlags & WHERE_ONEPASS_DESIRED );
006344 while( ALWAYS(pJ) && pJ!=pIx ){
006345 iIndexCur++;
006346 pJ = pJ->pNext;
006347 }
006348 op = OP_OpenWrite;
006349 pWInfo->aiCurOnePass[1] = iIndexCur;
006350 }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){
006351 iIndexCur = iAuxArg;
006352 op = OP_ReopenIdx;
006353 }else{
006354 iIndexCur = pParse->nTab++;
006355 if( pIx->bHasExpr && OptimizationEnabled(db, SQLITE_IndexedExpr) ){
006356 whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem);
006357 }
006358 }
006359 pLevel->iIdxCur = iIndexCur;
006360 assert( pIx!=0 );
006361 assert( pIx->pSchema==pTab->pSchema );
006362 assert( iIndexCur>=0 );
006363 if( op ){
006364 sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb);
006365 sqlite3VdbeSetP4KeyInfo(pParse, pIx);
006366 if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0
006367 && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0
006368 && (pLoop->wsFlags & WHERE_BIGNULL_SORT)==0
006369 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0
006370 && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0
006371 && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED
006372 ){
006373 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ);
006374 }
006375 VdbeComment((v, "%s", pIx->zName));
006376 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
006377 {
006378 u64 colUsed = 0;
006379 int ii, jj;
006380 for(ii=0; ii<pIx->nColumn; ii++){
006381 jj = pIx->aiColumn[ii];
006382 if( jj<0 ) continue;
006383 if( jj>63 ) jj = 63;
006384 if( (pTabItem->colUsed & MASKBIT(jj))==0 ) continue;
006385 colUsed |= ((u64)1)<<(ii<63 ? ii : 63);
006386 }
006387 sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, iIndexCur, 0, 0,
006388 (u8*)&colUsed, P4_INT64);
006389 }
006390 #endif /* SQLITE_ENABLE_COLUMN_USED_MASK */
006391 }
006392 }
006393 if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb);
006394 if( (pTabItem->fg.jointype & JT_RIGHT)!=0
006395 && (pLevel->pRJ = sqlite3WhereMalloc(pWInfo, sizeof(WhereRightJoin)))!=0
006396 ){
006397 WhereRightJoin *pRJ = pLevel->pRJ;
006398 pRJ->iMatch = pParse->nTab++;
006399 pRJ->regBloom = ++pParse->nMem;
006400 sqlite3VdbeAddOp2(v, OP_Blob, 65536, pRJ->regBloom);
006401 pRJ->regReturn = ++pParse->nMem;
006402 sqlite3VdbeAddOp2(v, OP_Null, 0, pRJ->regReturn);
006403 assert( pTab==pTabItem->pTab );
006404 if( HasRowid(pTab) ){
006405 KeyInfo *pInfo;
006406 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, 1);
006407 pInfo = sqlite3KeyInfoAlloc(pParse->db, 1, 0);
006408 if( pInfo ){
006409 pInfo->aColl[0] = 0;
006410 pInfo->aSortFlags[0] = 0;
006411 sqlite3VdbeAppendP4(v, pInfo, P4_KEYINFO);
006412 }
006413 }else{
006414 Index *pPk = sqlite3PrimaryKeyIndex(pTab);
006415 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, pPk->nKeyCol);
006416 sqlite3VdbeSetP4KeyInfo(pParse, pPk);
006417 }
006418 pLoop->wsFlags &= ~WHERE_IDX_ONLY;
006419 /* The nature of RIGHT JOIN processing is such that it messes up
006420 ** the output order. So omit any ORDER BY/GROUP BY elimination
006421 ** optimizations. We need to do an actual sort for RIGHT JOIN. */
006422 pWInfo->nOBSat = 0;
006423 pWInfo->eDistinct = WHERE_DISTINCT_UNORDERED;
006424 }
006425 }
006426 pWInfo->iTop = sqlite3VdbeCurrentAddr(v);
006427 if( db->mallocFailed ) goto whereBeginError;
006428
006429 /* Generate the code to do the search. Each iteration of the for
006430 ** loop below generates code for a single nested loop of the VM
006431 ** program.
006432 */
006433 for(ii=0; ii<nTabList; ii++){
006434 int addrExplain;
006435 int wsFlags;
006436 SrcItem *pSrc;
006437 if( pParse->nErr ) goto whereBeginError;
006438 pLevel = &pWInfo->a[ii];
006439 wsFlags = pLevel->pWLoop->wsFlags;
006440 pSrc = &pTabList->a[pLevel->iFrom];
006441 if( pSrc->fg.isMaterialized ){
006442 if( pSrc->fg.isCorrelated ){
006443 sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub);
006444 }else{
006445 int iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
006446 sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub);
006447 sqlite3VdbeJumpHere(v, iOnce);
006448 }
006449 }
006450 assert( pTabList == pWInfo->pTabList );
006451 if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){
006452 if( (wsFlags & WHERE_AUTO_INDEX)!=0 ){
006453 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
006454 constructAutomaticIndex(pParse, &pWInfo->sWC, notReady, pLevel);
006455 #endif
006456 }else{
006457 sqlite3ConstructBloomFilter(pWInfo, ii, pLevel, notReady);
006458 }
006459 if( db->mallocFailed ) goto whereBeginError;
006460 }
006461 addrExplain = sqlite3WhereExplainOneScan(
006462 pParse, pTabList, pLevel, wctrlFlags
006463 );
006464 pLevel->addrBody = sqlite3VdbeCurrentAddr(v);
006465 notReady = sqlite3WhereCodeOneLoopStart(pParse,v,pWInfo,ii,pLevel,notReady);
006466 pWInfo->iContinue = pLevel->addrCont;
006467 if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){
006468 sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain);
006469 }
006470 }
006471
006472 /* Done. */
006473 VdbeModuleComment((v, "Begin WHERE-core"));
006474 pWInfo->iEndWhere = sqlite3VdbeCurrentAddr(v);
006475 return pWInfo;
006476
006477 /* Jump here if malloc fails */
006478 whereBeginError:
006479 if( pWInfo ){
006480 pParse->nQueryLoop = pWInfo->savedNQueryLoop;
006481 whereInfoFree(db, pWInfo);
006482 }
006483 return 0;
006484 }
006485
006486 /*
006487 ** Part of sqlite3WhereEnd() will rewrite opcodes to reference the
006488 ** index rather than the main table. In SQLITE_DEBUG mode, we want
006489 ** to trace those changes if PRAGMA vdbe_addoptrace=on. This routine
006490 ** does that.
006491 */
006492 #ifndef SQLITE_DEBUG
006493 # define OpcodeRewriteTrace(D,K,P) /* no-op */
006494 #else
006495 # define OpcodeRewriteTrace(D,K,P) sqlite3WhereOpcodeRewriteTrace(D,K,P)
006496 static void sqlite3WhereOpcodeRewriteTrace(
006497 sqlite3 *db,
006498 int pc,
006499 VdbeOp *pOp
006500 ){
006501 if( (db->flags & SQLITE_VdbeAddopTrace)==0 ) return;
006502 sqlite3VdbePrintOp(0, pc, pOp);
006503 }
006504 #endif
006505
006506 #ifdef SQLITE_DEBUG
006507 /*
006508 ** Return true if cursor iCur is opened by instruction k of the
006509 ** bytecode. Used inside of assert() only.
006510 */
006511 static int cursorIsOpen(Vdbe *v, int iCur, int k){
006512 while( k>=0 ){
006513 VdbeOp *pOp = sqlite3VdbeGetOp(v,k--);
006514 if( pOp->p1!=iCur ) continue;
006515 if( pOp->opcode==OP_Close ) return 0;
006516 if( pOp->opcode==OP_OpenRead ) return 1;
006517 if( pOp->opcode==OP_OpenWrite ) return 1;
006518 if( pOp->opcode==OP_OpenDup ) return 1;
006519 if( pOp->opcode==OP_OpenAutoindex ) return 1;
006520 if( pOp->opcode==OP_OpenEphemeral ) return 1;
006521 }
006522 return 0;
006523 }
006524 #endif /* SQLITE_DEBUG */
006525
006526 /*
006527 ** Generate the end of the WHERE loop. See comments on
006528 ** sqlite3WhereBegin() for additional information.
006529 */
006530 void sqlite3WhereEnd(WhereInfo *pWInfo){
006531 Parse *pParse = pWInfo->pParse;
006532 Vdbe *v = pParse->pVdbe;
006533 int i;
006534 WhereLevel *pLevel;
006535 WhereLoop *pLoop;
006536 SrcList *pTabList = pWInfo->pTabList;
006537 sqlite3 *db = pParse->db;
006538 int iEnd = sqlite3VdbeCurrentAddr(v);
006539 int nRJ = 0;
006540
006541 /* Generate loop termination code.
006542 */
006543 VdbeModuleComment((v, "End WHERE-core"));
006544 for(i=pWInfo->nLevel-1; i>=0; i--){
006545 int addr;
006546 pLevel = &pWInfo->a[i];
006547 if( pLevel->pRJ ){
006548 /* Terminate the subroutine that forms the interior of the loop of
006549 ** the RIGHT JOIN table */
006550 WhereRightJoin *pRJ = pLevel->pRJ;
006551 sqlite3VdbeResolveLabel(v, pLevel->addrCont);
006552 pLevel->addrCont = 0;
006553 pRJ->endSubrtn = sqlite3VdbeCurrentAddr(v);
006554 sqlite3VdbeAddOp3(v, OP_Return, pRJ->regReturn, pRJ->addrSubrtn, 1);
006555 VdbeCoverage(v);
006556 nRJ++;
006557 }
006558 pLoop = pLevel->pWLoop;
006559 if( pLevel->op!=OP_Noop ){
006560 #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
006561 int addrSeek = 0;
006562 Index *pIdx;
006563 int n;
006564 if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED
006565 && i==pWInfo->nLevel-1 /* Ticket [ef9318757b152e3] 2017-10-21 */
006566 && (pLoop->wsFlags & WHERE_INDEXED)!=0
006567 && (pIdx = pLoop->u.btree.pIndex)->hasStat1
006568 && (n = pLoop->u.btree.nDistinctCol)>0
006569 && pIdx->aiRowLogEst[n]>=36
006570 ){
006571 int r1 = pParse->nMem+1;
006572 int j, op;
006573 for(j=0; j<n; j++){
006574 sqlite3VdbeAddOp3(v, OP_Column, pLevel->iIdxCur, j, r1+j);
006575 }
006576 pParse->nMem += n+1;
006577 op = pLevel->op==OP_Prev ? OP_SeekLT : OP_SeekGT;
006578 addrSeek = sqlite3VdbeAddOp4Int(v, op, pLevel->iIdxCur, 0, r1, n);
006579 VdbeCoverageIf(v, op==OP_SeekLT);
006580 VdbeCoverageIf(v, op==OP_SeekGT);
006581 sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2);
006582 }
006583 #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */
006584 /* The common case: Advance to the next row */
006585 if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont);
006586 sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3);
006587 sqlite3VdbeChangeP5(v, pLevel->p5);
006588 VdbeCoverage(v);
006589 VdbeCoverageIf(v, pLevel->op==OP_Next);
006590 VdbeCoverageIf(v, pLevel->op==OP_Prev);
006591 VdbeCoverageIf(v, pLevel->op==OP_VNext);
006592 if( pLevel->regBignull ){
006593 sqlite3VdbeResolveLabel(v, pLevel->addrBignull);
006594 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, pLevel->regBignull, pLevel->p2-1);
006595 VdbeCoverage(v);
006596 }
006597 #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
006598 if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek);
006599 #endif
006600 }else if( pLevel->addrCont ){
006601 sqlite3VdbeResolveLabel(v, pLevel->addrCont);
006602 }
006603 if( (pLoop->wsFlags & WHERE_IN_ABLE)!=0 && pLevel->u.in.nIn>0 ){
006604 struct InLoop *pIn;
006605 int j;
006606 sqlite3VdbeResolveLabel(v, pLevel->addrNxt);
006607 for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){
006608 assert( sqlite3VdbeGetOp(v, pIn->addrInTop+1)->opcode==OP_IsNull
006609 || pParse->db->mallocFailed );
006610 sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
006611 if( pIn->eEndLoopOp!=OP_Noop ){
006612 if( pIn->nPrefix ){
006613 int bEarlyOut =
006614 (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0
006615 && (pLoop->wsFlags & WHERE_IN_EARLYOUT)!=0;
006616 if( pLevel->iLeftJoin ){
006617 /* For LEFT JOIN queries, cursor pIn->iCur may not have been
006618 ** opened yet. This occurs for WHERE clauses such as
006619 ** "a = ? AND b IN (...)", where the index is on (a, b). If
006620 ** the RHS of the (a=?) is NULL, then the "b IN (...)" may
006621 ** never have been coded, but the body of the loop run to
006622 ** return the null-row. So, if the cursor is not open yet,
006623 ** jump over the OP_Next or OP_Prev instruction about to
006624 ** be coded. */
006625 sqlite3VdbeAddOp2(v, OP_IfNotOpen, pIn->iCur,
006626 sqlite3VdbeCurrentAddr(v) + 2 + bEarlyOut);
006627 VdbeCoverage(v);
006628 }
006629 if( bEarlyOut ){
006630 sqlite3VdbeAddOp4Int(v, OP_IfNoHope, pLevel->iIdxCur,
006631 sqlite3VdbeCurrentAddr(v)+2,
006632 pIn->iBase, pIn->nPrefix);
006633 VdbeCoverage(v);
006634 /* Retarget the OP_IsNull against the left operand of IN so
006635 ** it jumps past the OP_IfNoHope. This is because the
006636 ** OP_IsNull also bypasses the OP_Affinity opcode that is
006637 ** required by OP_IfNoHope. */
006638 sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
006639 }
006640 }
006641 sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop);
006642 VdbeCoverage(v);
006643 VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Prev);
006644 VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Next);
006645 }
006646 sqlite3VdbeJumpHere(v, pIn->addrInTop-1);
006647 }
006648 }
006649 sqlite3VdbeResolveLabel(v, pLevel->addrBrk);
006650 if( pLevel->pRJ ){
006651 sqlite3VdbeAddOp3(v, OP_Return, pLevel->pRJ->regReturn, 0, 1);
006652 VdbeCoverage(v);
006653 }
006654 if( pLevel->addrSkip ){
006655 sqlite3VdbeGoto(v, pLevel->addrSkip);
006656 VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName));
006657 sqlite3VdbeJumpHere(v, pLevel->addrSkip);
006658 sqlite3VdbeJumpHere(v, pLevel->addrSkip-2);
006659 }
006660 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
006661 if( pLevel->addrLikeRep ){
006662 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, (int)(pLevel->iLikeRepCntr>>1),
006663 pLevel->addrLikeRep);
006664 VdbeCoverage(v);
006665 }
006666 #endif
006667 if( pLevel->iLeftJoin ){
006668 int ws = pLoop->wsFlags;
006669 addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v);
006670 assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 );
006671 if( (ws & WHERE_IDX_ONLY)==0 ){
006672 assert( pLevel->iTabCur==pTabList->a[pLevel->iFrom].iCursor );
006673 sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur);
006674 }
006675 if( (ws & WHERE_INDEXED)
006676 || ((ws & WHERE_MULTI_OR) && pLevel->u.pCoveringIdx)
006677 ){
006678 if( ws & WHERE_MULTI_OR ){
006679 Index *pIx = pLevel->u.pCoveringIdx;
006680 int iDb = sqlite3SchemaToIndex(db, pIx->pSchema);
006681 sqlite3VdbeAddOp3(v, OP_ReopenIdx, pLevel->iIdxCur, pIx->tnum, iDb);
006682 sqlite3VdbeSetP4KeyInfo(pParse, pIx);
006683 }
006684 sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur);
006685 }
006686 if( pLevel->op==OP_Return ){
006687 sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst);
006688 }else{
006689 sqlite3VdbeGoto(v, pLevel->addrFirst);
006690 }
006691 sqlite3VdbeJumpHere(v, addr);
006692 }
006693 VdbeModuleComment((v, "End WHERE-loop%d: %s", i,
006694 pWInfo->pTabList->a[pLevel->iFrom].pTab->zName));
006695 }
006696
006697 assert( pWInfo->nLevel<=pTabList->nSrc );
006698 for(i=0, pLevel=pWInfo->a; i<pWInfo->nLevel; i++, pLevel++){
006699 int k, last;
006700 VdbeOp *pOp, *pLastOp;
006701 Index *pIdx = 0;
006702 SrcItem *pTabItem = &pTabList->a[pLevel->iFrom];
006703 Table *pTab = pTabItem->pTab;
006704 assert( pTab!=0 );
006705 pLoop = pLevel->pWLoop;
006706
006707 /* Do RIGHT JOIN processing. Generate code that will output the
006708 ** unmatched rows of the right operand of the RIGHT JOIN with
006709 ** all of the columns of the left operand set to NULL.
006710 */
006711 if( pLevel->pRJ ){
006712 sqlite3WhereRightJoinLoop(pWInfo, i, pLevel);
006713 continue;
006714 }
006715
006716 /* For a co-routine, change all OP_Column references to the table of
006717 ** the co-routine into OP_Copy of result contained in a register.
006718 ** OP_Rowid becomes OP_Null.
006719 */
006720 if( pTabItem->fg.viaCoroutine ){
006721 testcase( pParse->db->mallocFailed );
006722 translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur,
006723 pTabItem->regResult, 0);
006724 continue;
006725 }
006726
006727 /* If this scan uses an index, make VDBE code substitutions to read data
006728 ** from the index instead of from the table where possible. In some cases
006729 ** this optimization prevents the table from ever being read, which can
006730 ** yield a significant performance boost.
006731 **
006732 ** Calls to the code generator in between sqlite3WhereBegin and
006733 ** sqlite3WhereEnd will have created code that references the table
006734 ** directly. This loop scans all that code looking for opcodes
006735 ** that reference the table and converts them into opcodes that
006736 ** reference the index.
006737 */
006738 if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){
006739 pIdx = pLoop->u.btree.pIndex;
006740 }else if( pLoop->wsFlags & WHERE_MULTI_OR ){
006741 pIdx = pLevel->u.pCoveringIdx;
006742 }
006743 if( pIdx
006744 && !db->mallocFailed
006745 ){
006746 if( pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable) ){
006747 last = iEnd;
006748 }else{
006749 last = pWInfo->iEndWhere;
006750 }
006751 if( pIdx->bHasExpr ){
006752 IndexedExpr *p = pParse->pIdxEpr;
006753 while( p ){
006754 if( p->iIdxCur==pLevel->iIdxCur ){
006755 #ifdef WHERETRACE_ENABLED
006756 if( sqlite3WhereTrace & 0x200 ){
006757 sqlite3DebugPrintf("Disable pParse->pIdxEpr term {%d,%d}\n",
006758 p->iIdxCur, p->iIdxCol);
006759 if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(p->pExpr);
006760 }
006761 #endif
006762 p->iDataCur = -1;
006763 p->iIdxCur = -1;
006764 }
006765 p = p->pIENext;
006766 }
006767 }
006768 k = pLevel->addrBody + 1;
006769 #ifdef SQLITE_DEBUG
006770 if( db->flags & SQLITE_VdbeAddopTrace ){
006771 printf("TRANSLATE cursor %d->%d in opcode range %d..%d\n",
006772 pLevel->iTabCur, pLevel->iIdxCur, k, last-1);
006773 }
006774 /* Proof that the "+1" on the k value above is safe */
006775 pOp = sqlite3VdbeGetOp(v, k - 1);
006776 assert( pOp->opcode!=OP_Column || pOp->p1!=pLevel->iTabCur );
006777 assert( pOp->opcode!=OP_Rowid || pOp->p1!=pLevel->iTabCur );
006778 assert( pOp->opcode!=OP_IfNullRow || pOp->p1!=pLevel->iTabCur );
006779 #endif
006780 pOp = sqlite3VdbeGetOp(v, k);
006781 pLastOp = pOp + (last - k);
006782 assert( pOp<=pLastOp );
006783 do{
006784 if( pOp->p1!=pLevel->iTabCur ){
006785 /* no-op */
006786 }else if( pOp->opcode==OP_Column
006787 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
006788 || pOp->opcode==OP_Offset
006789 #endif
006790 ){
006791 int x = pOp->p2;
006792 assert( pIdx->pTable==pTab );
006793 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
006794 if( pOp->opcode==OP_Offset ){
006795 /* Do not need to translate the column number */
006796 }else
006797 #endif
006798 if( !HasRowid(pTab) ){
006799 Index *pPk = sqlite3PrimaryKeyIndex(pTab);
006800 x = pPk->aiColumn[x];
006801 assert( x>=0 );
006802 }else{
006803 testcase( x!=sqlite3StorageColumnToTable(pTab,x) );
006804 x = sqlite3StorageColumnToTable(pTab,x);
006805 }
006806 x = sqlite3TableColumnToIndex(pIdx, x);
006807 if( x>=0 ){
006808 pOp->p2 = x;
006809 pOp->p1 = pLevel->iIdxCur;
006810 OpcodeRewriteTrace(db, k, pOp);
006811 }else{
006812 /* Unable to translate the table reference into an index
006813 ** reference. Verify that this is harmless - that the
006814 ** table being referenced really is open.
006815 */
006816 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
006817 assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
006818 || cursorIsOpen(v,pOp->p1,k)
006819 || pOp->opcode==OP_Offset
006820 );
006821 #else
006822 assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
006823 || cursorIsOpen(v,pOp->p1,k)
006824 );
006825 #endif
006826 }
006827 }else if( pOp->opcode==OP_Rowid ){
006828 pOp->p1 = pLevel->iIdxCur;
006829 pOp->opcode = OP_IdxRowid;
006830 OpcodeRewriteTrace(db, k, pOp);
006831 }else if( pOp->opcode==OP_IfNullRow ){
006832 pOp->p1 = pLevel->iIdxCur;
006833 OpcodeRewriteTrace(db, k, pOp);
006834 }
006835 #ifdef SQLITE_DEBUG
006836 k++;
006837 #endif
006838 }while( (++pOp)<pLastOp );
006839 #ifdef SQLITE_DEBUG
006840 if( db->flags & SQLITE_VdbeAddopTrace ) printf("TRANSLATE complete\n");
006841 #endif
006842 }
006843 }
006844
006845 /* The "break" point is here, just past the end of the outer loop.
006846 ** Set it.
006847 */
006848 sqlite3VdbeResolveLabel(v, pWInfo->iBreak);
006849
006850 /* Final cleanup
006851 */
006852 pParse->nQueryLoop = pWInfo->savedNQueryLoop;
006853 whereInfoFree(db, pWInfo);
006854 pParse->withinRJSubrtn -= nRJ;
006855 return;
006856 }