SQLite

Changes On Branch analyze-with-limit
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch analyze-with-limit Excluding Merge-Ins

This is equivalent to a diff from c20d4fde to 7e666b60

2020-04-09
15:01
Merge trunk enhancements into the approximate-analyze branch. (check-in: 17901ea6 user: drh tags: approximate-analyze)
2020-04-01
17:10
Add support for the "ANALYZE WITH LIMIT=N" syntax. (Leaf check-in: 7e666b60 user: drh tags: analyze-with-limit)
01:15
Simple fixes to PRAGMA analysis_limit. (check-in: c20d4fde user: drh tags: approximate-analyze)
2020-03-31
20:57
Add "PRAGMA analysis_limit=N;" to limit the number of rows visited by ANALYZE when N is positive. Positive N also disables collecting stat4. (check-in: a279b151 user: drh tags: approximate-analyze)

Changes to src/analyze.c.

181
182
183
184
185
186
187

188
189
190
191
192
193
194
195
196
197
198
199
200



201
202
203
204
205
206
207
    { "sqlite_stat4", 0 },
#endif
    { "sqlite_stat3", 0 },
  };
  int i;
  sqlite3 *db = pParse->db;
  Db *pDb;

  Vdbe *v = sqlite3GetVdbe(pParse);
  int aRoot[ArraySize(aTable)];
  u8 aCreateTbl[ArraySize(aTable)];
#ifdef SQLITE_ENABLE_STAT4
  const int nToOpen = OptimizationEnabled(db,SQLITE_Stat4) ? 2 : 1;
#else
  const int nToOpen = 1;
#endif

  if( v==0 ) return;
  assert( sqlite3BtreeHoldsAllMutexes(db) );
  assert( sqlite3VdbeDb(v)==db );
  pDb = &db->aDb[iDb];




  /* Create new statistic tables if they do not exist, or clear them
  ** if they do already exist.
  */
  for(i=0; i<ArraySize(aTable); i++){
    const char *zTab = aTable[i].zName;
    Table *pStat;







>













>
>
>







181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
    { "sqlite_stat4", 0 },
#endif
    { "sqlite_stat3", 0 },
  };
  int i;
  sqlite3 *db = pParse->db;
  Db *pDb;
  int nRowLimit;
  Vdbe *v = sqlite3GetVdbe(pParse);
  int aRoot[ArraySize(aTable)];
  u8 aCreateTbl[ArraySize(aTable)];
#ifdef SQLITE_ENABLE_STAT4
  const int nToOpen = OptimizationEnabled(db,SQLITE_Stat4) ? 2 : 1;
#else
  const int nToOpen = 1;
#endif

  if( v==0 ) return;
  assert( sqlite3BtreeHoldsAllMutexes(db) );
  assert( sqlite3VdbeDb(v)==db );
  pDb = &db->aDb[iDb];
  assert( pParse->eAux1==2 );
  nRowLimit = pParse->aux1.nRowLimit;
  pParse->eAux1 = 0;

  /* Create new statistic tables if they do not exist, or clear them
  ** if they do already exist.
  */
  for(i=0; i<ArraySize(aTable); i++){
    const char *zTab = aTable[i].zName;
    Table *pStat;
235
236
237
238
239
240
241


242
243
244
245
246
247
248
#endif
      }else{
        /* The sqlite_stat[134] table already exists.  Delete all rows. */
        sqlite3VdbeAddOp2(v, OP_Clear, aRoot[i], iDb);
      }
    }
  }



  /* Open the sqlite_stat[134] tables for writing. */
  for(i=0; i<nToOpen; i++){
    assert( i<ArraySize(aTable) );
    sqlite3VdbeAddOp4Int(v, OP_OpenWrite, iStatCur+i, aRoot[i], iDb, 3);
    sqlite3VdbeChangeP5(v, aCreateTbl[i]);
    VdbeComment((v, aTable[i].zName));







>
>







239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
#endif
      }else{
        /* The sqlite_stat[134] table already exists.  Delete all rows. */
        sqlite3VdbeAddOp2(v, OP_Clear, aRoot[i], iDb);
      }
    }
  }
  pParse->eAux1 = 2;
  pParse->aux1.nRowLimit = nRowLimit;

  /* Open the sqlite_stat[134] tables for writing. */
  for(i=0; i<nToOpen; i++){
    assert( i<ArraySize(aTable) );
    sqlite3VdbeAddOp4Int(v, OP_OpenWrite, iStatCur+i, aRoot[i], iDb, 3);
    sqlite3VdbeChangeP5(v, aCreateTbl[i]);
    VdbeComment((v, aTable[i].zName));
1119
1120
1121
1122
1123
1124
1125

1126
1127
1128
1129
1130
1131
1132
1133
#endif
    {
      addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur);
      VdbeCoverage(v);
      sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, 1);
    }
    assert( regTemp2==regStat+4 );

    sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2);
    sqlite3VdbeAddFunctionCall(pParse, 0, regStat+1, regStat, 4,
                               &statInitFuncdef, 0);

    /* Implementation of the following:
    **
    **   Rewind csr
    **   if eof(csr) goto end_of_scan;







>
|







1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
#endif
    {
      addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur);
      VdbeCoverage(v);
      sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, 1);
    }
    assert( regTemp2==regStat+4 );
    assert( pParse->eAux1==2 );
    sqlite3VdbeAddOp2(v, OP_Integer, pParse->aux1.nRowLimit, regTemp2);
    sqlite3VdbeAddFunctionCall(pParse, 0, regStat+1, regStat, 4,
                               &statInitFuncdef, 0);

    /* Implementation of the following:
    **
    **   Rewind csr
    **   if eof(csr) goto end_of_scan;
1221
1222
1223
1224
1225
1226
1227

1228
1229
1230
1231
1232
1233
1234
1235
      }
    }
#endif
    assert( regChng==(regStat+1) );
    {
      sqlite3VdbeAddFunctionCall(pParse, 1, regStat, regTemp, 2+IsStat4,
                                 &statPushFuncdef, 0);

      if( db->nAnalysisLimit ){
        int j1, j2, j3;
        j1 = sqlite3VdbeAddOp1(v, OP_IsNull, regTemp); VdbeCoverage(v);
        j2 = sqlite3VdbeAddOp1(v, OP_If, regTemp); VdbeCoverage(v);
        j3 = sqlite3VdbeAddOp4Int(v, OP_SeekGT, iIdxCur, 0, regPrev, 1);
        sqlite3VdbeJumpHere(v, j1);
        sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v);
        sqlite3VdbeJumpHere(v, j2);







>
|







1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
      }
    }
#endif
    assert( regChng==(regStat+1) );
    {
      sqlite3VdbeAddFunctionCall(pParse, 1, regStat, regTemp, 2+IsStat4,
                                 &statPushFuncdef, 0);
      assert( pParse->eAux1==2 );
      if( pParse->aux1.nRowLimit ){
        int j1, j2, j3;
        j1 = sqlite3VdbeAddOp1(v, OP_IsNull, regTemp); VdbeCoverage(v);
        j2 = sqlite3VdbeAddOp1(v, OP_If, regTemp); VdbeCoverage(v);
        j3 = sqlite3VdbeAddOp4Int(v, OP_SeekGT, iIdxCur, 0, regPrev, 1);
        sqlite3VdbeJumpHere(v, j1);
        sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v);
        sqlite3VdbeJumpHere(v, j2);
1248
1249
1250
1251
1252
1253
1254

1255
1256
1257
1258
1259
1260
1261
1262
#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
    sqlite3VdbeChangeP4(v, -1, (char*)pStat1, P4_TABLE);
#endif
    sqlite3VdbeChangeP5(v, OPFLAG_APPEND);

    /* Add the entries to the stat4 table. */
#ifdef SQLITE_ENABLE_STAT4

    if( OptimizationEnabled(db, SQLITE_Stat4) && db->nAnalysisLimit==0 ){
      int regEq = regStat1;
      int regLt = regStat1+1;
      int regDLt = regStat1+2;
      int regSample = regStat1+3;
      int regCol = regStat1+4;
      int regSampleRowid = regCol + nCol;
      int addrNext;







>
|







1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
    sqlite3VdbeChangeP4(v, -1, (char*)pStat1, P4_TABLE);
#endif
    sqlite3VdbeChangeP5(v, OPFLAG_APPEND);

    /* Add the entries to the stat4 table. */
#ifdef SQLITE_ENABLE_STAT4
    assert( pParse->eAux1==2 );
    if( OptimizationEnabled(db, SQLITE_Stat4) && pParse->aux1.nRowLimit==0 ){
      int regEq = regStat1;
      int regLt = regStat1+1;
      int regDLt = regStat1+2;
      int regSample = regStat1+3;
      int regCol = regStat1+4;
      int regSampleRowid = regCol + nCol;
      int addrNext;
1395
1396
1397
1398
1399
1400
1401





1402
1403
1404
1405
1406
1407
1408
  Vdbe *v;

  /* Read the database schema. If an error occurs, leave an error message
  ** and code in pParse and return NULL. */
  assert( sqlite3BtreeHoldsAllMutexes(pParse->db) );
  if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){
    return;





  }

  assert( pName2!=0 || pName1==0 );
  if( pName1==0 ){
    /* Form 1:  Analyze everything */
    for(i=0; i<db->nDb; i++){
      if( i==1 ) continue;  /* Do not analyze the TEMP database */







>
>
>
>
>







1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
  Vdbe *v;

  /* Read the database schema. If an error occurs, leave an error message
  ** and code in pParse and return NULL. */
  assert( sqlite3BtreeHoldsAllMutexes(pParse->db) );
  if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){
    return;
  }
  assert( pParse->eAux1==0 || pParse->eAux1==2 );
  if( pParse->eAux1==0 ){
    pParse->aux1.nRowLimit = db->nAnalysisLimit;
    pParse->eAux1 = 2;
  }

  assert( pName2!=0 || pName1==0 );
  if( pName1==0 ){
    /* Form 1:  Analyze everything */
    for(i=0; i<db->nDb; i++){
      if( i==1 ) continue;  /* Do not analyze the TEMP database */

Changes to src/build.c.

1155
1156
1157
1158
1159
1160
1161

1162
1163

1164
1165
1166
1167
1168
1169
1170
    */
#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE)
    if( isView || isVirtual ){
      sqlite3VdbeAddOp2(v, OP_Integer, 0, reg2);
    }else
#endif
    {

      pParse->addrCrTab =
         sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, reg2, BTREE_INTKEY);

    }
    sqlite3OpenMasterTable(pParse, iDb);
    sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1);
    sqlite3VdbeAddOp4(v, OP_Blob, 6, reg3, 0, nullRow, P4_STATIC);
    sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1);
    sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
    sqlite3VdbeAddOp0(v, OP_Close);







>
|

>







1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
    */
#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE)
    if( isView || isVirtual ){
      sqlite3VdbeAddOp2(v, OP_Integer, 0, reg2);
    }else
#endif
    {
      assert( pParse->eAux1==0 || pParse->eAux1==1 );
      pParse->aux1.addrCrTab =
         sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, reg2, BTREE_INTKEY);
      pParse->eAux1 = 1;
    }
    sqlite3OpenMasterTable(pParse, iDb);
    sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1);
    sqlite3VdbeAddOp4(v, OP_Blob, 6, reg3, 0, nullRow, P4_STATIC);
    sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1);
    sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
    sqlite3VdbeAddOp0(v, OP_Close);
2004
2005
2006
2007
2008
2009
2010

2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
    }
    pTab->tabFlags |= TF_HasNotNull;
  }

  /* Convert the P3 operand of the OP_CreateBtree opcode from BTREE_INTKEY
  ** into BTREE_BLOBKEY.
  */

  if( pParse->addrCrTab ){
    assert( v );
    sqlite3VdbeChangeP3(v, pParse->addrCrTab, BTREE_BLOBKEY);
  }

  /* Locate the PRIMARY KEY index.  Or, if this table was originally
  ** an INTEGER PRIMARY KEY table, create a new PRIMARY KEY index. 
  */
  if( pTab->iPKey>=0 ){
    ExprList *pList;







>
|

|







2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
    }
    pTab->tabFlags |= TF_HasNotNull;
  }

  /* Convert the P3 operand of the OP_CreateBtree opcode from BTREE_INTKEY
  ** into BTREE_BLOBKEY.
  */
  assert( pParse->eAux1==0 || pParse->eAux1==1 );
  if( pParse->eAux1 ){
    assert( v );
    sqlite3VdbeChangeP3(v, pParse->aux1.addrCrTab, BTREE_BLOBKEY);
  }

  /* Locate the PRIMARY KEY index.  Or, if this table was originally
  ** an INTEGER PRIMARY KEY table, create a new PRIMARY KEY index. 
  */
  if( pTab->iPKey>=0 ){
    ExprList *pList;

Changes to src/parse.y.

1552
1553
1554
1555
1556
1557
1558
1559
1560












1561
1562
1563
1564
1565
1566
1567
%ifndef SQLITE_OMIT_REINDEX
cmd ::= REINDEX.                {sqlite3Reindex(pParse, 0, 0);}
cmd ::= REINDEX nm(X) dbnm(Y).  {sqlite3Reindex(pParse, &X, &Y);}
%endif  SQLITE_OMIT_REINDEX

/////////////////////////////////// ANALYZE ///////////////////////////////////
%ifndef SQLITE_OMIT_ANALYZE
cmd ::= ANALYZE.                {sqlite3Analyze(pParse, 0, 0);}
cmd ::= ANALYZE nm(X) dbnm(Y).  {sqlite3Analyze(pParse, &X, &Y);}












%endif

//////////////////////// ALTER TABLE table ... ////////////////////////////////
%ifndef SQLITE_OMIT_ALTERTABLE
cmd ::= ALTER TABLE fullname(X) RENAME TO nm(Z). {
  sqlite3AlterRenameTable(pParse,X,&Z);
}







|
|
>
>
>
>
>
>
>
>
>
>
>
>







1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
%ifndef SQLITE_OMIT_REINDEX
cmd ::= REINDEX.                {sqlite3Reindex(pParse, 0, 0);}
cmd ::= REINDEX nm(X) dbnm(Y).  {sqlite3Reindex(pParse, &X, &Y);}
%endif  SQLITE_OMIT_REINDEX

/////////////////////////////////// ANALYZE ///////////////////////////////////
%ifndef SQLITE_OMIT_ANALYZE
cmd ::= ANALYZE anlyzcfg.                {sqlite3Analyze(pParse, 0, 0);}
cmd ::= ANALYZE nm(X) dbnm(Y) anlyzcfg.  {sqlite3Analyze(pParse, &X, &Y);}
anlyzcfg ::= .
anlyzcfg ::= WITH LIMIT EQ INTEGER(X). {
  char *z = sqlite3NameFromToken(pParse->db, &X);
  if( z ){
    i64 x;
    sqlite3DecOrHexToI64(z, &x);
    pParse->eAux1 = 2;
    pParse->aux1.nRowLimit = x>0x7ffffff0 ? 0 : (int)x;
    sqlite3DbFree(pParse->db, z);
  }
}
   
%endif

//////////////////////// ALTER TABLE table ... ////////////////////////////////
%ifndef SQLITE_OMIT_ALTERTABLE
cmd ::= ALTER TABLE fullname(X) RENAME TO nm(Z). {
  sqlite3AlterRenameTable(pParse,X,&Z);
}

Changes to src/sqliteInt.h.

3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326





3327
3328
3329
3330
3331
3332
3333
  int nTableLock;        /* Number of locks in aTableLock */
  TableLock *aTableLock; /* Required table locks for shared-cache mode */
#endif
  AutoincInfo *pAinc;  /* Information about AUTOINCREMENT counters */
  Parse *pToplevel;    /* Parse structure for main program (or NULL) */
  Table *pTriggerTab;  /* Table triggers are being coded for */
  Parse *pParentParse; /* Parent parser if this parser is nested */
  int addrCrTab;       /* Address of OP_CreateBtree opcode on CREATE TABLE */
  u32 nQueryLoop;      /* Est number of iterations of a query (10*log2(N)) */
  u32 oldmask;         /* Mask of old.* columns referenced */
  u32 newmask;         /* Mask of new.* columns referenced */





  u8 eTriggerOp;       /* TK_UPDATE, TK_INSERT or TK_DELETE */
  u8 eOrconf;          /* Default ON CONFLICT policy for trigger steps */
  u8 disableTriggers;  /* True to disable triggers */

  /**************************************************************************
  ** Fields above must be initialized to zero.  The fields that follow,
  ** down to the beginning of the recursive section, do not need to be







<



>
>
>
>
>







3316
3317
3318
3319
3320
3321
3322

3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
  int nTableLock;        /* Number of locks in aTableLock */
  TableLock *aTableLock; /* Required table locks for shared-cache mode */
#endif
  AutoincInfo *pAinc;  /* Information about AUTOINCREMENT counters */
  Parse *pToplevel;    /* Parse structure for main program (or NULL) */
  Table *pTriggerTab;  /* Table triggers are being coded for */
  Parse *pParentParse; /* Parent parser if this parser is nested */

  u32 nQueryLoop;      /* Est number of iterations of a query (10*log2(N)) */
  u32 oldmask;         /* Mask of old.* columns referenced */
  u32 newmask;         /* Mask of new.* columns referenced */
  union {
    int addrCrTab;       /* (1) Addr of OP_CreateBtree opcode on CREATE TABLE */
    int nRowLimit;       /* (2) LIMIT= value for ANALYZE */
  } aux1;
  u8 eAux1;            /* Which field of the aux1 union is in use */
  u8 eTriggerOp;       /* TK_UPDATE, TK_INSERT or TK_DELETE */
  u8 eOrconf;          /* Default ON CONFLICT policy for trigger steps */
  u8 disableTriggers;  /* True to disable triggers */

  /**************************************************************************
  ** Fields above must be initialized to zero.  The fields that follow,
  ** down to the beginning of the recursive section, do not need to be