SQLite

Changes On Branch parse-cleanup
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch parse-cleanup Excluding Merge-Ins

This is equivalent to a diff from 49dfce46 to 38ef8ab9

2021-01-12
16:26
Merge the ParseCleanup enhancement to trunk. (check-in: 35824c1b user: drh tags: trunk)
15:30
Fix a potential use-after-free following an OOM in sqlite3ParserAddCleanup() and add a mechanism to detect situations where this might occur in the future. (Closed-Leaf check-in: 38ef8ab9 user: drh tags: parse-cleanup)
14:19
Fix for the top-level configure script so that it works with tcl 8.7. (check-in: 4810f814 user: dan tags: trunk)
2021-01-11
20:37
Add a linked list of ParseCleanup objects to the end of a Parse object and use that list as a place to put other sub-objects that need to be deallocated. Have a single such list for infrequently used sub-objects is more efficient than doing an a separate check for each kind of sub-object. Edit: Now possible to have use-after-free following an OOM. (check-in: affa2b7b user: drh tags: parse-cleanup)
2021-01-09
19:10
New CLI command: ".stats vmstep" enables the display of the virtual-machine step count only, after each command. Useful for optimization problems. (check-in: 49dfce46 user: drh tags: trunk)
18:24
More detailed compile-time testing before attempting to use atomic load intrinsics. See forum post fc0237a39b30ac0a. (check-in: 5204c2c4 user: drh tags: trunk)

Changes to src/insert.c.

366
367
368
369
370
371
372


373

374
375
376
377
378
379
380
366
367
368
369
370
371
372
373
374

375
376
377
378
379
380
381
382







+
+
-
+







      return 0;
    }

    pInfo = pToplevel->pAinc;
    while( pInfo && pInfo->pTab!=pTab ){ pInfo = pInfo->pNext; }
    if( pInfo==0 ){
      pInfo = sqlite3DbMallocRawNN(pParse->db, sizeof(*pInfo));
      sqlite3ParserAddCleanup(pToplevel, sqlite3DbFree, pInfo);
      testcase( pParse->earlyCleanup );
      if( pInfo==0 ) return 0;
      if( pParse->db->mallocFailed ) return 0;
      pInfo->pNext = pToplevel->pAinc;
      pToplevel->pAinc = pInfo;
      pInfo->pTab = pTab;
      pInfo->iDb = iDb;
      pToplevel->nMem++;                  /* Register to hold name of table */
      pInfo->regCtr = ++pToplevel->nMem;  /* Max rowid register */
      pToplevel->nMem +=2;       /* Rowid in sqlite_sequence + orig max val */

Changes to src/prepare.c.

566
567
568
569
570
571
572






573

574


575
576
577
578
579
580
581







































582
583
584
585
586
587
588
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580

581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635







+
+
+
+
+
+

+
-
+
+







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







  sqlite3 *db = pParse->db;
  AggInfo *pThis = pParse->pAggList;
  while( pThis ){
    AggInfo *pNext = pThis->pNext;
    agginfoFree(db, pThis);
    pThis = pNext;
  }
  while( pParse->pCleanup ){
    ParseCleanup *pThis = pParse->pCleanup;
    pParse->pCleanup = pThis->pNext;
    pThis->xCleanup(db, pThis->pPtr);
    sqlite3DbFree(db, pThis);
  }
  sqlite3DbFree(db, pParse->aLabel);
  if( pParse->pConstExpr ){
  sqlite3ExprListDelete(db, pParse->pConstExpr);
    sqlite3ExprListDelete(db, pParse->pConstExpr);
  }
  if( db ){
    assert( db->lookaside.bDisable >= pParse->disableLookaside );
    db->lookaside.bDisable -= pParse->disableLookaside;
    db->lookaside.sz = db->lookaside.bDisable ? 0 : db->lookaside.szTrue;
  }
  pParse->disableLookaside = 0;
}

/*
** Add a new cleanup operation to a Parser.  The cleanup should happen when
** the parser object is destroyed.  But, beware: the cleanup might happen
** immediately.
**
** Use this mechanism for uncommon cleanups.  There is a higher setup
** cost for this mechansim (an extra malloc), so it should not be used
** for common cleanups that happen on most calls.  But for less
** common cleanups, we save a single NULL-pointer comparison in
** sqlite3ParserReset(), which reduces the total CPU cycle count.
**
** If a memory allocation error occurs, then the cleanup happens immediately.
** When eithr SQLITE_DEBUG or SQLITE_COVERAGE_TEST are defined, the
** pParse->earlyCleanup flag is set in that case.  Calling code show verify
** that test cases exist for which this happens, to guard against possible
** use-after-free errors following an OOM.  The preferred way to do this is
** to immediately follow the call to this routine with:
**
**       testcase( pParse->earlyCleanup );
*/
void sqlite3ParserAddCleanup(
  Parse *pParse,                      /* Destroy when this Parser finishes */
  void (*xCleanup)(sqlite3*,void*),   /* The cleanup routine */
  void *pPtr                          /* Pointer to object to be cleaned up */
){
  ParseCleanup *pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup));
  if( pCleanup ){
    pCleanup->pNext = pParse->pCleanup;
    pParse->pCleanup = pCleanup;
    pCleanup->pPtr = pPtr;
    pCleanup->xCleanup = xCleanup;
  }else{
    xCleanup(pParse->db, pPtr);
#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
    pParse->earlyCleanup = 1;
#endif
  }
}

/*
** Compile the UTF-8 encoded SQL statement zSql into a statement handle.
*/
static int sqlite3Prepare(
  sqlite3 *db,              /* Database handle. */
  const char *zSql,         /* UTF-8 encoded SQL statement. */

Changes to src/select.c.

4136
4137
4138
4139
4140
4141
4142


4143
4144


4145
4146
4147
4148
4149
4150
4151
4136
4137
4138
4139
4140
4141
4142
4143
4144


4145
4146
4147
4148
4149
4150
4151
4152
4153







+
+
-
-
+
+







  **
  ** pSubitem->pTab is always non-NULL by test restrictions and tests above.
  */
  if( ALWAYS(pSubitem->pTab!=0) ){
    Table *pTabToDel = pSubitem->pTab;
    if( pTabToDel->nTabRef==1 ){
      Parse *pToplevel = sqlite3ParseToplevel(pParse);
      sqlite3ParserAddCleanup(pToplevel, 
         (void(*)(sqlite3*,void*))sqlite3DeleteTable,
      pTabToDel->pNextZombie = pToplevel->pZombieTab;
      pToplevel->pZombieTab = pTabToDel;
         pTabToDel);
      testcase( pToplevel->earlyCleanup );
    }else{
      pTabToDel->nTabRef--;
    }
    pSubitem->pTab = 0;
  }

  /* The following loop runs once for each term in a compound-subquery
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864






4865
4866
4867
4868
4869
4870
4871
4854
4855
4856
4857
4858
4859
4860

4861
4862
4863
4864

4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877







-




-
+
+
+
+
+
+







** onto the top of the stack. If argument bFree is true, then this
** WITH clause will never be popped from the stack. In this case it
** should be freed along with the Parse object. In other cases, when
** bFree==0, the With object will be freed along with the SELECT 
** statement with which it is associated.
*/
void sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){
  assert( bFree==0 || (pParse->pWith==0 && pParse->pWithToFree==0) );
  if( pWith ){
    assert( pParse->pWith!=pWith );
    pWith->pOuter = pParse->pWith;
    pParse->pWith = pWith;
    if( bFree ) pParse->pWithToFree = pWith;
    if( bFree ){
      sqlite3ParserAddCleanup(pParse, 
         (void(*)(sqlite3*,void*))sqlite3WithDelete,
         pWith);
      testcase( pParse->earlyCleanup );
    }
  }
}

/*
** This function checks if argument pFrom refers to a CTE declared by 
** a WITH clause on the stack currently maintained by the parser. And,
** if currently processing a CTE expression, if it is a recursive

Changes to src/sqliteInt.h.

1150
1151
1152
1153
1154
1155
1156

1157
1158
1159
1160
1161
1162
1163
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164







+







typedef struct KeyClass KeyClass;
typedef struct KeyInfo KeyInfo;
typedef struct Lookaside Lookaside;
typedef struct LookasideSlot LookasideSlot;
typedef struct Module Module;
typedef struct NameContext NameContext;
typedef struct Parse Parse;
typedef struct ParseCleanup ParseCleanup;
typedef struct PreUpdate PreUpdate;
typedef struct PrintfArguments PrintfArguments;
typedef struct RenameToken RenameToken;
typedef struct RowSet RowSet;
typedef struct Savepoint Savepoint;
typedef struct Select Select;
typedef struct SQLiteThread SQLiteThread;
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2183
2184
2185
2186
2187
2188
2189

2190
2191
2192
2193
2194
2195
2196







-







#ifndef SQLITE_OMIT_VIRTUALTABLE
  int nModuleArg;      /* Number of arguments to the module */
  char **azModuleArg;  /* 0: module 1: schema 2: vtab name 3...: args */
  VTable *pVTable;     /* List of VTable objects. */
#endif
  Trigger *pTrigger;   /* List of triggers stored in pSchema */
  Schema *pSchema;     /* Schema that contains this table */
  Table *pNextZombie;  /* Next on the Parse.pZombieTab list */
};

/*
** Allowed values for Table.tabFlags.
**
** TF_OOOHidden applies to tables or view that have hidden columns that are
** followed by non-hidden columns.  Example:  "CREATE VIRTUAL TABLE x USING
3344
3345
3346
3347
3348
3349
3350











3351
3352
3353
3354
3355
3356
3357
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368







+
+
+
+
+
+
+
+
+
+
+







# define DbMaskTest(M,I)    (((M)&(((yDbMask)1)<<(I)))!=0)
# define DbMaskZero(M)      (M)=0
# define DbMaskSet(M,I)     (M)|=(((yDbMask)1)<<(I))
# define DbMaskAllZero(M)   (M)==0
# define DbMaskNonZero(M)   (M)!=0
#endif

/*
** An instance of the ParseCleanup object specifies an operation that
** should be performed after parsing to deallocation resources obtained
** during the parse and which are no longer needed.
*/
struct ParseCleanup {
  ParseCleanup *pNext;               /* Next cleanup task */
  void *pPtr;                        /* Pointer to object to deallocate */
  void (*xCleanup)(sqlite3*,void*);  /* Deallocation routine */
};

/*
** An SQL parser context.  A copy of this structure is passed through
** the parser and down into all the parser action routine in order to
** carry around information that is global to the entire parse.
**
** The structure is divided into two parts.  When the parser and code
** generate call themselves recursively, the first part of the structure
3375
3376
3377
3378
3379
3380
3381



3382
3383
3384
3385
3386
3387
3388
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402







+
+
+







  u8 nTempReg;         /* Number of temporary registers in aTempReg[] */
  u8 isMultiWrite;     /* True if statement may modify/insert multiple rows */
  u8 mayAbort;         /* True if statement may throw an ABORT exception */
  u8 hasCompound;      /* Need to invoke convertCompoundSelectToSubquery() */
  u8 okConstFactor;    /* OK to factor out constants */
  u8 disableLookaside; /* Number of times lookaside has been disabled */
  u8 disableVtab;      /* Disable all virtual tables for this parse */
#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
  u8 earlyCleanup;     /* OOM inside sqlite3ParserAddCleanup() */
#endif
  int nRangeReg;       /* Size of the temporary register block */
  int iRangeReg;       /* First register in temporary register block */
  int nErr;            /* Number of errors seen */
  int nTab;            /* Number of previously allocated VDBE cursors */
  int nMem;            /* Number of memory cells used so far */
  int szOpAlloc;       /* Bytes of memory space allocated for Vdbe.aOp[] */
  int iSelfTab;        /* Table associated with an index on expr, or negative
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463

3464
3465
3466
3467
3468
3469
3470
3467
3468
3469
3470
3471
3472
3473

3474
3475

3476
3477
3478
3479
3480
3481
3482
3483







-


-
+







                            ** during a RENAME COLUMN */
  Trigger *pNewTrigger;     /* Trigger under construct by a CREATE TRIGGER */
  const char *zAuthContext; /* The 6th parameter to db->xAuth callbacks */
#ifndef SQLITE_OMIT_VIRTUALTABLE
  Token sArg;               /* Complete text of a module argument */
  Table **apVtabLock;       /* Pointer to virtual tables needing locking */
#endif
  Table *pZombieTab;        /* List of Table objects to delete after code gen */
  TriggerPrg *pTriggerPrg;  /* Linked list of coded triggers */
  With *pWith;              /* Current WITH clause, or NULL */
  With *pWithToFree;        /* Free this WITH object at the end of the parse */
  ParseCleanup *pCleanup;   /* List of cleanup operations to run after parse */
#ifndef SQLITE_OMIT_ALTERTABLE
  RenameToken *pRename;     /* Tokens subject to renaming by ALTER TABLE */
#endif
};

#define PARSE_MODE_NORMAL        0
#define PARSE_MODE_DECLARE_VTAB  1
4823
4824
4825
4826
4827
4828
4829

4830
4831
4832
4833
4834
4835
4836
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850







+







int sqlite3VtabCallDestroy(sqlite3*, int, const char *);
int sqlite3VtabBegin(sqlite3 *, VTable *);
FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*);
sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*);
int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
void sqlite3ParserReset(Parse*);
void sqlite3ParserAddCleanup(Parse*,void(*)(sqlite3*,void*),void*);
#ifdef SQLITE_ENABLE_NORMALIZE
char *sqlite3Normalize(Vdbe*, const char*);
#endif
int sqlite3Reprepare(Vdbe*);
void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*);
CollSeq *sqlite3ExprCompareCollSeq(Parse*,const Expr*);
CollSeq *sqlite3BinaryCompareCollSeq(Parse *, const Expr*, const Expr*);

Changes to src/tokenize.c.

709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
709
710
711
712
713
714
715


716










717
718
719
720
721
722
723







-
-

-
-
-
-
-
-
-
-
-
-







    ** will take responsibility for freeing the Table structure.
    */
    sqlite3DeleteTable(db, pParse->pNewTable);
  }
  if( !IN_RENAME_OBJECT ){
    sqlite3DeleteTrigger(db, pParse->pNewTrigger);
  }

  if( pParse->pWithToFree ) sqlite3WithDelete(db, pParse->pWithToFree);
  sqlite3DbFree(db, pParse->pVList);
  while( pParse->pAinc ){
    AutoincInfo *p = pParse->pAinc;
    pParse->pAinc = p->pNext;
    sqlite3DbFreeNN(db, p);
  }
  while( pParse->pZombieTab ){
    Table *p = pParse->pZombieTab;
    pParse->pZombieTab = p->pNextZombie;
    sqlite3DeleteTable(db, p);
  }
  db->pParse = pParse->pParentParse;
  pParse->pParentParse = 0;
  assert( nErr==0 || pParse->rc!=SQLITE_OK );
  return nErr;
}


Changes to src/trigger.c.

996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
996
997
998
999
1000
1001
1002

1003
1004
1005
1006
1007
1008
1009







-







    pProgram->nCsr = pSubParse->nTab;
    pProgram->token = (void *)pTrigger;
    pPrg->aColmask[0] = pSubParse->oldmask;
    pPrg->aColmask[1] = pSubParse->newmask;
    sqlite3VdbeDelete(v);
  }

  assert( !pSubParse->pAinc       && !pSubParse->pZombieTab );
  assert( !pSubParse->pTriggerPrg && !pSubParse->nMaxArg );
  sqlite3ParserReset(pSubParse);
  sqlite3StackFree(db, pSubParse);

  return pPrg;
}