/ Check-in [48ca30f9]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem causing a corrupt pager-cache if an OOM or IO error was encountered while committing a concurrent transacation.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | begin-concurrent
Files: files | file ages | folders
SHA3-256: 48ca30f9d7817d87a5e9a069fdc51b1a34e00585f8a35771895dd743c7bfe07c
User & Date: dan 2018-12-29 20:42:12
Wiki:begin-concurrent
Context
2019-01-02
16:08
Merge latest trunk changes into this branch. check-in: 5bf212f1 user: dan tags: begin-concurrent
2018-12-29
20:47
Merge latest begin-concurrent changes with this branch. check-in: 1625887c user: dan tags: begin-concurrent-wal2
20:42
Fix a problem causing a corrupt pager-cache if an OOM or IO error was encountered while committing a concurrent transacation. check-in: 48ca30f9 user: dan tags: begin-concurrent
2018-12-18
17:20
Merge latest trunk changes into this branch. check-in: 123cbb33 user: dan tags: begin-concurrent
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/pager.c.

3209
3210
3211
3212
3213
3214
3215
3216






3217
3218
3219
3220
3221
3222
3223
  ** been written (but not committed) to the log file, do one of the 
  ** following:
  **
  **   + Discard the cached page (if refcount==0), or
  **   + Reload page content from the database (if refcount>0).
  */
  pPager->dbSize = pPager->dbOrigSize;
  rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager);






  pList = sqlite3PcacheDirtyList(pPager->pPCache);

#ifndef SQLITE_OMIT_CONCURRENT
  /* If this is an CONCURRENT transaction, then page 1 must be reread from 
  ** the db file, even if it is not dirty. This is because the b-tree layer 
  ** may have already zeroed the nFree and iTrunk header fields.  */
  if( rc==SQLITE_OK && (pList==0 || pList->pgno!=1) && pPager->pAllRead ){







|
>
>
>
>
>
>







3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
  ** been written (but not committed) to the log file, do one of the 
  ** following:
  **
  **   + Discard the cached page (if refcount==0), or
  **   + Reload page content from the database (if refcount>0).
  */
  pPager->dbSize = pPager->dbOrigSize;
  rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager, 
#ifdef SQLITE_OMIT_CONCURRENT
      0
#else
      pPager->pAllRead!=0
#endif
  );
  pList = sqlite3PcacheDirtyList(pPager->pPCache);

#ifndef SQLITE_OMIT_CONCURRENT
  /* If this is an CONCURRENT transaction, then page 1 must be reread from 
  ** the db file, even if it is not dirty. This is because the b-tree layer 
  ** may have already zeroed the nFree and iTrunk header fields.  */
  if( rc==SQLITE_OK && (pList==0 || pList->pgno!=1) && pPager->pAllRead ){

Changes to src/wal.c.

3252
3253
3254
3255
3256
3257
3258


3259



3260
3261
3262
3263
3264
3265
3266
3267
3268







3269
3270
3271
3272
3273
3274
3275
** to the WAL since the start of the transaction. If the callback returns
** other than SQLITE_OK, it is not invoked again and the error code is
** returned to the caller.
**
** Otherwise, if the callback function does not return an error, this
** function returns SQLITE_OK.
*/


int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){



  int rc = SQLITE_OK;
  if( pWal->writeLock ){
    Pgno iMax = pWal->hdr.mxFrame;
    Pgno iFrame;
  
    /* Restore the clients cache of the wal-index header to the state it
    ** was in before the client began writing to the database. 
    */
    memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr));








    for(iFrame=pWal->hdr.mxFrame+1; 
        ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; 
        iFrame++
    ){
      /* This call cannot fail. Unless the page for which the page number
      ** is passed as the second argument is (a) in the cache and 







>
>
|
>
>
>









>
>
>
>
>
>
>







3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
** to the WAL since the start of the transaction. If the callback returns
** other than SQLITE_OK, it is not invoked again and the error code is
** returned to the caller.
**
** Otherwise, if the callback function does not return an error, this
** function returns SQLITE_OK.
*/
int sqlite3WalUndo(
  Wal *pWal, 
  int (*xUndo)(void *, Pgno), 
  void *pUndoCtx,
  int bConcurrent                 /* True if this is a CONCURRENT transaction */
){
  int rc = SQLITE_OK;
  if( pWal->writeLock ){
    Pgno iMax = pWal->hdr.mxFrame;
    Pgno iFrame;
  
    /* Restore the clients cache of the wal-index header to the state it
    ** was in before the client began writing to the database. 
    */
    memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr));
#ifndef SQLITE_OMIT_CONCURRENT
    if( bConcurrent ){
      pWal->hdr.aCksum[0]++;
    }
#else
    UNUSED_PARAMETER(bConcurrent);
#endif

    for(iFrame=pWal->hdr.mxFrame+1; 
        ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; 
        iFrame++
    ){
      /* This call cannot fail. Unless the page for which the page number
      ** is passed as the second argument is (a) in the cache and 

Changes to src/wal.h.

30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
..
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
# define sqlite3WalLimit(x,y)
# define sqlite3WalClose(v,w,x,y,z)              0
# define sqlite3WalBeginReadTransaction(y,z)     0
# define sqlite3WalEndReadTransaction(z)
# define sqlite3WalDbsize(y)                     0
# define sqlite3WalBeginWriteTransaction(y)      0
# define sqlite3WalEndWriteTransaction(x)        0
# define sqlite3WalUndo(x,y,z)                   0
# define sqlite3WalSavepoint(y,z)
# define sqlite3WalSavepointUndo(y,z)            0
# define sqlite3WalFrames(u,v,w,x,y,z)           0
# define sqlite3WalCheckpoint(q,r,s,t,u,v,w,x,y,z) 0
# define sqlite3WalCallback(z)                   0
# define sqlite3WalExclusiveMode(y,z)            0
# define sqlite3WalHeapMemory(z)                 0
................................................................................
Pgno sqlite3WalDbsize(Wal *pWal);

/* Obtain or release the WRITER lock. */
int sqlite3WalBeginWriteTransaction(Wal *pWal);
int sqlite3WalEndWriteTransaction(Wal *pWal);

/* Undo any frames written (but not committed) to the log */
int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx);

/* Return an integer that records the current (uncommitted) write
** position in the WAL */
void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData);

/* Move the write position of the WAL back to iFrame.  Called in
** response to a ROLLBACK TO command. */







|







 







|







30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
..
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
# define sqlite3WalLimit(x,y)
# define sqlite3WalClose(v,w,x,y,z)              0
# define sqlite3WalBeginReadTransaction(y,z)     0
# define sqlite3WalEndReadTransaction(z)
# define sqlite3WalDbsize(y)                     0
# define sqlite3WalBeginWriteTransaction(y)      0
# define sqlite3WalEndWriteTransaction(x)        0
# define sqlite3WalUndo(w,x,y,z)                 0
# define sqlite3WalSavepoint(y,z)
# define sqlite3WalSavepointUndo(y,z)            0
# define sqlite3WalFrames(u,v,w,x,y,z)           0
# define sqlite3WalCheckpoint(q,r,s,t,u,v,w,x,y,z) 0
# define sqlite3WalCallback(z)                   0
# define sqlite3WalExclusiveMode(y,z)            0
# define sqlite3WalHeapMemory(z)                 0
................................................................................
Pgno sqlite3WalDbsize(Wal *pWal);

/* Obtain or release the WRITER lock. */
int sqlite3WalBeginWriteTransaction(Wal *pWal);
int sqlite3WalEndWriteTransaction(Wal *pWal);

/* Undo any frames written (but not committed) to the log */
int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx, int);

/* Return an integer that records the current (uncommitted) write
** position in the WAL */
void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData);

/* Move the write position of the WAL back to iFrame.  Called in
** response to a ROLLBACK TO command. */

Changes to test/concfault.test.

78
79
80
81
82
83
84









































85
86
  }
} -test {
  faultsim_test_result {0 {}} 
  catchsql { ROLLBACK }
  faultsim_integrity_check
}










































finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
  }
} -test {
  faultsim_test_result {0 {}} 
  catchsql { ROLLBACK }
  faultsim_integrity_check
}


#-------------------------------------------------------------------------
reset_db

do_execsql_test 2.0 {
  PRAGMA auto_vacuum = 0;
  PRAGMA journal_mode = wal;
  CREATE TABLE t1(a PRIMARY KEY, b);
  CREATE TABLE t2(a PRIMARY KEY, b);
  INSERT INTO t1 VALUES(randomblob(1000), randomblob(100));
  INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
  INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
  INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
  INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
  DELETE FROM t1 WHERE rowid%2;
} {wal}

faultsim_save_and_close
do_faultsim_test 1 -prep {
  faultsim_restore_and_reopen
  execsql {
    SELECT * FROM t1;
    BEGIN CONCURRENT;
      INSERT INTO t2 VALUES(1, 2);
  }
  sqlite3 db2 test.db
  execsql {
    PRAGMA journal_size_limit = 10000;
    INSERT INTO t1 VALUES(randomblob(1000), randomblob(1000));
  } db2
  db2 close
} -body {
  execsql { COMMIT }
} -test {
  faultsim_test_result {0 {}} 
  catchsql { ROLLBACK }
  set res [catchsql { SELECT count(*) FROM t1 }]
  if {$res!="0 9"} { error "expected {0 9} got {$res}" }
  faultsim_integrity_check
}

finish_test