SQLite

Check-in [681d96eb82]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem that could cause a spurious SQLITE_NOMEM error when attempting to resume an RBU operation if the previous client failed right after completing the incremental checkpoint. Also a "cannot vacuum wal db" error that could occur when resuming an RBU vacuum if an error (OOM or IO error) occurs during the incremental checkpoint.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 681d96eb822e606da53700867191d4738bda20c8
User & Date: dan 2017-01-17 10:41:42.780
Context
2017-01-18
22:16
Fix handling of initial hidden and/or system files in the opendir() implementation for Windows. No changes to non-test code. (check-in: 26dd42b462 user: mistachkin tags: trunk)
20:14
Add temporary code to record and report on the set of b-tree pages read and written by the current transaction. This is likely still buggy. (check-in: 2a8f6c890c user: dan tags: transaction-pages)
17:20
Baseline interface definition for the experimental sqlite3_kv accessor object. (check-in: a435841e71 user: drh tags: sqlite3_kv)
2017-01-17
10:41
Fix a problem that could cause a spurious SQLITE_NOMEM error when attempting to resume an RBU operation if the previous client failed right after completing the incremental checkpoint. Also a "cannot vacuum wal db" error that could occur when resuming an RBU vacuum if an error (OOM or IO error) occurs during the incremental checkpoint. (check-in: 681d96eb82 user: dan tags: trunk)
00:10
Disable intrinsic functions for Windows using Clang, due to reports of linkage errors. This causes a 0.6% performance reduction. We will want to revisit this change in the future. (check-in: 7fd560c6d2 user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Added ext/rbu/rbufault4.test.




































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
# 2014 October 22
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#

if {![info exists testdir]} {
  set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
source $testdir/malloc_common.tcl
set ::testprefix rbufault4

for {set tn 1} {1} {incr tn} {
  reset_db
  do_execsql_test 1.0 {
    CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
    CREATE INDEX i1b ON t1(b);
    CREATE INDEX i1c ON t1(c);
    INSERT INTO t1 VALUES(1, 2, 3);
    INSERT INTO t1 VALUES(4, 5, 6);
  }

  forcedelete test.db2
  sqlite3rbu_vacuum rbu test.db test.db2
  for {set i 0} {$i < $tn} {incr i} { rbu step }
  set rc [rbu close]
  if {$rc!="SQLITE_OK"} { 
    if {$rc!="SQLITE_DONE"} {error $rc}
    break
  }
  faultsim_save

  do_faultsim_test $tn -faults oom-t* -prep {
    faultsim_restore
  } -body {
    sqlite3rbu_vacuum rbu test.db test.db2
    while 1 {
      set rc [rbu step]
      if {$rc=="SQLITE_DONE"} break
      if {$rc!="SQLITE_OK"} { error $rc }
    }
  } -test {
    catch {rbu close}
    faultsim_test_result {0 {}} {1 SQLITE_NOMEM} {1 SQLITE_IOERR_NOMEM}

    sqlite3rbu_vacuum rbu test.db test.db2
    while {[rbu step]=="SQLITE_OK"} {}
    set trc [rbu close]
    if {$trc!="SQLITE_DONE"} { error "Got $trc instead of SQLITE_DONE!" }

    set rc [db one {PRAGMA integrity_check}]
    if {$rc!="ok"} { error "Got $rc instead of ok!" }
  }
}



finish_test

Changes to ext/rbu/rbuvacuum2.test.
195
196
197
198
199
200
201































202
203
204

    do_test 5.$tn.2 { file exists test.db-vacuum } 1
    do_test 5.$tn.3 { file attributes test.db-vacuum -permissions} $perm
    rbu close
  }
}

































finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235

    do_test 5.$tn.2 { file exists test.db-vacuum } 1
    do_test 5.$tn.3 { file attributes test.db-vacuum -permissions} $perm
    rbu close
  }
}

#-------------------------------------------------------------------------
# Test the outcome of some other connection running a checkpoint while
# the incremental checkpoint is suspended.
#
reset_db
do_execsql_test 6.0 {
  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
  CREATE INDEX i1b ON t1(b);
  CREATE INDEX i1c ON t1(c);
  INSERT INTO t1 VALUES(1, 2, 3);
  INSERT INTO t1 VALUES(4, 5, 6);
}
forcedelete test.db2

do_test 6.1 {
  sqlite3rbu_vacuum rbu test.db test.db2
  while {[rbu state]!="checkpoint"} { rbu step }
  rbu close
} {SQLITE_OK}

do_execsql_test 6.2 {
  SELECT 1 FROM sqlite_master LIMIT 1;
  PRAGMA wal_checkpoint;
} {1 0 4 4}

do_test 6.3 {
  sqlite3rbu_vacuum rbu test.db test.db2
  while {[rbu step]!="SQLITE_DONE"} { rbu step }
  rbu close
  execsql { PRAGMA integrity_check }
} {ok}

finish_test

Changes to ext/rbu/sqlite3rbu.c.
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
}


/*
** Open the database handle and attach the RBU database as "rbu". If an
** error occurs, leave an error code and message in the RBU handle.
*/
static void rbuOpenDatabase(sqlite3rbu *p){
  assert( p->rc || (p->dbMain==0 && p->dbRbu==0) );
  assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 );

  /* Open the RBU database */
  p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1);

  if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){







|







2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
}


/*
** Open the database handle and attach the RBU database as "rbu". If an
** error occurs, leave an error code and message in the RBU handle.
*/
static void rbuOpenDatabase(sqlite3rbu *p, int *pbRetry){
  assert( p->rc || (p->dbMain==0 && p->dbRbu==0) );
  assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 );

  /* Open the RBU database */
  p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1);

  if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){
2416
2417
2418
2419
2420
2421
2422









2423
2424
2425
2426
2427
2428
2429
  }

  p->eStage = 0;
  if( p->rc==SQLITE_OK && p->dbMain==0 ){
    if( !rbuIsVacuum(p) ){
      p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1);
    }else if( p->pRbuFd->pWalFd ){









      p->rc = SQLITE_ERROR;
      p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database");
    }else{
      char *zTarget;
      char *zExtra = 0;
      if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){
        zExtra = &p->zRbu[5];







>
>
>
>
>
>
>
>
>







2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
  }

  p->eStage = 0;
  if( p->rc==SQLITE_OK && p->dbMain==0 ){
    if( !rbuIsVacuum(p) ){
      p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1);
    }else if( p->pRbuFd->pWalFd ){
      if( pbRetry ){
        p->pRbuFd->bNolock = 0;
        sqlite3_close(p->dbRbu);
        sqlite3_close(p->dbMain);
        p->dbMain = 0;
        p->dbRbu = 0;
        *pbRetry = 1;
        return;
      }
      p->rc = SQLITE_ERROR;
      p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database");
    }else{
      char *zTarget;
      char *zExtra = 0;
      if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){
        zExtra = &p->zRbu[5];
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610

2611
2612

2613
2614
2615
2616
2617
2618
2619
  if( p->rc==SQLITE_OK ){
    int rc2;
    p->eStage = RBU_STAGE_CAPTURE;
    rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0);
    if( rc2!=SQLITE_INTERNAL ) p->rc = rc2;
  }

  if( p->rc==SQLITE_OK ){
    p->eStage = RBU_STAGE_CKPT;
    p->nStep = (pState ? pState->nRow : 0);
    p->aBuf = rbuMalloc(p, p->pgsz);
    p->iWalCksum = rbuShmChecksum(p);
  }

  if( p->rc==SQLITE_OK && pState && pState->iWalCksum!=p->iWalCksum ){

    p->rc = SQLITE_DONE;
    p->eStage = RBU_STAGE_DONE;

  }
}

/*
** Called when iAmt bytes are read from offset iOff of the wal file while
** the rbu object is in capture mode. Record the frame number of the frame
** being read in the aFrame[] array.







|






|
>
|
|
>







2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
  if( p->rc==SQLITE_OK ){
    int rc2;
    p->eStage = RBU_STAGE_CAPTURE;
    rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0);
    if( rc2!=SQLITE_INTERNAL ) p->rc = rc2;
  }

  if( p->rc==SQLITE_OK && p->nFrame>0 ){
    p->eStage = RBU_STAGE_CKPT;
    p->nStep = (pState ? pState->nRow : 0);
    p->aBuf = rbuMalloc(p, p->pgsz);
    p->iWalCksum = rbuShmChecksum(p);
  }

  if( p->rc==SQLITE_OK ){
    if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){
      p->rc = SQLITE_DONE;
      p->eStage = RBU_STAGE_DONE;
    }
  }
}

/*
** Called when iAmt bytes are read from offset iOff of the wal file while
** the rbu object is in capture mode. Record the frame number of the frame
** being read in the aFrame[] array.
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
        }
      }
#else
      p->rc = rename(zOal, zWal) ? SQLITE_IOERR : SQLITE_OK;
#endif

      if( p->rc==SQLITE_OK ){
        rbuOpenDatabase(p);
        rbuSetupCheckpoint(p, 0);
      }
    }
  }

  sqlite3_free(zWal);
  sqlite3_free(zOal);







|







2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
        }
      }
#else
      p->rc = rename(zOal, zWal) ? SQLITE_IOERR : SQLITE_OK;
#endif

      if( p->rc==SQLITE_OK ){
        rbuOpenDatabase(p, 0);
        rbuSetupCheckpoint(p, 0);
      }
    }
  }

  sqlite3_free(zWal);
  sqlite3_free(zOal);
3489
3490
3491
3492
3493
3494
3495

3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506








3507



3508
3509
3510
3511
3512
3513
3514
    /* Create the custom VFS. */
    memset(p, 0, sizeof(sqlite3rbu));
    rbuCreateVfs(p);

    /* Open the target, RBU and state databases */
    if( p->rc==SQLITE_OK ){
      char *pCsr = (char*)&p[1];

      if( zTarget ){
        p->zTarget = pCsr;
        memcpy(p->zTarget, zTarget, nTarget+1);
        pCsr += nTarget+1;
      }
      p->zRbu = pCsr;
      memcpy(p->zRbu, zRbu, nRbu+1);
      pCsr += nRbu+1;
      if( zState ){
        p->zState = rbuMPrintf(p, "%s", zState);
      }








      rbuOpenDatabase(p);



    }

    if( p->rc==SQLITE_OK ){
      pState = rbuLoadState(p);
      assert( pState || p->rc!=SQLITE_OK );
      if( p->rc==SQLITE_OK ){








>











>
>
>
>
>
>
>
>
|
>
>
>







3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
    /* Create the custom VFS. */
    memset(p, 0, sizeof(sqlite3rbu));
    rbuCreateVfs(p);

    /* Open the target, RBU and state databases */
    if( p->rc==SQLITE_OK ){
      char *pCsr = (char*)&p[1];
      int bRetry = 0;
      if( zTarget ){
        p->zTarget = pCsr;
        memcpy(p->zTarget, zTarget, nTarget+1);
        pCsr += nTarget+1;
      }
      p->zRbu = pCsr;
      memcpy(p->zRbu, zRbu, nRbu+1);
      pCsr += nRbu+1;
      if( zState ){
        p->zState = rbuMPrintf(p, "%s", zState);
      }

      /* If the first attempt to open the database file fails and the bRetry
      ** flag it set, this means that the db was not opened because it seemed
      ** to be a wal-mode db. But, this may have happened due to an earlier
      ** RBU vacuum operation leaving an old wal file in the directory.
      ** If this is the case, it will have been checkpointed and deleted
      ** when the handle was closed and a second attempt to open the 
      ** database may succeed.  */
      rbuOpenDatabase(p, &bRetry);
      if( bRetry ){
        rbuOpenDatabase(p, 0);
      }
    }

    if( p->rc==SQLITE_OK ){
      pState = rbuLoadState(p);
      assert( pState || p->rc!=SQLITE_OK );
      if( p->rc==SQLITE_OK ){