SQLite: Check-in [681d96eb]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview

Comment:	Fix a problem that could cause a spurious SQLITE_NOMEM error when attempting to resume an RBU operation if the previous client failed right after completing the incremental checkpoint. Also a "cannot vacuum wal db" error that could occur when resuming an RBU vacuum if an error (OOM or IO error) occurs during the incremental checkpoint.
Downloads:	Tarball \| ZIP archive \| SQL archive
Timelines:	family \| ancestors \| descendants \| both \| trunk
Files:	files \| file ages \| folders
SHA1:	681d96eb822e606da53700867191d4738bda20c8
User & Date:	dan 2017-01-17 10:41:42

Context

2017-01-18
22:16		Fix handling of initial hidden and/or system files in the opendir() implementation for Windows. No changes to non-test code. (check-in: 26dd42b4 user: mistachkin tags: trunk)
20:14		Add temporary code to record and report on the set of b-tree pages read and written by the current transaction. This is likely still buggy. (check-in: 2a8f6c89 user: dan tags: transaction-pages)
17:20		Baseline interface definition for the experimental sqlite3_kv accessor object. (check-in: a435841e user: drh tags: sqlite3_kv)
2017-01-17
10:41		Fix a problem that could cause a spurious SQLITE_NOMEM error when attempting to resume an RBU operation if the previous client failed right after completing the incremental checkpoint. Also a "cannot vacuum wal db" error that could occur when resuming an RBU vacuum if an error (OOM or IO error) occurs during the incremental checkpoint. (check-in: 681d96eb user: dan tags: trunk)
00:10		Disable intrinsic functions for Windows using Clang, due to reports of linkage errors. This causes a 0.6% performance reduction. We will want to revisit this change in the future. (check-in: 7fd560c6 user: drh tags: trunk)

Changes

Hide Diffs Unified Diffs Ignore Whitespace Patch

Added ext/rbu/rbufault4.test.

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

# 2014 October 22
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#

if {![info exists testdir]} {
  set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
source $testdir/malloc_common.tcl
set ::testprefix rbufault4

for {set tn 1} {1} {incr tn} {
  reset_db
  do_execsql_test 1.0 {
    CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
    CREATE INDEX i1b ON t1(b);
    CREATE INDEX i1c ON t1(c);
    INSERT INTO t1 VALUES(1, 2, 3);
    INSERT INTO t1 VALUES(4, 5, 6);
  }

  forcedelete test.db2
  sqlite3rbu_vacuum rbu test.db test.db2
  for {set i 0} {$i < $tn} {incr i} { rbu step }
  set rc [rbu close]
  if {$rc!="SQLITE_OK"} { 
    if {$rc!="SQLITE_DONE"} {error $rc}
    break
  }
  faultsim_save

  do_faultsim_test $tn -faults oom-t* -prep {
    faultsim_restore
  } -body {
    sqlite3rbu_vacuum rbu test.db test.db2
    while 1 {
      set rc [rbu step]
      if {$rc=="SQLITE_DONE"} break
      if {$rc!="SQLITE_OK"} { error $rc }
    }
  } -test {
    catch {rbu close}
    faultsim_test_result {0 {}} {1 SQLITE_NOMEM} {1 SQLITE_IOERR_NOMEM}

    sqlite3rbu_vacuum rbu test.db test.db2
    while {[rbu step]=="SQLITE_OK"} {}
    set trc [rbu close]
    if {$trc!="SQLITE_DONE"} { error "Got $trc instead of SQLITE_DONE!" }

    set rc [db one {PRAGMA integrity_check}]
    if {$rc!="ok"} { error "Got $rc instead of ok!" }
  }
}



finish_test

Changes to ext/rbu/rbuvacuum2.test.


    do_test 5.$tn.2 { file exists test.db-vacuum } 1
    do_test 5.$tn.3 { file attributes test.db-vacuum -permissions} $perm
    rbu close
  }
}

































finish_test








>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


    do_test 5.$tn.2 { file exists test.db-vacuum } 1
    do_test 5.$tn.3 { file attributes test.db-vacuum -permissions} $perm
    rbu close
  }
}

#-------------------------------------------------------------------------
# Test the outcome of some other connection running a checkpoint while
# the incremental checkpoint is suspended.
#
reset_db
do_execsql_test 6.0 {
  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
  CREATE INDEX i1b ON t1(b);
  CREATE INDEX i1c ON t1(c);
  INSERT INTO t1 VALUES(1, 2, 3);
  INSERT INTO t1 VALUES(4, 5, 6);
}
forcedelete test.db2

do_test 6.1 {
  sqlite3rbu_vacuum rbu test.db test.db2
  while {[rbu state]!="checkpoint"} { rbu step }
  rbu close
} {SQLITE_OK}

do_execsql_test 6.2 {
  SELECT 1 FROM sqlite_master LIMIT 1;
  PRAGMA wal_checkpoint;
} {1 0 4 4}

do_test 6.3 {
  sqlite3rbu_vacuum rbu test.db test.db2
  while {[rbu step]!="SQLITE_DONE"} { rbu step }
  rbu close
  execsql { PRAGMA integrity_check }
} {ok}

finish_test

Changes to ext/rbu/sqlite3rbu.c.

}


/*
** Open the database handle and attach the RBU database as "rbu". If an
** error occurs, leave an error code and message in the RBU handle.
*/
static void rbuOpenDatabase(sqlite3rbu *p){
  assert( p->rc || (p->dbMain==0 && p->dbRbu==0) );
  assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 );

  /* Open the RBU database */
  p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1);

  if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){
................................................................................
  }

  p->eStage = 0;
  if( p->rc==SQLITE_OK && p->dbMain==0 ){
    if( !rbuIsVacuum(p) ){
      p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1);
    }else if( p->pRbuFd->pWalFd ){









      p->rc = SQLITE_ERROR;
      p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database");
    }else{
      char *zTarget;
      char *zExtra = 0;
      if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){
        zExtra = &p->zRbu[5];
................................................................................
  if( p->rc==SQLITE_OK ){
    int rc2;
    p->eStage = RBU_STAGE_CAPTURE;
    rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0);
    if( rc2!=SQLITE_INTERNAL ) p->rc = rc2;
  }

  if( p->rc==SQLITE_OK ){
    p->eStage = RBU_STAGE_CKPT;
    p->nStep = (pState ? pState->nRow : 0);
    p->aBuf = rbuMalloc(p, p->pgsz);
    p->iWalCksum = rbuShmChecksum(p);
  }


  if( p->rc==SQLITE_OK && pState && pState->iWalCksum!=p->iWalCksum ){
    p->rc = SQLITE_DONE;
    p->eStage = RBU_STAGE_DONE;

  }
}

/*
** Called when iAmt bytes are read from offset iOff of the wal file while
** the rbu object is in capture mode. Record the frame number of the frame
** being read in the aFrame[] array.
................................................................................
        }
      }
#else
      p->rc = rename(zOal, zWal) ? SQLITE_IOERR : SQLITE_OK;
#endif

      if( p->rc==SQLITE_OK ){
        rbuOpenDatabase(p);
        rbuSetupCheckpoint(p, 0);
      }
    }
  }

  sqlite3_free(zWal);
  sqlite3_free(zOal);
................................................................................
    /* Create the custom VFS. */
    memset(p, 0, sizeof(sqlite3rbu));
    rbuCreateVfs(p);

    /* Open the target, RBU and state databases */
    if( p->rc==SQLITE_OK ){
      char *pCsr = (char*)&p[1];

      if( zTarget ){
        p->zTarget = pCsr;
        memcpy(p->zTarget, zTarget, nTarget+1);
        pCsr += nTarget+1;
      }
      p->zRbu = pCsr;
      memcpy(p->zRbu, zRbu, nRbu+1);
      pCsr += nRbu+1;
      if( zState ){
        p->zState = rbuMPrintf(p, "%s", zState);
      }










      rbuOpenDatabase(p);

    }

    if( p->rc==SQLITE_OK ){
      pState = rbuLoadState(p);
      assert( pState || p->rc!=SQLITE_OK );
      if( p->rc==SQLITE_OK ){








|







 







>
>
>
>
>
>
>
>
>







 







|






>
|
|
|
>







 







|







 







>











>
>
>
>
>
>
>
>
>
>
|
>

}


/*
** Open the database handle and attach the RBU database as "rbu". If an
** error occurs, leave an error code and message in the RBU handle.
*/
static void rbuOpenDatabase(sqlite3rbu *p, int *pbRetry){
  assert( p->rc || (p->dbMain==0 && p->dbRbu==0) );
  assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 );

  /* Open the RBU database */
  p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1);

  if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){
................................................................................
  }

  p->eStage = 0;
  if( p->rc==SQLITE_OK && p->dbMain==0 ){
    if( !rbuIsVacuum(p) ){
      p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1);
    }else if( p->pRbuFd->pWalFd ){
      if( pbRetry ){
        p->pRbuFd->bNolock = 0;
        sqlite3_close(p->dbRbu);
        sqlite3_close(p->dbMain);
        p->dbMain = 0;
        p->dbRbu = 0;
        *pbRetry = 1;
        return;
      }
      p->rc = SQLITE_ERROR;
      p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database");
    }else{
      char *zTarget;
      char *zExtra = 0;
      if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){
        zExtra = &p->zRbu[5];
................................................................................
  if( p->rc==SQLITE_OK ){
    int rc2;
    p->eStage = RBU_STAGE_CAPTURE;
    rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0);
    if( rc2!=SQLITE_INTERNAL ) p->rc = rc2;
  }

  if( p->rc==SQLITE_OK && p->nFrame>0 ){
    p->eStage = RBU_STAGE_CKPT;
    p->nStep = (pState ? pState->nRow : 0);
    p->aBuf = rbuMalloc(p, p->pgsz);
    p->iWalCksum = rbuShmChecksum(p);
  }

  if( p->rc==SQLITE_OK ){
    if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){
      p->rc = SQLITE_DONE;
      p->eStage = RBU_STAGE_DONE;
    }
  }
}

/*
** Called when iAmt bytes are read from offset iOff of the wal file while
** the rbu object is in capture mode. Record the frame number of the frame
** being read in the aFrame[] array.
................................................................................
        }
      }
#else
      p->rc = rename(zOal, zWal) ? SQLITE_IOERR : SQLITE_OK;
#endif

      if( p->rc==SQLITE_OK ){
        rbuOpenDatabase(p, 0);
        rbuSetupCheckpoint(p, 0);
      }
    }
  }

  sqlite3_free(zWal);
  sqlite3_free(zOal);
................................................................................
    /* Create the custom VFS. */
    memset(p, 0, sizeof(sqlite3rbu));
    rbuCreateVfs(p);

    /* Open the target, RBU and state databases */
    if( p->rc==SQLITE_OK ){
      char *pCsr = (char*)&p[1];
      int bRetry = 0;
      if( zTarget ){
        p->zTarget = pCsr;
        memcpy(p->zTarget, zTarget, nTarget+1);
        pCsr += nTarget+1;
      }
      p->zRbu = pCsr;
      memcpy(p->zRbu, zRbu, nRbu+1);
      pCsr += nRbu+1;
      if( zState ){
        p->zState = rbuMPrintf(p, "%s", zState);
      }

      /* If the first attempt to open the database file fails and the bRetry
      ** flag it set, this means that the db was not opened because it seemed
      ** to be a wal-mode db. But, this may have happened due to an earlier
      ** RBU vacuum operation leaving an old wal file in the directory.
      ** If this is the case, it will have been checkpointed and deleted
      ** when the handle was closed and a second attempt to open the 
      ** database may succeed.  */
      rbuOpenDatabase(p, &bRetry);
      if( bRetry ){
        rbuOpenDatabase(p, 0);
      }
    }

    if( p->rc==SQLITE_OK ){
      pState = rbuLoadState(p);
      assert( pState || p->rc!=SQLITE_OK );
      if( p->rc==SQLITE_OK ){