SQLite

Check-in [4012bb3aa9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix another RBU case similar to the previous. This one for systems where the sector-size is larger than the page-size.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 4012bb3aa91927156ba149caa4e5c622b0729d79
User & Date: dan 2017-03-02 16:56:48.085
References
2017-03-07
14:46
Fix another RBU case similar to the previous. This one for systems where the sector-size is larger than the page-size. Cherrypick of [4012bb3a]. (check-in: 59a11b7f1f user: dan tags: version-3.17.0-rbu-fixes)
Context
2017-03-07
14:46
Fix another RBU case similar to the previous. This one for systems where the sector-size is larger than the page-size. Cherrypick of [4012bb3a]. (check-in: 59a11b7f1f user: dan tags: version-3.17.0-rbu-fixes)
2017-03-02
23:40
Fix a bug in the 'start of ...' date/time modifiers when they follow a julian day number. Fix for ticket [6097cb92745327a1]. (check-in: 081dbcfb6d user: drh tags: trunk)
16:56
Fix another RBU case similar to the previous. This one for systems where the sector-size is larger than the page-size. (check-in: 4012bb3aa9 user: dan tags: trunk)
14:51
When saving the state of an RBU update in the incremental-checkpoint phase, sync the database file. Otherwise, if a power failure occurs and the RBU update resumed following system recovery, the database may become corrupt. (check-in: edee6a80e1 user: dan tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/rbu/rbucrash2.test.
41
42
43
44
45
46
47

48




49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
  INSERT INTO data_t1 VALUES('five', randomblob(3500), NULL, 0);
  INSERT INTO data_t1 VALUES('six', randomblob(3500), NULL, 0);
}
db_save_and_close

proc do_rbu_crash_test2 {tn script} {


  foreach f {test.db test.db2} {




    set bDone 0
    for {set iDelay 1} {$bDone==0} {incr iDelay} {
      forcedelete test.db2 test.db2-journal test.db test.db-oal test.db-wal
      db_restore
  
      set res [
        crashsql -file $f -delay $iDelay -tclbody $script -dflt 1 -opendb {} \
            -blocksize 512 {}
      ]
  
      set bDone 1
      if {$res == "1 {child process exited abnormally}"} {
        set bDone 0
      } elseif {$res != "0 {}"} {
        error "unexected catchsql result: $res"
      }
  
      sqlite3rbu rbu test.db test.db2
      while {[rbu step]=="SQLITE_OK"} {}
      rbu close
  
      sqlite3 db test.db
      do_execsql_test $tn.delay=$iDelay.f=$f {
        PRAGMA integrity_check;
      } {ok}
      db close
    }
  }
}








>
|
>
>
>
>







|














|







41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
  INSERT INTO data_t1 VALUES('five', randomblob(3500), NULL, 0);
  INSERT INTO data_t1 VALUES('six', randomblob(3500), NULL, 0);
}
db_save_and_close

proc do_rbu_crash_test2 {tn script} {

  foreach {f blksz} {
    test.db   512
    test.db2  512
    test.db   4096
    test.db2  4096
  } {
    set bDone 0
    for {set iDelay 1} {$bDone==0} {incr iDelay} {
      forcedelete test.db2 test.db2-journal test.db test.db-oal test.db-wal
      db_restore
  
      set res [
        crashsql -file $f -delay $iDelay -tclbody $script -dflt 1 -opendb {} \
            -blocksize $blksz {}
      ]
  
      set bDone 1
      if {$res == "1 {child process exited abnormally}"} {
        set bDone 0
      } elseif {$res != "0 {}"} {
        error "unexected catchsql result: $res"
      }
  
      sqlite3rbu rbu test.db test.db2
      while {[rbu step]=="SQLITE_OK"} {}
      rbu close
  
      sqlite3 db test.db
      do_execsql_test $tn.delay=$iDelay.f=$f.blksz=$blksz {
        PRAGMA integrity_check;
      } {ok}
      db close
    }
  }
}

Changes to ext/rbu/rbuprogress.test.
36
37
38
39
40
41
42

43
44
45
46
47
48
49
    INSERT INTO rbu_count VALUES('data_t1', 3);
  }
  return $filename
}


do_execsql_test 1.0 {

  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
}

do_test 1.1 {
  create_rbu1 rbu.db
  sqlite3rbu rbu test.db rbu.db
  rbu bp_progress







>







36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
    INSERT INTO rbu_count VALUES('data_t1', 3);
  }
  return $filename
}


do_execsql_test 1.0 {
  PRAGMA page_size = 4096;
  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
}

do_test 1.1 {
  create_rbu1 rbu.db
  sqlite3rbu rbu test.db rbu.db
  rbu bp_progress
262
263
264
265
266
267
268

269
270
271
272
273
274
275
  }] {SQLITE_DONE}]
}

foreach bReopen {0 1} {
  do_test 3.$bReopen.1.0 {
    reset_db
    execsql {

      CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
      CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
      CREATE TABLE t3(a INTEGER PRIMARY KEY, b);
      CREATE TABLE t4(a INTEGER PRIMARY KEY, b);
    }
    create_db_file rbu.db {
      CREATE TABLE data_t1(a, b, rbu_control);







>







263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
  }] {SQLITE_DONE}]
}

foreach bReopen {0 1} {
  do_test 3.$bReopen.1.0 {
    reset_db
    execsql {
      PRAGMA page_size = 4096;
      CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
      CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
      CREATE TABLE t3(a INTEGER PRIMARY KEY, b);
      CREATE TABLE t4(a INTEGER PRIMARY KEY, b);
    }
    create_db_file rbu.db {
      CREATE TABLE data_t1(a, b, rbu_control);
Changes to ext/rbu/sqlite3rbu.c.
352
353
354
355
356
357
358

359
360
361
362
363
364
365
  int rc;                         /* Value returned by last rbu_step() call */
  char *zErrmsg;                  /* Error message if rc!=SQLITE_OK */
  int nStep;                      /* Rows processed for current object */
  int nProgress;                  /* Rows processed for all objects */
  RbuObjIter objiter;             /* Iterator for skipping through tbl/idx */
  const char *zVfsName;           /* Name of automatically created rbu vfs */
  rbu_file *pTargetFd;            /* File handle open on target db */

  i64 iOalSz;
  i64 nPhaseOneStep;

  /* The following state variables are used as part of the incremental
  ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding
  ** function rbuSetupCheckpoint() for details.  */
  u32 iMaxFrame;                  /* Largest iWalFrame value in aFrame[] */







>







352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
  int rc;                         /* Value returned by last rbu_step() call */
  char *zErrmsg;                  /* Error message if rc!=SQLITE_OK */
  int nStep;                      /* Rows processed for current object */
  int nProgress;                  /* Rows processed for all objects */
  RbuObjIter objiter;             /* Iterator for skipping through tbl/idx */
  const char *zVfsName;           /* Name of automatically created rbu vfs */
  rbu_file *pTargetFd;            /* File handle open on target db */
  int nPagePerSector;             /* Pages per sector for pTargetFd */
  i64 iOalSz;
  i64 nPhaseOneStep;

  /* The following state variables are used as part of the incremental
  ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding
  ** function rbuSetupCheckpoint() for details.  */
  u32 iMaxFrame;                  /* Largest iWalFrame value in aFrame[] */
2616
2617
2618
2619
2620
2621
2622










2623
2624
2625
2626
2627
2628
2629
    p->iWalCksum = rbuShmChecksum(p);
  }

  if( p->rc==SQLITE_OK ){
    if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){
      p->rc = SQLITE_DONE;
      p->eStage = RBU_STAGE_DONE;










    }
  }
}

/*
** Called when iAmt bytes are read from offset iOff of the wal file while
** the rbu object is in capture mode. Record the frame number of the frame







>
>
>
>
>
>
>
>
>
>







2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
    p->iWalCksum = rbuShmChecksum(p);
  }

  if( p->rc==SQLITE_OK ){
    if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){
      p->rc = SQLITE_DONE;
      p->eStage = RBU_STAGE_DONE;
    }else{
      int nSectorSize;
      sqlite3_file *pDb = p->pTargetFd->pReal;
      assert( p->nPagePerSector==0 );
      nSectorSize = pDb->pMethods->xSectorSize(pDb);
      if( nSectorSize>p->pgsz ){
        p->nPagePerSector = nSectorSize / p->pgsz;
      }else{
        p->nPagePerSector = 1;
      }
    }
  }
}

/*
** Called when iAmt bytes are read from offset iOff of the wal file while
** the rbu object is in capture mode. Record the frame number of the frame
3271
3272
3273
3274
3275
3276
3277












3278

3279
3280




3281
3282
3283
3284
3285
3286
3287
            }
  
            if( p->rc==SQLITE_OK ){
              p->eStage = RBU_STAGE_DONE;
              p->rc = SQLITE_DONE;
            }
          }else{












            RbuFrame *pFrame = &p->aFrame[p->nStep];

            rbuCheckpointFrame(p, pFrame);
            p->nStep++;




          }
          p->nProgress++;
        }
        break;
      }

      default:







>
>
>
>
>
>
>
>
>
>
>
>
|
>
|
|
>
>
>
>







3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
            }
  
            if( p->rc==SQLITE_OK ){
              p->eStage = RBU_STAGE_DONE;
              p->rc = SQLITE_DONE;
            }
          }else{
            /* At one point the following block copied a single frame from the
            ** wal file to the database file. So that one call to sqlite3rbu_step()
            ** checkpointed a single frame. 
            **
            ** However, if the sector-size is larger than the page-size, and the
            ** application calls sqlite3rbu_savestate() or close() immediately
            ** after this step, then rbu_step() again, then a power failure occurs,
            ** then the database page written here may be damaged. Work around
            ** this by checkpointing frames until the next page in the aFrame[]
            ** lies on a different disk sector to the current one. */
            u32 iSector;
            do{
              RbuFrame *pFrame = &p->aFrame[p->nStep];
              iSector = (pFrame->iDbPage-1) / p->nPagePerSector;
              rbuCheckpointFrame(p, pFrame);
              p->nStep++;
            }while( p->nStep<p->nFrame 
                 && iSector==((p->aFrame[p->nStep].iDbPage-1) / p->nPagePerSector)
                 && p->rc==SQLITE_OK
            );
          }
          p->nProgress++;
        }
        break;
      }

      default: