/ Check-in [00b2f4b0]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Detect attempts to use rbu vacuum on a wal mode database (not allowed). And attempts to write to a database in the middle of an rbu vacuum (which prevents the vacuum from resuming).
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | rbu-vacuum
Files: files | file ages | folders
SHA1: 00b2f4b09ffca5156e43c4db2bfe0b2c979c49b0
User & Date: dan 2016-04-19 16:20:24
Context
2016-04-19
17:11
When an RBU vacuum is started on a db identified using a URI filename, pass the same URI parameters when creating the new version of the db. This ensures that RBU vacuum works with password protected databases. check-in: ca021ba8 user: dan tags: rbu-vacuum
16:20
Detect attempts to use rbu vacuum on a wal mode database (not allowed). And attempts to write to a database in the middle of an rbu vacuum (which prevents the vacuum from resuming). check-in: 00b2f4b0 user: dan tags: rbu-vacuum
2016-04-18
21:00
Another fix to rbu vacuum for a zipvfs case. check-in: 29407d70 user: dan tags: rbu-vacuum
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/rbu/rbuvacuum.test.

    13     13   # contains tests to ensure that the sqlite3rbu_vacuum() API works as
    14     14   # expected.
    15     15   #
    16     16   
    17     17   source [file join [file dirname [info script]] rbu_common.tcl]
    18     18   set ::testprefix rbuvacuum
    19     19   
    20         -proc do_rbu_vacuum_test {tn} {
           20  +proc do_rbu_vacuum_test {tn step} {
    21     21     uplevel [list do_test $tn.1 {
    22     22       forcedelete state.db
    23         -    if {$::step==0} { sqlite3rbu_vacuum rbu test.db state.db }
           23  +    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
    24     24       while 1 {
    25         -      if {$::step==1} { sqlite3rbu_vacuum rbu test.db state.db }
           25  +      if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
    26     26         set rc [rbu step]
    27     27         if {$rc!="SQLITE_OK"} break
    28         -      if {$::step==1} { rbu close }
           28  +      if {$step==1} { rbu close }
    29     29       }
    30     30       rbu close
    31     31     } {SQLITE_DONE}]
    32     32   
    33     33     uplevel [list do_execsql_test $tn.2 {
    34     34       PRAGMA integrity_check
    35     35     } ok]
................................................................................
    45     45       PRAGMA page_size = 1024;
    46     46       CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
    47     47       INSERT INTO t1 VALUES(1, 2, 3);
    48     48       INSERT INTO t1 VALUES(4, 5, 6);
    49     49       INSERT INTO t1 VALUES(7, 8, 9);
    50     50       PRAGMA integrity_check;
    51     51     } {ok}
    52         -  do_rbu_vacuum_test 1.1
           52  +  do_rbu_vacuum_test 1.1 $step
    53     53   
    54     54     # A vacuum that actually reclaims space.
    55     55     do_execsql_test 1.2.1 {
    56     56       INSERT INTO t1 VALUES(8, randomblob(900), randomblob(900));
    57     57       INSERT INTO t1 VALUES(9, randomblob(900), randomblob(900));
    58     58       INSERT INTO t1 VALUES(10, randomblob(900), randomblob(900));
    59     59       INSERT INTO t1 VALUES(11, randomblob(900), randomblob(900));
................................................................................
    60     60       INSERT INTO t1 VALUES(12, randomblob(900), randomblob(900));
    61     61       PRAGMA page_count;
    62     62     } {12}
    63     63     do_execsql_test 1.2.2 {
    64     64       DELETE FROM t1 WHERE rowid BETWEEN 8 AND 11;
    65     65       PRAGMA page_count;
    66     66     } {12}
    67         -  do_rbu_vacuum_test 1.2.3
           67  +  do_rbu_vacuum_test 1.2.3 $step
    68     68     do_execsql_test 1.2.4 {
    69     69       PRAGMA page_count;
    70     70     } {3}
    71     71     
    72     72     # Add an index to the table.
    73     73     do_execsql_test 1.3.1 {
    74     74       CREATE INDEX t1b ON t1(b);
................................................................................
    78     78       INSERT INTO t1 VALUES(16, randomblob(900), randomblob(900));
    79     79       PRAGMA page_count;
    80     80     } {18}
    81     81     do_execsql_test 1.3.2 {
    82     82       DELETE FROM t1 WHERE rowid BETWEEN 12 AND 15;
    83     83       PRAGMA page_count;
    84     84     } {18}
    85         -  do_rbu_vacuum_test 1.3.3
           85  +  do_rbu_vacuum_test 1.3.3 $step
    86     86     do_execsql_test 1.3.4 {
    87     87       PRAGMA page_count;
    88     88     } {5}
    89     89   
    90     90     # WITHOUT ROWID table.
    91     91     do_execsql_test 1.4.1 {
    92     92       CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID;
................................................................................
    98     98       INSERT INTO t2 VALUES(randomblob(900), 6, randomblob(900));
    99     99       INSERT INTO t2 VALUES(randomblob(900), 7, randomblob(900));
   100    100       INSERT INTO t2 VALUES(randomblob(900), 8, randomblob(900));
   101    101   
   102    102       DELETE FROM t2 WHERE b BETWEEN 2 AND 7;
   103    103       PRAGMA page_count;
   104    104     } {20}
   105         -  do_rbu_vacuum_test 1.4.2
          105  +  do_rbu_vacuum_test 1.4.2 $step
   106    106     do_execsql_test 1.4.3 {
   107    107       PRAGMA page_count;
   108    108     } {10}
   109    109     
   110    110     # WITHOUT ROWID table with an index.
   111    111     do_execsql_test 1.4.1 {
   112    112       CREATE INDEX t2c ON t2(c);
................................................................................
   116    116       INSERT INTO t2 VALUES(randomblob(900), 11, randomblob(900));
   117    117       INSERT INTO t2 VALUES(randomblob(900), 12, randomblob(900));
   118    118       INSERT INTO t2 VALUES(randomblob(900), 13, randomblob(900));
   119    119   
   120    120       DELETE FROM t2 WHERE b BETWEEN 8 AND 12;
   121    121       PRAGMA page_count;
   122    122     } {35}
   123         -  do_rbu_vacuum_test 1.4.2
          123  +  do_rbu_vacuum_test 1.4.2 $step
   124    124     do_execsql_test 1.4.3 {
   125    125       PRAGMA page_count;
   126    126     } {15}
   127    127     do_execsql_test 1.4.4 {
   128    128       VACUUM;
   129    129       PRAGMA page_count;
   130    130     } {15}
................................................................................
   131    131   
   132    132     do_execsql_test 1.5.1 {
   133    133       CREATE TABLE t3(a, b, c);
   134    134       INSERT INTO t3 VALUES('a', 'b', 'c');
   135    135       INSERT INTO t3 VALUES('d', 'e', 'f');
   136    136       INSERT INTO t3 VALUES('g', 'h', 'i');
   137    137     }
   138         -  do_rbu_vacuum_test 1.5.2
          138  +  do_rbu_vacuum_test 1.5.2 $step
   139    139     do_execsql_test 1.5.3 {
   140    140       SELECT * FROM t3
   141    141     } {a b c d e f g h i}
   142    142     do_execsql_test 1.5.4 {
   143    143       CREATE INDEX t3a ON t3(a);
   144    144       CREATE INDEX t3b ON t3(b);
   145    145       CREATE INDEX t3c ON t3(c);
   146    146       INSERT INTO t3 VALUES('j', 'k', 'l');
   147    147       DELETE FROM t3 WHERE a = 'g';
   148    148     }
   149         -  do_rbu_vacuum_test 1.5.5
          149  +  do_rbu_vacuum_test 1.5.5 $step
   150    150     do_execsql_test 1.5.6 {
   151    151       SELECT rowid, * FROM t3 ORDER BY b
   152    152     } {1 a b c 2 d e f 4 j k l}
   153    153   
   154    154     do_execsql_test 1.6.1 {
   155    155       CREATE TABLE t4(a PRIMARY KEY, b, c);
   156    156       INSERT INTO t4 VALUES('a', 'b', 'c');
   157    157       INSERT INTO t4 VALUES('d', 'e', 'f');
   158    158       INSERT INTO t4 VALUES('g', 'h', 'i');
   159    159     }
   160         -  do_rbu_vacuum_test 1.6.2
          160  +  do_rbu_vacuum_test 1.6.2 $step
   161    161     do_execsql_test 1.6.3 {
   162    162       SELECT * FROM t4
   163    163     } {a b c d e f g h i}
   164    164     do_execsql_test 1.6.4 {
   165    165       CREATE INDEX t4a ON t4(a);
   166    166       CREATE INDEX t4b ON t4(b);
   167    167       CREATE INDEX t4c ON t4(c);
   168    168       
   169    169       INSERT INTO t4 VALUES('j', 'k', 'l');
   170    170       DELETE FROM t4 WHERE a='g';
   171    171     }
   172         -  do_rbu_vacuum_test 1.6.5
          172  +  do_rbu_vacuum_test 1.6.5 $step
   173    173     do_execsql_test 1.6.6 {
   174    174       SELECT * FROM t4 ORDER BY b
   175    175     } {a b c d e f j k l}
   176    176   
          177  +}
          178  +
          179  +#-------------------------------------------------------------------------
          180  +# Test some error cases:
          181  +#
          182  +#   2.1.* the db being vacuumed being in wal mode already.
          183  +#   2.2.* database modified mid vacuum.
          184  +#
          185  +reset_db
          186  +do_execsql_test 2.1.0 {
          187  +  CREATE TABLE t1(a, b);
          188  +  INSERT INTO t1 VALUES(1, 2);
          189  +  INSERT INTO t1 VALUES(3, 4);
          190  +  INSERT INTO t1 VALUES(5, 6);
          191  +  INSERT INTO t1 VALUES(7, 8);
          192  +  PRAGMA journal_mode = wal;
          193  +  INSERT INTO t1 VALUES(9, 10);
          194  +} wal
          195  +do_test 2.1.1 {
          196  +  forcedelete state.db
          197  +  sqlite3rbu_vacuum rbu test.db state.db
          198  +  rbu step
          199  +} {SQLITE_ERROR}
          200  +do_test 2.1.2 {
          201  +  list [catch { rbu close } msg] $msg
          202  +} {1 {SQLITE_ERROR - cannot vacuum wal mode database}}
          203  +
          204  +reset_db
          205  +do_execsql_test 2.2.0 {
          206  +  CREATE TABLE tx(a PRIMARY KEY, b BLOB);
          207  +  INSERT INTO tx VALUES(1, randomblob(900));
          208  +  INSERT INTO tx SELECT a+1, randomblob(900) FROM tx;
          209  +  INSERT INTO tx SELECT a+2, randomblob(900) FROM tx;
          210  +  INSERT INTO tx SELECT a+4, randomblob(900) FROM tx;
          211  +  INSERT INTO tx SELECT a+8, randomblob(900) FROM tx;
          212  +}
          213  +db_save_and_close
          214  +for {set i 1} 1 {incr i} {
          215  +  forcedelete state.db
          216  +  db_restore_and_reopen
          217  +
          218  +  sqlite3rbu_vacuum rbu test.db state.db
          219  +  for {set step 0} {$step<$i} {incr step} { rbu step }
          220  +  rbu close
          221  +  if {[file exists test.db-wal]} break
          222  +
          223  +  execsql { INSERT INTO tx VALUES(20, 20) }
          224  +
          225  +  do_test 2.2.$i.1 {
          226  +    sqlite3rbu_vacuum rbu test.db state.db 
          227  +    rbu step
          228  +  } {SQLITE_BUSY}
          229  +  do_test 2.2.$i.2 {
          230  +    list [catch { rbu close } msg] $msg
          231  +  } {1 {SQLITE_BUSY - database modified during rbu vacuum}}
          232  +
   177    233   }
   178    234   
   179    235   catch { db close }
   180    236   finish_test
   181    237   

Changes to ext/rbu/sqlite3rbu.c.

   395    395     sqlite3_file *pReal;            /* Underlying file handle */
   396    396     rbu_vfs *pRbuVfs;               /* Pointer to the rbu_vfs object */
   397    397     sqlite3rbu *pRbu;               /* Pointer to rbu object (rbu target only) */
   398    398   
   399    399     int openFlags;                  /* Flags this file was opened with */
   400    400     u32 iCookie;                    /* Cookie value for main db files */
   401    401     u8 iWriteVer;                   /* "write-version" value for main db files */
   402         -  u8 bNolock;
          402  +  u8 bNolock;                     /* True to fail EXCLUSIVE locks */
   403    403   
   404    404     int nShm;                       /* Number of entries in apShm[] array */
   405    405     char **apShm;                   /* Array of mmap'd *-shm regions */
   406    406     char *zDel;                     /* Delete this when closing file */
   407    407   
   408    408     const char *zWal;               /* Wal filename for this main db file */
   409    409     rbu_file *pWalFd;               /* Wal file descriptor for this main db */
................................................................................
  2369   2369           rbuFreeState(pState);
  2370   2370         }
  2371   2371       }
  2372   2372       if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1);
  2373   2373     }
  2374   2374   
  2375   2375     p->eStage = 0;
  2376         -  if( p->dbMain==0 ){
         2376  +  if( p->rc==SQLITE_OK && p->dbMain==0 ){
  2377   2377       if( !rbuIsVacuum(p) ){
  2378   2378         p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1);
         2379  +    }else if( p->pRbuFd->pWalFd ){
         2380  +      p->rc = SQLITE_ERROR;
         2381  +      p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database");
  2379   2382       }else{
  2380   2383         char *zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1", p->zRbu);
  2381   2384         if( zTarget==0 ){
  2382   2385           p->rc = SQLITE_NOMEM;
  2383   2386           return;
  2384   2387         }
  2385   2388         p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1);
................................................................................
  2983   2986   ** Update the contents of the rbu_state table within the rbu database. The
  2984   2987   ** value stored in the RBU_STATE_STAGE column is eStage. All other values
  2985   2988   ** are determined by inspecting the rbu handle passed as the first argument.
  2986   2989   */
  2987   2990   static void rbuSaveState(sqlite3rbu *p, int eStage){
  2988   2991     if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){
  2989   2992       sqlite3_stmt *pInsert = 0;
         2993  +    rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd);
  2990   2994       int rc;
  2991   2995   
  2992   2996       assert( p->zErrmsg==0 );
  2993   2997       rc = prepareFreeAndCollectError(p->dbRbu, &pInsert, &p->zErrmsg, 
  2994   2998           sqlite3_mprintf(
  2995   2999             "INSERT OR REPLACE INTO %s.rbu_state(k, v) VALUES "
  2996   3000             "(%d, %d), "
................................................................................
  3005   3009             p->zStateDb,
  3006   3010             RBU_STATE_STAGE, eStage,
  3007   3011             RBU_STATE_TBL, p->objiter.zTbl, 
  3008   3012             RBU_STATE_IDX, p->objiter.zIdx, 
  3009   3013             RBU_STATE_ROW, p->nStep, 
  3010   3014             RBU_STATE_PROGRESS, p->nProgress,
  3011   3015             RBU_STATE_CKPT, p->iWalCksum,
  3012         -          RBU_STATE_COOKIE, (i64)p->pTargetFd->iCookie,
         3016  +          RBU_STATE_COOKIE, (i64)pFd->iCookie,
  3013   3017             RBU_STATE_OALSZ, p->iOalSz,
  3014   3018             RBU_STATE_PHASEONESTEP, p->nPhaseOneStep
  3015   3019         )
  3016   3020       );
  3017   3021       assert( pInsert==0 || rc==SQLITE_OK );
  3018   3022   
  3019   3023       if( rc==SQLITE_OK ){
................................................................................
  3427   3431           p->zErrmsg = sqlite3_mprintf("cannot update wal mode database");
  3428   3432         }else if( p->eStage==RBU_STAGE_MOVE ){
  3429   3433           p->eStage = RBU_STAGE_CKPT;
  3430   3434           p->nStep = 0;
  3431   3435         }
  3432   3436       }
  3433   3437   
  3434         -    if( p->rc==SQLITE_OK
  3435         -     && !rbuIsVacuum(p)
         3438  +    if( p->rc==SQLITE_OK 
  3436   3439        && (p->eStage==RBU_STAGE_OAL || p->eStage==RBU_STAGE_MOVE)
  3437         -     && pState->eStage!=0 && p->pTargetFd->iCookie!=pState->iCookie
  3438         -    ){   
  3439         -      /* At this point (pTargetFd->iCookie) contains the value of the
  3440         -      ** change-counter cookie (the thing that gets incremented when a 
  3441         -      ** transaction is committed in rollback mode) currently stored on 
  3442         -      ** page 1 of the database file. */
  3443         -      p->rc = SQLITE_BUSY;
  3444         -      p->zErrmsg = sqlite3_mprintf("database modified during rbu update");
         3440  +     && pState->eStage!=0
         3441  +    ){
         3442  +      rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd);
         3443  +      if( pFd->iCookie!=pState->iCookie ){   
         3444  +        /* At this point (pTargetFd->iCookie) contains the value of the
         3445  +        ** change-counter cookie (the thing that gets incremented when a 
         3446  +        ** transaction is committed in rollback mode) currently stored on 
         3447  +        ** page 1 of the database file. */
         3448  +        p->rc = SQLITE_BUSY;
         3449  +        p->zErrmsg = sqlite3_mprintf("database modified during rbu %s",
         3450  +            (rbuIsVacuum(p) ? "vacuum" : "update")
         3451  +        );
         3452  +      }
  3445   3453       }
  3446   3454   
  3447   3455       if( p->rc==SQLITE_OK ){
  3448   3456         if( p->eStage==RBU_STAGE_OAL ){
  3449   3457           sqlite3 *db = p->dbMain;
  3450   3458   
  3451   3459           /* Open transactions both databases. The *-oal file is opened or
................................................................................
  3837   3845           rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst);
  3838   3846           if( rc==SQLITE_OK ){
  3839   3847             u8 *aBuf = (u8*)zBuf;
  3840   3848             rbuPutU32(&aBuf[52], 0);          /* largest root page number */
  3841   3849             rbuPutU32(&aBuf[36], 0);          /* number of free pages */
  3842   3850             rbuPutU32(&aBuf[32], 0);          /* first page on free list trunk */
  3843   3851             rbuPutU32(&aBuf[28], 1);          /* size of db file in pages */
         3852  +          rbuPutU32(&aBuf[24], pRbu->pRbuFd->iCookie+1);  /* Change counter */
  3844   3853   
  3845   3854             if( iAmt>100 ){
  3846   3855               assert( iAmt>=101 );
  3847   3856               memset(&aBuf[101], 0, iAmt-101);
  3848   3857               rbuPutU16(&aBuf[105], iAmt & 0xFFFF);
  3849   3858             }
  3850   3859           }
................................................................................
  3853   3862       }
  3854   3863       if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){
  3855   3864         /* These look like magic numbers. But they are stable, as they are part
  3856   3865          ** of the definition of the SQLite file format, which may not change. */
  3857   3866         u8 *pBuf = (u8*)zBuf;
  3858   3867         p->iCookie = rbuGetU32(&pBuf[24]);
  3859   3868         p->iWriteVer = pBuf[19];
  3860         -      if( pRbu && rbuIsVacuum(p->pRbu) ){
  3861         -        rbu_file *pRbuFd = 0;
  3862         -        sqlite3_file_control(pRbu->dbRbu, "main", 
  3863         -            SQLITE_FCNTL_FILE_POINTER, (void*)&pRbuFd
  3864         -        );
  3865         -        rbuPutU32(&pBuf[24], pRbuFd->iCookie+1);
  3866         -      }
  3867   3869       }
  3868   3870     }
  3869   3871     return rc;
  3870   3872   }
  3871   3873   
  3872   3874   /*
  3873   3875   ** Write data to an rbuVfs-file.