Index: ext/rbu/rbu1.test ================================================================== --- ext/rbu/rbu1.test +++ ext/rbu/rbu1.test @@ -609,11 +609,11 @@ } {SQLITE_ERROR - invalid rbu_control value} 9 { CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID; CREATE TABLE rbu.data_t1(a, b, rbu_control); - INSERT INTO rbu.data_t1 VALUES(1, 2, 2); + INSERT INTO rbu.data_t1 VALUES(1, 2, 3); } {SQLITE_ERROR - invalid rbu_control value} 10 { CREATE TABLE t2(a, b); CREATE TABLE rbu.data_t1(a, b, rbu_control); Index: ext/rbu/rbudiff.test ================================================================== --- ext/rbu/rbudiff.test +++ ext/rbu/rbudiff.test @@ -34,16 +34,38 @@ } set rc } proc apply_rbudiff {sql target} { + test_rbucount $sql forcedelete rbu.db sqlite3 rbudb rbu.db rbudb eval $sql rbudb close step_rbu $target rbu.db } + +# The only argument is the output of an [sqldiff -rbu] run. This command +# tests that the contents of the rbu_count table is correct. An exception +# is thrown if it is not. +# +proc test_rbucount {sql} { + sqlite3 tmpdb "" + tmpdb eval $sql + tmpdb eval { + SELECT name FROM sqlite_master WHERE name LIKE 'data%' AND type='table' + } { + set a [tmpdb eval "SELECT count(*) FROM $name"] + set b [tmpdb eval {SELECT cnt FROM rbu_count WHERE tbl = $name}] + if {$a != $b} { + tmpdb close + error "rbu_count error - tbl = $name" + } + } + tmpdb close + return "" +} proc rbudiff_cksum {db1} { set txt "" sqlite3 dbtmp $db1 ADDED ext/rbu/rbuprogress.test Index: ext/rbu/rbuprogress.test ================================================================== --- /dev/null +++ ext/rbu/rbuprogress.test @@ -0,0 +1,415 @@ +# 2016 March 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rbuprogress + + +proc create_db_file {filename sql} { + forcedelete $filename + sqlite3 tmpdb $filename + tmpdb eval $sql + tmpdb close +} + +# Create a simple RBU database. That expects to write to a table: +# +# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +# +proc create_rbu1 {filename} { + create_db_file $filename { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t1 VALUES(2, 'two', 'three', 0); + INSERT INTO data_t1 VALUES(3, NULL, 8.2, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 3); + } + return $filename +} + + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +} + +do_test 1.1 { + create_rbu1 rbu.db + sqlite3rbu rbu test.db rbu.db + rbu bp_progress +} {0 0} +do_test 1.2 { rbu step ; rbu bp_progress } {3333 0} +do_test 1.3 { rbu step ; rbu bp_progress } {6666 0} +do_test 1.4 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.5 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.6 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.7 { rbu step ; rbu bp_progress } {10000 5000} +do_test 1.8 { rbu step ; rbu bp_progress } {10000 10000} +do_test 1.9 { rbu step ; rbu bp_progress } {10000 10000} + +do_test 1.10 { + rbu close +} {SQLITE_DONE} + +#------------------------------------------------------------------------- +# +proc do_sp_test {tn bReopen target rbu reslist} { + uplevel [list do_test $tn [subst -nocommands { + if {$bReopen==0} { sqlite3rbu rbu $target $rbu } + set res [list] + while 1 { + if {$bReopen} { sqlite3rbu rbu $target $rbu } + set rc [rbu step] + if {[set rc] != "SQLITE_OK"} { rbu close ; error "error 1" } + lappend res [lindex [rbu bp_progress] 0] + if {[lindex [set res] end]==10000} break + if {$bReopen} { rbu close } + } + if {[set res] != [list $reslist]} { + rbu close + error "1. reslist incorrect (expect=$reslist got=[set res])" + } + + # One step to clean up the temporary tables used to update the only + # target table in the rbu database. And one more to move the *-oal + # file to *-wal. After each of these steps, the progress remains + # at "10000 0". + # + if {[lindex [list $reslist] 0]!=-1} { + rbu step + set res [rbu bp_progress] + if {[set res] != [list 10000 0]} { + rbu close + error "2. reslist incorrect (expect=10000 0 got=[set res])" + } + } + + rbu step + set res [rbu bp_progress] + if {[set res] != [list 10000 0]} { + rbu close + error "3. reslist incorrect (expect=10000 0 got=[set res])" + } + + # Do the checkpoint. + while {[rbu step]=="SQLITE_OK"} { + foreach {a b} [rbu bp_progress] {} + if {[set a]!=10000 || [set b]<=0 || [set b]>10000} { + rbu close + error "4. reslist incorrect (expect=10000 1..10000 got=[set a] [set b])" + } + } + + set res [rbu bp_progress] + if {[set res] != [list 10000 10000]} { + rbu close + error "5. reslist is incorrect (expect=10000 10000 got=[set res])" + } + + rbu close + }] {SQLITE_DONE}] +} + +foreach {bReopen} { 0 1 } { + reset_db + do_test 2.$bReopen.1.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(5, 5, 5, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 2); + } + } {} + do_sp_test 2.$bReopen.1.1 $bReopen test.db rbu.db {5000 10000} + + reset_db + do_test 2.$bReopen.2.0 { + execsql { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) } + create_rbu1 rbu.db + } {rbu.db} + do_sp_test 2.$bReopen.2.1 $bReopen test.db rbu.db {3333 6666 10000} + + reset_db + do_test 2.$bReopen.3.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(5, NULL, NULL, 1); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 3); + } + } {} + do_sp_test 2.$bReopen.3.1 $bReopen test.db rbu.db {1666 3333 6000 8000 10000} + + reset_db + do_test 2.$bReopen.4.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(2, 4, 4, '.xx'); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + } + } {} + do_sp_test 2.$bReopen.4.1 $bReopen test.db rbu.db {3333 6666 10000} + + reset_db + do_test 2.$bReopen.5.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, NULL, 4, '.xx'); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + } + } {} + do_sp_test 2.$bReopen.5.1 $bReopen test.db rbu.db {10000} + + reset_db + do_test 2.$bReopen.6.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(5, NULL, NULL, 1); + } + } {} + do_sp_test 2.$bReopen.6.1 $bReopen test.db rbu.db {-1 -1 -1 -1 -1 10000} +} + +#------------------------------------------------------------------------- +# The following tests verify that the API works when resuming an update +# during the incremental checkpoint stage. +# +proc do_phase2_test {tn bReopen target rbu nStep} { + uplevel [list do_test $tn [subst -nocommands { + + # Build the OAL/WAL file: + sqlite3rbu rbu $target $rbu + while {[lindex [rbu bp_progress] 0]<10000} { + set rc [rbu step] + if {"SQLITE_OK" != [set rc]} { rbu close } + } + + # Clean up the temp tables and move the *-oal file to *-wal. + rbu step + rbu step + + for {set i 0} {[set i] < $nStep} {incr i} { + if {$bReopen} { + rbu close + sqlite3rbu rbu $target $rbu + } + rbu step + set res [rbu bp_progress] + set expect [expr (1 + [set i]) * 10000 / $nStep] + if {[lindex [set res] 1] != [set expect]} { + error "Have [set res], expected 10000 [set expect]" + } + } + + set rc [rbu step] + if {[set rc] != "SQLITE_DONE"} { + error "Have [set rc], expected SQLITE_DONE" + } + + rbu close + }] {SQLITE_DONE}] +} + +foreach bReopen {0 1} { + do_test 3.$bReopen.1.0 { + reset_db + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, rbu_control); + CREATE TABLE data_t2(a, b, rbu_control); + CREATE TABLE data_t3(a, b, rbu_control); + CREATE TABLE data_t4(a, b, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 0); + INSERT INTO data_t2 VALUES(1, 2, 0); + INSERT INTO data_t3 VALUES(1, 2, 0); + INSERT INTO data_t4 VALUES(1, 2, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + INSERT INTO rbu_count VALUES('data_t2', 1); + INSERT INTO rbu_count VALUES('data_t3', 1); + INSERT INTO rbu_count VALUES('data_t4', 1); + } + } {} + do_phase2_test 3.$bReopen.1.1 $bReopen test.db rbu.db 5 +} + + +foreach {bReopen} { 0 1 } { + foreach {tn tbl} { + ipk { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) } + wr { CREATE TABLE t1(a INT PRIMARY KEY, b, c) WITHOUT ROWID } + pk { CREATE TABLE t1(a INT PRIMARY KEY, b, c) } + } { + + foreach {tn2 rbusql r1 r3} { + 1 { + CREATE TABLE data0_t1(a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(15, 15, 15, 0); + INSERT INTO data0_t1 VALUES(20, 20, 20, 0); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 5000 7500 10000} + {1666 3333 5000 6666 8333 10000} + + 2 { + CREATE TABLE data0_t1(a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(10, 10, 10, 2); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 1); + } + {3333 6666 10000} + {2000 4000 6000 8000 10000} + + 3 { + CREATE TABLE data0_t1(a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(7, 7, 7, 2); + INSERT INTO data0_t1 VALUES(10, 10, 10, 2); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 4000 6000 8000 10000} + {1666 2500 3750 5000 6250 7500 8750 10000} + + } { + + reset_db ; execsql $tbl + do_test 4.$tn.$bReopen.$tn2.0 { + execsql { + CREATE INDEX t1c ON t1(c); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(5, 5, 5); + INSERT INTO t1 VALUES(10, 10, 10); + } + create_db_file rbu.db $rbusql + } {} + + set R(ipk) $r1 + set R(wr) $r1 + set R(pk) $r3 + do_sp_test 4.$tn.$bReopen.$tn2.1 $bReopen test.db rbu.db $R($tn) + } + } +} + +foreach {bReopen} { 0 1 } { + foreach {tn tbl} { + nopk { + CREATE TABLE t1(a, b, c); + CREATE INDEX t1c ON t1(c); + } + vtab { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); + } + } { + + foreach {tn2 rbusql r1 r2} { + 1 { + CREATE TABLE data0_t1(a, b, c, rbu_rowid, rbu_control); + INSERT INTO data0_t1 VALUES(15, 15, 15, 4, 0); + INSERT INTO data0_t1 VALUES(20, 20, 20, 5, 0); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 5000 7500 10000} + {5000 10000} + + 2 { + CREATE TABLE data0_t1(rbu_rowid, a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(0, 7, 7, 7, 2); + INSERT INTO data0_t1 VALUES(2, 10, 10, 10, 2); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 4000 6000 8000 10000} + {5000 10000} + + 3 { + CREATE TABLE data0_t1(rbu_rowid, a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(1, NULL, NULL, NULL, 1); + INSERT INTO data0_t1 VALUES(2, NULL, NULL, 7, '..x'); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 4000 6000 8000 10000} + {5000 10000} + } { + + reset_db ; execsql $tbl + do_test 5.$tn.$bReopen.$tn2.0 { + execsql { + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(5, 5, 5); + INSERT INTO t1 VALUES(10, 10, 10); + } + create_db_file rbu.db $rbusql + } {} + + set R(nopk) $r1 + set R(vtab) $r2 + do_sp_test 5.$tn.$bReopen.$tn2.1 $bReopen test.db rbu.db $R($tn) + } + } +} + + +finish_test + Index: ext/rbu/sqlite3rbu.c ================================================================== --- ext/rbu/sqlite3rbu.c +++ ext/rbu/sqlite3rbu.c @@ -145,18 +145,19 @@ ** target db file. ** ** RBU_STATE_OALSZ: ** Valid if STAGE==1. The size in bytes of the *-oal file. */ -#define RBU_STATE_STAGE 1 -#define RBU_STATE_TBL 2 -#define RBU_STATE_IDX 3 -#define RBU_STATE_ROW 4 -#define RBU_STATE_PROGRESS 5 -#define RBU_STATE_CKPT 6 -#define RBU_STATE_COOKIE 7 -#define RBU_STATE_OALSZ 8 +#define RBU_STATE_STAGE 1 +#define RBU_STATE_TBL 2 +#define RBU_STATE_IDX 3 +#define RBU_STATE_ROW 4 +#define RBU_STATE_PROGRESS 5 +#define RBU_STATE_CKPT 6 +#define RBU_STATE_COOKIE 7 +#define RBU_STATE_OALSZ 8 +#define RBU_STATE_PHASEONESTEP 9 #define RBU_STAGE_OAL 1 #define RBU_STAGE_MOVE 2 #define RBU_STAGE_CAPTURE 3 #define RBU_STAGE_CKPT 4 @@ -198,10 +199,11 @@ i64 iWalCksum; int nRow; i64 nProgress; u32 iCookie; i64 iOalSz; + i64 nPhaseOneStep; }; struct RbuUpdateStmt { char *zMask; /* Copy of update mask used with pUpdate */ sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */ @@ -242,10 +244,11 @@ const char *zDataTbl; /* Name of rbu db table (or null) */ const char *zIdx; /* Name of target db index (or null) */ int iTnum; /* Root page of current object */ int iPkTnum; /* If eType==EXTERNAL, root of PK index */ int bUnique; /* Current index is unique */ + int nIndex; /* Number of aux. indexes on table zTbl */ /* Statements created by rbuObjIterPrepareAll() */ int nCol; /* Number of columns in current object */ sqlite3_stmt *pSelect; /* Source data */ sqlite3_stmt *pInsert; /* Statement for INSERT operations */ @@ -295,10 +298,47 @@ u32 iWalFrame; }; /* ** RBU handle. +** +** nPhaseOneStep: +** If the RBU database contains an rbu_count table, this value is set to +** a running estimate of the number of b-tree operations required to +** finish populating the *-oal file. This allows the sqlite3_bp_progress() +** API to calculate the permyriadage progress of populating the *-oal file +** using the formula: +** +** permyriadage = (10000 * nProgress) / nPhaseOneStep +** +** nPhaseOneStep is initialized to the sum of: +** +** nRow * (nIndex + 1) +** +** for all source tables in the RBU database, where nRow is the number +** of rows in the source table and nIndex the number of indexes on the +** corresponding target database table. +** +** This estimate is accurate if the RBU update consists entirely of +** INSERT operations. However, it is inaccurate if: +** +** * the RBU update contains any UPDATE operations. If the PK specified +** for an UPDATE operation does not exist in the target table, then +** no b-tree operations are required on index b-trees. Or if the +** specified PK does exist, then (nIndex*2) such operations are +** required (one delete and one insert on each index b-tree). +** +** * the RBU update contains any DELETE operations for which the specified +** PK does not exist. In this case no operations are required on index +** b-trees. +** +** * the RBU update contains REPLACE operations. These are similar to +** UPDATE operations. +** +** nPhaseOneStep is updated to account for the conditions above during the +** first pass of each source table. The updated nPhaseOneStep value is +** stored in the rbu_state table if the RBU update is suspended. */ struct sqlite3rbu { int eStage; /* Value of RBU_STATE_STAGE field */ sqlite3 *dbMain; /* target database handle */ sqlite3 *dbRbu; /* rbu database handle */ @@ -312,10 +352,11 @@ int nProgress; /* Rows processed for all objects */ RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ const char *zVfsName; /* Name of automatically created rbu vfs */ rbu_file *pTargetFd; /* File handle open on target db */ i64 iOalSz; + i64 nPhaseOneStep; /* The following state variables are used as part of the incremental ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding ** function rbuSetupCheckpoint() for details. */ u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */ @@ -1142,10 +1183,11 @@ p->rc = prepareFreeAndCollectError(p->dbMain, &pList, &p->zErrmsg, sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) ); } + pIter->nIndex = 0; while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ const char *zIdx = (const char*)sqlite3_column_text(pList, 1); sqlite3_stmt *pXInfo = 0; if( zIdx==0 ) break; p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, @@ -1155,10 +1197,16 @@ int iCid = sqlite3_column_int(pXInfo, 1); if( iCid>=0 ) pIter->abIndexed[iCid] = 1; } rbuFinalize(p, pXInfo); bIndex = 1; + pIter->nIndex++; + } + + if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ + /* "PRAGMA index_list" includes the main PK b-tree */ + pIter->nIndex--; } rbuFinalize(p, pList); if( bIndex==0 ) pIter->abIndexed = 0; } @@ -1268,10 +1316,11 @@ } rbuFinalize(p, pStmt); rbuObjIterCacheIndexedCols(p, pIter); assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 ); + assert( pIter->eType!=RBU_PK_VTAB || pIter->nIndex==0 ); } return p->rc; } @@ -1820,10 +1869,18 @@ sqlite3_value **apVal ){ sqlite3rbu *p = sqlite3_user_data(pCtx); int rc = SQLITE_OK; int i; + + assert( sqlite3_value_int(apVal[0])!=0 + || p->objiter.eType==RBU_PK_EXTERNAL + || p->objiter.eType==RBU_PK_NONE + ); + if( sqlite3_value_int(apVal[0])!=0 ){ + p->nPhaseOneStep += p->objiter.nIndex; + } for(i=0; rc==SQLITE_OK && iobjiter.pTmpInsert, i+1, apVal[i]); } if( rc==SQLITE_OK ){ @@ -2565,10 +2622,21 @@ sqlite3_stmt *pWriter; int i; assert( p->rc==SQLITE_OK ); assert( eType!=RBU_DELETE || pIter->zIdx==0 ); + assert( eType==RBU_DELETE || eType==RBU_IDX_DELETE + || eType==RBU_INSERT || eType==RBU_IDX_INSERT + ); + + /* If this is a delete, decrement nPhaseOneStep by nIndex. If the DELETE + ** statement below does actually delete a row, nPhaseOneStep will be + ** incremented by the same amount when SQL function rbu_tmp_insert() + ** is invoked by the trigger. */ + if( eType==RBU_DELETE ){ + p->nPhaseOneStep -= p->objiter.nIndex; + } if( eType==RBU_IDX_DELETE || eType==RBU_DELETE ){ pWriter = pIter->pDelete; }else{ pWriter = pIter->pInsert; @@ -2636,24 +2704,28 @@ || eType==RBU_REPLACE || eType==RBU_IDX_DELETE || eType==RBU_IDX_INSERT || eType==RBU_UPDATE ); assert( eType!=RBU_UPDATE || pIter->zIdx==0 ); - if( pIter->zIdx==0 && eType==RBU_IDX_DELETE ){ + if( pIter->zIdx==0 && (eType==RBU_IDX_DELETE || eType==RBU_IDX_INSERT) ){ rbuBadControlError(p); } else if( eType==RBU_REPLACE ){ - if( pIter->zIdx==0 ) rbuStepOneOp(p, RBU_DELETE); + if( pIter->zIdx==0 ){ + p->nPhaseOneStep += p->objiter.nIndex; + rbuStepOneOp(p, RBU_DELETE); + } if( p->rc==SQLITE_OK ) rbuStepOneOp(p, RBU_INSERT); } else if( eType!=RBU_UPDATE ){ rbuStepOneOp(p, eType); } else{ sqlite3_value *pVal; sqlite3_stmt *pUpdate = 0; assert( eType==RBU_UPDATE ); + p->nPhaseOneStep -= p->objiter.nIndex; rbuGetUpdateStmt(p, pIter, zMask, &pUpdate); if( pUpdate ){ int i; for(i=0; p->rc==SQLITE_OK && inCol; i++){ char c = zMask[pIter->aiSrcOrder[i]]; @@ -2727,20 +2799,22 @@ "(%d, %Q), " "(%d, %d), " "(%d, %d), " "(%d, %lld), " "(%d, %lld), " + "(%d, %lld), " "(%d, %lld) ", p->zStateDb, RBU_STATE_STAGE, eStage, RBU_STATE_TBL, p->objiter.zTbl, RBU_STATE_IDX, p->objiter.zIdx, RBU_STATE_ROW, p->nStep, RBU_STATE_PROGRESS, p->nProgress, RBU_STATE_CKPT, p->iWalCksum, RBU_STATE_COOKIE, (i64)p->pTargetFd->iCookie, - RBU_STATE_OALSZ, p->iOalSz + RBU_STATE_OALSZ, p->iOalSz, + RBU_STATE_PHASEONESTEP, p->nPhaseOneStep ) ); assert( pInsert==0 || rc==SQLITE_OK ); if( rc==SQLITE_OK ){ @@ -2923,10 +2997,14 @@ case RBU_STATE_OALSZ: pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); break; + case RBU_STATE_PHASEONESTEP: + pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); + break; + default: rc = SQLITE_CORRUPT; break; } } @@ -3029,10 +3107,104 @@ if( p->zVfsName ){ sqlite3rbu_destroy_vfs(p->zVfsName); p->zVfsName = 0; } } + +/* +** This user-defined SQL function is invoked with a single argument - the +** name of a table expected to appear in the target database. It returns +** the number of auxilliary indexes on the table. +*/ +static void rbuIndexCntFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + sqlite3rbu *p = (sqlite3rbu*)sqlite3_user_data(pCtx); + sqlite3_stmt *pStmt = 0; + char *zErrmsg = 0; + int rc; + + assert( nVal==1 ); + + rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &zErrmsg, + sqlite3_mprintf("SELECT count(*) FROM sqlite_master " + "WHERE type='index' AND tbl_name = %Q", sqlite3_value_text(apVal[0])) + ); + if( rc!=SQLITE_OK ){ + sqlite3_result_error(pCtx, zErrmsg, -1); + }else{ + int nIndex = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + nIndex = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + sqlite3_result_int(pCtx, nIndex); + }else{ + sqlite3_result_error(pCtx, sqlite3_errmsg(p->dbMain), -1); + } + } + + sqlite3_free(zErrmsg); +} + +/* +** If the RBU database contains the rbu_count table, use it to initialize +** the sqlite3rbu.nPhaseOneStep variable. The schema of the rbu_count table +** is assumed to contain the same columns as: +** +** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; +** +** There should be one row in the table for each data_xxx table in the +** database. The 'tbl' column should contain the name of a data_xxx table, +** and the cnt column the number of rows it contains. +** +** sqlite3rbu.nPhaseOneStep is initialized to the sum of (1 + nIndex) * cnt +** for all rows in the rbu_count table, where nIndex is the number of +** indexes on the corresponding target database table. +*/ +static void rbuInitPhaseOneSteps(sqlite3rbu *p){ + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + int bExists = 0; /* True if rbu_count exists */ + + p->nPhaseOneStep = -1; + + p->rc = sqlite3_create_function(p->dbRbu, + "rbu_index_cnt", 1, SQLITE_UTF8, (void*)p, rbuIndexCntFunc, 0, 0 + ); + + /* Check for the rbu_count table. If it does not exist, or if an error + ** occurs, nPhaseOneStep will be left set to -1. */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + "SELECT 1 FROM sqlite_master WHERE tbl_name = 'rbu_count'" + ); + } + if( p->rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + bExists = 1; + } + p->rc = sqlite3_finalize(pStmt); + } + + if( p->rc==SQLITE_OK && bExists ){ + p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + "SELECT sum(cnt * (1 + rbu_index_cnt(rbu_target_name(tbl))))" + "FROM rbu_count" + ); + if( p->rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->nPhaseOneStep = sqlite3_column_int64(pStmt, 0); + } + p->rc = sqlite3_finalize(pStmt); + } + } + } +} /* ** Open and return a new RBU handle. */ sqlite3rbu *sqlite3rbu_open( @@ -3075,13 +3247,15 @@ assert( pState || p->rc!=SQLITE_OK ); if( p->rc==SQLITE_OK ){ if( pState->eStage==0 ){ rbuDeleteOalFile(p); + rbuInitPhaseOneSteps(p); p->eStage = RBU_STAGE_OAL; }else{ p->eStage = pState->eStage; + p->nPhaseOneStep = pState->nPhaseOneStep; } p->nProgress = pState->nProgress; p->iOalSz = pState->iOalSz; } } @@ -3240,10 +3414,46 @@ ** current RBU update was started. */ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu){ return pRbu->nProgress; } + +/* +** Return permyriadage progress indications for the two main stages of +** an RBU update. +*/ +void sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){ + const int MAX_PROGRESS = 10000; + switch( p->eStage ){ + case RBU_STAGE_OAL: + if( p->nPhaseOneStep>0 ){ + *pnOne = (int)(MAX_PROGRESS * (i64)p->nProgress/(i64)p->nPhaseOneStep); + }else{ + *pnOne = -1; + } + *pnTwo = 0; + break; + + case RBU_STAGE_MOVE: + *pnOne = MAX_PROGRESS; + *pnTwo = 0; + break; + + case RBU_STAGE_CKPT: + *pnOne = MAX_PROGRESS; + *pnTwo = (int)(MAX_PROGRESS * (i64)p->nStep / (i64)p->nFrame); + break; + + case RBU_STAGE_DONE: + *pnOne = MAX_PROGRESS; + *pnTwo = MAX_PROGRESS; + break; + + default: + assert( 0 ); + } +} int sqlite3rbu_savestate(sqlite3rbu *p){ int rc = p->rc; if( rc==SQLITE_DONE ) return SQLITE_OK; Index: ext/rbu/sqlite3rbu.h ================================================================== --- ext/rbu/sqlite3rbu.h +++ ext/rbu/sqlite3rbu.h @@ -398,10 +398,52 @@ ** updates) that have been performed on the target database since the ** current RBU update was started. */ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu); +/* +** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100) +** progress indications for the two stages of an RBU update. This API may +** be useful for driving GUI progress indicators and similar. +** +** An RBU update is divided into two stages: +** +** * Stage 1, in which changes are accumulated in an oal/wal file, and +** * Stage 2, in which the contents of the wal file are copied into the +** main database. +** +** The update is visible to non-RBU clients during stage 2. During stage 1 +** non-RBU reader clients may see the original database. +** +** If this API is called during stage 2 of the update, output variable +** (*pnOne) is set to 10000 to indicate that stage 1 has finished and (*pnTwo) +** to a value between 0 and 10000 to indicate the permyriadage progress of +** stage 2. A value of 5000 indicates that stage 2 is half finished, +** 9000 indicates that it is 90% finished, and so on. +** +** If this API is called during stage 1 of the update, output variable +** (*pnTwo) is set to 0 to indicate that stage 2 has not yet started. The +** value to which (*pnOne) is set depends on whether or not the RBU +** database contains an "rbu_count" table. The rbu_count table, if it +** exists, must contain the same columns as the following: +** +** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; +** +** There must be one row in the table for each source (data_xxx) table within +** the RBU database. The 'tbl' column should contain the name of the source +** table. The 'cnt' column should contain the number of rows within the +** source table. +** +** If the rbu_count table is present and populated correctly and this +** API is called during stage 1, the *pnOne output variable is set to the +** permyriadage progress of the same stage. If the rbu_count table does +** not exist, then (*pnOne) is set to -1 during stage 1. If the rbu_count +** table exists but is not correctly populated, the value of the *pnOne +** output variable during stage 1 is undefined. +*/ +void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo); + /* ** Create an RBU VFS named zName that accesses the underlying file-system ** via existing VFS zParent. Or, if the zParent parameter is passed NULL, ** then the new RBU VFS uses the default system VFS to access the file-system. ** The new object is registered as a non-default VFS with SQLite before Index: ext/rbu/test_rbu.c ================================================================== --- ext/rbu/test_rbu.c +++ ext/rbu/test_rbu.c @@ -64,10 +64,11 @@ {"step", 2, ""}, /* 0 */ {"close", 2, ""}, /* 1 */ {"create_rbu_delta", 2, ""}, /* 2 */ {"savestate", 2, ""}, /* 3 */ {"dbMain_eval", 3, "SQL"}, /* 4 */ + {"bp_progress", 2, ""}, /* 5 */ {0,0,0} }; int iCmd; if( objc<2 ){ @@ -133,10 +134,22 @@ Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3_errmsg(db), -1)); ret = TCL_ERROR; } break; } + + case 5: /* bp_progress */ { + int one, two; + Tcl_Obj *pObj; + sqlite3rbu_bp_progress(pRbu, &one, &two); + + pObj = Tcl_NewObj(); + Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(one)); + Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(two)); + Tcl_SetObjResult(interp, pObj); + break; + } default: /* seems unlikely */ assert( !"cannot happen" ); break; } Index: tool/sqldiff.c ================================================================== --- tool/sqldiff.c +++ tool/sqldiff.c @@ -1242,10 +1242,11 @@ int nCol; Str ct = {0, 0, 0}; /* The "CREATE TABLE data_xxx" statement */ Str sql = {0, 0, 0}; /* Query to find differences */ Str insert = {0, 0, 0}; /* First part of output INSERT statement */ sqlite3_stmt *pStmt = 0; + int nRow = 0; /* Total rows in data_xxx table */ /* --rbu mode must use real primary keys. */ g.bSchemaPK = 1; /* Check that the schemas of the two tables match. Exit early otherwise. */ @@ -1287,10 +1288,11 @@ strFree(&ct); } /* Output the first part of the INSERT statement */ fprintf(out, "%s", insert.z); + nRow++; if( sqlite3_column_type(pStmt, nCol)==SQLITE_INTEGER ){ for(i=0; i<=nCol; i++){ if( i>0 ) fprintf(out, ", "); printQuoted(out, sqlite3_column_value(pStmt, i)); @@ -1340,10 +1342,16 @@ /* And the closing bracket of the insert statement */ fprintf(out, ");\n"); } sqlite3_finalize(pStmt); + if( nRow>0 ){ + Str cnt = {0, 0, 0}; + strPrintf(&cnt, "INSERT INTO rbu_count VALUES('data_%q', %d);", zTab, nRow); + fprintf(out, "%s\n", cnt.z); + strFree(&cnt); + } strFree(&ct); strFree(&sql); strFree(&insert); } @@ -1854,11 +1862,17 @@ if( rc || zErrMsg ){ cmdlineError("\"%s\" does not appear to be a valid SQLite database", zDb2); } if( neverUseTransaction ) useTransaction = 0; - if( useTransaction ) printf("BEGIN TRANSACTION;\n"); + if( useTransaction ) fprintf(out, "BEGIN TRANSACTION;\n"); + if( xDiff==rbudiff_one_table ){ + fprintf(out, "CREATE TABLE IF NOT EXISTS rbu_count" + "(tbl TEXT PRIMARY KEY COLLATE NOCASE, cnt INTEGER) " + "WITHOUT ROWID;\n" + ); + } if( zTab ){ xDiff(zTab, out); }else{ /* Handle tables one by one */ pStmt = db_prepare(