/ Check-in [144bb29f]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem with OTA updates in the presence of database readers.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | ota-update
Files: files | file ages | folders
SHA1: 144bb29ffcbfe96dc10c0224113e73a80e89314b
User & Date: dan 2015-02-18 17:40:05
Context
2015-02-18
20:16
Add ota tests to increase code coverage. Fix some minor issues in error handling within the ota code. check-in: 2b10c5d2 user: dan tags: ota-update
17:40
Fix a problem with OTA updates in the presence of database readers. check-in: 144bb29f user: dan tags: ota-update
2015-02-17
20:49
Improve test coverage of ota code a bit. check-in: a438fa6c user: dan tags: ota-update
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/ota/ota3.test.

    22     22   proc run_ota {target ota} {
    23     23     sqlite3ota ota $target $ota
    24     24     while { [ota step]=="SQLITE_OK" } {}
    25     25     ota close
    26     26   }
    27     27   
    28     28   forcedelete test.db-oal ota.db
           29  +db close
           30  +sqlite3_shutdown
           31  +sqlite3_config_uri 1
           32  +reset_db
    29     33   
    30     34   #--------------------------------------------------------------------
    31     35   # Test that for an OTA to be applied, no corruption results if the
    32     36   # affinities on the source and target table do not match.
    33     37   #
    34     38   do_execsql_test 1.0 {
    35     39     CREATE TABLE x1(a INTEGER PRIMARY KEY, b TEXT, c REAL);
................................................................................
   123    127     forcedelete ota.db
   124    128     sqlite3 db2 ota.db
   125    129     db2 eval {
   126    130       CREATE TABLE data_x2(a, b, c, d, ota_control);
   127    131       INSERT INTO data_x2 VALUES(1, 'a', 2, 3, 0);
   128    132     }
   129    133     db2 close
   130         -  breakpoint
   131    134     list [catch { run_ota test.db ota.db } msg] $msg
   132    135   } {1 SQLITE_ERROR}
   133    136   
   134    137   do_execsql_test 2.5 {
   135    138     PRAGMA integrity_check;
   136    139   } {ok}
   137    140   
................................................................................
   149    152   
   150    153   do_test 3.2 {
   151    154     sqlite3ota_destroy_vfs nosuchvfs
   152    155     sqlite3ota_destroy_vfs unix
   153    156     sqlite3ota_destroy_vfs win32
   154    157   } {}
   155    158   
          159  +#-------------------------------------------------------------------------
          160  +# Test that it is an error to specify an explicit VFS that does not 
          161  +# include ota VFS functionality.
          162  +#
          163  +do_test 4.1 {
          164  +  testvfs tvfs
          165  +  sqlite3ota ota file:test.db?vfs=tvfs ota.db 
          166  +  list [catch { ota step } msg] $msg
          167  +} {0 SQLITE_ERROR}
          168  +do_test 4.2 {
          169  +  list [catch { ota close } msg] $msg
          170  +} {1 {SQLITE_ERROR - ota vfs not found}}
          171  +tvfs delete
          172  +
   156    173   finish_test
          174  +
   157    175   

Changes to ext/ota/ota6.test.

    41     41     ota close
    42     42   }
    43     43   
    44     44   # Test the outcome of some other client writing the db while the *-oal 
    45     45   # file is being generated. Once this has happened, the update cannot be
    46     46   # progressed.
    47     47   #
    48         -for {set nStep 1} {$nStep < 7} {incr nStep} {
           48  +for {set nStep 1} {$nStep < 8} {incr nStep} {
    49     49     do_test 1.$nStep.1 {
    50     50       setup_test
    51     51       sqlite3ota ota test.db ota.db
    52     52       for {set i 0} {$i<$nStep} {incr i} {ota step}
    53     53   
    54     54       ota close
    55     55       sqlite3 db test.db
................................................................................
    64     64       list [file exists test.db-oal] [file exists test.db-wal]
    65     65     } {1 0}
    66     66     do_test 1.$nStep.4 {
    67     67       list [catch { ota close } msg] $msg
    68     68     } {1 {SQLITE_BUSY - database modified during ota update}}
    69     69   }
    70     70   
    71         -
    72     71   # Test the outcome of some other client writing the db after the *-oal
    73     72   # file has been copied to the *-wal path. Once this has happened, any
    74     73   # other client writing to the db causes OTA to consider its job finished.
    75     74   #
    76         -for {set nStep 7} {$nStep < 20} {incr nStep} {
           75  +for {set nStep 8} {$nStep < 20} {incr nStep} {
    77     76     do_test 1.$nStep.1 {
    78     77       setup_test
    79     78       sqlite3ota ota test.db ota.db
    80     79       for {set i 0} {$i<$nStep} {incr i} {ota step}
    81     80       ota close
    82     81       sqlite3 db test.db
    83     82       execsql { INSERT INTO t1 VALUES(5, 'hello') }

Changes to ext/ota/otafault.test.

    33     33       INSERT INTO t1 VALUES(2, 2, 2);
    34     34       INSERT INTO t1 VALUES(3, 3, 3);
    35     35   
    36     36       CREATE TABLE ota.data_t1(a, b, c, ota_control);
    37     37       INSERT INTO data_t1 VALUES(2, NULL, NULL, 1);
    38     38       INSERT INTO data_t1 VALUES(3, 'three', NULL, '.x.');
    39     39       INSERT INTO data_t1 VALUES(4, 4, 4, 0);
    40         -  } {SELECT * FROM t1} {1 1 1   3 three 3   4 4 4}
           40  +  } {
           41  +    SELECT * FROM t1
           42  +  } {1 1 1   3 three 3   4 4 4}
    41     43   
    42     44     2 {
    43     45       CREATE TABLE t2(a PRIMARY KEY, b, c) WITHOUT ROWID;
    44     46       CREATE INDEX t2cb ON t2(c, b);
    45     47       INSERT INTO t2 VALUES('a', 'a', 'a');
    46     48       INSERT INTO t2 VALUES('b', 'b', 'b');
    47     49       INSERT INTO t2 VALUES('c', 'c', 'c');
    48     50   
    49     51       CREATE TABLE ota.data_t2(a, b, c, ota_control);
    50     52       INSERT INTO data_t2 VALUES('b', NULL, NULL, 1);
    51     53       INSERT INTO data_t2 VALUES('c', 'see', NULL, '.x.');
    52     54       INSERT INTO data_t2 VALUES('d', 'd', 'd', 0);
    53         -  } {SELECT * FROM t2} {a a a   c see c     d d d}
           55  +  } {
           56  +    SELECT * FROM t2
           57  +  } {a a a   c see c     d d d}
    54     58   
    55     59     3 {
    56     60       CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
    57     61       CREATE TABLE t2(a PRIMARY KEY, b, c) WITHOUT ROWID;
    58     62       CREATE INDEX t1cb ON t1(c, b);
    59     63       CREATE INDEX t2cb ON t2(c, b);
    60     64   
    61     65       CREATE TABLE ota.data_t1(a, b, c, ota_control);
    62     66       CREATE TABLE ota.data_t2(a, b, c, ota_control);
    63     67       INSERT INTO data_t1 VALUES(1, 2, 3, 0);
    64     68       INSERT INTO data_t2 VALUES(4, 5, 6, 0);
    65         -  } {SELECT * FROM t1 UNION ALL SELECT * FROM t2} {1 2 3 4 5 6}
           69  +  } {
           70  +    SELECT * FROM t1 UNION ALL SELECT * FROM t2
           71  +  } {1 2 3 4 5 6}
           72  +
           73  +  4 {
           74  +    CREATE TABLE t1(a PRIMARY KEY, b, c);
           75  +    CREATE INDEX t1c ON t1(c);
           76  +    INSERT INTO t1 VALUES('A', 'B', 'C');
           77  +    INSERT INTO t1 VALUES('D', 'E', 'F');
           78  +
           79  +    CREATE TABLE ota.data_t1(a, b, c, ota_control);
           80  +    INSERT INTO data_t1 VALUES('D', NULL, NULL, 1);
           81  +    INSERT INTO data_t1 VALUES('A', 'Z', NULL, '.x.');
           82  +    INSERT INTO data_t1 VALUES('G', 'H', 'I', 0);
           83  +  } {
           84  +    SELECT * FROM t1 ORDER BY a;
           85  +  } {A Z C G H I}
    66     86   
    67     87   } {
    68     88     catch {db close}
    69     89     forcedelete ota.db test.db
    70     90     sqlite3 db test.db
    71     91     execsql {
    72     92       PRAGMA encoding = utf16;
................................................................................
   133    153       catch {db close}
   134    154       sqlite3_shutdown
   135    155       sqlite3_config_lookaside {*}$lookaside_config
   136    156       sqlite3_initialize
   137    157       autoinstall_test_functions
   138    158   
   139    159   
   140         -  }
   141         -
   142         -  for {set iStep 0} {$iStep<=21} {incr iStep} {
   143         -  
   144         -    forcedelete test.db-journal test.db-wal ota.db-journal ota.db-wal
   145         -  
   146         -    copy_if_exists test.db.bak test.db
   147         -    copy_if_exists ota.db.bak ota.db
   148         -  
   149         -    sqlite3ota ota test.db ota.db
   150         -    for {set x 0} {$x < $::iStep} {incr x} { ota step }
   151         -    ota close
          160  +    for {set iStep 0} {$iStep<=21} {incr iStep} {
          161  +    
          162  +      forcedelete test.db-journal test.db-wal ota.db-journal ota.db-wal
          163  +    
          164  +      copy_if_exists test.db.bak test.db
          165  +      copy_if_exists ota.db.bak ota.db
          166  +    
          167  +      sqlite3ota ota test.db ota.db
          168  +      for {set x 0} {$x < $::iStep} {incr x} { ota step }
          169  +      ota close
   152    170     
   153         -    copy_if_exists test.db test.db.bak.2
   154         -    copy_if_exists test.db-wal test.db.bak.2-wal
   155         -    copy_if_exists test.db-oal test.db.bak.2-oal
   156         -    copy_if_exists ota.db ota.db.bak.2
   157         -  
   158         -    do_faultsim_test 3.$tn.$iStep -faults $::f -prep {
   159         -      catch { db close }
   160         -      forcedelete test.db-journal test.db-wal ota.db-journal ota.db-wal
   161         -      copy_if_exists test.db.bak.2 test.db
   162         -      copy_if_exists test.db.bak.2-wal test.db-wal
   163         -      copy_if_exists test.db.bak.2-oal test.db-oal
   164         -      copy_if_exists ota.db.bak.2  ota.db
   165         -    } -body {
   166         -      sqlite3ota ota test.db ota.db
   167         -      while {[ota step] == "SQLITE_OK"} {}
   168         -      ota close
   169         -    } -test {
   170         -      faultsim_test_result {*}$::reslist
          171  +# sqlite3 x ota.db ; puts "XYZ [x eval { SELECT * FROM ota_state } ]" ; x close
          172  +    
          173  +      copy_if_exists test.db     test.db.bak.2
          174  +      copy_if_exists test.db-wal test.db.bak.2-wal
          175  +      copy_if_exists test.db-oal test.db.bak.2-oal
          176  +      copy_if_exists ota.db      ota.db.bak.2
   171    177       
   172         -      if {$testrc==0} {
   173         -        sqlite3 db test.db
   174         -        faultsim_integrity_check
   175         -        set res [db eval $::sql]
   176         -        if {$res != [list {*}$::expect]} {
   177         -          puts ""
   178         -          puts "res: $res"
   179         -          puts "exp: $expected"
   180         -          error "data not as expected!"
          178  +      do_faultsim_test 3.$tn.$iStep -faults $::f -prep {
          179  +        catch { db close }
          180  +        forcedelete test.db-journal test.db-wal ota.db-journal ota.db-wal
          181  +        copy_if_exists test.db.bak.2 test.db
          182  +        copy_if_exists test.db.bak.2-wal test.db-wal
          183  +        copy_if_exists test.db.bak.2-oal test.db-oal
          184  +        copy_if_exists ota.db.bak.2  ota.db
          185  +      } -body {
          186  +        sqlite3ota ota test.db ota.db
          187  +        ota step
          188  +        ota close
          189  +      } -test {
          190  +
          191  +        if {$testresult=="SQLITE_OK"} {set testresult "SQLITE_DONE"}
          192  +        faultsim_test_result {*}$::reslist
          193  +      
          194  +        if {$testrc==0} {
          195  +          # No error occurred. If the OTA has not already been fully applied,
          196  +          # apply the rest of it now. Then ensure that the final state of the
          197  +          # target db is as expected. And that "PRAGMA integrity_check"
          198  +          # passes.
          199  +          sqlite3ota ota test.db ota.db
          200  +          while {[ota step] == "SQLITE_OK"} {}
          201  +          ota close
          202  +
          203  +          sqlite3 db test.db
          204  +          faultsim_integrity_check
          205  +
          206  +          set res [db eval $::sql]
          207  +          if {$res != [list {*}$::expect]} {
          208  +            puts ""
          209  +            puts "res: $res"
          210  +            puts "exp: $::expect"
          211  +            error "data not as expected!"
          212  +          }
   181    213           }
   182    214         }
   183    215       }
   184    216     }
   185    217   }
   186    218   
   187    219   finish_test
   188    220   

Changes to ext/ota/sqlite3ota.c.

    71     71   #define OTA_STATE_IDX         3
    72     72   #define OTA_STATE_ROW         4
    73     73   #define OTA_STATE_PROGRESS    5
    74     74   #define OTA_STATE_CKPT        6
    75     75   #define OTA_STATE_COOKIE      7
    76     76   
    77     77   #define OTA_STAGE_OAL         1
    78         -#define OTA_STAGE_CAPTURE     2
    79         -#define OTA_STAGE_CKPT        3
    80         -#define OTA_STAGE_DONE        4
           78  +#define OTA_STAGE_MOVE        2
           79  +#define OTA_STAGE_CAPTURE     3
           80  +#define OTA_STAGE_CKPT        4
           81  +#define OTA_STAGE_DONE        5
    81     82   
    82     83   
    83     84   #define OTA_CREATE_STATE "CREATE TABLE IF NOT EXISTS ota.ota_state"        \
    84     85                                "(k INTEGER PRIMARY KEY, v)"
    85     86   
    86     87   typedef struct OtaState OtaState;
    87     88   typedef struct OtaObjIter OtaObjIter;
................................................................................
   945    946           p->rc = SQLITE_NOMEM;
   946    947           break;
   947    948         }
   948    949       }
   949    950   
   950    951       /* For a table with implicit rowids, append "old._rowid_" to the list. */
   951    952       if( pIter->eType==OTA_PK_EXTERNAL || pIter->eType==OTA_PK_NONE ){
   952         -      zList = sqlite3_mprintf("%z, %s._rowid_", zList, zObj);
          953  +      zList = otaMPrintf(p, "%z, %s._rowid_", zList, zObj);
   953    954       }
   954    955     }
   955    956     return zList;
   956    957   }
   957    958   
   958    959   /*
   959    960   ** Return an expression that can be used in a WHERE clause to match the
................................................................................
  1111   1112       }
  1112   1113       z = otaMPrintf(p, "%z)", z);
  1113   1114       rc = sqlite3_finalize(pXInfo);
  1114   1115       if( p->rc==SQLITE_OK ) p->rc = rc;
  1115   1116     }
  1116   1117     return z;
  1117   1118   }
         1119  +
         1120  +static void otaFinalize(sqlite3ota *p, sqlite3_stmt *pStmt){
         1121  +  int rc = sqlite3_finalize(pStmt);
         1122  +  if( p->rc==SQLITE_OK ) p->rc = rc;
         1123  +}
  1118   1124   
  1119   1125   /*
  1120   1126   ** This function creates the second imposter table used when writing to
  1121   1127   ** a table b-tree where the table has an external primary key. If the
  1122   1128   ** iterator passed as the second argument does not currently point to
  1123   1129   ** a table (not index) with an external primary key, this function is a
  1124   1130   ** no-op. 
................................................................................
  1156   1162           "SELECT name FROM sqlite_master WHERE rootpage = ?"
  1157   1163       );
  1158   1164       if( p->rc==SQLITE_OK ){
  1159   1165         sqlite3_bind_int(pQuery, 1, tnum);
  1160   1166         if( SQLITE_ROW==sqlite3_step(pQuery) ){
  1161   1167           zIdx = (const char*)sqlite3_column_text(pQuery, 0);
  1162   1168         }
  1163         -      if( zIdx==0 ){
  1164         -        p->rc = SQLITE_CORRUPT;
  1165         -      }
  1166   1169       }
  1167         -    assert( (zIdx==0)==(p->rc!=SQLITE_OK) );
  1168         -
  1169         -    if( p->rc==SQLITE_OK ){
         1170  +    if( zIdx ){
  1170   1171         p->rc = prepareFreeAndCollectError(p->db, &pXInfo, &p->zErrmsg,
  1171   1172             sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx)
  1172   1173         );
  1173   1174       }
  1174         -    sqlite3_finalize(pQuery);
         1175  +    otaFinalize(p, pQuery);
  1175   1176   
  1176   1177       while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
  1177   1178         int bKey = sqlite3_column_int(pXInfo, 5);
  1178   1179         if( bKey ){
  1179   1180           int iCid = sqlite3_column_int(pXInfo, 1);
  1180   1181           int bDesc = sqlite3_column_int(pXInfo, 3);
  1181   1182           const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4);
................................................................................
  1676   1677       if( p->rc ) return;
  1677   1678   
  1678   1679       iOff = (i64)(pFrame->iDbPage-1) * p->pgsz;
  1679   1680       p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff);
  1680   1681     }
  1681   1682   }
  1682   1683   
         1684  +
         1685  +/*
         1686  +** Take an EXCLUSIVE lock on the database file.
         1687  +*/
         1688  +static void otaLockDatabase(sqlite3ota *p){
         1689  +  if( p->rc==SQLITE_OK ){
         1690  +    sqlite3_file *pReal = p->pTargetFd->pReal;
         1691  +    p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_SHARED);
         1692  +    if( p->rc==SQLITE_OK ){
         1693  +      p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_EXCLUSIVE);
         1694  +    }
         1695  +  }
         1696  +}
         1697  +
  1683   1698   /*
  1684   1699   ** The OTA handle is currently in OTA_STAGE_OAL state, with a SHARED lock
  1685   1700   ** on the database file. This proc moves the *-oal file to the *-wal path,
  1686   1701   ** then reopens the database file (this time in vanilla, non-oal, WAL mode).
  1687   1702   ** If an error occurs, leave an error code and error message in the ota 
  1688   1703   ** handle.
  1689   1704   */
  1690   1705   static void otaMoveOalFile(sqlite3ota *p){
  1691   1706     const char *zBase = sqlite3_db_filename(p->db, "main");
  1692   1707   
  1693   1708     char *zWal = sqlite3_mprintf("%s-wal", zBase);
  1694   1709     char *zOal = sqlite3_mprintf("%s-oal", zBase);
  1695   1710   
  1696         -  assert( p->eStage==OTA_STAGE_OAL );
         1711  +  assert( p->eStage==OTA_STAGE_MOVE );
  1697   1712     assert( p->rc==SQLITE_OK && p->zErrmsg==0 );
  1698   1713     if( zWal==0 || zOal==0 ){
  1699   1714       p->rc = SQLITE_NOMEM;
  1700   1715     }else{
  1701   1716       /* Move the *-oal file to *-wal. At this point connection p->db is
  1702   1717       ** holding a SHARED lock on the target database file (because it is
  1703         -    ** in WAL mode). So no other connection may be writing the db.  */
  1704         -    otaFileSuffix3(zBase, zWal);
  1705         -    otaFileSuffix3(zBase, zOal);
  1706         -    rename(zOal, zWal);
         1718  +    ** in WAL mode). So no other connection may be writing the db. 
         1719  +    **
         1720  +    ** In order to ensure that there are no database readers, an EXCLUSIVE
         1721  +    ** lock is obtained here before the *-oal is moved to *-wal.
         1722  +    */
         1723  +    otaLockDatabase(p);
         1724  +    if( p->rc==SQLITE_OK ){
         1725  +      otaFileSuffix3(zBase, zWal);
         1726  +      otaFileSuffix3(zBase, zOal);
         1727  +      rename(zOal, zWal);
  1707   1728   
  1708         -    /* Re-open the databases. */
  1709         -    otaObjIterFinalize(&p->objiter);
  1710         -    sqlite3_close(p->db);
  1711         -    p->db = 0;
  1712         -    otaOpenDatabase(p);
  1713         -    otaSetupCheckpoint(p, 0);
         1729  +      /* Re-open the databases. */
         1730  +      otaObjIterFinalize(&p->objiter);
         1731  +      sqlite3_close(p->db);
         1732  +      p->db = 0;
         1733  +      otaOpenDatabase(p);
         1734  +      otaSetupCheckpoint(p, 0);
         1735  +    }
  1714   1736     }
  1715   1737   
  1716   1738     sqlite3_free(zWal);
  1717   1739     sqlite3_free(zOal);
  1718   1740   }
  1719   1741   
  1720   1742   /*
................................................................................
  1900   1922     return p->rc;
  1901   1923   }
  1902   1924   
  1903   1925   /*
  1904   1926   ** Increment the schema cookie of the main database opened by p->db.
  1905   1927   */
  1906   1928   static void otaIncrSchemaCookie(sqlite3ota *p){
  1907         -  int iCookie = 1000000;
  1908         -  sqlite3_stmt *pStmt;
  1909         -
  1910         -  assert( p->rc==SQLITE_OK && p->zErrmsg==0 );
  1911         -  p->rc = prepareAndCollectError(p->db, &pStmt, &p->zErrmsg, 
  1912         -      "PRAGMA schema_version"
  1913         -  );
  1914   1929     if( p->rc==SQLITE_OK ){
  1915         -    if( SQLITE_ROW==sqlite3_step(pStmt) ){
  1916         -      iCookie = sqlite3_column_int(pStmt, 0);
  1917         -    }
  1918         -    p->rc = sqlite3_finalize(pStmt);
  1919         -  }
  1920         -  if( p->rc==SQLITE_OK ){
  1921         -    otaMPrintfExec(p, "PRAGMA schema_version = %d", iCookie+1);
  1922         -  }
  1923         -}
         1930  +    int iCookie = 1000000;
         1931  +    sqlite3_stmt *pStmt;
         1932  +
         1933  +    p->rc = prepareAndCollectError(p->db, &pStmt, &p->zErrmsg, 
         1934  +        "PRAGMA schema_version"
         1935  +    );
         1936  +    if( p->rc==SQLITE_OK ){
         1937  +      if( SQLITE_ROW==sqlite3_step(pStmt) ){
         1938  +        iCookie = sqlite3_column_int(pStmt, 0);
         1939  +      }
         1940  +      p->rc = sqlite3_finalize(pStmt);
         1941  +    }
         1942  +    if( p->rc==SQLITE_OK ){
         1943  +      otaMPrintfExec(p, "PRAGMA schema_version = %d", iCookie+1);
         1944  +    }
         1945  +  }
         1946  +}
         1947  +
         1948  +static void otaSaveState(sqlite3ota *p, int eStage){
         1949  +  if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){
         1950  +    sqlite3_stmt *pInsert = 0;
         1951  +    int rc;
         1952  +
         1953  +    assert( p->zErrmsg==0 );
         1954  +    rc = prepareFreeAndCollectError(p->db, &pInsert, &p->zErrmsg, 
         1955  +        sqlite3_mprintf(
         1956  +          "INSERT OR REPLACE INTO ota.ota_state(k, v) VALUES "
         1957  +          "(%d, %d), "
         1958  +          "(%d, %Q), "
         1959  +          "(%d, %Q), "
         1960  +          "(%d, %d), "
         1961  +          "(%d, %lld), "
         1962  +          "(%d, %lld), "
         1963  +          "(%d, %lld) ",
         1964  +          OTA_STATE_STAGE, eStage,
         1965  +          OTA_STATE_TBL, p->objiter.zTbl, 
         1966  +          OTA_STATE_IDX, p->objiter.zIdx, 
         1967  +          OTA_STATE_ROW, p->nStep, 
         1968  +          OTA_STATE_PROGRESS, p->nProgress,
         1969  +          OTA_STATE_CKPT, p->iWalCksum,
         1970  +          OTA_STATE_COOKIE, (i64)p->pTargetFd->iCookie
         1971  +          )
         1972  +        );
         1973  +    assert( pInsert==0 || rc==SQLITE_OK );
         1974  +
         1975  +    if( rc==SQLITE_OK ){
         1976  +      sqlite3_step(pInsert);
         1977  +      rc = sqlite3_finalize(pInsert);
         1978  +    }else{
         1979  +      sqlite3_finalize(pInsert);
         1980  +    }
         1981  +
         1982  +    if( rc!=SQLITE_OK ){
         1983  +      p->rc = rc;
         1984  +    }
         1985  +  }
         1986  +}
         1987  +
  1924   1988   
  1925   1989   /*
  1926   1990   ** Step the OTA object.
  1927   1991   */
  1928   1992   int sqlite3ota_step(sqlite3ota *p){
  1929   1993     if( p ){
  1930   1994       switch( p->eStage ){
................................................................................
  1955   2019               }
  1956   2020             }
  1957   2021   
  1958   2022             otaObjIterNext(p, pIter);
  1959   2023           }
  1960   2024   
  1961   2025           if( p->rc==SQLITE_OK && pIter->zTbl==0 ){
  1962         -          p->nProgress++;
         2026  +          otaSaveState(p, OTA_STAGE_MOVE);
  1963   2027             otaIncrSchemaCookie(p);
  1964   2028             if( p->rc==SQLITE_OK ){
  1965   2029               p->rc = sqlite3_exec(p->db, "COMMIT", 0, 0, &p->zErrmsg);
  1966   2030             }
  1967         -          if( p->rc==SQLITE_OK ){
  1968         -            otaMoveOalFile(p);
  1969         -          }
         2031  +          p->eStage = OTA_STAGE_MOVE;
         2032  +        }
         2033  +        break;
         2034  +      }
         2035  +
         2036  +      case OTA_STAGE_MOVE: {
         2037  +        if( p->rc==SQLITE_OK ){
         2038  +          otaMoveOalFile(p);
         2039  +          p->nProgress++;
  1970   2040           }
  1971   2041           break;
  1972   2042         }
  1973   2043   
  1974   2044         case OTA_STAGE_CKPT: {
  1975   2045           if( p->nStep>=p->nFrame ){
  1976   2046             sqlite3_file *pDb = p->pTargetFd->pReal;
................................................................................
  2006   2076       }
  2007   2077       return p->rc;
  2008   2078     }else{
  2009   2079       return SQLITE_NOMEM;
  2010   2080     }
  2011   2081   }
  2012   2082   
  2013         -static void otaSaveState(sqlite3ota *p){
  2014         -  sqlite3_stmt *pInsert;
  2015         -  int rc;
  2016         -
  2017         -  assert( (p->rc==SQLITE_OK || p->rc==SQLITE_DONE) && p->zErrmsg==0 );
  2018         -  rc = prepareFreeAndCollectError(p->db, &pInsert, &p->zErrmsg, 
  2019         -      sqlite3_mprintf(
  2020         -        "INSERT OR REPLACE INTO ota.ota_state(k, v) VALUES "
  2021         -        "(%d, %d), "
  2022         -        "(%d, %Q), "
  2023         -        "(%d, %Q), "
  2024         -        "(%d, %d), "
  2025         -        "(%d, %lld), "
  2026         -        "(%d, %lld), "
  2027         -        "(%d, %lld) ",
  2028         -        OTA_STATE_STAGE, p->eStage,
  2029         -        OTA_STATE_TBL, p->objiter.zTbl, 
  2030         -        OTA_STATE_IDX, p->objiter.zIdx, 
  2031         -        OTA_STATE_ROW, p->nStep, 
  2032         -        OTA_STATE_PROGRESS, p->nProgress,
  2033         -        OTA_STATE_CKPT, p->iWalCksum,
  2034         -        OTA_STATE_COOKIE, (i64)p->pTargetFd->iCookie
  2035         -      )
  2036         -  );
  2037         -  assert( pInsert==0 || rc==SQLITE_OK );
  2038         -
  2039         -  if( rc==SQLITE_OK ){
  2040         -    sqlite3_step(pInsert);
  2041         -    rc = sqlite3_finalize(pInsert);
  2042         -  }else{
  2043         -    sqlite3_finalize(pInsert);
  2044         -  }
  2045         -
  2046         -  if( rc!=SQLITE_OK ){
  2047         -    p->rc = rc;
  2048         -  }
  2049         -}
  2050         -
  2051   2083   static void otaFreeState(OtaState *p){
  2052   2084     if( p ){
  2053   2085       sqlite3_free(p->zTbl);
  2054   2086       sqlite3_free(p->zIdx);
  2055   2087       sqlite3_free(p);
  2056   2088     }
  2057   2089   }
................................................................................
  2077   2109   
  2078   2110     rc = prepareAndCollectError(p->db, &pStmt, &p->zErrmsg, zSelect);
  2079   2111     while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
  2080   2112       switch( sqlite3_column_int(pStmt, 0) ){
  2081   2113         case OTA_STATE_STAGE:
  2082   2114           pRet->eStage = sqlite3_column_int(pStmt, 1);
  2083   2115           if( pRet->eStage!=OTA_STAGE_OAL
         2116  +         && pRet->eStage!=OTA_STAGE_MOVE
  2084   2117            && pRet->eStage!=OTA_STAGE_CKPT
  2085   2118           ){
  2086   2119             p->rc = SQLITE_CORRUPT;
  2087   2120           }
  2088   2121           break;
  2089   2122   
  2090   2123         case OTA_STATE_TBL:
................................................................................
  2231   2264           }else{
  2232   2265             p->eStage = pState->eStage;
  2233   2266           }
  2234   2267           p->nProgress = pState->nProgress;
  2235   2268         }
  2236   2269       }
  2237   2270       assert( p->rc!=SQLITE_OK || p->eStage!=0 );
         2271  +
         2272  +    if( p->rc==SQLITE_OK 
         2273  +     && (p->eStage==OTA_STAGE_OAL || p->eStage==OTA_STAGE_MOVE)
         2274  +    ){   
         2275  +      /* Check that this is not a wal mode database. If it is, it cannot 
         2276  +      ** be updated.  */
         2277  +      if( p->pTargetFd->pWalFd ){
         2278  +        p->rc = SQLITE_ERROR;
         2279  +        p->zErrmsg = sqlite3_mprintf("cannot update wal mode database");
         2280  +      }
         2281  +
         2282  +      /* At this point (pTargetFd->iCookie) contains the value of the
         2283  +      ** change-counter cookie (the thing that gets incremented when a 
         2284  +      ** transaction is committed in rollback mode) currently stored on 
         2285  +      ** page 1 of the database file. */
         2286  +      else if( pState->eStage!=0 && p->pTargetFd->iCookie!=pState->iCookie ){
         2287  +        p->rc = SQLITE_BUSY;
         2288  +        p->zErrmsg = sqlite3_mprintf("database modified during ota update");
         2289  +      }
         2290  +    }
  2238   2291   
  2239   2292       if( p->rc==SQLITE_OK ){
  2240   2293         if( p->eStage==OTA_STAGE_OAL ){
  2241   2294   
  2242         -        /* Check that this is not a wal mode database. If it is, it cannot 
  2243         -        ** be updated.  */
  2244         -        if( p->pTargetFd->pWalFd ){
  2245         -          p->rc = SQLITE_ERROR;
  2246         -          p->zErrmsg = sqlite3_mprintf("cannot update wal mode database");
  2247         -        }
  2248         -
  2249         -        /* At this point (pTargetFd->iCookie) contains the value of the
  2250         -        ** change-counter cookie (the thing that gets incremented when a 
  2251         -        ** transaction is committed in rollback mode) currently stored on 
  2252         -        ** page 1 of the database file. */
  2253         -        else if( pState->eStage==OTA_STAGE_OAL 
  2254         -         && p->pTargetFd->iCookie!=pState->iCookie 
  2255         -        ){
  2256         -          p->rc = SQLITE_BUSY;
  2257         -          p->zErrmsg = sqlite3_mprintf("database modified during ota update");
  2258         -        }
  2259         -
  2260   2295           /* Open the transaction */
  2261   2296           if( p->rc==SQLITE_OK ){
  2262   2297             p->rc = sqlite3_exec(p->db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
  2263   2298           }
  2264   2299     
  2265   2300           /* Point the object iterator at the first object */
  2266   2301           if( p->rc==SQLITE_OK ){
  2267   2302             p->rc = otaObjIterFirst(p, &p->objiter);
  2268   2303           }
  2269   2304     
  2270   2305           if( p->rc==SQLITE_OK ){
  2271   2306             otaLoadTransactionState(p, pState);
  2272   2307           }
         2308  +      }else if( p->eStage==OTA_STAGE_MOVE ){
         2309  +        /* no-op */
  2273   2310         }else if( p->eStage==OTA_STAGE_CKPT ){
  2274   2311           otaSetupCheckpoint(p, pState);
  2275   2312           p->nStep = pState->nRow;
  2276   2313         }else if( p->eStage==OTA_STAGE_DONE ){
  2277   2314           p->rc = SQLITE_DONE;
  2278   2315         }else{
  2279   2316           p->rc = SQLITE_CORRUPT;
................................................................................
  2321   2358     int rc;
  2322   2359     if( p ){
  2323   2360   
  2324   2361       /* If the update has not been fully applied, save the state in 
  2325   2362       ** the ota db. If successful, this call also commits the open 
  2326   2363       ** transaction on the ota db. */
  2327   2364       assert( p->rc!=SQLITE_ROW );
  2328         -    if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){
  2329         -      assert( p->zErrmsg==0 );
  2330         -      otaSaveState(p);
  2331         -    }
         2365  +    otaSaveState(p, p->eStage);
  2332   2366   
  2333   2367       /* Close any open statement handles. */
  2334   2368       otaObjIterFinalize(&p->objiter);
  2335   2369   
  2336   2370       /* Commit the transaction to the *-oal file. */
  2337   2371       if( p->rc==SQLITE_OK && p->eStage==OTA_STAGE_OAL ){
  2338   2372         p->rc = sqlite3_exec(p->db, "COMMIT", 0, 0, &p->zErrmsg);
................................................................................
  2623   2657     int rc = SQLITE_OK;
  2624   2658   
  2625   2659   #ifdef SQLITE_AMALGAMATION
  2626   2660       assert( WAL_CKPT_LOCK==1 );
  2627   2661   #endif
  2628   2662   
  2629   2663     assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
  2630         -  if( pOta && pOta->eStage==OTA_STAGE_OAL ){
         2664  +  if( pOta && (pOta->eStage==OTA_STAGE_OAL || pOta->eStage==OTA_STAGE_MOVE) ){
  2631   2665       /* Magic number 1 is the WAL_CKPT_LOCK lock. Preventing SQLite from
  2632   2666       ** taking this lock also prevents any checkpoints from occurring. 
  2633   2667       ** todo: really, it's not clear why this might occur, as 
  2634   2668       ** wal_autocheckpoint ought to be turned off.  */
  2635   2669       if( ofst==WAL_LOCK_CKPT && n==1 ) rc = SQLITE_BUSY;
  2636   2670     }else{
  2637   2671       int bCapture = 0;
  2638   2672       if( n==1 && (flags & SQLITE_SHM_EXCLUSIVE)
  2639         -     && p->pOta && p->pOta->eStage==OTA_STAGE_CAPTURE
         2673  +     && pOta && pOta->eStage==OTA_STAGE_CAPTURE
  2640   2674        && (ofst==WAL_LOCK_WRITE || ofst==WAL_LOCK_CKPT || ofst==WAL_LOCK_READ0)
  2641   2675       ){
  2642   2676         bCapture = 1;
  2643   2677       }
  2644   2678   
  2645   2679       if( bCapture==0 || 0==(flags & SQLITE_SHM_UNLOCK) ){
  2646   2680         rc = p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags);
  2647   2681         if( bCapture && rc==SQLITE_OK ){
  2648         -        p->pOta->mLock |= (1 << ofst);
         2682  +        pOta->mLock |= (1 << ofst);
  2649   2683         }
  2650   2684       }
  2651   2685     }
  2652   2686   
  2653   2687     return rc;
  2654   2688   }
  2655   2689   
................................................................................
  2658   2692     int iRegion, 
  2659   2693     int szRegion, 
  2660   2694     int isWrite, 
  2661   2695     void volatile **pp
  2662   2696   ){
  2663   2697     ota_file *p = (ota_file*)pFile;
  2664   2698     int rc = SQLITE_OK;
         2699  +  int eStage = (p->pOta ? p->pOta->eStage : 0);
  2665   2700   
  2666   2701     /* If not in OTA_STAGE_OAL, allow this call to pass through. Or, if this
  2667   2702     ** ota is in the OTA_STAGE_OAL state, use heap memory for *-shm space 
  2668   2703     ** instead of a file on disk.  */
  2669   2704     assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
  2670         -  if( p->pOta && p->pOta->eStage==OTA_STAGE_OAL ){
         2705  +  if( eStage==OTA_STAGE_OAL || eStage==OTA_STAGE_MOVE ){
  2671   2706       if( iRegion<=p->nShm ){
  2672   2707         int nByte = (iRegion+1) * sizeof(char*);
  2673   2708         char **apNew = (char**)sqlite3_realloc(p->apShm, nByte);
  2674   2709         if( apNew==0 ){
  2675   2710           rc = SQLITE_NOMEM;
  2676   2711         }else{
  2677   2712           memset(&apNew[p->nShm], 0, sizeof(char*) * (1 + iRegion - p->nShm));
................................................................................
  2710   2745     ota_file *p = (ota_file *)pFile;
  2711   2746     p->pReal->pMethods->xShmBarrier(p->pReal);
  2712   2747   }
  2713   2748   
  2714   2749   static int otaVfsShmUnmap(sqlite3_file *pFile, int delFlag){
  2715   2750     ota_file *p = (ota_file*)pFile;
  2716   2751     int rc = SQLITE_OK;
         2752  +  int eStage = (p->pOta ? p->pOta->eStage : 0);
  2717   2753   
  2718   2754     assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
  2719         -  if( p->pOta && p->pOta->eStage==OTA_STAGE_OAL ){
         2755  +  if( eStage==OTA_STAGE_OAL || eStage==OTA_STAGE_MOVE ){
  2720   2756       /* no-op */
  2721   2757     }else{
  2722   2758       rc = p->pReal->pMethods->xShmUnmap(p->pReal, delFlag);
  2723   2759     }
  2724   2760     return rc;
  2725   2761   }
  2726   2762