SQLite

proc run_ota {target ota} {
  sqlite3ota ota $target $ota
  while { [ota step]=="SQLITE_OK" } {}
  ota close
}

forcedelete test.db-oal ota.db
db close
sqlite3_shutdown
sqlite3_config_uri 1
reset_db

#--------------------------------------------------------------------
# Test that for an OTA to be applied, no corruption results if the
# affinities on the source and target table do not match.
#
do_execsql_test 1.0 {
  CREATE TABLE x1(a INTEGER PRIMARY KEY, b TEXT, c REAL);

  forcedelete ota.db
  sqlite3 db2 ota.db
  db2 eval {
    CREATE TABLE data_x2(a, b, c, d, ota_control);
    INSERT INTO data_x2 VALUES(1, 'a', 2, 3, 0);
  }
  db2 close

  list [catch { run_ota test.db ota.db } msg] $msg
} {1 SQLITE_ERROR}

do_execsql_test 2.5 {
  PRAGMA integrity_check;
} {ok}

do_test 3.2 {
  sqlite3ota_destroy_vfs nosuchvfs
  sqlite3ota_destroy_vfs unix
  sqlite3ota_destroy_vfs win32
} {}

#-------------------------------------------------------------------------
# Test that it is an error to specify an explicit VFS that does not 
# include ota VFS functionality.
#
do_test 4.1 {
  testvfs tvfs
  sqlite3ota ota file:test.db?vfs=tvfs ota.db 
  list [catch { ota step } msg] $msg
} {0 SQLITE_ERROR}
do_test 4.2 {
  list [catch { ota close } msg] $msg
} {1 {SQLITE_ERROR - ota vfs not found}}
tvfs delete

finish_test

  ota close
}

# Test the outcome of some other client writing the db while the *-oal 
# file is being generated. Once this has happened, the update cannot be
# progressed.
#
for {set nStep 1} {$nStep < 8} {incr nStep} {
  do_test 1.$nStep.1 {
    setup_test
    sqlite3ota ota test.db ota.db
    for {set i 0} {$i<$nStep} {incr i} {ota step}

    ota close
    sqlite3 db test.db

    list [file exists test.db-oal] [file exists test.db-wal]
  } {1 0}
  do_test 1.$nStep.4 {
    list [catch { ota close } msg] $msg
  } {1 {SQLITE_BUSY - database modified during ota update}}
}


# Test the outcome of some other client writing the db after the *-oal
# file has been copied to the *-wal path. Once this has happened, any
# other client writing to the db causes OTA to consider its job finished.
#
for {set nStep 8} {$nStep < 20} {incr nStep} {
  do_test 1.$nStep.1 {
    setup_test
    sqlite3ota ota test.db ota.db
    for {set i 0} {$i<$nStep} {incr i} {ota step}
    ota close
    sqlite3 db test.db
    execsql { INSERT INTO t1 VALUES(5, 'hello') }

    INSERT INTO t1 VALUES(2, 2, 2);
    INSERT INTO t1 VALUES(3, 3, 3);

    CREATE TABLE ota.data_t1(a, b, c, ota_control);
    INSERT INTO data_t1 VALUES(2, NULL, NULL, 1);
    INSERT INTO data_t1 VALUES(3, 'three', NULL, '.x.');
    INSERT INTO data_t1 VALUES(4, 4, 4, 0);
  } {
    SELECT * FROM t1
  } {1 1 1   3 three 3   4 4 4}

  2 {
    CREATE TABLE t2(a PRIMARY KEY, b, c) WITHOUT ROWID;
    CREATE INDEX t2cb ON t2(c, b);
    INSERT INTO t2 VALUES('a', 'a', 'a');
    INSERT INTO t2 VALUES('b', 'b', 'b');
    INSERT INTO t2 VALUES('c', 'c', 'c');

    CREATE TABLE ota.data_t2(a, b, c, ota_control);
    INSERT INTO data_t2 VALUES('b', NULL, NULL, 1);
    INSERT INTO data_t2 VALUES('c', 'see', NULL, '.x.');
    INSERT INTO data_t2 VALUES('d', 'd', 'd', 0);
  } {
    SELECT * FROM t2
  } {a a a   c see c     d d d}

  3 {
    CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
    CREATE TABLE t2(a PRIMARY KEY, b, c) WITHOUT ROWID;
    CREATE INDEX t1cb ON t1(c, b);
    CREATE INDEX t2cb ON t2(c, b);

    CREATE TABLE ota.data_t1(a, b, c, ota_control);
    CREATE TABLE ota.data_t2(a, b, c, ota_control);
    INSERT INTO data_t1 VALUES(1, 2, 3, 0);
    INSERT INTO data_t2 VALUES(4, 5, 6, 0);
  } {
    SELECT * FROM t1 UNION ALL SELECT * FROM t2
  } {1 2 3 4 5 6}

  4 {
    CREATE TABLE t1(a PRIMARY KEY, b, c);
    CREATE INDEX t1c ON t1(c);
    INSERT INTO t1 VALUES('A', 'B', 'C');
    INSERT INTO t1 VALUES('D', 'E', 'F');

    CREATE TABLE ota.data_t1(a, b, c, ota_control);
    INSERT INTO data_t1 VALUES('D', NULL, NULL, 1);
    INSERT INTO data_t1 VALUES('A', 'Z', NULL, '.x.');
    INSERT INTO data_t1 VALUES('G', 'H', 'I', 0);
  } {
    SELECT * FROM t1 ORDER BY a;
  } {A Z C G H I}

} {
  catch {db close}
  forcedelete ota.db test.db
  sqlite3 db test.db
  execsql {
    PRAGMA encoding = utf16;

    catch {db close}
    sqlite3_shutdown
    sqlite3_config_lookaside {*}$lookaside_config
    sqlite3_initialize
    autoinstall_test_functions




  for {set iStep 0} {$iStep<=21} {incr iStep} {
  
    forcedelete test.db-journal test.db-wal ota.db-journal ota.db-wal
  
    copy_if_exists test.db.bak test.db
    copy_if_exists ota.db.bak ota.db
  
    sqlite3ota ota test.db ota.db
    for {set x 0} {$x < $::iStep} {incr x} { ota step }
    ota close
  
# sqlite3 x ota.db ; puts "XYZ [x eval { SELECT * FROM ota_state } ]" ; x close
  
    copy_if_exists test.db test.db.bak.2
    copy_if_exists test.db-wal test.db.bak.2-wal
    copy_if_exists test.db-oal test.db.bak.2-oal
    copy_if_exists ota.db ota.db.bak.2
  
    do_faultsim_test 3.$tn.$iStep -faults $::f -prep {
      catch { db close }
      forcedelete test.db-journal test.db-wal ota.db-journal ota.db-wal
      copy_if_exists test.db.bak.2 test.db
      copy_if_exists test.db.bak.2-wal test.db-wal
      copy_if_exists test.db.bak.2-oal test.db-oal
      copy_if_exists ota.db.bak.2  ota.db
    } -body {
      sqlite3ota ota test.db ota.db
        ota step
      ota close
    } -test {

        if {$testresult=="SQLITE_OK"} {set testresult "SQLITE_DONE"}
      faultsim_test_result {*}$::reslist
    
      if {$testrc==0} {
          # No error occurred. If the OTA has not already been fully applied,
          # apply the rest of it now. Then ensure that the final state of the
          # target db is as expected. And that "PRAGMA integrity_check"
          # passes.
          sqlite3ota ota test.db ota.db
          while {[ota step] == "SQLITE_OK"} {}
          ota close

        sqlite3 db test.db
        faultsim_integrity_check

        set res [db eval $::sql]
        if {$res != [list {*}$::expect]} {
          puts ""
          puts "res: $res"
            puts "exp: $::expect"
          error "data not as expected!"
        }
      }
    }
  }
}
}

finish_test

#define OTA_STATE_IDX         3
#define OTA_STATE_ROW         4
#define OTA_STATE_PROGRESS    5
#define OTA_STATE_CKPT        6
#define OTA_STATE_COOKIE      7

#define OTA_STAGE_OAL         1
#define OTA_STAGE_MOVE        2
#define OTA_STAGE_CAPTURE     3
#define OTA_STAGE_CKPT        4
#define OTA_STAGE_DONE        5


#define OTA_CREATE_STATE "CREATE TABLE IF NOT EXISTS ota.ota_state"        \
                             "(k INTEGER PRIMARY KEY, v)"

typedef struct OtaState OtaState;
typedef struct OtaObjIter OtaObjIter;

        p->rc = SQLITE_NOMEM;
        break;
      }
    }

    /* For a table with implicit rowids, append "old._rowid_" to the list. */
    if( pIter->eType==OTA_PK_EXTERNAL || pIter->eType==OTA_PK_NONE ){
      zList = otaMPrintf(p, "%z, %s._rowid_", zList, zObj);
    }
  }
  return zList;
}

/*
** Return an expression that can be used in a WHERE clause to match the

    }
    z = otaMPrintf(p, "%z)", z);
    rc = sqlite3_finalize(pXInfo);
    if( p->rc==SQLITE_OK ) p->rc = rc;
  }
  return z;
}

static void otaFinalize(sqlite3ota *p, sqlite3_stmt *pStmt){
  int rc = sqlite3_finalize(pStmt);
  if( p->rc==SQLITE_OK ) p->rc = rc;
}

/*
** This function creates the second imposter table used when writing to
** a table b-tree where the table has an external primary key. If the
** iterator passed as the second argument does not currently point to
** a table (not index) with an external primary key, this function is a
** no-op. 

        "SELECT name FROM sqlite_master WHERE rootpage = ?"
    );
    if( p->rc==SQLITE_OK ){
      sqlite3_bind_int(pQuery, 1, tnum);
      if( SQLITE_ROW==sqlite3_step(pQuery) ){
        zIdx = (const char*)sqlite3_column_text(pQuery, 0);
      }


      }


    if( zIdx ){

      p->rc = prepareFreeAndCollectError(p->db, &pXInfo, &p->zErrmsg,
          sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx)
      );
    }
    otaFinalize(p, pQuery);

    while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
      int bKey = sqlite3_column_int(pXInfo, 5);
      if( bKey ){
        int iCid = sqlite3_column_int(pXInfo, 1);
        int bDesc = sqlite3_column_int(pXInfo, 3);
        const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4);

    if( p->rc ) return;

    iOff = (i64)(pFrame->iDbPage-1) * p->pgsz;
    p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff);
  }
}


/*
** Take an EXCLUSIVE lock on the database file.
*/
static void otaLockDatabase(sqlite3ota *p){
  if( p->rc==SQLITE_OK ){
    sqlite3_file *pReal = p->pTargetFd->pReal;
    p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_SHARED);
    if( p->rc==SQLITE_OK ){
      p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_EXCLUSIVE);
    }
  }
}

/*
** The OTA handle is currently in OTA_STAGE_OAL state, with a SHARED lock
** on the database file. This proc moves the *-oal file to the *-wal path,
** then reopens the database file (this time in vanilla, non-oal, WAL mode).
** If an error occurs, leave an error code and error message in the ota 
** handle.
*/
static void otaMoveOalFile(sqlite3ota *p){
  const char *zBase = sqlite3_db_filename(p->db, "main");

  char *zWal = sqlite3_mprintf("%s-wal", zBase);
  char *zOal = sqlite3_mprintf("%s-oal", zBase);

  assert( p->eStage==OTA_STAGE_MOVE );
  assert( p->rc==SQLITE_OK && p->zErrmsg==0 );
  if( zWal==0 || zOal==0 ){
    p->rc = SQLITE_NOMEM;
  }else{
    /* Move the *-oal file to *-wal. At this point connection p->db is
    ** holding a SHARED lock on the target database file (because it is
    ** in WAL mode). So no other connection may be writing the db. 
    **
    ** In order to ensure that there are no database readers, an EXCLUSIVE
    ** lock is obtained here before the *-oal is moved to *-wal.
    */
    otaLockDatabase(p);
    if( p->rc==SQLITE_OK ){
    otaFileSuffix3(zBase, zWal);
    otaFileSuffix3(zBase, zOal);
    rename(zOal, zWal);

    /* Re-open the databases. */
    otaObjIterFinalize(&p->objiter);
    sqlite3_close(p->db);
    p->db = 0;
    otaOpenDatabase(p);
    otaSetupCheckpoint(p, 0);
  }
  }

  sqlite3_free(zWal);
  sqlite3_free(zOal);
}

/*
** The SELECT statement iterating through the keys for the current object

  return p->rc;
}

/*
** Increment the schema cookie of the main database opened by p->db.
*/
static void otaIncrSchemaCookie(sqlite3ota *p){
  if( p->rc==SQLITE_OK ){
  int iCookie = 1000000;
  sqlite3_stmt *pStmt;


  p->rc = prepareAndCollectError(p->db, &pStmt, &p->zErrmsg, 
      "PRAGMA schema_version"
  );
  if( p->rc==SQLITE_OK ){
    if( SQLITE_ROW==sqlite3_step(pStmt) ){
      iCookie = sqlite3_column_int(pStmt, 0);
    }
    p->rc = sqlite3_finalize(pStmt);
  }
  if( p->rc==SQLITE_OK ){
    otaMPrintfExec(p, "PRAGMA schema_version = %d", iCookie+1);
  }
}
}

static void otaSaveState(sqlite3ota *p, int eStage){
  if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){
    sqlite3_stmt *pInsert = 0;
    int rc;

    assert( p->zErrmsg==0 );
    rc = prepareFreeAndCollectError(p->db, &pInsert, &p->zErrmsg, 
        sqlite3_mprintf(
          "INSERT OR REPLACE INTO ota.ota_state(k, v) VALUES "
          "(%d, %d), "
          "(%d, %Q), "
          "(%d, %Q), "
          "(%d, %d), "
          "(%d, %lld), "
          "(%d, %lld), "
          "(%d, %lld) ",
          OTA_STATE_STAGE, eStage,
          OTA_STATE_TBL, p->objiter.zTbl, 
          OTA_STATE_IDX, p->objiter.zIdx, 
          OTA_STATE_ROW, p->nStep, 
          OTA_STATE_PROGRESS, p->nProgress,
          OTA_STATE_CKPT, p->iWalCksum,
          OTA_STATE_COOKIE, (i64)p->pTargetFd->iCookie
          )
        );
    assert( pInsert==0 || rc==SQLITE_OK );

    if( rc==SQLITE_OK ){
      sqlite3_step(pInsert);
      rc = sqlite3_finalize(pInsert);
    }else{
      sqlite3_finalize(pInsert);
    }

    if( rc!=SQLITE_OK ){
      p->rc = rc;
    }
  }
}


/*
** Step the OTA object.
*/
int sqlite3ota_step(sqlite3ota *p){
  if( p ){
    switch( p->eStage ){

            }
          }

          otaObjIterNext(p, pIter);
        }

        if( p->rc==SQLITE_OK && pIter->zTbl==0 ){
          otaSaveState(p, OTA_STAGE_MOVE);
          otaIncrSchemaCookie(p);
          if( p->rc==SQLITE_OK ){
            p->rc = sqlite3_exec(p->db, "COMMIT", 0, 0, &p->zErrmsg);
          }
          p->eStage = OTA_STAGE_MOVE;
        }
        break;
      }

      case OTA_STAGE_MOVE: {
          if( p->rc==SQLITE_OK ){
            otaMoveOalFile(p);

          p->nProgress++;
        }
        break;
      }

      case OTA_STAGE_CKPT: {
        if( p->nStep>=p->nFrame ){
          sqlite3_file *pDb = p->pTargetFd->pReal;

    }
    return p->rc;
  }else{
    return SQLITE_NOMEM;
  }
}







































static void otaFreeState(OtaState *p){
  if( p ){
    sqlite3_free(p->zTbl);
    sqlite3_free(p->zIdx);
    sqlite3_free(p);
  }
}

  rc = prepareAndCollectError(p->db, &pStmt, &p->zErrmsg, zSelect);
  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
    switch( sqlite3_column_int(pStmt, 0) ){
      case OTA_STATE_STAGE:
        pRet->eStage = sqlite3_column_int(pStmt, 1);
        if( pRet->eStage!=OTA_STAGE_OAL
         && pRet->eStage!=OTA_STAGE_MOVE
         && pRet->eStage!=OTA_STAGE_CKPT
        ){
          p->rc = SQLITE_CORRUPT;
        }
        break;

      case OTA_STATE_TBL:

          p->eStage = pState->eStage;
        }
        p->nProgress = pState->nProgress;
      }
    }
    assert( p->rc!=SQLITE_OK || p->eStage!=0 );

    if( p->rc==SQLITE_OK 
     && (p->eStage==OTA_STAGE_OAL || p->eStage==OTA_STAGE_MOVE)
    ){   
        /* Check that this is not a wal mode database. If it is, it cannot 
        ** be updated.  */
        if( p->pTargetFd->pWalFd ){
          p->rc = SQLITE_ERROR;
          p->zErrmsg = sqlite3_mprintf("cannot update wal mode database");
        }

        /* At this point (pTargetFd->iCookie) contains the value of the
        ** change-counter cookie (the thing that gets incremented when a 
        ** transaction is committed in rollback mode) currently stored on 
        ** page 1 of the database file. */

      else if( pState->eStage!=0 && p->pTargetFd->iCookie!=pState->iCookie ){

          p->rc = SQLITE_BUSY;
          p->zErrmsg = sqlite3_mprintf("database modified during ota update");
        }
    }

    if( p->rc==SQLITE_OK ){
      if( p->eStage==OTA_STAGE_OAL ){

        /* Open the transaction */
        if( p->rc==SQLITE_OK ){
          p->rc = sqlite3_exec(p->db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
        }
  
        /* Point the object iterator at the first object */
        if( p->rc==SQLITE_OK ){
          p->rc = otaObjIterFirst(p, &p->objiter);
        }
  
        if( p->rc==SQLITE_OK ){
          otaLoadTransactionState(p, pState);
        }
      }else if( p->eStage==OTA_STAGE_MOVE ){
        /* no-op */
      }else if( p->eStage==OTA_STAGE_CKPT ){
        otaSetupCheckpoint(p, pState);
        p->nStep = pState->nRow;
      }else if( p->eStage==OTA_STAGE_DONE ){
        p->rc = SQLITE_DONE;
      }else{
        p->rc = SQLITE_CORRUPT;

  int rc;
  if( p ){

    /* If the update has not been fully applied, save the state in 
    ** the ota db. If successful, this call also commits the open 
    ** transaction on the ota db. */
    assert( p->rc!=SQLITE_ROW );


    otaSaveState(p, p->eStage);


    /* Close any open statement handles. */
    otaObjIterFinalize(&p->objiter);

    /* Commit the transaction to the *-oal file. */
    if( p->rc==SQLITE_OK && p->eStage==OTA_STAGE_OAL ){
      p->rc = sqlite3_exec(p->db, "COMMIT", 0, 0, &p->zErrmsg);

  int rc = SQLITE_OK;

#ifdef SQLITE_AMALGAMATION
    assert( WAL_CKPT_LOCK==1 );
#endif

  assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
  if( pOta && (pOta->eStage==OTA_STAGE_OAL || pOta->eStage==OTA_STAGE_MOVE) ){
    /* Magic number 1 is the WAL_CKPT_LOCK lock. Preventing SQLite from
    ** taking this lock also prevents any checkpoints from occurring. 
    ** todo: really, it's not clear why this might occur, as 
    ** wal_autocheckpoint ought to be turned off.  */
    if( ofst==WAL_LOCK_CKPT && n==1 ) rc = SQLITE_BUSY;
  }else{
    int bCapture = 0;
    if( n==1 && (flags & SQLITE_SHM_EXCLUSIVE)
     && pOta && pOta->eStage==OTA_STAGE_CAPTURE
     && (ofst==WAL_LOCK_WRITE || ofst==WAL_LOCK_CKPT || ofst==WAL_LOCK_READ0)
    ){
      bCapture = 1;
    }

    if( bCapture==0 || 0==(flags & SQLITE_SHM_UNLOCK) ){
      rc = p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags);
      if( bCapture && rc==SQLITE_OK ){
        pOta->mLock |= (1 << ofst);
      }
    }
  }

  return rc;
}

static int otaVfsShmMap(
  sqlite3_file *pFile, 
  int iRegion, 
  int szRegion, 
  int isWrite, 
  void volatile **pp
){
  ota_file *p = (ota_file*)pFile;
  int rc = SQLITE_OK;
  int eStage = (p->pOta ? p->pOta->eStage : 0);

  /* If not in OTA_STAGE_OAL, allow this call to pass through. Or, if this
  ** ota is in the OTA_STAGE_OAL state, use heap memory for *-shm space 
  ** instead of a file on disk.  */
  assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
  if( eStage==OTA_STAGE_OAL || eStage==OTA_STAGE_MOVE ){
    if( iRegion<=p->nShm ){
      int nByte = (iRegion+1) * sizeof(char*);
      char **apNew = (char**)sqlite3_realloc(p->apShm, nByte);
      if( apNew==0 ){
        rc = SQLITE_NOMEM;
      }else{
        memset(&apNew[p->nShm], 0, sizeof(char*) * (1 + iRegion - p->nShm));

  ota_file *p = (ota_file *)pFile;
  p->pReal->pMethods->xShmBarrier(p->pReal);
}

static int otaVfsShmUnmap(sqlite3_file *pFile, int delFlag){
  ota_file *p = (ota_file*)pFile;
  int rc = SQLITE_OK;
  int eStage = (p->pOta ? p->pOta->eStage : 0);

  assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
  if( eStage==OTA_STAGE_OAL || eStage==OTA_STAGE_MOVE ){
    /* no-op */
  }else{
    rc = p->pReal->pMethods->xShmUnmap(p->pReal, delFlag);
  }
  return rc;
}