SQLite

# 2014 August 30
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#

source [file join [file dirname [info script]] rbu_common.tcl]
set ::testprefix rbutemplimit

db close
sqlite3_shutdown
sqlite3_config_uri 1

proc setup_databases {} {
  forcedelete test.db2
  forcedelete test.db
  sqlite3 db test.db
  execsql {
    -- Create target database schema.
    --
    CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB(100), c BLOB(100));
    CREATE TABLE t2(a INTEGER PRIMARY KEY, b BLOB(100), c BLOB(100));
    CREATE INDEX i1b ON t1(b);
    CREATE INDEX i1c ON t1(c);
    CREATE INDEX i2b ON t2(b);
    CREATE INDEX i2c ON t2(c);
  
    -- Create a large RBU database.
    --
    ATTACH 'test.db2' AS rbu;
    CREATE TABLE rbu.data_t1(a, b, c, rbu_control);
    WITH s(i) AS (
      VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<10000
    )
    INSERT INTO data_t1 SELECT i, randomblob(100), randomblob(100), 0 FROM s;
    CREATE TABLE rbu.data_t2(a, b, c, rbu_control);
    WITH s(i) AS (
      VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<15000
    )
    INSERT INTO data_t2 SELECT i, randomblob(100), randomblob(100), 0 FROM s;
  }
  db close
}

proc run_rbu_cachesize {target rbu cachesize temp_limit} {
  sqlite3rbu rbu $target $rbu
  rbu temp_size_limit $temp_limit
  sqlite3_exec_nr [rbu db 1] "PRAGMA cache_size = $cachesize"
  while 1 {
    set rc [rbu step]
    set ::A([rbu temp_size]) 1
    if {$rc!="SQLITE_OK"} break
  }
  list [catch {rbu close} msg] $msg
}

proc step_rbu_cachesize {target rbu stepsize cachesize temp_limit} {
  set res ""
  while 1 {
    sqlite3rbu rbu $target $rbu
    rbu temp_size_limit $temp_limit
    sqlite3_exec_nr [rbu db 1] "PRAGMA cache_size = $cachesize"
    for {set i 0} {$i < $stepsize} {incr i} {
      set rc [rbu step]
      set ::A([rbu temp_size]) 1
      if {$rc!="SQLITE_OK"} break
    }
    set res [list [catch {rbu close} msg] $msg]
    if {$res != "0 SQLITE_OK"} break
  }
  set res
}

do_test 1.1.0 { setup_databases } {}

do_test 1.1.1 {
  unset -nocomplain ::A
  run_rbu_cachesize test.db test.db2 10 0
} {0 SQLITE_DONE}

do_test 1.1.2 { llength [array names ::A] } 3

do_test 1.1.3 { 
  foreach {a0 a1 a2} [lsort -integer [array names ::A]] {}
  list [expr $a0==0]                         \
       [expr $a1>1048576] [expr $a1<1200000] \
       [expr $a2>1500000] [expr $a2<1700000]
} {1 1 1 1 1}

do_test 1.2.1 {
  setup_databases
  run_rbu_cachesize test.db test.db2 10 1000000
} {1 SQLITE_FULL}
do_test 1.2.2 { info commands rbu } {}

do_test 1.3.1 {
  setup_databases
  run_rbu_cachesize test.db test.db2 10 1300000
} {1 SQLITE_FULL}
do_test 1.3.2 { info commands rbu } {}

do_test 1.4.1 {
  setup_databases
  run_rbu_cachesize test.db test.db2 10 1800000
} {0 SQLITE_DONE}
do_test 1.4.2 { info commands rbu } {}

do_test 1.5.1 {
  setup_databases
  unset -nocomplain ::A
  step_rbu_cachesize test.db test.db2 1000 10 2400000
} {0 SQLITE_DONE}
do_test 1.5.2 { info commands rbu } {}

do_test 1.6.1 {
  setup_databases
  unset -nocomplain ::A
  step_rbu_cachesize test.db test.db2 1000 10 1400000
} {1 SQLITE_FULL}
do_test 1.6.2 { info commands rbu } {}

finish_test

  u32 mLock;
  int nFrame;                     /* Entries in aFrame[] array */
  int nFrameAlloc;                /* Allocated size of aFrame[] array */
  RbuFrame *aFrame;
  int pgsz;
  u8 *aBuf;
  i64 iWalCksum;
  i64 szTemp;                     /* Current size of all temp files in use */
  i64 szTempLimit;                /* Total size limit for temp files */

  /* Used in RBU vacuum mode only */
  int nRbu;                       /* Number of RBU VFS in the stack */
  rbu_file *pRbuFd;               /* Fd for main db of dbRbu */
};

/*
** An rbu VFS is implemented using an instance of this structure.
**
** Variable pRbu is only non-NULL for automatically created RBU VFS objects.
** It is NULL for RBU VFS objects created explicitly using
** sqlite3rbu_create_vfs(). It is used to track the total amount of temp
** space used by the RBU handle.
*/
struct rbu_vfs {
  sqlite3_vfs base;               /* rbu VFS shim methods */
  sqlite3_vfs *pRealVfs;          /* Underlying VFS */
  sqlite3_mutex *mutex;           /* Mutex to protect pMain */
  sqlite3rbu *pRbu;               /* Owner RBU object */
  rbu_file *pMain;                /* Linked list of main db files */
};

/*
** Each file opened by an rbu VFS is represented by an instance of
** the following structure.
**
** If this is a temporary file (pRbu!=0 && flags&DELETE_ON_CLOSE), variable
** "sz" is set to the current size of the database file.
*/
struct rbu_file {
  sqlite3_file base;              /* sqlite3_file methods */
  sqlite3_file *pReal;            /* Underlying file handle */
  rbu_vfs *pRbuVfs;               /* Pointer to the rbu_vfs object */
  sqlite3rbu *pRbu;               /* Pointer to rbu object (rbu target only) */
  i64 sz;                         /* Size of file in bytes (temp only) */

  int openFlags;                  /* Flags this file was opened with */
  u32 iCookie;                    /* Cookie value for main db files */
  u8 iWriteVer;                   /* "write-version" value for main db files */
  u8 bNolock;                     /* True to fail EXCLUSIVE locks */

  int nShm;                       /* Number of entries in apShm[] array */

  sqlite3_randomness(sizeof(int), (void*)&rnd);
  sqlite3_snprintf(sizeof(zRnd), zRnd, "rbu_vfs_%d", rnd);
  p->rc = sqlite3rbu_create_vfs(zRnd, 0);
  if( p->rc==SQLITE_OK ){
    sqlite3_vfs *pVfs = sqlite3_vfs_find(zRnd);
    assert( pVfs );
    p->zVfsName = pVfs->zName;
    ((rbu_vfs*)pVfs)->pRbu = p;
  }
}

/*
** Destroy the private VFS created for the rbu handle passed as the only
** argument by an earlier call to rbuCreateVfs().
*/

      int rc2 = sqlite3_exec(p->dbRbu, "DELETE FROM stat.rbu_state", 0, 0, 0);
      if( p->rc==SQLITE_DONE && rc2!=SQLITE_OK ) p->rc = rc2;
    }

    /* Close the open database handle and VFS object. */
    sqlite3_close(p->dbRbu);
    sqlite3_close(p->dbMain);
    assert( p->szTemp==0 );
    rbuDeleteVfs(p);
    sqlite3_free(p->aBuf);
    sqlite3_free(p->aFrame);

    rbuEditErrmsg(p);
    rc = p->rc;
    if( pzErrmsg ){

**     database file are recorded. xShmLock() calls to unlock the same
**     locks are no-ops (so that once obtained, these locks are never
**     relinquished). Finally, calls to xSync() on the target database
**     file fail with SQLITE_INTERNAL errors.
*/

static void rbuUnlockShm(rbu_file *p){
  assert( p->openFlags & SQLITE_OPEN_MAIN_DB );
  if( p->pRbu ){
    int (*xShmLock)(sqlite3_file*,int,int,int) = p->pReal->pMethods->xShmLock;
    int i;
    for(i=0; i<SQLITE_SHM_NLOCK;i++){
      if( (1<<i) & p->pRbu->mLock ){
        xShmLock(p->pReal, i, 1, SQLITE_SHM_UNLOCK|SQLITE_SHM_EXCLUSIVE);
      }
    }
    p->pRbu->mLock = 0;
  }
}

/*
*/
static int rbuUpdateTempSize(rbu_file *pFd, sqlite3_int64 nNew){
  sqlite3rbu *pRbu = pFd->pRbu;
  i64 nDiff = nNew - pFd->sz;
  pRbu->szTemp += nDiff;
  pFd->sz = nNew;
  assert( pRbu->szTemp>=0 );
  if( pRbu->szTempLimit && pRbu->szTemp>pRbu->szTempLimit ) return SQLITE_FULL;
  return SQLITE_OK;
}

/*
** Close an rbu file.
*/
static int rbuVfsClose(sqlite3_file *pFile){
  rbu_file *p = (rbu_file*)pFile;
  int rc;

    rbu_file **pp;
    sqlite3_mutex_enter(p->pRbuVfs->mutex);
    for(pp=&p->pRbuVfs->pMain; *pp!=p; pp=&((*pp)->pMainNext));
    *pp = p->pMainNext;
    sqlite3_mutex_leave(p->pRbuVfs->mutex);
    rbuUnlockShm(p);
    p->pReal->pMethods->xShmUnmap(p->pReal, 0);
  }
  else if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){
    rbuUpdateTempSize(p, 0);
  }

  /* Close the underlying file handle */
  rc = p->pReal->pMethods->xClose(p->pReal);
  return rc;
}

  sqlite3rbu *pRbu = p->pRbu;
  int rc;

  if( pRbu && pRbu->eStage==RBU_STAGE_CAPTURE ){
    assert( p->openFlags & SQLITE_OPEN_MAIN_DB );
    rc = rbuCaptureDbWrite(p->pRbu, iOfst);
  }else{
    if( pRbu ){
      if( pRbu->eStage==RBU_STAGE_OAL 
       && (p->openFlags & SQLITE_OPEN_WAL) 
       && iOfst>=pRbu->iOalSz
      ){
        pRbu->iOalSz = iAmt + iOfst;
      }else if( p->openFlags & SQLITE_OPEN_DELETEONCLOSE ){
        i64 szNew = iAmt+iOfst;
        if( szNew>p->sz ){
          rc = rbuUpdateTempSize(p, szNew);
          if( rc!=SQLITE_OK ) return rc;
        }
      }
    }
    rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst);
    if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){
      /* These look like magic numbers. But they are stable, as they are part
      ** of the definition of the SQLite file format, which may not change. */
      u8 *pBuf = (u8*)zBuf;
      p->iCookie = rbuGetU32(&pBuf[24]);
      p->iWriteVer = pBuf[19];
    }
  }
  return rc;
}

/*
** Truncate an rbuVfs-file.
*/
static int rbuVfsTruncate(sqlite3_file *pFile, sqlite_int64 size){
  rbu_file *p = (rbu_file*)pFile;
  if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){
    int rc = rbuUpdateTempSize(p, size);
    if( rc!=SQLITE_OK ) return rc;
  }
  return p->pReal->pMethods->xTruncate(p->pReal, size);
}

/*
** Sync an rbuVfs-file.
*/
static int rbuVfsSync(sqlite3_file *pFile, int flags){

            rc = SQLITE_NOMEM;
          }
          pFd->pRbu = pDb->pRbu;
        }
        pDb->pWalFd = pFd;
      }
    }
  }else{
    pFd->pRbu = pRbuVfs->pRbu;
  }

  if( oflags & SQLITE_OPEN_MAIN_DB 
   && sqlite3_uri_boolean(zName, "rbu_memory", 0) 
  ){
    assert( oflags & SQLITE_OPEN_MAIN_DB );
    oflags =  SQLITE_OPEN_TEMP_DB | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE |

      sqlite3_mutex_free(pNew->mutex);
      sqlite3_free(pNew);
    }
  }

  return rc;
}

/*
** Configure the aggregate temp file size limit for this RBU handle.
*/
sqlite3_int64 sqlite3rbu_temp_size_limit(sqlite3rbu *pRbu, sqlite3_int64 n){
  if( n>=0 ){
    pRbu->szTempLimit = n;
  }
  return pRbu->szTempLimit;
}

sqlite3_int64 sqlite3rbu_temp_size(sqlite3rbu *pRbu){
  return pRbu->szTemp;
}


/**************************************************************************/

#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) */

** zipvfs databases.
*/
SQLITE_API sqlite3rbu *sqlite3rbu_vacuum(
  const char *zTarget, 
  const char *zState
);

/*
** Configure a limit for the amount of temp space that may be used by
** the RBU handle passed as the first argument. The new limit is specified
** in bytes by the second parameter. If it is positive, the limit is updated.
** If the second parameter to this function is passed zero, then the limit
** is removed entirely. If the second parameter is negative, the limit is
** not modified (this is useful for querying the current limit).
**
** In all cases the returned value is the current limit in bytes (zero 
** indicates unlimited).
**
** If the temp space limit is exceeded during operation, an SQLITE_FULL
** error is returned.
*/
SQLITE_API sqlite3_int64 sqlite3rbu_temp_size_limit(sqlite3rbu*, sqlite3_int64);

/*
** Return the current amount of temp file space, in bytes, currently used by 
** the RBU handle passed as the only argument.
*/
SQLITE_API sqlite3_int64 sqlite3rbu_temp_size(sqlite3rbu*);

/*
** Internally, each RBU connection uses a separate SQLite database 
** connection to access the target and rbu update databases. This
** API allows the application direct access to these database handles.
**
** The first argument passed to this function must be a valid, open, RBU
** handle. The second argument should be passed zero to access the target

  int ret = TCL_OK;
  sqlite3rbu *pRbu = (sqlite3rbu*)clientData;
  struct RbuCmd {
    const char *zName;
    int nArg;
    const char *zUsage;
  } aCmd[] = {
    {"step", 2, ""},                 /* 0 */
    {"close", 2, ""},                /* 1 */
    {"create_rbu_delta", 2, ""},     /* 2 */
    {"savestate", 2, ""},            /* 3 */
    {"dbMain_eval", 3, "SQL"},       /* 4 */
    {"bp_progress", 2, ""},          /* 5 */
    {"db", 3, "RBU"},                /* 6 */
    {"state", 2, ""},                /* 7 */
    {"progress", 2, ""},             /* 8 */
    {"close_no_error", 2, ""},       /* 9 */
    {"temp_size_limit", 3, "LIMIT"}, /* 10 */
    {"temp_size", 2, ""},            /* 11 */
    {0,0,0}
  };
  int iCmd;

  if( objc<2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "METHOD");
    return TCL_ERROR;

      Tcl_SetResult(interp, (char*)aRes[eState], TCL_STATIC);
      break;
    }
    case 8: /* progress */ {
      sqlite3_int64 nStep =  sqlite3rbu_progress(pRbu);
      Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nStep));
      break;
    }
                           
    case 10: /* temp_size_limit */ {
      sqlite3_int64 nLimit;
      if( Tcl_GetWideIntFromObj(interp, objv[2], &nLimit) ){
        ret = TCL_ERROR;
      }else{
        nLimit = sqlite3rbu_temp_size_limit(pRbu, nLimit);
        Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nLimit));
      }
      break;
    }
    case 11: /* temp_size */ {
      sqlite3_int64 sz = sqlite3rbu_temp_size(pRbu);
      Tcl_SetObjResult(interp, Tcl_NewWideIntObj(sz));
      break;
    }

    default: /* seems unlikely */
      assert( !"cannot happen" );
      break;
  }