SQLite

/*
** 2007 August 22
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
**
** @(#) $Id: journal.c,v 1.2 2007/08/23 08:06:45 danielk1977 Exp $
** @(#) $Id: journal.c,v 1.3 2007/08/24 08:15:54 danielk1977 Exp $
*/

#ifdef SQLITE_ENABLE_ATOMIC_WRITE

/*
** This file implements a special kind of sqlite3_file object used
** by SQLite to create journal files if the atomic-write optimization

  p->pMethod = &JournalFileMethods;
  p->nBuf = nBuf;
  p->flags = flags;
  p->zJournal = zName;
  p->pVfs = pVfs;
  return SQLITE_OK;
}

/*
** If the argument p points to a JournalFile structure, and the underlying
** file has not yet been created, create it now.
*/
int sqlite3JournalCreate(sqlite3_file *p){
  if( p->pMethods!=&JournalFileMethods ){
    return SQLITE_OK;
  }
  return createFile((JournalFile *)p);
}

/* 
** Return the number of bytes required to store a JournalFile that uses vfs
** pVfs to create the underlying on-disk files.
*/
int sqlite3JournalSize(sqlite3_vfs *pVfs){
  return (pVfs->szOsFile+sizeof(JournalFile));
}
#endif

** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.372 2007/08/24 03:51:34 drh Exp $
** @(#) $Id: pager.c,v 1.373 2007/08/24 08:15:54 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include <assert.h>
#include <string.h>

/*

  pagerEnter(pPager);

  /* If this is an in-memory db, or no pages have been written to, or this
  ** function has already been called, it is a no-op.
  */
  if( pPager->state!=PAGER_SYNCED && !MEMDB && pPager->dirtyCache ){
    PgHdr *pPg;
    assert( pPager->journalOpen );

#ifdef SQLITE_ENABLE_ATOMIC_WRITE
    /* The atomic-write optimization can be used if all of the
    ** following are true:
    **
    **    + The file-system supports the atomic-write property for
    **      blocks of size page-size, and
    **    + This commit is not part of a multi-file transaction, and
    **    + Exactly one page has been modified and store in the journal file.
    **
    ** If the optimization can be used, then the journal file will never
    ** be created for this transaction.
    */
    int useAtomicWrite = (
        !zMaster && 
    if( !zMaster && pPager->journalOff==jrnlBufferSize(pPager) && nTrunc==0
      && (0==pPager->pDirty || 0==pPager->pDirty->pDirty)
    ){
        pPager->journalOff==jrnlBufferSize(pPager) && 
        nTrunc==0 && 
        (0==pPager->pDirty || 0==pPager->pDirty->pDirty)
    );
    if( useAtomicWrite ){
      /* Update the nRec field in the journal file. */
      int offset = pPager->journalHdr + sizeof(aJournalMagic);
      assert(pPager->nRec==1);
      rc = write32bits(pPager->jfd, offset, pPager->nRec);

      /* Update the db file change counter. The following call will modify
      ** the in-memory representation of page 1 to include the updated
      ** change counter and then write page 1 directly to the database
      ** file. Because of the atomic-write property of the host file-system, 
      ** this is safe.
      */
      rc = pager_incr_changecounter(pPager, 1);
    }else 
    }else{
      rc = sqlite3JournalCreate(pPager->jfd);
      if( rc!=SQLITE_OK ) goto sync_exit;
    }

    if( !useAtomicWrite )
#endif

    /* If a master journal file name has already been written to the
    ** journal file, then no sync is required. This happens when it is
    ** written, then the process fails to upgrade from a RESERVED to an
    ** EXCLUSIVE lock. The next time the process tries to commit the
    ** transaction the m-j name will have already been written.
    */
    if( !pPager->setMaster ){
      assert( pPager->journalOpen );
      rc = pager_incr_changecounter(pPager, 0);
      if( rc!=SQLITE_OK ) goto sync_exit;
#ifndef SQLITE_OMIT_AUTOVACUUM
      if( nTrunc!=0 ){
        /* If this transaction has made the database smaller, then all pages
        ** being discarded by the truncation must be written to the journal
        ** file.

/*
** 2001 September 15
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** Internal interface definitions for SQLite.
**
** @(#) $Id: sqliteInt.h,v 1.598 2007/08/23 02:47:53 drh Exp $
** @(#) $Id: sqliteInt.h,v 1.599 2007/08/24 08:15:54 danielk1977 Exp $
*/
#ifndef _SQLITEINT_H_
#define _SQLITEINT_H_
#include "sqliteLimit.h"


#if defined(SQLITE_TCL) || defined(TCLSH)

int sqlite3Reprepare(Vdbe*);
void sqlite3ExprListCheckLength(Parse*, ExprList*, int, const char*);
CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *);

#ifdef SQLITE_ENABLE_ATOMIC_WRITE
  int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int);
  int sqlite3JournalSize(sqlite3_vfs *);
  int sqlite3JournalCreate(sqlite3_file *);
#else
  #define sqlite3JournalSize(pVfs) ((pVfs)->szOsFile)
#endif

#if SQLITE_MAX_EXPR_DEPTH>0
  void sqlite3ExprSetHeight(Expr *);
  int sqlite3SelectExprHeight(Select *);

**
******************************************************************************
**
** This file contains code that modified the OS layer in order to simulate
** the effect on the database file of an OS crash or power failure.  This
** is used to test the ability of SQLite to recover from those situations.
*/
#if SQLITE_TEST          /* This file is used for the testing only */
#if SQLITE_TEST          /* This file is used for testing only */
#include "sqliteInt.h"
#include "tcl.h"

#ifndef SQLITE_OMIT_DISKIO  /* This file is a no-op if disk I/O is disabled */

/* #define TRACE_CRASHTEST */

/*
** Flush the write-list as if xSync() had been called on file handle
** pFile. If isCrash is true, simulate a crash.
*/
static int writeListSync(CrashFile *pFile, int isCrash){
  int rc = SQLITE_OK;
  int iDc = g.iDeviceCharacteristics;
  i64 iSize;

  WriteBuffer *pWrite;
  WriteBuffer **ppPtr;

  /* If this is not a crash simulation, set pFinal to point to the 
  /* Set pFinal to point to the last element of the write-list that
  ** is associated with file handle pFile.
  ** last element of the write-list that is associated with file handle
  ** pFile.
  **
  ** If this is a crash simulation, set pFinal to an arbitrarily selected
  ** element of the write-list.
  */
  WriteBuffer *pFinal = 0;
  if( !isCrash ){
    for(pWrite=g.pWriteList; pWrite; pWrite=pWrite->pNext){
      if( pWrite->pFile==pFile ){
        pFinal = pWrite;
      }
    }
  }else if( iDc&(SQLITE_IOCAP_SEQUENTIAL|SQLITE_IOCAP_SAFE_APPEND) ){
    int nWrite = 0;
    int iFinal;
    for(pWrite=g.pWriteList; pWrite; pWrite=pWrite->pNext) nWrite++;
    sqlite3Randomness(sizeof(int), &iFinal);
    iFinal = ((iFinal<0)?-1*iFinal:iFinal)%nWrite;
    for(pWrite=g.pWriteList; iFinal>0; pWrite=pWrite->pNext) iFinal--;
    pFinal = pWrite;
  }

#ifdef TRACE_CRASHTEST
  printf("Sync %s (is %s crash)\n", pFile->zName, (isCrash?"a":"not a"));
#endif

  sqlite3OsFileSize((sqlite3_file *)pFile, &iSize);

  ppPtr = &g.pWriteList;
  for(pWrite=*ppPtr; rc==SQLITE_OK && pWrite; pWrite=*ppPtr){
    sqlite3_file *pRealFile = pWrite->pFile->pRealFile;

    /* (eAction==1)      -> write block out normally,
    ** (eAction==2)      -> do nothing,
    ** (eAction==3)      -> trash sectors.
    */
    int eAction = 0;
    if( !isCrash ){
      eAction = 2;
      if( (pWrite->pFile==pFile || iDc&SQLITE_IOCAP_SEQUENTIAL) ){
        eAction = 1;
      }
    }else{
      char random;
      sqlite3Randomness(1, &random);

      /* Do not select option 3 (sector trashing) if the IOCAP_ATOMIC flag 
      ** is set or this is an OsTruncate(), not an Oswrite().
      */
      if( iDc&SQLITE_IOCAP_ATOMIC || pWrite->zBuf==0 ){
      if( (iDc&SQLITE_IOCAP_ATOMIC) || (pWrite->zBuf==0) ){
        random &= 0x01;
      }

      /* If IOCAP_SEQUENTIAL is set and this is not the final entry
      ** in the truncated write-list, always select option 1 (write
      ** out correctly).
      */
      if( (iDc&SQLITE_IOCAP_SEQUENTIAL && pWrite!=pFinal) ){
        random = 0;
      }

      /* If IOCAP_SAFE_APPEND is set and this OsWrite() operation is
      ** an append (first byte of the written region is 1 byte past the
      ** current EOF), always select option 1 (write out correctly).
      */
      if( iDc&SQLITE_IOCAP_SAFE_APPEND && pWrite->zBuf ){
        i64 iSize;
        sqlite3OsFileSize(pRealFile, &iSize);
        if( iSize==pWrite->iOffset ){
          random = 0;
        }
      }

      if( (random&0x06)==0x06 ){
        eAction = 3;
      }else{
        eAction = ((random&0x01)?2:1);
      }
    }

    switch( eAction ){
      case 1: {               /* Write out correctly */
        if( pWrite->zBuf ){
          rc = sqlite3OsWrite(
              pRealFile, pWrite->zBuf, pWrite->nBuf, pWrite->iOffset
          );
        }else{
          rc = sqlite3OsTruncate(pRealFile, pWrite->iOffset);
        }
        *ppPtr = pWrite->pNext;
#ifdef TRACE_CRASHTEST
        if( isCrash ){
          printf("Writing %d bytes @ %d\n", pWrite->nBuf, (int)pWrite->iOffset);
          printf("Writing %d bytes @ %d (%s)\n", 
            pWrite->nBuf, (int)pWrite->iOffset, pWrite->pFile->zName
          );
        }
#endif
        sqlite3_free(pWrite);
        break;
      }
      case 2: {               /* Do nothing */
        ppPtr = &pWrite->pNext;
#ifdef TRACE_CRASHTEST
        if( isCrash ){
          printf("Omiting %d bytes @ %d\n", pWrite->nBuf, (int)pWrite->iOffset);
          printf("Omiting %d bytes @ %d (%s)\n", 
            pWrite->nBuf, (int)pWrite->iOffset, pWrite->pFile->zName
          );
        }
#endif
        break;
      }
      case 3: {               /* Trash sectors */
        u8 *zGarbage;
        int iFirst = (pWrite->iOffset/g.iSectorSize);
        int iLast = (pWrite->iOffset+pWrite->nBuf-1)/g.iSectorSize;

        assert(pWrite->zBuf);

#ifdef TRACE_CRASHTEST
        printf("Trashing %d sectors @ sector %d\n", 1+iLast-iFirst, iFirst);
        printf("Trashing %d sectors @ sector %d (%s)\n", 
            1+iLast-iFirst, iFirst, pWrite->pFile->zName
        );
#endif

        zGarbage = sqlite3_malloc(g.iSectorSize);
        if( zGarbage ){
          sqlite3_int64 i;
          for(i=iFirst; rc==SQLITE_OK && i<=iLast; i++){
            sqlite3Randomness(g.iSectorSize, zGarbage); 

** The caller will have allocated pVfs->szOsFile bytes of space
** at pFile. This file uses this space for the CrashFile structure
** and allocates space for the "real" file structure using 
** sqlite3_malloc(). The assumption here is (pVfs->szOsFile) is
** equal or greater than sizeof(CrashFile).
*/
static int cfOpen(
  void *pAppData,
  sqlite3_vfs *pCfVfs,
  const char *zName,
  sqlite3_file *pFile,
  int flags,
  int *pOutFlags
){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pAppData;
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData;
  int rc;
  CrashFile *pWrapper = (CrashFile *)pFile;
  sqlite3_file *pReal = (sqlite3_file*)&pWrapper[1];

  memset(pWrapper, 0, sizeof(CrashFile));
  rc = sqlite3OsOpen(pVfs, zName, pReal, flags, pOutFlags);

  }
  if( rc!=SQLITE_OK && pWrapper->pMethod ){
    sqlite3OsClose(pFile);
  }
  return rc;
}

static int cfDelete(void *pAppData, const char *zPath, int dirSync){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pAppData;
  return pVfs->xDelete(pVfs->pAppData, zPath, dirSync);
static int cfDelete(sqlite3_vfs *pCfVfs, const char *zPath, int dirSync){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData;
  return pVfs->xDelete(pVfs, zPath, dirSync);
}
static int cfAccess(void *pAppData, const char *zPath, int flags){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pAppData;
  return pVfs->xAccess(pVfs->pAppData, zPath, flags);
static int cfAccess(sqlite3_vfs *pCfVfs, const char *zPath, int flags){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData;
  return pVfs->xAccess(pVfs, zPath, flags);
}
static int cfGetTempName(void *pAppData, char *zBufOut){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pAppData;
  return pVfs->xGetTempName(pVfs->pAppData, zBufOut);
static int cfGetTempName(sqlite3_vfs *pCfVfs, char *zBufOut){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData;
  return pVfs->xGetTempName(pVfs, zBufOut);
}
static int cfFullPathname(void *pAppData, const char *zPath, char *zPathOut){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pAppData;
  return pVfs->xFullPathname(pVfs->pAppData, zPath, zPathOut);
static int cfFullPathname(sqlite3_vfs *pCfVfs, const char *zPath, char *zPathOut){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData;
  return pVfs->xFullPathname(pVfs, zPath, zPathOut);
}
static void *cfDlOpen(void *pAppData, const char *zPath){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pAppData;
  return pVfs->xDlOpen(pVfs->pAppData, zPath);
static void *cfDlOpen(sqlite3_vfs *pCfVfs, const char *zPath){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData;
  return pVfs->xDlOpen(pVfs, zPath);
}
static int cfRandomness(void *pAppData, int nByte, char *zBufOut){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pAppData;
  return pVfs->xRandomness(pVfs->pAppData, nByte, zBufOut);
static int cfRandomness(sqlite3_vfs *pCfVfs, int nByte, char *zBufOut){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData;
  return pVfs->xRandomness(pVfs, nByte, zBufOut);
}
static int cfSleep(void *pAppData, int nMicro){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pAppData;
  return pVfs->xSleep(pVfs->pAppData, nMicro);
static int cfSleep(sqlite3_vfs *pCfVfs, int nMicro){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData;
  return pVfs->xSleep(pVfs, nMicro);
}
static int cfCurrentTime(void *pAppData, double *pTimeOut){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pAppData;
  return pVfs->xCurrentTime(pVfs->pAppData, pTimeOut);
static int cfCurrentTime(sqlite3_vfs *pCfVfs, double *pTimeOut){
  sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData;
  return pVfs->xCurrentTime(pVfs, pTimeOut);
}

static int processDevSymArgs(
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[],
  int *piDeviceChar,

# 2007 August 23
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains tests that verify that SQLite can correctly rollback
# databases after crashes when using the special IO modes triggered 
# by device IOCAP flags.
#
# $Id: crash3.test,v 1.1 2007/08/23 11:07:10 danielk1977 Exp $
# $Id: crash3.test,v 1.2 2007/08/24 08:15:54 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

ifcapable !crashtest {
  finish_test
  return

      }
      set res
    } {{$res1} or {$res2}}
  }]
  uplevel $script
}

# This block tests crash-recovery when the IOCAP_ATOMIC flags is set.
#
# Each iteration of the following loop sets up the database to contain
# the following schema and data:
#
#    CREATE TABLE abc(a, b, c);
#    INSERT INTO abc VALUES(1, 2, 3);
#
# Then execute the SQL statement, scheduling a crash for part-way through

    do_test2 crash3-1.$tn.3 {
      execsql { SELECT * FROM abc }
    } {1 2 3} $res2

    incr tn
  }
}

# This block tests both the IOCAP_SEQUENTIAL and IOCAP_SAFE_APPEND flags.
#
db close
file delete -force test.db test.db-journal
sqlite3 db test.db
do_test crash3-2.0 {
  execsql {
    BEGIN;
    CREATE TABLE abc(a PRIMARY KEY, b, c);
    CREATE TABLE def(d PRIMARY KEY, e, f);
    PRAGMA default_cache_size = 10;
    INSERT INTO abc VALUES(randstr(10,1000),randstr(10,1000),randstr(10,1000));
    INSERT INTO abc 
      SELECT randstr(10,1000),randstr(10,1000),randstr(10,1000) FROM abc;
    INSERT INTO abc 
      SELECT randstr(10,1000),randstr(10,1000),randstr(10,1000) FROM abc;
    INSERT INTO abc 
      SELECT randstr(10,1000),randstr(10,1000),randstr(10,1000) FROM abc;
    INSERT INTO abc 
      SELECT randstr(10,1000),randstr(10,1000),randstr(10,1000) FROM abc;
    INSERT INTO abc 
      SELECT randstr(10,1000),randstr(10,1000),randstr(10,1000) FROM abc;
    INSERT INTO abc 
      SELECT randstr(10,1000),randstr(10,1000),randstr(10,1000) FROM abc;
    COMMIT;
  }
} {}

set tn 1
foreach {::crashfile ::delay ::char} {
  test.db         1 sequential
  test.db         1 safe_append
  test.db-journal 1 sequential
  test.db-journal 1 safe_append
  test.db-journal 2 safe_append
  test.db-journal 2 sequential
  test.db-journal 3 sequential
  test.db-journal 3 safe_append
} {
  for {set ii 0} {$ii < 100} {incr ii} {
    set ::SQL [subst {
      SELECT randstr($ii,$ii+10);
      BEGIN;
      DELETE FROM abc WHERE random()%5;
      INSERT INTO abc 
        SELECT randstr(10,1000),randstr(10,1000),randstr(10,1000) 
        FROM abc
        WHERE (random()%5)==0;
      DELETE FROM def WHERE random()%5;
      INSERT INTO def 
        SELECT randstr(10,1000),randstr(10,1000),randstr(10,1000) 
        FROM def
        WHERE (random()%5)==0;
      COMMIT;
    }]

    do_test crash3-2.$tn.$ii {
      crashsql -file $::crashfile -delay $::delay -char $::char $::SQL
      db close
      sqlite3 db test.db
      execsql {PRAGMA integrity_check}
    } {ok}
  }
  incr tn
}

# The following block tests an interaction between IOCAP_ATOMIC and
# IOCAP_SEQUENTIAL. At one point, if both flags were set, small
# journal files that contained only a single page, but were required 
# for some other reason (i.e. nTrunk) were not being written to
# disk.
#
for {set ii 0} {$ii < 10} {incr ii} {
  db close
  file delete -force test.db test.db-journal
  crashsql -file test.db -char {sequential atomic} {
    CREATE TABLE abc(a, b, c);
  }
  sqlite3 db test.db
  do_test crash3-3.$ii {
    execsql {PRAGMA integrity_check}
  } {ok}
}

finish_test