SQLite

** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.237 2006/01/15 20:28:28 drh Exp $
** @(#) $Id: pager.c,v 1.238 2006/01/16 11:29:19 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>

/*
** Hash a page number
*/
#define pager_hash(PN)  ((PN)&(N_PG_HASH-1))

/*
** A open page cache is an instance of the following structure.
**
** Pager.errCode may be set to SQLITE_IOERR, SQLITE_CORRUPT, SQLITE_PROTOCOL
** or SQLITE_FULL. Once one of the first three errors occurs, it persists
** and is returned as the result of every major pager API call.  The
** SQLITE_FULL return code is slightly different. It persists only until the
** next successful rollback is performed on the pager cache. Also,
** SQLITE_FULL does not affect the sqlite3pager_get() and sqlite3pager_lookup()
** APIs, they may still be used successfully.
*/
struct Pager {
  u8 journalOpen;             /* True if journal file descriptors is valid */
  u8 journalStarted;          /* True if header of journal is synced */
  u8 useJournal;              /* Use a rollback journal on this file */
  u8 noReadlock;              /* Do not bother to obtain readlocks */
  u8 stmtOpen;                /* True if the statement subjournal is open */
  u8 stmtInUse;               /* True we are in a statement subtransaction */
  u8 stmtAutoopen;            /* Open stmt journal when main journal is opened*/
  u8 noSync;                  /* Do not sync the journal if true */
  u8 fullSync;                /* Do extra syncs of the journal for robustness */
  u8 state;                   /* PAGER_UNLOCK, _SHARED, _RESERVED, etc. */
  u8 errMask;                 /* One of several kinds of errors */
  u8 errCode;                 /* One of several kinds of errors */
  u8 tempFile;                /* zFilename is a temporary file */
  u8 readOnly;                /* True for a read-only database */
  u8 needSync;                /* True if an fsync() is needed on the journal */
  u8 dirtyCache;              /* True if cached pages have changed */
  u8 alwaysRollback;          /* Disable dont_rollback() for all pages */
  u8 memDb;                   /* True to inhibit all file I/O */
  u8 setMaster;               /* True if a m-j name has been written to jrnl */

*/
#ifdef SQLITE_TEST
# define TEST_INCR(x)  x++
#else
# define TEST_INCR(x)
#endif

/*
** These are bits that can be set in Pager.errMask.
**
** TODO: Maybe we just want a variable - Pager.errCode. Can we really 
**       have two simultaneous error conditions?
**
** Recovering from an SQLITE_FULL, SQLITE_LOCK, SQLITE_CORRUPT or 
** SQLITE_IOERR error is not a simple matter, particularly if the pager 
** cache is shared between multiple connections.
**
** SQLITE_FULL (PAGER_ERR_FULL):
**     Cleared when the transaction is rolled back.
**
** SQLITE_CORRUPT (PAGER_ERR_CORRUPT):
**     Cannot be cleared. The upper layer must close the current pager 
**     and open a new one on the same file to continue.
**
** SQLITE_PROTOCOL (PAGER_ERR_LOCK):
**     This error only occurs if an internal error occurs or another process 
**     is not following the sqlite locking protocol (i.e. someone is 
**     manipulating the database file using something other than sqlite).
**     This is handled in the same way as database corruption - the error 
**     cannot be cleared except by closing the current pager and opening 
**     a brand new one on the same file.
**
** SQLITE_IOERR (PAGER_ERR_DISK):
**     Cleared when the transaction is rolled back.
*/
#define PAGER_ERR_FULL     0x01  /* a write() failed */
#define PAGER_ERR_LOCK     0x02  /* error in the locking protocol */
#define PAGER_ERR_CORRUPT  0x04  /* database or journal corruption */
#define PAGER_ERR_DISK     0x08  /* general disk I/O error - bad hard drive? */

/*
** Journal files begin with the following magic string.  The data
** was obtained from /dev/random.  It is used only as a sanity check.
**
** Since version 2.8.0, the journal format contains additional sanity
** checking information.  If the power fails while the journal is begin
** written, semi-random garbage data might appear in the journal

static u32 retrieve32bits(PgHdr *p, int offset){
  unsigned char *ac;
  ac = &((unsigned char*)PGHDR_TO_DATA(p))[offset];
  return (ac[0]<<24) | (ac[1]<<16) | (ac[2]<<8) | ac[3];
}


/*
** Convert the bits in the pPager->errMask into an approprate
** return code.
*/
static int pager_errcode(Pager *pPager){
  int rc = SQLITE_OK;
  if( pPager->errMask & PAGER_ERR_LOCK )    rc = SQLITE_PROTOCOL;
  if( pPager->errMask & PAGER_ERR_DISK )    rc = SQLITE_IOERR;
  if( pPager->errMask & PAGER_ERR_FULL )    rc = SQLITE_FULL;
  if( pPager->errMask & PAGER_ERR_CORRUPT ) rc = SQLITE_CORRUPT;
  return rc;
}

/*
** This function should be called when an error occurs within the pager
** code to set the appropriate bits in Pager.errMask.
** code to set Pager.errCode.
*/
static int pager_error(Pager *pPager, int rc){
  assert( pPager->errCode==SQLITE_FULL || pPager->errCode==SQLITE_OK );
  switch( rc ){
    case SQLITE_PROTOCOL:
  if( 
    rc==SQLITE_FULL || 
      pPager->errMask |= PAGER_ERR_LOCK;
      break;
    case SQLITE_IOERR:
    rc==SQLITE_IOERR || 
      pPager->errMask |= PAGER_ERR_DISK;
      break;
    case SQLITE_FULL:
    rc==SQLITE_CORRUPT ||
      pPager->errMask |= PAGER_ERR_FULL;
      break;
    case SQLITE_CORRUPT:
      pPager->errMask |= PAGER_ERR_CORRUPT;
    rc==SQLITE_PROTOCOL
  ){
    pPager->errCode = rc;
      break;
  }
  return rc;
}

#ifdef SQLITE_CHECK_PAGES
/*
** Return a 32-bit hash of the page data for pPage.

** The CHECK_PAGE macro takes a PgHdr* as an argument. If SQLITE_CHECK_PAGES
** is defined, and NDEBUG is not defined, an assert() statement checks
** that the page is either dirty or still matches the calculated page-hash.
*/
#define CHECK_PAGE(x) checkPage(x)
static void checkPage(PgHdr *pPg){
  Pager *pPager = pPg->pPager;
  assert( !pPg->pageHash || pPager->errMask || MEMDB || pPg->dirty || 
  assert( !pPg->pageHash || pPager->errCode || MEMDB || pPg->dirty || 
      pPg->pageHash==pager_pagehash(pPg) );
}

#else
#define CHECK_PAGE(x)
#endif

** Unlock the database and clear the in-memory cache.  This routine
** sets the state of the pager back to what it was when it was first
** opened.  Any outstanding pages are invalidated and subsequent attempts
** to access those pages will likely result in a coredump.
*/
static void pager_reset(Pager *pPager){
  PgHdr *pPg, *pNext;
  if( pPager->errMask ) return;
  if( pPager->errCode ) return;
  for(pPg=pPager->pAll; pPg; pPg=pNext){
    pNext = pPg->pNextAll;
    sqliteFree(pPg);
  }
  pPager->pFirst = 0;
  pPager->pFirstSynced = 0;
  pPager->pLast = 0;

    }
  }

  pPager->journalOff = szJ;
  
end_stmt_playback:
  if( rc!=SQLITE_OK ){
    pPager->errMask |= PAGER_ERR_CORRUPT;
    rc = SQLITE_CORRUPT;
    rc = pager_error(pPager, SQLITE_CORRUPT);
  }else{
    pPager->journalOff = szJ;
    /* pager_reload_cache(pPager); */
  }
  return rc;
}

int sqlite3pager_pagecount(Pager *pPager){
  i64 n;
  assert( pPager!=0 );
  if( pPager->dbSize>=0 ){
    n = pPager->dbSize;
  } else {
    if( sqlite3OsFileSize(pPager->fd, &n)!=SQLITE_OK ){
      pPager->errMask |= PAGER_ERR_DISK;
      pager_error(pPager, SQLITE_IOERR);
      return 0;
    }
    if( n>0 && n<pPager->pageSize ){
      n = 1;
    }else{
      n /= pPager->pageSize;
    }

/*
** Truncate the file to the number of pages specified.
*/
int sqlite3pager_truncate(Pager *pPager, Pgno nPage){
  int rc;
  sqlite3pager_pagecount(pPager);
  if( pPager->errMask!=0 ){
    rc = pager_errcode(pPager);
  if( pPager->errCode ){
    rc = pPager->errCode;
    return rc;
  }
  if( nPage>=(unsigned)pPager->dbSize ){
    return SQLITE_OK;
  }
  if( MEMDB ){
    pPager->dbSize = nPage;

      */
      disable_simulated_io_errors();
      sqlite3pager_rollback(pPager);
      enable_simulated_io_errors();
      if( !MEMDB ){
        sqlite3OsUnlock(pPager->fd, NO_LOCK);
      }
      assert( pPager->errMask || pPager->journalOpen==0 );
      assert( pPager->errCode || pPager->journalOpen==0 );
      break;
    }
    case PAGER_SHARED: {
      if( !MEMDB ){
        sqlite3OsUnlock(pPager->fd, NO_LOCK);
      }
      break;

      assert( !pHist->pStmt );
    }
#endif
    pNext = pPg->pNextAll;
    sqliteFree(pPg);
  }
  TRACE2("CLOSE %d\n", PAGERID(pPager));
  assert( pPager->errMask || (pPager->journalOpen==0 && pPager->stmtOpen==0) );
  assert( pPager->errCode || (pPager->journalOpen==0 && pPager->stmtOpen==0) );
  if( pPager->journalOpen ){
    sqlite3OsClose(&pPager->jfd);
  }
  sqliteFree(pPager->aInJournal);
  if( pPager->stmtOpen ){
    sqlite3OsClose(&pPager->stfd);
  }

  }
}

/*
** Try to find a page in the cache that can be recycled. 
**
** This routine may return SQLITE_IOERR, SQLITE_FULL or SQLITE_OK. It 
** does not set the pPager->errMask variable.
** does not set the pPager->errCode variable.
*/
static int pager_recycle(Pager *pPager, int syncOk, PgHdr **ppPg){
  PgHdr *pPg;
  *ppPg = 0;

  /* Find a page to recycle.  Try to locate a page that does not
  ** require us to do an fsync() on the journal.

        nReleased += sqliteAllocSize(pPg);
        sqliteFree(pPg);
      }

      if( rc!=SQLITE_OK ){
        /* An error occured whilst writing to the database file or 
        ** journal in pager_recycle(). The error is not returned to the 
        ** caller of this function. Instead, set the Pager.errMask variable.
        ** caller of this function. Instead, set the Pager.errCode variable.
        ** The error will be returned to the user (or users, in the case 
        ** of a shared pager cache) of the pager for which the error occured.
        */
        assert( rc==SQLITE_IOERR || rc==SQLITE_FULL );
        assert( p->state>=PAGER_RESERVED );
        pager_error(p, rc);
      }

    return SQLITE_CORRUPT_BKPT;
  }

  /* Make sure we have not hit any critical errors.
  */ 
  assert( pPager!=0 );
  *ppPage = 0;
  if( pPager->errMask & ~(PAGER_ERR_FULL) ){
    return pager_errcode(pPager);
  if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){
    return pPager->errCode;
  }

  /* If this is the first page accessed, then get a SHARED lock
  ** on the database file.
  */
  if( pPager->nRef==0 && !MEMDB ){
    if( !pPager->noReadlock ){

    if( pPg->pNextHash ){
      assert( pPg->pNextHash->pPrevHash==0 );
      pPg->pNextHash->pPrevHash = pPg;
    }
    if( pPager->nExtra>0 ){
      memset(PGHDR_TO_EXTRA(pPg, pPager), 0, pPager->nExtra);
    }
    if( pPager->errMask!=0 ){
    if( pPager->errCode ){
      sqlite3pager_unref(PGHDR_TO_DATA(pPg));
      rc = pager_errcode(pPager);
      rc = pPager->errCode;
      return rc;
    }
    if( sqlite3pager_pagecount(pPager)<(int)pgno ){
      memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize);
    }else{
      int rc;
      assert( MEMDB==0 );

** has ever happened.
*/
void *sqlite3pager_lookup(Pager *pPager, Pgno pgno){
  PgHdr *pPg;

  assert( pPager!=0 );
  assert( pgno!=0 );
  if( pPager->errMask & ~(PAGER_ERR_FULL) ){
  if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){
    return 0;
  }
  pPg = pager_lookup(pPager, pgno);
  if( pPg==0 ) return 0;
  page_ref(pPg);
  return PGHDR_TO_DATA(pPg);
}

  sqlite3OsSetFullSync(pPager->fd, pPager->fullSync);
  sqlite3OsOpenDirectory(pPager->jfd, pPager->zDirectory);
  pPager->journalOpen = 1;
  pPager->journalStarted = 0;
  pPager->needSync = 0;
  pPager->alwaysRollback = 0;
  pPager->nRec = 0;
  if( pPager->errMask!=0 ){
    rc = pager_errcode(pPager);
  if( pPager->errCode ){
    rc = pPager->errCode;
    goto failed_to_open_journal;
  }
  pPager->origDbSize = pPager->dbSize;

  rc = writeJournalHdr(pPager);

  if( pPager->stmtAutoopen && rc==SQLITE_OK ){

int sqlite3pager_write(void *pData){
  PgHdr *pPg = DATA_TO_PGHDR(pData);
  Pager *pPager = pPg->pPager;
  int rc = SQLITE_OK;

  /* Check for errors
  */
  if( pPager->errMask ){ 
    return pager_errcode(pPager);
  if( pPager->errCode ){ 
    return pPager->errCode;
  }
  if( pPager->readOnly ){
    return SQLITE_PERM;
  }

  assert( !pPager->setMaster );

          pPager->journalOff += szPg;
          TRACE4("JOURNAL %d page %d needSync=%d\n",
                  PAGERID(pPager), pPg->pgno, pPg->needSync);
          CODEC(pPager, pData, pPg->pgno, 0);
          *(u32*)PGHDR_TO_EXTRA(pPg, pPager) = saved;
          if( rc!=SQLITE_OK ){
            sqlite3pager_rollback(pPager);
            pPager->errMask |= PAGER_ERR_FULL;
            if( !pPager->errCode ){
              pager_error(pPager, SQLITE_FULL);
            }
            return rc;
          }
          pPager->nRec++;
          assert( pPager->aInJournal!=0 );
          pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7);
          pPg->needSync = !pPager->noSync;
          if( pPager->stmtInUse ){

        CODEC(pPager, pData, pPg->pgno, 7);
        rc = sqlite3OsWrite(pPager->stfd,((char*)pData)-4,
                               pPager->pageSize+4);
        TRACE3("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno);
        CODEC(pPager, pData, pPg->pgno, 0);
        if( rc!=SQLITE_OK ){
          sqlite3pager_rollback(pPager);
          pPager->errMask |= PAGER_ERR_FULL;
          if( !pPager->errCode ){
            pager_error(pPager, SQLITE_FULL);
          }
          return rc;
        }
        pPager->stmtNRec++;
        assert( pPager->aInStmt!=0 );
        pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
      }
      page_add_to_stmt_list(pPg);

** and an error code is returned.  If the commit worked, SQLITE_OK
** is returned.
*/
int sqlite3pager_commit(Pager *pPager){
  int rc;
  PgHdr *pPg;

  if( pPager->errMask==PAGER_ERR_FULL ){
  if( pPager->errCode==SQLITE_FULL ){
    rc = sqlite3pager_rollback(pPager);
    if( rc==SQLITE_OK ){
      rc = SQLITE_FULL;
    }
    return rc;
  }
  if( pPager->errMask!=0 ){
    rc = pager_errcode(pPager);
  if( pPager->errCode ){
    rc = pPager->errCode;
    return rc;
  }
  if( pPager->state<PAGER_RESERVED ){
    return SQLITE_ERROR;
  }
  TRACE2("COMMIT %d\n", PAGERID(pPager));
  if( MEMDB ){

  if( !pPager->dirtyCache || !pPager->journalOpen ){
    rc = pager_unwritelock(pPager);
    pPager->dbSize = -1;
    return rc;
  }

  if( pPager->errMask!=0 && pPager->errMask!=PAGER_ERR_FULL ){
  if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){
    if( pPager->state>=PAGER_EXCLUSIVE ){
      pager_playback(pPager);
    }
    return pager_errcode(pPager);
    return pPager->errCode;
  }
  if( pPager->state==PAGER_RESERVED ){
    int rc2;
    rc = pager_reload_cache(pPager);
    rc2 = pager_unwritelock(pPager);
    if( rc==SQLITE_OK ){
      rc = rc2;
    }
  }else{
    rc = pager_playback(pPager);
  }
  if( rc!=SQLITE_OK ){
    rc = SQLITE_CORRUPT_BKPT;
    pPager->errMask |= PAGER_ERR_CORRUPT;
    pager_error(pPager, SQLITE_CORRUPT);
  }
  pPager->dbSize = -1;
  return rc;
}

/*
** Return TRUE if the database file is opened read-only.  Return FALSE

int *sqlite3pager_stats(Pager *pPager){
  static int a[11];
  a[0] = pPager->nRef;
  a[1] = pPager->nPage;
  a[2] = pPager->mxPage;
  a[3] = pPager->dbSize;
  a[4] = pPager->state;
  a[5] = pPager->errMask;
  a[5] = pPager->errCode;
#ifdef SQLITE_TEST
  a[6] = pPager->nHit;
  a[7] = pPager->nMiss;
  a[8] = pPager->nOvfl;
  a[9] = pPager->nRead;
  a[10] = pPager->nWrite;
#endif

**    May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains the implementation of the sqlite3_prepare()
** interface, and routines that contribute to loading the database schema
** from disk.
**
** $Id: prepare.c,v 1.22 2006/01/13 06:33:24 danielk1977 Exp $
** $Id: prepare.c,v 1.23 2006/01/16 11:29:20 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include <ctype.h>

/*
** Fill the InitData structure with an error message that indicates

  const char** pzTail       /* OUT: End of parsed string */
){
  Parse sParse;
  char *zErrMsg = 0;
  int rc = SQLITE_OK;
  int i;

  /* Assert that malloc() has not failed */
  assert( !sqlite3ThreadDataReadOnly()->mallocFailed );

  assert( ppStmt );
  *ppStmt = 0;
  if( sqlite3SafetyOn(db) ){
    return SQLITE_MISUSE;
  }

/*
** 2001 September 15
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** Internal interface definitions for SQLite.
**
** @(#) $Id: sqliteInt.h,v 1.467 2006/01/13 06:33:24 danielk1977 Exp $
** @(#) $Id: sqliteInt.h,v 1.468 2006/01/16 11:29:20 danielk1977 Exp $
*/
#ifndef _SQLITEINT_H_
#define _SQLITEINT_H_

/*
** Extra interface definitions for those who need them
*/

#ifndef SQLITE_OMIT_SHARED_CACHE
  void sqlite3TableLock(Parse *, int, int, u8, const char *);
#else
  #define sqlite3TableLock(v,w,x,y,z)
#endif

void sqlite3MallocClearFailed();
#ifdef NDEBUG
#ifndef SQLITE_MEMDEBUG
  #define sqlite3MallocDisallow()
  #define sqlite3MallocAllow()
#else
  void sqlite3MallocDisallow();
  void sqlite3MallocAllow();
#endif

#ifdef SQLITE_SSE
#include "sseInt.h"
#endif

#endif

** statement is completely executed or an error occurs.
*/
int sqlite3_step(sqlite3_stmt *pStmt){
  Vdbe *p = (Vdbe*)pStmt;
  sqlite3 *db;
  int rc;

  /* Assert that malloc() has not failed */
  assert(!sqlite3ThreadDataReadOnly()->mallocFailed);
  assert( !sqlite3ThreadDataReadOnly()->mallocFailed );

  if( p==0 || p->magic!=VDBE_MAGIC_RUN ){
    return SQLITE_MISUSE;
  }
  if( p->aborted ){
    return SQLITE_ABORT;
  }