SQLite

/*
** 2004 April 6
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.213 2004/11/06 12:26:08 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
**     "Sorting And Searching", pages 473-480. Addison-Wesley
**     Publishing Company, Reading, Massachusetts.

  int nCell;
  u8 *data;
  BtCursor cur;
  Btree *pBt;
  int maxLocal, usableSize;
  char zContext[100];
  char *hit;

  sprintf(zContext, "Page %d: ", iPage);

  /* Check that the page exists
  */
  cur.pBt = pBt = pCheck->pBt;
  usableSize = pBt->usableSize;
  if( iPage==0 ) return 0;
  if( checkRef(pCheck, iPage, zParentContext) ) return 0;

        rc = sqlite3RealSeek(&pFile->fd, BLOCK_OFFSET(i));
      }
      if( rc==SQLITE_OK && !skip ){
        int len = BLOCKSIZE;
        if( BLOCK_OFFSET(i+1)>nMax ){
          len = nMax-BLOCK_OFFSET(i);
        }
        if( len>0 ){
          if( trash ){
            sqlite3Randomness(len, p);
          }
          rc = sqlite3RealWrite(&pFile->fd, p, len);
        }
      }
      sqliteFree(p);
    }
  }
  sqliteFree(pFile->apBlk);
  pFile->nBlk = 0;
  pFile->apBlk = 0;

** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.174 2004/11/06 12:26:08 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>

#else
#define TRACE1(X)
#define TRACE2(X,Y)
#define TRACE3(X,Y,Z)
#define TRACE4(X,Y,Z,W)
#endif

#ifdef OS_TEST
#define PAGERID(p) (p->fd->fd.h)
#define FILEHANDLEID(fd) (fd->fd.h)
#else
#define PAGERID(p) (p->fd.h)
#define FILEHANDLEID(fd) (fd.h)
#endif

/*
** The page cache as a whole is always in one of the following
** states:
**
**   PAGER_UNLOCK        The page cache is not currently reading or 
**                       writing the database file.  There is no

  ** not the database file. The page is left marked dirty in this case.
  **
  ** If in EXCLUSIVE state, then we update the pager cache if it exists
  ** and the main file. The page is then marked not dirty.
  */
  pPg = pager_lookup(pPager, pgno);
  assert( pPager->state>=PAGER_EXCLUSIVE || pPg );
  TRACE3("PLAYBACK %d page %d\n", PAGERID(pPager), pgno);
  if( pPager->state>=PAGER_EXCLUSIVE ){
    sqlite3OsSeek(&pPager->fd, (pgno-1)*(i64)pPager->pageSize);
    rc = sqlite3OsWrite(&pPager->fd, aData, pPager->pageSize);
  }
  if( pPg ){
    /* No page should ever be rolled back that is in use, except for page
    ** 1 which is held in use in order to keep the lock on the database

  int rc = SQLITE_OK;
  for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
    char zBuf[SQLITE_MAX_PAGE_SIZE];
    if( !pPg->dirty ) continue;
    if( (int)pPg->pgno <= pPager->origDbSize ){
      sqlite3OsSeek(&pPager->fd, pPager->pageSize*(i64)(pPg->pgno-1));
      rc = sqlite3OsRead(&pPager->fd, zBuf, pPager->pageSize);
      TRACE3("REFETCH %d page %d\n", PAGERID(pPager), pPg->pgno);
      if( rc ) break;
      CODEC(pPager, zBuf, pPg->pgno, 2);
    }else{
      memset(zBuf, 0, pPager->pageSize);
    }
    if( pPg->nRef==0 || memcmp(zBuf, PGHDR_TO_DATA(pPg), pPager->pageSize) ){
      memcpy(PGHDR_TO_DATA(pPg), zBuf, pPager->pageSize);

  nameLen = strlen(zFullPathname);
  pPager = sqliteMalloc( sizeof(*pPager) + nameLen*3 + 30 );
  if( pPager==0 ){
    sqlite3OsClose(&fd);
    sqliteFree(zFullPathname);
    return SQLITE_NOMEM;
  }
  TRACE3("OPEN %d %s\n", FILEHANDLEID(fd), zFullPathname);
  pPager->zFilename = (char*)&pPager[1];
  pPager->zDirectory = &pPager->zFilename[nameLen+1];
  pPager->zJournal = &pPager->zDirectory[nameLen+1];
  strcpy(pPager->zFilename, zFullPathname);
  strcpy(pPager->zDirectory, zFullPathname);
  for(i=nameLen; i>0 && pPager->zDirectory[i-1]!='/'; i--){}
  if( i>0 ) pPager->zDirectory[i-1] = 0;

    assert( pPager->pLast==pPg );
    pPager->pLast = pPg->pPrevFree;
  }
  pPg->pNextFree = pPg->pPrevFree = 0;

  /* Unlink from the pgno hash table */
  unlinkHashChain(pPager, pPg);













}

/*
** This routine is used to truncate an in-memory database.  Delete
** all pages whose pgno is larger than pPager->dbSize and is unreferenced.
** Referenced pages larger than pPager->dbSize are zeroed.
*/

      assert( !pHist->pOrig );
      assert( !pHist->pStmt );
    }
#endif
    pNext = pPg->pNextAll;
    sqliteFree(pPg);
  }
  TRACE2("CLOSE %d\n", PAGERID(pPager));
  sqlite3OsClose(&pPager->fd);
  assert( pPager->journalOpen==0 );
  /* Temp files are automatically deleted by the OS
  ** if( pPager->tempFile ){
  **   sqlite3OsDelete(pPager->zFilename);
  ** }
  */

      {
        /* Write the nRec value into the journal file header. If in
        ** full-synchronous mode, sync the journal first. This ensures that
        ** all data has really hit the disk before nRec is updated to mark
        ** it as a candidate for rollback. 
        */
        if( pPager->fullSync ){
          TRACE2("SYNC journal of %d\n", PAGERID(pPager));
          rc = sqlite3OsSync(&pPager->jfd);
          if( rc!=0 ) return rc;
        }
        sqlite3OsSeek(&pPager->jfd, pPager->journalHdr + sizeof(aJournalMagic));
        rc = write32bits(&pPager->jfd, pPager->nRec);
        if( rc ) return rc;

        sqlite3OsSeek(&pPager->jfd, pPager->journalOff);
      }
      TRACE2("SYNC journal of %d\n", PAGERID(pPager));
      rc = sqlite3OsSync(&pPager->jfd);
      if( rc!=0 ) return rc;
      pPager->journalStarted = 1;
    }
    pPager->needSync = 0;

    /* Erase the needSync flag from every page.

    /* If there are dirty pages in the page cache with page numbers greater
    ** than Pager.dbSize, this means sqlite3pager_truncate() was called to
    ** make the file smaller (presumably by auto-vacuum code). Do not write
    ** any such pages to the file.
    */
    if( pList->pgno<=pPager->dbSize ){
      CODEC(pPager, PGHDR_TO_DATA(pList), pList->pgno, 6);
      TRACE3("STORE %d page %d\n", PAGERID(pPager), pList->pgno);
      rc = sqlite3OsWrite(&pPager->fd, PGHDR_TO_DATA(pList), pPager->pageSize);
      CODEC(pPager, PGHDR_TO_DATA(pList), pList->pgno, 0);
    }
#ifndef NDEBUG
    else{
      TRACE3("NOSTORE %d page %d\n", PAGERID(pPager), pList->pgno);
    }
#endif
    if( rc ) return rc;
    pList->dirty = 0;
    pList = pList->pDirty;
  }
  return SQLITE_OK;

    if( pPager->dbSize<(int)pgno ){
      memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize);
    }else{
      int rc;
      assert( MEMDB==0 );
      sqlite3OsSeek(&pPager->fd, (pgno-1)*(i64)pPager->pageSize);
      rc = sqlite3OsRead(&pPager->fd, PGHDR_TO_DATA(pPg), pPager->pageSize);
      TRACE3("FETCH %d page %d\n", PAGERID(pPager), pPg->pgno);
      CODEC(pPager, PGHDR_TO_DATA(pPg), pPg->pgno, 3);
      if( rc!=SQLITE_OK ){
        i64 fileSize;
        if( sqlite3OsFileSize(&pPager->fd,&fileSize)!=SQLITE_OK
               || fileSize>=pgno*pPager->pageSize ){
          sqlite3pager_unref(PGHDR_TO_DATA(pPg));
          return rc;

          rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK);
        }
      }
      if( rc!=SQLITE_OK ){
        return rc;
      }
      pPager->dirtyCache = 0;
      TRACE2("TRANSACTION %d\n", PAGERID(pPager));
      if( pPager->useJournal && !pPager->tempFile ){
        rc = pager_open_journal(pPager);
      }
    }
  }
  return rc;
}

    */
    if( !pPg->inJournal && (pPager->useJournal || MEMDB) ){
      if( (int)pPg->pgno <= pPager->origDbSize ){
        int szPg;
        u32 saved;
        if( MEMDB ){
          PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
          TRACE3("JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno);
          assert( pHist->pOrig==0 );
          pHist->pOrig = sqliteMallocRaw( pPager->pageSize );
          if( pHist->pOrig ){
            memcpy(pHist->pOrig, PGHDR_TO_DATA(pPg), pPager->pageSize);
          }
        }else{
          u32 cksum;
          CODEC(pPager, pData, pPg->pgno, 7);
          cksum = pager_cksum(pPager, pPg->pgno, pData);
          saved = *(u32*)PGHDR_TO_EXTRA(pPg, pPager);
          store32bits(cksum, pPg, pPager->pageSize);
          szPg = pPager->pageSize+8;
          store32bits(pPg->pgno, pPg, -4);
          rc = sqlite3OsWrite(&pPager->jfd, &((char*)pData)[-4], szPg);
          pPager->journalOff += szPg;
          TRACE4("JOURNAL %d page %d needSync=%d\n",
                  PAGERID(pPager), pPg->pgno, pPg->needSync);
          CODEC(pPager, pData, pPg->pgno, 0);
          *(u32*)PGHDR_TO_EXTRA(pPg, pPager) = saved;
          if( rc!=SQLITE_OK ){
            sqlite3pager_rollback(pPager);
            pPager->errMask |= PAGER_ERR_FULL;
            return rc;
          }
          pPager->nRec++;
          assert( pPager->aInJournal!=0 );
          pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7);
          pPg->needSync = !pPager->noSync;
          if( pPager->stmtInUse ){
            pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
            page_add_to_stmt_list(pPg);
          }
        }
      }else{
        pPg->needSync = !pPager->journalStarted && !pPager->noSync;
        TRACE4("APPEND %d page %d needSync=%d\n",
                PAGERID(pPager), pPg->pgno, pPg->needSync);
      }
      if( pPg->needSync ){
        pPager->needSync = 1;
      }
      pPg->inJournal = 1;
    }
  

      if( MEMDB ){
        PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
        assert( pHist->pStmt==0 );
        pHist->pStmt = sqliteMallocRaw( pPager->pageSize );
        if( pHist->pStmt ){
          memcpy(pHist->pStmt, PGHDR_TO_DATA(pPg), pPager->pageSize);
        }
        TRACE3("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno);
      }else{
        store32bits(pPg->pgno, pPg, -4);
        CODEC(pPager, pData, pPg->pgno, 7);
        rc = sqlite3OsWrite(&pPager->stfd,((char*)pData)-4, pPager->pageSize+4);
        TRACE3("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno);
        CODEC(pPager, pData, pPg->pgno, 0);
        if( rc!=SQLITE_OK ){
          sqlite3pager_rollback(pPager);
          pPager->errMask |= PAGER_ERR_FULL;
          return rc;
        }
        pPager->stmtNRec++;

      ** When the database file grows, we must make sure that the last page
      ** gets written at least once so that the disk file will be the correct
      ** size. If you do not write this page and the size of the file
      ** on the disk ends up being too small, that can lead to database
      ** corruption during the next transaction.
      */
    }else{
      TRACE3("DONT_WRITE page %d of %d\n", pgno, PAGERID(pPager));
      pPg->dirty = 0;
    }
  }
}

/*
** A call to this routine tells the pager that if a rollback occurs,

    assert( pPager->aInJournal!=0 );
    pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7);
    pPg->inJournal = 1;
    if( pPager->stmtInUse ){
      pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
      page_add_to_stmt_list(pPg);
    }
    TRACE3("DONT_ROLLBACK page %d of %d\n", pPg->pgno, PAGERID(pPager));
  }
  if( pPager->stmtInUse && !pPg->inStmt && (int)pPg->pgno<=pPager->stmtSize ){
    assert( pPg->inJournal || (int)pPg->pgno>pPager->origDbSize );
    assert( pPager->aInStmt!=0 );
    pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
    page_add_to_stmt_list(pPg);
  }

  if( pPager->errMask!=0 ){
    rc = pager_errcode(pPager);
    return rc;
  }
  if( pPager->state<PAGER_RESERVED ){
    return SQLITE_ERROR;
  }
  TRACE2("COMMIT %d\n", PAGERID(pPager));
  if( MEMDB ){
    pPg = pager_get_all_dirty_pages(pPager);
    while( pPg ){
      clearHistory(PGHDR_TO_HIST(pPg, pPager));
      pPg->dirty = 0;
      pPg->inJournal = 0;
      pPg->inStmt = 0;

** process is writing trash into the journal file (SQLITE_CORRUPT) or
** unless a prior malloc() failed (SQLITE_NOMEM).  Appropriate error
** codes are returned for all these occasions.  Otherwise,
** SQLITE_OK is returned.
*/
int sqlite3pager_rollback(Pager *pPager){
  int rc;
  TRACE2("ROLLBACK %d\n", PAGERID(pPager));
  if( MEMDB ){
    PgHdr *p;
    for(p=pPager->pAll; p; p=p->pNextAll){
      PgHistory *pHist;
      assert( !p->alwaysRollback );
      if( !p->dirty ){
        assert( !((PgHistory *)PGHDR_TO_HIST(p, pPager))->pOrig );
        assert( !((PgHistory *)PGHDR_TO_HIST(p, pPager))->pStmt );
        continue;
      }

      pHist = PGHDR_TO_HIST(p, pPager);
      if( pHist->pOrig ){
        memcpy(PGHDR_TO_DATA(p), pHist->pOrig, pPager->pageSize);
        TRACE3("ROLLBACK-PAGE %d of %d\n", p->pgno, PAGERID(pPager));
      }else{
        TRACE3("PAGE %d is clean on %d\n", p->pgno, PAGERID(pPager));
      }
      clearHistory(pHist);
      p->dirty = 0;
      p->inJournal = 0;
      p->inStmt = 0;
      p->pPrevStmt = p->pNextStmt = 0;

** changes of a single SQL command within a larger transaction.
*/
int sqlite3pager_stmt_begin(Pager *pPager){
  int rc;
  char zTemp[SQLITE_TEMPNAME_SIZE];
  assert( !pPager->stmtInUse );
  assert( pPager->dbSize>=0 );
  TRACE2("STMT-BEGIN %d\n", PAGERID(pPager));
  if( MEMDB ){
    pPager->stmtInUse = 1;
    pPager->stmtSize = pPager->dbSize;
    return SQLITE_OK;
  }
  if( !pPager->journalOpen ){
    pPager->stmtAutoopen = 1;

/*
** Commit a statement.
*/
int sqlite3pager_stmt_commit(Pager *pPager){
  if( pPager->stmtInUse ){
    PgHdr *pPg, *pNext;
    TRACE2("STMT-COMMIT %d\n", PAGERID(pPager));
    if( !MEMDB ){
      sqlite3OsSeek(&pPager->stfd, 0);
      /* sqlite3OsTruncate(&pPager->stfd, 0); */
      sqliteFree( pPager->aInStmt );
      pPager->aInStmt = 0;
    }
    for(pPg=pPager->pStmt; pPg; pPg=pNext){

/*
** Rollback a statement.
*/
int sqlite3pager_stmt_rollback(Pager *pPager){
  int rc;
  if( pPager->stmtInUse ){
    TRACE2("STMT-ROLLBACK %d\n", PAGERID(pPager));
    if( MEMDB ){
      PgHdr *pPg;
      for(pPg=pPager->pStmt; pPg; pPg=pPg->pNextStmt){
        PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
        if( pHist->pStmt ){
          memcpy(PGHDR_TO_DATA(pPg), pHist->pStmt, pPager->pageSize);
          sqliteFree(pHist->pStmt);

** passed to an sqlite3pager_sync() call.
**
** If parameter nTrunc is non-zero, then the pager file is truncated to
** nTrunc pages (this is used by auto-vacuum databases).
*/
int sqlite3pager_sync(Pager *pPager, const char *zMaster, Pgno nTrunc){
  int rc = SQLITE_OK;

  TRACE4("DATABASE SYNC: File=%s zMaster=%s nTrunc=%d\n", 
      pPager->zFilename, zMaster, nTrunc);

  /* If this is an in-memory db, or no pages have been written to, or this
  ** function has already been called, it is a no-op.
  */
  if( pPager->state!=PAGER_SYNCED && !MEMDB && pPager->dirtyCache ){
    PgHdr *pPg;
    assert( pPager->journalOpen );

    /* If a master journal file name has already been written to the
    ** journal file, then no sync is required. This happens when it is
    ** written, then the process fails to upgrade from a RESERVED to an
    ** EXCLUSIVE lock. The next time the process tries to commit the
    ** transaction the m-j name will have already been written.
    */
    if( !pPager->setMaster ){
      rc = pager_incr_changecounter(pPager);
      if( rc!=SQLITE_OK ) goto sync_exit;
#ifndef SQLITE_OMIT_AUTOVACUUM
      if( nTrunc!=0 ){
        /* If this transaction has made the database smaller, then all pages
        ** being discarded by the truncation must be written to the journal
        ** file.
        */
        Pgno i;
        void *pPage;
        for( i=nTrunc+1; i<=pPager->origDbSize; i++ ){
          if( !(pPager->aInJournal[i/8] & (1<<(i&7))) ){
            rc = sqlite3pager_get(pPager, i, &pPage);
            if( rc!=SQLITE_OK ) goto sync_exit;
            rc = sqlite3pager_write(pPage);
            sqlite3pager_unref(pPage);
            if( rc!=SQLITE_OK ) goto sync_exit;
          }
        } 
      }
#endif
      rc = writeMasterJournal(pPager, zMaster);
      if( rc!=SQLITE_OK ) goto sync_exit;
      rc = syncJournal(pPager);
      if( rc!=SQLITE_OK ) goto sync_exit;
    }

#ifndef SQLITE_OMIT_AUTOVACUUM

  PgHdr *pPg = DATA_TO_PGHDR(pData);
  PgHdr *pPgOld; 
  int h;

  assert( !pPager->stmtInUse );
  assert( pPg->nRef>0 );

  TRACE4("MOVE %d page %d moves to %d\n", PAGERID(pPager), pPg->pgno, pgno);

  /* Unlink pPg from it's hash-chain */
  unlinkHashChain(pPager, pPg);

  /* If the cache contains a page with page-number pgno, remove it
  ** from it's hash chain.
  */
  pPgOld = pager_lookup(pPager, pgno);
  if( pPgOld ){
    assert( pPgOld->nRef==0 );
    unlinkHashChain(pPager, pPgOld);
    pPgOld->dirty = 0;
    if( pPgOld->needSync ){
      pPg->needSync = 1;
    }
  }

  /* Change the page number for pPg and insert it into the new hash-chain. */
  pPg->pgno = pgno;
  h = pager_hash(pgno);
  if( pPager->aHash[h] ){
    assert( pPager->aHash[h]->pPrevHash==0 );