SQLite

/*
** 2004 April 6
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.553 2008/12/27 15:23:13 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** See the header comment on "btreeInt.h" for additional information.
** Including a description of file format and an overview of operation.
*/
#include "btreeInt.h"

**
** If the nFin parameter is non-zero, the implementation assumes
** that the caller will keep calling incrVacuumStep() until
** it returns SQLITE_DONE or an error, and that nFin is the
** number of pages the database file will contain after this 
** process is complete.
*/
static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){

  Pgno nFreeList;           /* Number of pages still on the free-list */

  assert( sqlite3_mutex_held(pBt->mutex) );





  if( !PTRMAP_ISPAGE(pBt, iLastPg) && iLastPg!=PENDING_BYTE_PAGE(pBt) ){
    int rc;
    u8 eType;
    Pgno iPtrPage;

    nFreeList = get4byte(&pBt->pPage1->aData[36]);

      releasePage(pLastPg);
      if( rc!=SQLITE_OK ){
        return rc;
      }
    }
  }

  if( nFin==0 ){
    iLastPg--;
    while( iLastPg==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, iLastPg) ){
      iLastPg--;
    }
    sqlite3PagerTruncateImage(pBt->pPager, iLastPg);
  }
  return SQLITE_OK;
}

/*
** A write-transaction must be opened before calling this function.
** It performs a single unit of work towards an incremental vacuum.

  sqlite3BtreeEnter(p);
  pBt->db = p->db;
  assert( pBt->inTransaction==TRANS_WRITE && p->inTrans==TRANS_WRITE );
  if( !pBt->autoVacuum ){
    rc = SQLITE_DONE;
  }else{
    invalidateAllOverflowCache(pBt);
    rc = incrVacuumStep(pBt, 0, sqlite3PagerImageSize(pBt->pPager));
  }
  sqlite3BtreeLeave(p);
  return rc;
}

/*
** This routine is called prior to sqlite3PagerCommit when a transaction
** is commited for an auto-vacuum database.
**
** If SQLITE_OK is returned, then *pnTrunc is set to the number of pages
** the database file should be truncated to during the commit process. 
** i.e. the database has been reorganized so that only the first *pnTrunc
** pages are in use.
*/
static int autoVacuumCommit(BtShared *pBt){
  int rc = SQLITE_OK;
  Pager *pPager = pBt->pPager;
  VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager) );

  assert( sqlite3_mutex_held(pBt->mutex) );
  invalidateAllOverflowCache(pBt);
  assert(pBt->autoVacuum);
  if( !pBt->incrVacuum ){
    Pgno nFin;


    Pgno nFree;
    Pgno nPtrmap;
    Pgno iFree;
    const int pgsz = pBt->pageSize;
    Pgno nOrig = pagerPagecount(pBt);

    if( PTRMAP_ISPAGE(pBt, nOrig) ){
      return SQLITE_CORRUPT_BKPT;
    }
    if( nOrig==PENDING_BYTE_PAGE(pBt) ){
      nOrig--;
    }
    nFree = get4byte(&pBt->pPage1->aData[36]);
    nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+pgsz/5)/(pgsz/5);
    nFin = nOrig - nFree - nPtrmap;
    if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){
      nFin--;
    }
    while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){
      nFin--;
    }


    for(iFree=nOrig; iFree>nFin && rc==SQLITE_OK; iFree--){
      rc = incrVacuumStep(pBt, nFin, iFree);
    }
    if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){

      rc = SQLITE_OK;

      rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
      put4byte(&pBt->pPage1->aData[32], 0);
      put4byte(&pBt->pPage1->aData[36], 0);
      sqlite3PagerTruncateImage(pBt->pPager, nFin);

    }
    if( rc!=SQLITE_OK ){
      sqlite3PagerRollback(pPager);
    }
  }





  assert( nRef==sqlite3PagerRefcount(pPager) );
  return rc;
}

#endif /* ifndef SQLITE_OMIT_AUTOVACUUM */

/*

  if( p->inTrans==TRANS_WRITE ){
    BtShared *pBt = p->pBt;
    Pgno nTrunc = 0;
    sqlite3BtreeEnter(p);
    pBt->db = p->db;
#ifndef SQLITE_OMIT_AUTOVACUUM
    if( pBt->autoVacuum ){
      rc = autoVacuumCommit(pBt);
      if( rc!=SQLITE_OK ){
        sqlite3BtreeLeave(p);
        return rc;
      }
    }
#endif
    rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, nTrunc, 0);

#endif
  btreeIntegrity(p);
  unlockAllTables(p);

  if( p->inTrans==TRANS_WRITE ){
    int rc2;





    assert( TRANS_WRITE==pBt->inTransaction );
    rc2 = sqlite3PagerRollback(pBt->pPager);
    if( rc2!=SQLITE_OK ){
      rc = rc2;
    }

    /* The rollback may have destroyed the pPage1->aData value.  So

  }else{
    /* There are no pages on the freelist, so create a new page at the
    ** end of the file */
    int nPage = pagerPagecount(pBt);
    *pPgno = nPage + 1;

#ifndef SQLITE_OMIT_AUTOVACUUM










    if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt, *pPgno) ){
      /* If *pPgno refers to a pointer-map page, allocate two new pages
      ** at the end of the file instead of one. The first allocated page
      ** becomes a new pointer-map page, the second is used by the caller.
      */
      TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", *pPgno));
      assert( *pPgno!=PENDING_BYTE_PAGE(pBt) );
      (*pPgno)++;
      if( *pPgno==PENDING_BYTE_PAGE(pBt) ){ (*pPgno)++; }
    }



#endif

    assert( *pPgno!=PENDING_BYTE_PAGE(pBt) );
    rc = sqlite3BtreeGetPage(pBt, *pPgno, ppPage, 0);
    if( rc ) return rc;
    rc = sqlite3PagerWrite((*ppPage)->pDbPage);
    if( rc!=SQLITE_OK ){

  sCheck.pBt = pBt;
  sCheck.pPager = pBt->pPager;
  sCheck.nPage = pagerPagecount(sCheck.pBt);
  sCheck.mxErr = mxErr;
  sCheck.nErr = 0;
  sCheck.mallocFailed = 0;
  *pnErr = 0;





  if( sCheck.nPage==0 ){
    unlockBtreeIfUnused(pBt);
    sqlite3BtreeLeave(p);
    return 0;
  }
  sCheck.anRef = sqlite3Malloc( (sCheck.nPage+1)*sizeof(sCheck.anRef[0]) );
  if( !sCheck.anRef ){

/*
** 2004 April 6
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btreeInt.h,v 1.38 2008/12/27 15:23:13 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
**     "Sorting And Searching", pages 473-480. Addison-Wesley
**     Publishing Company, Reading, Massachusetts.

  MemPage *pPage1;      /* First page of the database */
  u8 inStmt;            /* True if we are in a statement subtransaction */
  u8 readOnly;          /* True if the underlying file is readonly */
  u8 pageSizeFixed;     /* True if the page size can no longer be changed */
#ifndef SQLITE_OMIT_AUTOVACUUM
  u8 autoVacuum;        /* True if auto-vacuum is enabled */
  u8 incrVacuum;        /* True if incr-vacuum is enabled */

#endif
  u16 pageSize;         /* Total number of bytes on a page */
  u16 usableSize;       /* Number of usable bytes on each page */
  u16 maxLocal;         /* Maximum local payload in non-LEAFDATA tables */
  u16 minLocal;         /* Minimum local payload in non-LEAFDATA tables */
  u16 maxLeaf;          /* Maximum local payload in a LEAFDATA table */
  u16 minLeaf;          /* Minimum local payload in a LEAFDATA table */

** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.524 2008/12/27 15:23:13 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"

/*
** Macros for troubleshooting.  Normally turned off
*/

** Pager.errCode may be set to SQLITE_IOERR, SQLITE_CORRUPT, or
** or SQLITE_FULL. Once one of the first three errors occurs, it persists
** and is returned as the result of every major pager API call.  The
** SQLITE_FULL return code is slightly different. It persists only until the
** next successful rollback is performed on the pager cache. Also,
** SQLITE_FULL does not affect the sqlite3PagerGet() and sqlite3PagerLookup()
** APIs, they may still be used successfully.
**
** Managing the size of the database file in pages is a little complicated.
** The variable Pager.dbSize contains the number of pages that the database
** image currently contains. As the database image grows or shrinks this
** variable is updated. The variable Pager.dbFileSize contains the number
** of pages in the database file. This may be different from Pager.dbSize
** if some pages have been appended to the database image but not yet written
** out from the cache to the actual file on disk. Or if the image has been
** truncated by an incremental-vacuum operation. The Pager.dbOrigSize variable
** contains the number of pages in the database image when the current
** transaction was opened. The contents of all three of these variables is
** only guaranteed to be correct if the boolean Pager.dbSizeValid is true.
*/
struct Pager {
  sqlite3_vfs *pVfs;          /* OS functions to use for IO */
  u8 journalOpen;             /* True if journal file descriptors is valid */
  u8 journalStarted;          /* True if header of journal is synced */
  u8 useJournal;              /* Use a rollback journal on this file */
  u8 noReadlock;              /* Do not bother to obtain readlocks */

  u8 setMaster;               /* True if a m-j name has been written to jrnl */
  u8 doNotSync;               /* Boolean. While true, do not spill the cache */
  u8 exclusiveMode;           /* Boolean. True if locking_mode==EXCLUSIVE */
  u8 journalMode;             /* On of the PAGER_JOURNALMODE_* values */
  u8 dbModified;              /* True if there are any changes to the Db */
  u8 changeCountDone;         /* Set after incrementing the change-counter */
  u8 dbSizeValid;             /* Set when dbSize is correct */
  Pgno dbSize;                /* Number of pages in the database */
  Pgno dbOrigSize;            /* dbSize before the current transaction */
  Pgno dbFileSize;            /* Number of pages in the database file */
  u32 vfsFlags;               /* Flags for sqlite3_vfs.xOpen() */
  int errCode;                /* One of several kinds of errors */


  int nRec;                   /* Number of pages written to the journal */
  u32 cksumInit;              /* Quasi-random value added to every checksum */
  int stmtNRec;               /* Number of records in stmt subjournal */
  int nExtra;                 /* Add this many bytes to each in-memory page */
  int pageSize;               /* Number of bytes in a page */
  int nPage;                  /* Total number of in-memory pages */
  int mxPage;                 /* Maximum number of pages to hold in cache */

/*
** The maximum legal page number is (2^31 - 1).
*/
#define PAGER_MAX_PGNO 2147483647

/*
** Return true if it is necessary to write page *pPg into the sub-journal.



** A page needs to be written into the sub-journal if there exists one
** or more open savepoints for which:
**
**   * The page-number is less than or equal to PagerSavepoint.nOrig, and


**   * The bit corresponding to the page-number is not set in
**     PagerSavepoint.pInSavepoint.
*/
static int subjRequiresPage(PgHdr *pPg){
  Pgno pgno = pPg->pgno;
  Pager *pPager = pPg->pPager;
  int i;
  for(i=0; i<pPager->nSavepoint; i++){
    PagerSavepoint *p = &pPager->aSavepoint[i];
    if( p->nOrig>=pgno && 0==sqlite3BitvecTest(p->pInSavepoint, pgno) ){
      return 1;
    }
  }
  return 0;


}

/*
** Return true if the page is already in the journal file.
*/
static int pageInJournal(PgHdr *pPg){
  return sqlite3BitvecTest(pPg->pPager->pInJournal, pPg->pgno);
}

/*
** Read a 32-bit integer from the given file descriptor.  Store the integer
** that is read in *pRes.  Return SQLITE_OK if everything worked, or an

    put32bits(&zHeader[sizeof(aJournalMagic)], 0);
  }

  /* The random check-hash initialiser */ 
  sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit);
  put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit);
  /* The initial database size */
  put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbOrigSize);
  /* The assumed sector size for this process */
  put32bits(&zHeader[sizeof(aJournalMagic)+12], pPager->sectorSize);

  /* Initializing the tail of the buffer is not necessary.  Everything
  ** works find if the following memset() is omitted.  But initializing
  ** the memory prevents valgrind from complaining, so we are willing to
  ** take the performance hit.

    */
    if( pPager->errCode ){
      if( rc==SQLITE_OK ) pPager->errCode = SQLITE_OK;
      pager_reset(pPager);
      releaseAllSavepoint(pPager);
      pPager->journalOff = 0;
      pPager->journalStarted = 0;
      pPager->dbOrigSize = 0;
    }

    pPager->state = PAGER_UNLOCK;
    pPager->changeCountDone = 0;
  }
}

  if( !pPager->exclusiveMode ){
    rc2 = osUnlock(pPager->fd, SHARED_LOCK);
    pPager->state = PAGER_SHARED;
  }else if( pPager->state==PAGER_SYNCED ){
    pPager->state = PAGER_EXCLUSIVE;
  }
  pPager->dbOrigSize = 0;
  pPager->setMaster = 0;
  pPager->needSync = 0;
  /* lruListSetFirstSynced(pPager); */
  if( !MEMDB ){
    pPager->dbSizeValid = 0;
  }
  pPager->dbModified = 0;

  );
  if( (pPager->state>=PAGER_EXCLUSIVE)
   && (pPg==0 || 0==(pPg->flags&PGHDR_NEED_SYNC))
   && (pPager->fd->pMethods)
  ){
    i64 ofst = (pgno-1)*(i64)pPager->pageSize;
    rc = sqlite3OsWrite(pPager->fd, aData, pPager->pageSize, ofst);
    if( pgno>pPager->dbFileSize ){
      pPager->dbFileSize = pgno;
    }
  }
  if( pPg ){
    /* No page should ever be explicitly rolled back that is in use, except
    ** for page 1 which is held in use in order to keep the lock on the
    ** database active. However such a page may be rolled back as a result
    ** of an internal error resulting in an automatic call to
    ** sqlite3PagerRollback().

    newSize = pPager->pageSize*(i64)nPage;
    if( rc==SQLITE_OK && currentSize!=newSize ){
      if( currentSize>newSize ){
        rc = sqlite3OsTruncate(pPager->fd, newSize);
      }else{
        rc = sqlite3OsWrite(pPager->fd, "", 1, newSize-1);
      }
      if( rc==SQLITE_OK ){
        pPager->dbFileSize = nPage;
      }
    }
  }
  if( rc==SQLITE_OK ){
    pPager->dbSize = nPage;
    pager_truncate_cache(pPager);
  }
  return rc;

      return SQLITE_NOMEM;
    }
  }

  /* Truncate the database back to the size it was before the 
  ** savepoint being reverted was opened.
  */
  rc = pager_truncate(pPager, pSavepoint?pSavepoint->nOrig:pPager->dbOrigSize);
  assert( pPager->state>=PAGER_SHARED );

  /* Now roll back all main journal file records that occur after byte
  ** byte offset PagerSavepoint.iOffset that have a page number less than
  ** or equal to PagerSavepoint.nOrig. As each record is played back,
  ** the corresponding bit in bitvec PagerSavepoint.pInSavepoint is 
  ** cleared.

    }
    if( n>0 && n<pPager->pageSize ){
      n = 1;
    }else{
      n /= pPager->pageSize;
    }
    if( pPager->state!=PAGER_UNLOCK ){
      pPager->dbSize = (Pgno)n;
      pPager->dbFileSize = (Pgno)n;
      pPager->dbSizeValid = 1;
    }
  }
  if( n==(PENDING_BYTE/pPager->pageSize) ){
    n++;
  }
  if( n>pPager->mxPgno ){

      IOTRACE(("LOCK %p %d\n", pPager, locktype))
    }
  }
  return rc;
}

/*
** Truncate the file to the number of pages specified. 
**
** Unless an IO error occurs, this function is guaranteed to modify the 
** database file itself. If an exclusive lock is not held when this function
** is called, one is obtained before truncating the file.
*/
int sqlite3PagerTruncate(Pager *pPager, Pgno nPage){
  int rc = SQLITE_OK;
  assert( pPager->state>=PAGER_SHARED );

  sqlite3PagerPagecount(pPager, 0);
  if( pPager->errCode ){
    rc = pPager->errCode;
  }else if( nPage<pPager->dbFileSize ){
    rc = syncJournal(pPager);
    if( rc==SQLITE_OK ){
      /* Get an exclusive lock on the database before truncating. */
      rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK);
    }
    if( rc==SQLITE_OK ){
      rc = pager_truncate(pPager, nPage);
    }
  }

  return rc;
}

#ifndef SQLITE_OMIT_AUTOVACUUM
/*
** Truncate the in-memory database file image to nPage pages. Unlike
** sqlite3PagerTruncate(), this function does not actually modify the
** database file on disk. It just sets the internal state of the pager
** object so that the truncation will be done when the current 
** transaction is committed.
*/
void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){
  assert( pPager->dbSizeValid );
  assert( pPager->dbSize>=nPage );
  pPager->dbSize = nPage;
}

/*
** Return the current size of the database file image in pages. This
** function differs from sqlite3PagerPagecount() in two ways:
**
**  a) It may only be called when at least one reference to a database
**     page is held. This guarantees that the database size is already
**     known and a call to sqlite3OsFileSize() is not required.
**
**  b) The return value is not adjusted for the locking page.
*/
Pgno sqlite3PagerImageSize(Pager *pPager){
  assert( pPager->dbSizeValid );
  return pPager->dbSize;
}
#endif  /* ifndef SQLITE_OMIT_AUTOVACUUM */

/*
** Shutdown the page cache.  Free all memory and close all files.
**
** If a transaction was in progress when this routine is called, that
** transaction is rolled back.  All outstanding pages are invalidated
** and their memory is freed.  Any attempt to use a page associated
** with this page cache after this function returns will likely

                   PAGERID(pPager), pList->pgno, pager_pagehash(pList));
      IOTRACE(("PGOUT %p %d\n", pPager, pList->pgno));
      rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset);
      PAGER_INCR(sqlite3_pager_writedb_count);
      PAGER_INCR(pPager->nWrite);
      if( pList->pgno==1 ){
        memcpy(&pPager->dbFileVers, &pData[24], sizeof(pPager->dbFileVers));
      }
      if( pList->pgno>pPager->dbFileSize ){
        pPager->dbFileSize = pList->pgno;
      }
    }
#ifndef NDEBUG
    else{
      PAGERTRACE3("NOSTORE %d page %d\n", PAGERID(pPager), pList->pgno);
    }
#endif

  pPager->journalStarted = 0;
  pPager->needSync = 0;
  pPager->nRec = 0;
  if( pPager->errCode ){
    rc = pPager->errCode;
    goto failed_to_open_journal;
  }
  pPager->dbOrigSize = pPager->dbSize;

  rc = writeJournalHdr(pPager);

  if( pPager->nSavepoint && rc==SQLITE_OK ){
    rc = openSubJournal(pPager);
  }
  if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM && rc!=SQLITE_IOERR_NOMEM ){

    /* This happens when the pager was in exclusive-access mode the last
    ** time a (read or write) transaction was successfully concluded
    ** by this connection. Instead of deleting the journal file it was 
    ** kept open and either was truncated to 0 bytes or its header was
    ** overwritten with zeros.
    */
    assert( pPager->nRec==0 );
    assert( pPager->dbOrigSize==0 );
    assert( pPager->pInJournal==0 );
    sqlite3PagerPagecount(pPager, 0);
    pPager->pInJournal = sqlite3BitvecCreate( pPager->dbSize );
    if( !pPager->pInJournal ){
      rc = SQLITE_NOMEM;
    }else{
      pPager->dbOrigSize = pPager->dbSize;
      rc = writeJournalHdr(pPager);
    }
  }
  assert( !pPager->journalOpen || pPager->journalOff>0 || rc!=SQLITE_OK );
  return rc;
}

    return rc;
  }

  /* Mark the page as dirty.  If the page has already been written
  ** to the journal then we can return right away.
  */
  sqlite3PcacheMakeDirty(pPg);
  if( pageInJournal(pPg) && !subjRequiresPage(pPg) ){
    pPager->dirtyCache = 1;
    pPager->dbModified = 1;
  }else{

    /* If we get this far, it means that the page needs to be
    ** written to the transaction journal or the ckeckpoint journal
    ** or both.

    pPager->dbModified = 1;
  
    /* The transaction journal now exists and we have a RESERVED or an
    ** EXCLUSIVE lock on the main database file.  Write the current page to
    ** the transaction journal if it is not there already.
    */
    if( !pageInJournal(pPg) && pPager->journalOpen ){
      if( pPg->pgno<=pPager->dbOrigSize ){
        u32 cksum;
        char *pData2;

        /* We should never write to the journal file the page that
        ** contains the database locks.  The following assert verifies
        ** that we do not. */
        assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) );

    }
  
    /* If the statement journal is open and the page is not in it,
    ** then write the current page to the statement journal.  Note that
    ** the statement journal format differs from the standard journal format
    ** in that it omits the checksums and the header.
    */
    if( subjRequiresPage(pPg) ){
      i64 offset = pPager->stmtNRec*(4+pPager->pageSize);
      char *pData2 = CODEC2(pPager, pData, pPg->pgno, 7);
      assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize );
      rc = write32bits(pPager->sjfd, offset, pPg->pgno);
      if( rc==SQLITE_OK ){
        rc = sqlite3OsWrite(pPager->sjfd, pData2, pPager->pageSize, offset+4);
      }
      PAGERTRACE3("STMT-JOURNAL %d page %d @ %d\n", PAGERID(pPager), pPg->pgno);
      if( rc!=SQLITE_OK ){
        return rc;

** rolled back in spite of the sqlite3PagerDontRollback() call.
*/
int sqlite3PagerDontWrite(DbPage *pDbPage){
  PgHdr *pPg = pDbPage;
  Pager *pPager = pPg->pPager;
  int rc;

  if( pPg->pgno>pPager->dbOrigSize ){
    return SQLITE_OK;
  }
  if( pPager->pAlwaysRollback==0 ){
    assert( pPager->pInJournal );
    pPager->pAlwaysRollback = sqlite3BitvecCreate(pPager->dbOrigSize);
    if( !pPager->pAlwaysRollback ){
      return SQLITE_NOMEM;
    }
  }
  rc = sqlite3BitvecSet(pPager->pAlwaysRollback, pPg->pgno);

  if( rc==SQLITE_OK && (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){
    assert( pPager->state>=PAGER_SHARED );
    if( pPager->dbSize==pPg->pgno && pPager->dbOrigSize<pPager->dbSize ){
      /* If this pages is the last page in the file and the file has grown
      ** during the current transaction, then do NOT mark the page as clean.
      ** When the database file grows, we must make sure that the last page
      ** gets written at least once so that the disk file will be the correct
      ** size. If you do not write this page and the size of the file
      ** on the disk ends up being too small, that can lead to database
      ** corruption during the next transaction.

  /* If the journal file is not open, or DontWrite() has been called on
  ** this page (DontWrite() sets the alwaysRollback flag), then this
  ** function is a no-op.
  */
  if( pPager->journalOpen==0 
   || sqlite3BitvecTest(pPager->pAlwaysRollback, pPg->pgno)
   || pPg->pgno>pPager->dbOrigSize
  ){
    return;
  }

#ifdef SQLITE_SECURE_DELETE
  if( sqlite3BitvecTest(pPager->pInJournal, pPg->pgno)!=0
   || pPg->pgno>pPager->dbOrigSize ){
    return;
  }
#endif

  /* If SECURE_DELETE is disabled, then there is no way that this
  ** routine can be called on a page for which sqlite3PagerDontWrite()
  ** has not been previously called during the same transaction.
  ** And if DontWrite() has previously been called, the following
  ** conditions must be met.
  **
  ** (Later:)  Not true.  If the database is corrupted by having duplicate
  ** pages on the freelist (ex: corrupt9.test) then the following is not
  ** necessarily true:
  */
  /* assert( !pPg->inJournal && (int)pPg->pgno <= pPager->dbOrigSize ); */

  assert( pPager->pInJournal!=0 );
  sqlite3BitvecSet(pPager->pInJournal, pPg->pgno);
  pPg->flags &= ~PGHDR_NEED_READ;
  addToSavepointBitvecs(pPager, pPg->pgno);
  PAGERTRACE3("DONT_ROLLBACK page %d of %d\n", pPg->pgno, PAGERID(pPager));
  IOTRACE(("GARBAGE %p %d\n", pPager, pPg->pgno))

    change_counter = sqlite3Get4byte((u8*)pPager->dbFileVers);
    change_counter++;
    put32bits(((char*)pPgHdr->pData)+24, change_counter);

#ifdef SQLITE_ENABLE_ATOMIC_WRITE
    if( isDirect && pPager->fd->pMethods ){
      const void *zBuf = pPgHdr->pData;
      assert( pPager->dbFileSize>0 );
      rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0);
    }
#endif

    /* Release the page reference. */
    sqlite3PagerUnref(pPgHdr);
    pPager->changeCountDone = 1;

  if( pPager->dbModified==0 &&
        (pPager->journalMode!=PAGER_JOURNALMODE_DELETE ||
          pPager->exclusiveMode!=0) ){
    assert( pPager->dirtyCache==0 || pPager->journalOpen==0 );
    return SQLITE_OK;
  }

  PAGERTRACE4("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", 
      pPager->zFilename, zMaster, pPager->dbSize);

  /* If this is an in-memory db, or no pages have been written to, or this
  ** function has already been called, it is a no-op.
  */
  if( pPager->state!=PAGER_SYNCED && !MEMDB && pPager->dirtyCache ){
    PgHdr *pPg;

    */
    int useAtomicWrite;
    pPg = sqlite3PcacheDirtyList(pPager->pPCache);
    useAtomicWrite = (
        !zMaster && 
        pPager->journalOpen &&
        pPager->journalOff==jrnlBufferSize(pPager) && 
        pPager->dbSize>=pPager->dbFileSize && 
        (pPg==0 || pPg->pDirty==0)
    );
    assert( pPager->journalOpen || pPager->journalMode==PAGER_JOURNALMODE_OFF );
    if( useAtomicWrite ){
      /* Update the nRec field in the journal file. */
      int offset = pPager->journalHdr + sizeof(aJournalMagic);
      assert(pPager->nRec==1);

    ** transaction the m-j name will have already been written.
    */
    if( !pPager->setMaster ){
      rc = pager_incr_changecounter(pPager, 0);
      if( rc!=SQLITE_OK ) goto sync_exit;
      if( pPager->journalMode!=PAGER_JOURNALMODE_OFF ){
#ifndef SQLITE_OMIT_AUTOVACUUM
        if( pPager->dbSize<pPager->dbOrigSize ){
          /* If this transaction has made the database smaller, then all pages
          ** being discarded by the truncation must be written to the journal
          ** file.
          */
          Pgno i;
          Pgno iSkip = PAGER_MJ_PGNO(pPager);
          Pgno dbSize = pPager->dbSize;
          for( i=pPager->dbSize+1; i<=pPager->dbOrigSize; i++ ){
            if( !sqlite3BitvecTest(pPager->pInJournal, i) && i!=iSkip ){
              rc = sqlite3PagerGet(pPager, i, &pPg);
              if( rc!=SQLITE_OK ) goto sync_exit;
              rc = sqlite3PagerWrite(pPg);
              sqlite3PagerUnref(pPg);
              if( rc!=SQLITE_OK ) goto sync_exit;
            }
          } 
          pPager->dbSize = dbSize;
        }
#endif
        rc = writeMasterJournal(pPager, zMaster);
        if( rc!=SQLITE_OK ) goto sync_exit;
        rc = syncJournal(pPager);
      }
    }
    if( rc!=SQLITE_OK ) goto sync_exit;

#ifndef SQLITE_OMIT_AUTOVACUUM
    if( pPager->dbSize<pPager->dbFileSize ){
      rc = sqlite3PagerTruncate(pPager, pPager->dbSize);
      if( rc!=SQLITE_OK ) goto sync_exit;
    }
#endif

    /* Write all dirty pages to the database file */
    pPg = sqlite3PcacheDirtyList(pPager->pPCache);
    rc = pager_write_pagelist(pPg);

    /* Sync the database file. */
    if( !pPager->noSync && !noSync ){
      rc = sqlite3OsSync(pPager->fd, pPager->sync_flags);
    }
    IOTRACE(("DBSYNC %p\n", pPager))

    pPager->state = PAGER_SYNCED;
  }else if( MEMDB && pPager->dbSize<pPager->dbFileSize ){
    rc = sqlite3PagerTruncate(pPager, pPager->dbSize);
  }

sync_exit:
  if( rc==SQLITE_IOERR_BLOCKED ){
    /* pager_incr_changecounter() may attempt to obtain an exclusive
     * lock to spill the cache and return IOERR_BLOCKED. But since 
     * there is no chance the cache is inconsistent, it is

  **
  ** If the isCommit flag is set, there is no need to remember that
  ** the journal needs to be sync()ed before database page pPg->pgno 
  ** can be written to. The caller has already promised not to write to it.
  */
  if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){
    needSyncPgno = pPg->pgno;
    assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize );
    assert( pPg->flags&PGHDR_DIRTY );
    assert( pPager->needSync );
  }

  /* If the cache contains a page with page-number pgno, remove it
  ** from its hash chain. Also, if the PgHdr.needSync was set for 
  ** page pgno before the 'move' operation, it needs to be retained 

    ** sure the Pager.needSync flag is set too.
    */
    int rc;
    PgHdr *pPgHdr;
    assert( pPager->needSync );
    rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr);
    if( rc!=SQLITE_OK ){
      if( pPager->pInJournal && needSyncPgno<=pPager->dbOrigSize ){
        sqlite3BitvecClear(pPager->pInJournal, needSyncPgno);
      }
      return rc;
    }
    pPager->needSync = 1;
    assert( pPager->noSync==0 && !MEMDB );
    pPgHdr->flags |= PGHDR_NEED_SYNC;

**    May you share freely, never taking more than you give.
**
*************************************************************************
** This header file defines the interface that the sqlite page cache
** subsystem.  The page cache subsystem reads and writes a file a page
** at a time and provides a journal for rollback.
**
** @(#) $Id: pager.h,v 1.90 2008/12/27 15:23:13 danielk1977 Exp $
*/

#ifndef _PAGER_H_
#define _PAGER_H_

/*
** If defined as non-zero, auto-vacuum is enabled by default. Otherwise

int sqlite3PagerJournalMode(Pager *, int);
i64 sqlite3PagerJournalSizeLimit(Pager *, i64);
void *sqlite3PagerTempSpace(Pager*);
int sqlite3PagerSync(Pager *pPager);

int sqlite3PagerOpenSavepoint(Pager *pPager, int n);
int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint);

#ifndef SQLITE_OMIT_AUTOVACUUM
  void sqlite3PagerTruncateImage(Pager*,Pgno);
  Pgno sqlite3PagerImageSize(Pager *);
#endif

#ifdef SQLITE_HAS_CODEC
  void sqlite3PagerSetCodec(Pager*,void*(*)(void*,void*,Pgno,int),void*);
#endif

#if !defined(NDEBUG) || defined(SQLITE_TEST)
  Pgno sqlite3PagerPagenumber(DbPage*);

#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the incremental vacuum feature.
#
# Note: There are also some tests for incremental vacuum and IO 
# errors in incrvacuum_ioerr.test.
#
# $Id: incrvacuum.test,v 1.21 2008/12/27 15:23:13 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If this build of the library does not support auto-vacuum, omit this
# whole file.
ifcapable {!autovacuum || !pragma} {

  puts $out "This is not an SQLite database file"
  close $out
  sqlite3 db3 invalid.db
  catchsql {
    PRAGMA incremental_vacuum(10);
  } db3
} {1 {file is encrypted or is not a database}}

do_test incrvacuum-15.1 {
  db close
  file delete -force test.db
  sqlite3 db test.db

  set str [string repeat "abcdefghij" 500]

  execsql {
    PRAGMA cache_size = 10;
    PRAGMA auto_vacuum = incremental;
    CREATE TABLE t1(x, y);
    INSERT INTO t1 VALUES('a', $str);
    INSERT INTO t1 VALUES('b', $str);
    INSERT INTO t1 VALUES('c', $str);
    INSERT INTO t1 VALUES('d', $str);
    INSERT INTO t1 VALUES('e', $str);
    INSERT INTO t1 VALUES('f', $str);
    INSERT INTO t1 VALUES('g', $str);
    INSERT INTO t1 VALUES('h', $str);
    INSERT INTO t1 VALUES('i', $str);
    INSERT INTO t1 VALUES('j', $str);
    INSERT INTO t1 VALUES('j', $str);

    CREATE TABLE t2(x PRIMARY KEY, y);
    INSERT INTO t2 VALUES('a', $str);
    INSERT INTO t2 VALUES('b', $str);
    INSERT INTO t2 VALUES('c', $str);
    INSERT INTO t2 VALUES('d', $str);

    BEGIN;
      DELETE FROM t2;
      PRAGMA incremental_vacuum;
  }

  catchsql {INSERT INTO t2 SELECT * FROM t1}

  execsql { 
    COMMIT;
    PRAGMA integrity_check;
  }
} {ok}

db2 close
db3 close
finish_test

# 2008 December 15
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# $Id: savepoint.test,v 1.4 2008/12/27 15:23:13 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl


#----------------------------------------------------------------------
# The following tests - savepoint-1.* - test that the SAVEPOINT, RELEASE

        ROLLBACK TO one;
      COMMIT;
    }
  } {}

  integrity_check savepoint-6.4
}

#-------------------------------------------------------------------------
# The following tests, savepoint-7.*, attempt to break the logic 
# surrounding savepoints by growing and shrinking the database file.
#
db close
file delete -force test.db
sqlite3 db test.db

do_test savepoint-7.1 {
  execsql {
    PRAGMA auto_vacuum = incremental;
    PRAGMA cache_size = 10;
    BEGIN;
    CREATE TABLE t1(a PRIMARY KEY, b);
      INSERT INTO t1(a) VALUES('alligator');
      INSERT INTO t1(a) VALUES('angelfish');
      INSERT INTO t1(a) VALUES('ant');
      INSERT INTO t1(a) VALUES('antelope');
      INSERT INTO t1(a) VALUES('ape');
      INSERT INTO t1(a) VALUES('baboon');
      INSERT INTO t1(a) VALUES('badger');
      INSERT INTO t1(a) VALUES('bear');
      INSERT INTO t1(a) VALUES('beetle');
      INSERT INTO t1(a) VALUES('bird');
      INSERT INTO t1(a) VALUES('bison');
      UPDATE t1 SET b =    randstr(1000,1000);
      UPDATE t1 SET b = b||randstr(1000,1000);
      UPDATE t1 SET b = b||randstr(1000,1000);
      UPDATE t1 SET b = b||randstr(10,1000);
    COMMIT;
  }
  expr ([execsql { PRAGMA page_count }] > 20)
} {1}
do_test savepoint-7.2.1 {
  execsql {
    BEGIN;
      SAVEPOINT one;
      CREATE TABLE t2(a, b);
      INSERT INTO t2 SELECT a, b FROM t1;
      ROLLBACK TO one;
  }
  execsql {
    PRAGMA integrity_check;
  }
} {ok}
do_test savepoint-7.2.2 {
  execsql {
    COMMIT;
    PRAGMA integrity_check;
  }
} {ok}

do_test savepoint-7.3.1 {
  execsql {
    CREATE TABLE t2(a, b);
    INSERT INTO t2 SELECT a, b FROM t1;
  }
} {}
do_test savepoint-7.3.2 {
  execsql {
    BEGIN;
      SAVEPOINT one;
        DELETE FROM t2;
        PRAGMA incremental_vacuum;
        SAVEPOINT two;
          INSERT INTO t2 SELECT a, b FROM t1;
        ROLLBACK TO two;
    COMMIT;
  }
  execsql { PRAGMA integrity_check }
} {ok}

do_test savepoint-7.4.1 {
  db close
  file delete -force test.db
  sqlite3 db test.db
  execsql {
    PRAGMA auto_vacuum = incremental;
    CREATE TABLE t1(a, b, PRIMARY KEY(a, b));
    INSERT INTO t1 VALUES(randstr(1000,1000), randstr(1000,1000));
    BEGIN;
      DELETE FROM t1;
      SAVEPOINT one;
      PRAGMA incremental_vacuum;
      ROLLBACK TO one;
    COMMIT;
  }

  execsql { PRAGMA integrity_check }
} {ok}



finish_test