/ Check-in [32bb2d58]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix multiple performance regressions (ticket #2298 among them) and add tests to make sure they do not come back. (CVS 3839)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 32bb2d5859906b4fb0f6083eedd7f3a81b9cf5e2
User & Date: drh 2007-04-13 02:14:30
Context
2007-04-13
03:23
Additional tests designed to detect future performance regressions. (CVS 3840) check-in: 764e7262 user: drh tags: trunk
02:14
Fix multiple performance regressions (ticket #2298 among them) and add tests to make sure they do not come back. (CVS 3839) check-in: 32bb2d58 user: drh tags: trunk
2007-04-12
21:25
Changes toward fixes for tickets #2296 and #2291. (CVS 3838) check-in: 0dd3e2e4 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to src/btree.c.

5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
....
1392
1393
1394
1395
1396
1397
1398







1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
....
3826
3827
3828
3829
3830
3831
3832

3833
3834
3835
3836
3837
3838
3839
....
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
....
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
....
6557
6558
6559
6560
6561
6562
6563




6564
6565
6566
6567
6568
6569









6570
6571
6572
6573
6574
6575

6576
6577
6578
6579
6580
6581
6582
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.354 2007/04/09 12:45:03 drh Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
**     "Sorting And Searching", pages 473-480. Addison-Wesley
**     Publishing Company, Reading, Massachusetts.
................................................................................
  pPage->nCell = 0;
  pPage->isInit = 1;
}

/*
** Get a page from the pager.  Initialize the MemPage.pBt and
** MemPage.aData elements if needed.







*/
static int getPage(BtShared *pBt, Pgno pgno, MemPage **ppPage, int clrFlag){
  int rc;
  MemPage *pPage;
  DbPage *pDbPage;

  rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, clrFlag);
  if( rc ) return rc;
  pPage = (MemPage *)sqlite3PagerGetExtra(pDbPage);
  pPage->aData = sqlite3PagerGetData(pDbPage);
  pPage->pDbPage = pDbPage;
  pPage->pBt = pBt;
  pPage->pgno = pgno;
  pPage->hdrOffset = pPage->pgno==1 ? 100 : 0;
  *ppPage = pPage;
  if( clrFlag ){
    sqlite3PagerDontRollback(pPage->pDbPage);
  }
  return SQLITE_OK;
}

/*
** Get a page from the pager and initialize it.  This routine
** is just a convenience wrapper around separate calls to
** getPage() and initPage().
................................................................................
                 *pPgno, closest+1, k, pTrunk->pgno, n-1));
          if( closest<k-1 ){
            memcpy(&aData[8+closest*4], &aData[4+k*4], 4);
          }
          put4byte(&aData[4], k-1);
          rc = getPage(pBt, *pPgno, ppPage, 1);
          if( rc==SQLITE_OK ){

            rc = sqlite3PagerWrite((*ppPage)->pDbPage);
            if( rc!=SQLITE_OK ){
              releasePage(*ppPage);
            }
          }
          searchList = 0;
        }
................................................................................
    }else{
      /* Add the newly freed page as a leaf on the current trunk */
      rc = sqlite3PagerWrite(pTrunk->pDbPage);
      if( rc==SQLITE_OK ){
        put4byte(&pTrunk->aData[4], k+1);
        put4byte(&pTrunk->aData[8+k*4], pPage->pgno);
#ifndef SQLITE_SECURE_DELETE
        sqlite3PagerDontWrite(pBt->pPager, pPage->pgno);
#endif
      }
      TRACE(("FREE-PAGE: %d leaf on trunk page %d\n",pPage->pgno,pTrunk->pgno));
    }
    releasePage(pTrunk);
  }
  return rc;
................................................................................
  nOvfl = (info.nPayload - info.nLocal + ovflPageSize - 1)/ovflPageSize;
  assert( ovflPgno==0 || nOvfl>0 );
  while( nOvfl-- ){
    MemPage *pOvfl;
    if( ovflPgno==0 || ovflPgno>sqlite3PagerPagecount(pBt->pPager) ){
      return SQLITE_CORRUPT_BKPT;
    }
    rc = getPage(pBt, ovflPgno, &pOvfl, 0);
    if( rc ) return rc;
    if( nOvfl ){
      ovflPgno = get4byte(pOvfl->aData);
    }
    rc = freePage(pOvfl);
    sqlite3PagerUnref(pOvfl->pDbPage);
    if( rc ) return rc;
................................................................................
    DbPage *pDbPage;
    if( i==iSkip ) continue;
    rc = sqlite3PagerGet(pBtFrom->pPager, i, &pDbPage);
    if( rc ) break;
    rc = sqlite3PagerOverwrite(pBtTo->pPager, i, sqlite3PagerGetData(pDbPage));
    sqlite3PagerUnref(pDbPage);
  }




  for(i=nPage+1; rc==SQLITE_OK && i<=nToPage; i++){
    DbPage *pDbPage;
    if( i==iSkip ) continue;
    rc = sqlite3PagerGet(pBtTo->pPager, i, &pDbPage);
    if( rc ) break;
    rc = sqlite3PagerWrite(pDbPage);









    sqlite3PagerUnref(pDbPage);
    sqlite3PagerDontWrite(pBtTo->pPager, i);
  }
  if( !rc && nPage<nToPage ){
    rc = sqlite3PagerTruncate(pBtTo->pPager, nPage);
  }

  if( rc ){
    sqlite3BtreeRollback(pTo);
  }
  return rc;  
}
#endif /* SQLITE_OMIT_VACUUM */








|







 







>
>
>
>
>
>
>

|




|








<
<
<







 







>







 







|







 







|







 







>
>
>
>






>
>
>
>
>
>
>
>
>

<




>







5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
....
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420



1421
1422
1423
1424
1425
1426
1427
....
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
....
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
....
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
....
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586
6587
6588

6589
6590
6591
6592
6593
6594
6595
6596
6597
6598
6599
6600
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.355 2007/04/13 02:14:30 drh Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
**     "Sorting And Searching", pages 473-480. Addison-Wesley
**     Publishing Company, Reading, Massachusetts.
................................................................................
  pPage->nCell = 0;
  pPage->isInit = 1;
}

/*
** Get a page from the pager.  Initialize the MemPage.pBt and
** MemPage.aData elements if needed.
**
** If the noContent flag is set, it means that we do not care about
** the content of the page at this time.  So do not go to the disk
** to fetch the content.  Just fill in the content with zeros for now.
** If in the future we call sqlite3PagerWrite() on this page, that
** means we have started to be concerned about content and the disk
** read should occur at that point.
*/
static int getPage(BtShared *pBt, Pgno pgno, MemPage **ppPage, int noContent){
  int rc;
  MemPage *pPage;
  DbPage *pDbPage;

  rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, noContent);
  if( rc ) return rc;
  pPage = (MemPage *)sqlite3PagerGetExtra(pDbPage);
  pPage->aData = sqlite3PagerGetData(pDbPage);
  pPage->pDbPage = pDbPage;
  pPage->pBt = pBt;
  pPage->pgno = pgno;
  pPage->hdrOffset = pPage->pgno==1 ? 100 : 0;
  *ppPage = pPage;



  return SQLITE_OK;
}

/*
** Get a page from the pager and initialize it.  This routine
** is just a convenience wrapper around separate calls to
** getPage() and initPage().
................................................................................
                 *pPgno, closest+1, k, pTrunk->pgno, n-1));
          if( closest<k-1 ){
            memcpy(&aData[8+closest*4], &aData[4+k*4], 4);
          }
          put4byte(&aData[4], k-1);
          rc = getPage(pBt, *pPgno, ppPage, 1);
          if( rc==SQLITE_OK ){
            sqlite3PagerDontRollback((*ppPage)->pDbPage);
            rc = sqlite3PagerWrite((*ppPage)->pDbPage);
            if( rc!=SQLITE_OK ){
              releasePage(*ppPage);
            }
          }
          searchList = 0;
        }
................................................................................
    }else{
      /* Add the newly freed page as a leaf on the current trunk */
      rc = sqlite3PagerWrite(pTrunk->pDbPage);
      if( rc==SQLITE_OK ){
        put4byte(&pTrunk->aData[4], k+1);
        put4byte(&pTrunk->aData[8+k*4], pPage->pgno);
#ifndef SQLITE_SECURE_DELETE
        sqlite3PagerDontWrite(pPage->pDbPage);
#endif
      }
      TRACE(("FREE-PAGE: %d leaf on trunk page %d\n",pPage->pgno,pTrunk->pgno));
    }
    releasePage(pTrunk);
  }
  return rc;
................................................................................
  nOvfl = (info.nPayload - info.nLocal + ovflPageSize - 1)/ovflPageSize;
  assert( ovflPgno==0 || nOvfl>0 );
  while( nOvfl-- ){
    MemPage *pOvfl;
    if( ovflPgno==0 || ovflPgno>sqlite3PagerPagecount(pBt->pPager) ){
      return SQLITE_CORRUPT_BKPT;
    }
    rc = getPage(pBt, ovflPgno, &pOvfl, nOvfl==0);
    if( rc ) return rc;
    if( nOvfl ){
      ovflPgno = get4byte(pOvfl->aData);
    }
    rc = freePage(pOvfl);
    sqlite3PagerUnref(pOvfl->pDbPage);
    if( rc ) return rc;
................................................................................
    DbPage *pDbPage;
    if( i==iSkip ) continue;
    rc = sqlite3PagerGet(pBtFrom->pPager, i, &pDbPage);
    if( rc ) break;
    rc = sqlite3PagerOverwrite(pBtTo->pPager, i, sqlite3PagerGetData(pDbPage));
    sqlite3PagerUnref(pDbPage);
  }

  /* If the file is shrinking, journal the pages that are being truncated
  ** so that they can be rolled back if the commit fails.
  */
  for(i=nPage+1; rc==SQLITE_OK && i<=nToPage; i++){
    DbPage *pDbPage;
    if( i==iSkip ) continue;
    rc = sqlite3PagerGet(pBtTo->pPager, i, &pDbPage);
    if( rc ) break;
    rc = sqlite3PagerWrite(pDbPage);
    sqlite3PagerDontWrite(pDbPage);
    /* Yeah.  It seems wierd to call DontWrite() right after Write().  But
    ** that is because the names of those procedures do not exactly 
    ** represent what they do.  Write() really means "put this page in the
    ** rollback journal and mark it as dirty so that it will be written
    ** to the database file later."  DontWrite() undoes the second part of
    ** that and prevents the page from being written to the database.  The
    ** page is still on the rollback journal, though.  And that is the whole
    ** point of this loop: to put pages on the rollback journal. */
    sqlite3PagerUnref(pDbPage);

  }
  if( !rc && nPage<nToPage ){
    rc = sqlite3PagerTruncate(pBtTo->pPager, nPage);
  }

  if( rc ){
    sqlite3BtreeRollback(pTo);
  }
  return rc;  
}
#endif /* SQLITE_OMIT_VACUUM */

Changes to src/os_common.h.

93
94
95
96
97
98
99

100
101
102
103
104
105
106
int sqlite3_diskfull = 0;
#define SimulateIOError(CODE)  \
  if( sqlite3_io_error_pending || sqlite3_io_error_hit ) \
     if( sqlite3_io_error_pending-- == 1 \
         || (sqlite3_io_error_persist && sqlite3_io_error_hit) ) \
                { local_ioerr(); CODE; }
static void local_ioerr(){

  sqlite3_io_error_hit = 1;
}
#define SimulateDiskfullError(CODE) \
   if( sqlite3_diskfull_pending ){ \
     if( sqlite3_diskfull_pending == 1 ){ \
       local_ioerr(); \
       sqlite3_diskfull = 1; \







>







93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
int sqlite3_diskfull = 0;
#define SimulateIOError(CODE)  \
  if( sqlite3_io_error_pending || sqlite3_io_error_hit ) \
     if( sqlite3_io_error_pending-- == 1 \
         || (sqlite3_io_error_persist && sqlite3_io_error_hit) ) \
                { local_ioerr(); CODE; }
static void local_ioerr(){
  IOTRACE(("IOERR\n"));
  sqlite3_io_error_hit = 1;
}
#define SimulateDiskfullError(CODE) \
   if( sqlite3_diskfull_pending ){ \
     if( sqlite3_diskfull_pending == 1 ){ \
       local_ioerr(); \
       sqlite3_diskfull = 1; \

Changes to src/pager.c.

14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
...
158
159
160
161
162
163
164

165
166
167
168
169
170
171
...
293
294
295
296
297
298
299
300
301

302
303
304





305
306
307


308
309
310
311
312
313
314
...
894
895
896
897
898
899
900


901
902
903
904
905
906
907
....
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
....
2045
2046
2047
2048
2049
2050
2051


2052
2053
2054
2055
2056
2057
2058
....
2466
2467
2468
2469
2470
2471
2472
2473
2474

2475
2476
2477
2478
2479
2480
2481
2482
....
2670
2671
2672
2673
2674
2675
2676


2677
2678
2679
2680
2681
2682
2683
....
2702
2703
2704
2705
2706
2707
2708


2709
2710
2711
2712
2713
2714
2715
2716
....
2835
2836
2837
2838
2839
2840
2841

2842
2843
2844
2845
2846
2847
2848
....
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965



2966
2967





2968
2969
2970
2971
2972
2973
2974
....
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
....
3032
3033
3034
3035
3036
3037
3038
3039
3040


3041
3042
3043
3044
3045
3046
3047
3048
3049




3050
3051
3052
3053
3054
3055
3056
....
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
....
3360
3361
3362
3363
3364
3365
3366

















3367
3368
3369
3370
3371
3372
3373
....
3421
3422
3423
3424
3425
3426
3427
3428

3429
3430
3431
3432
3433
3434
3435
....
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
....
3626
3627
3628
3629
3630
3631
3632
3633
3634

3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667





3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678

3679
3680
3681
3682
3683
3684
3685
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.326 2007/04/09 11:20:54 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>
................................................................................
  PgHdr *pNextHash, *pPrevHash;  /* Hash collision chain for PgHdr.pgno */
  PgHdr *pNextFree, *pPrevFree;  /* Freelist of pages where nRef==0 */
  PgHdr *pNextAll;               /* A list of all pages */
  u8 inJournal;                  /* TRUE if has been written to journal */
  u8 dirty;                      /* TRUE if we need to write back changes */
  u8 needSync;                   /* Sync journal before writing this page */
  u8 alwaysRollback;             /* Disable DontRollback() for this page */

  short int nRef;                /* Number of users of this page */
  PgHdr *pDirty, *pPrevDirty;    /* Dirty pages */
  u32 notUsed;                   /* Buffer space */
#ifdef SQLITE_CHECK_PAGES
  u32 pageHash;
#endif
  /* pPager->pageSize bytes of page data follow this header */
................................................................................
  Pager *pNext;               /* Linked list of pagers in this thread */
#endif
  char *pTmpSpace;            /* Pager.pageSize bytes of space for tmp use */
  u32 iChangeCount;           /* Db change-counter for which cache is valid */
};

/*
** If SQLITE_TEST is defined then increment the variable given in
** the argument

*/
#ifdef SQLITE_TEST
# define TEST_INCR(x)  x++





#else
# define TEST_INCR(x)
#endif



/*
** Journal files begin with the following magic string.  The data
** was obtained from /dev/random.  It is used only as a sanity check.
**
** Since version 2.8.0, the journal format contains additional sanity
** checking information.  If the power fails while the journal is begin
................................................................................
** opened.  Any outstanding pages are invalidated and subsequent attempts
** to access those pages will likely result in a coredump.
*/
static void pager_reset(Pager *pPager){
  PgHdr *pPg, *pNext;
  if( pPager->errCode ) return;
  for(pPg=pPager->pAll; pPg; pPg=pNext){


    pNext = pPg->pNextAll;
    sqliteFree(pPg);
  }
  pPager->pStmt = 0;
  pPager->pFirst = 0;
  pPager->pFirstSynced = 0;
  pPager->pLast = 0;
................................................................................
  }  
  if( master_open ){
    sqlite3OsClose(&master);
  }
  return rc;
}

#if 0
/*
** Make every page in the cache agree with what is on disk.  In other words,
** reread the disk to reset the state of the cache.
**
** This routine is called after a rollback in which some of the dirty cache
** pages had never been written out to disk.  We need to roll back the
** cache content and the easiest way to do that is to reread the old content
** back from the disk.
*/
static int pager_reload_cache(Pager *pPager){
  PgHdr *pPg;
  int rc = SQLITE_OK;
  for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
    char *zBuf = pPager->pTmpSpace;        /* Temp storage for one page */
    if( !pPg->dirty ) continue;
    if( (int)pPg->pgno <= pPager->origDbSize ){
      rc = sqlite3OsSeek(pPager->fd, pPager->pageSize*(i64)(pPg->pgno-1));
      if( rc==SQLITE_OK ){
        rc = sqlite3OsRead(pPager->fd, zBuf, pPager->pageSize);
      }
      PAGERTRACE3("REFETCH %d page %d\n", PAGERID(pPager), pPg->pgno);
      if( rc ) break;
      CODEC1(pPager, zBuf, pPg->pgno, 2);
    }else{
      memset(zBuf, 0, pPager->pageSize);
    }
    if( pPg->nRef==0 || memcmp(zBuf, PGHDR_TO_DATA(pPg), pPager->pageSize) ){
      memcpy(PGHDR_TO_DATA(pPg), zBuf, pPager->pageSize);
      if( pPager->xReiniter ){
        pPager->xReiniter(pPg, pPager->pageSize);
      }else{
        memset(PGHDR_TO_EXTRA(pPg, pPager), 0, pPager->nExtra);
      }
    }
    pPg->needSync = 0;
    pPg->dirty = 0;
#ifdef SQLITE_CHECK_PAGES
    pPg->pageHash = pager_pagehash(pPg);
#endif
  }
  pPager->pDirty = 0;
  return rc;
}
#endif

static void pager_truncate_cache(Pager *pPager);

/*
** Truncate the main file of the given pager to the number of pages
** indicated. Also truncate the cached representation of the file.
*/
................................................................................
    if( pPg->pgno<=dbSize ){
      ppPg = &pPg->pNextAll;
    }else if( pPg->nRef>0 ){
      memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize);
      ppPg = &pPg->pNextAll;
    }else{
      *ppPg = pPg->pNextAll;


      unlinkPage(pPg);
      makeClean(pPg);
      sqliteFree(pPg);
      pPager->nPage--;
    }
  }
}
................................................................................
    ** than Pager.dbSize, this means sqlite3PagerTruncate() was called to
    ** make the file smaller (presumably by auto-vacuum code). Do not write
    ** any such pages to the file.
    */
    if( pList->pgno<=pPager->dbSize ){
      char *pData = CODEC2(pPager, PGHDR_TO_DATA(pList), pList->pgno, 6);
      PAGERTRACE3("STORE %d page %d\n", PAGERID(pPager), pList->pgno);
      IOTRACE(("PGOUT %p %d\n", pPager, pList->pgno))
      rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize);

      TEST_INCR(pPager->nWrite);
    }
#ifndef NDEBUG
    else{
      PAGERTRACE3("NOSTORE %d page %d\n", PAGERID(pPager), pList->pgno);
    }
#endif
    if( rc ) return rc;
................................................................................
        if( pPg==pPager->pAll ){
           pPager->pAll = pPg->pNextAll;
        }else{
          for( pTmp=pPager->pAll; pTmp->pNextAll!=pPg; pTmp=pTmp->pNextAll ){}
          pTmp->pNextAll = pPg->pNextAll;
        }
        nReleased += sqliteAllocSize(pPg);


        sqliteFree(pPg);
      }

      if( rc!=SQLITE_OK ){
        /* An error occured whilst writing to the database file or 
        ** journal in pager_recycle(). The error is not returned to the 
        ** caller of this function. Instead, set the Pager.errCode variable.
................................................................................
  int rc;
  assert( MEMDB==0 );
  rc = sqlite3OsSeek(pPager->fd, (pgno-1)*(i64)pPager->pageSize);
  if( rc==SQLITE_OK ){
    rc = sqlite3OsRead(pPager->fd, PGHDR_TO_DATA(pPg),
                          pPager->pageSize);
  }


  IOTRACE(("PGIN %p %d\n", pPager, pgno))
  PAGERTRACE3("FETCH %d page %d\n", PAGERID(pPager), pPg->pgno);
  CODEC1(pPager, PGHDR_TO_DATA(pPg), pPg->pgno, 3);
  return rc;
}


/*
................................................................................
            return rc;
          }
          iChangeCounter = (zC[0]<<24) + (zC[1]<<16) + (zC[2]<<8) + zC[3];
        }

        if( iChangeCounter!=pPager->iChangeCount ){
          pager_reset(pPager);

        }
      }
    }
    assert( pPager->exclusiveMode || pPager->state<=PAGER_SHARED );
    if( pPager->state==PAGER_UNLOCK ){
      pPager->state = PAGER_SHARED;
    }
................................................................................
** to find a page in the in-memory cache first.  If the page is not already
** in memory, this routine goes to disk to read it in whereas _lookup()
** just returns 0.  This routine acquires a read-lock the first time it
** has to go to disk, and could also playback an old journal if necessary.
** Since _lookup() never goes to disk, it never has to deal with locks
** or journal files.
**
** If clrFlag is false, the page contents are actually read from disk.
** If clfFlag is true, it means the page is about to be erased and
** rewritten without first being read so there is no point it doing
** the disk I/O.



*/
int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag){





  PgHdr *pPg;
  int rc;

  assert( pPager->state==PAGER_UNLOCK || pPager->nRef>0 || pgno==1 );

  /* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page
  ** number greater than this, or zero, is requested.
................................................................................
  assert( pPager->state!=PAGER_UNLOCK );

  pPg = pager_lookup(pPager, pgno);
  if( pPg==0 ){
    /* The requested page is not in the page cache. */
    int nMax;
    int h;
    TEST_INCR(pPager->nMiss);
    rc = pagerAllocatePage(pPager, &pPg);
    if( rc!=SQLITE_OK ){
      return rc;
    }

    pPg->pgno = pgno;
    assert( !MEMDB || pgno>pPager->stmtSize );
................................................................................
      rc = pPager->errCode;
      return rc;
    }

    /* Populate the page with data, either by reading from the database
    ** file, or by setting the entire page to zero.
    */
    if( nMax<(int)pgno || MEMDB || (clrFlag && !pPager->alwaysRollback) ){
      memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize);


    }else{
      rc = readDbPage(pPager, pPg, pgno);
      if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){
        pPg->pgno = 0;
        sqlite3PagerUnref(pPg);
        return rc;
      }else{
        TEST_INCR(pPager->nRead);
      }




    }

    /* Link the page into the page hash table */
    h = pgno & (pPager->nHash-1);
    assert( pgno!=0 );
    pPg->pNextHash = pPager->aHash[h];
    pPager->aHash[h] = pPg;
................................................................................

#ifdef SQLITE_CHECK_PAGES
    pPg->pageHash = pager_pagehash(pPg);
#endif
  }else{
    /* The requested page is in the page cache. */
    assert(pPager->nRef>0 || pgno==1);
    TEST_INCR(pPager->nHit);
    page_ref(pPg);
  }
  *ppPage = pPg;
  return SQLITE_OK;
}

/*
................................................................................
  if( pPager->readOnly ){
    return SQLITE_PERM;
  }

  assert( !pPager->setMaster );

  CHECK_PAGE(pPg);


















  /* Mark the page as dirty.  If the page has already been written
  ** to the journal then we can return right away.
  */
  makeDirty(pPg);
  if( pPg->inJournal && (pageInStatement(pPg) || pPager->stmtInUse==0) ){
    pPager->dirtyCache = 1;
................................................................................
          pData2 -= 4;
          saved = *(u32*)pEnd;
          put32bits(pEnd, cksum);
          szPg = pPager->pageSize+8;
          put32bits(pData2, pPg->pgno);
          rc = sqlite3OsWrite(pPager->jfd, pData2, szPg);
          IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno,
                   pPager->journalOff, szPg))

          pPager->journalOff += szPg;
          PAGERTRACE4("JOURNAL %d page %d needSync=%d\n",
                  PAGERID(pPager), pPg->pgno, pPg->needSync);
          *(u32*)pEnd = saved;

	  /* An error has occured writing to the journal file. The 
          ** transaction will be rolled back by the layer above.
................................................................................
  }
  return rc;
}
#endif

/*
** A call to this routine tells the pager that it is not necessary to
** write the information on page "pgno" back to the disk, even though
** that page might be marked as dirty.
**
** The overlying software layer calls this routine when all of the data
** on the given page is unused.  The pager marks the page as clean so
** that it does not get written to disk.
**
** Tests show that this optimization, together with the
................................................................................
** a transaction then removed from the freelist during a later part
** of the same transaction and reused for some other purpose.  When it
** is first added to the freelist, this routine is called.  When reused,
** the sqlite3PagerDontRollback() routine is called.  But because the
** page contains critical data, we still need to be sure it gets
** rolled back in spite of the sqlite3PagerDontRollback() call.
*/
void sqlite3PagerDontWrite(Pager *pPager, Pgno pgno){
  PgHdr *pPg;


  if( MEMDB ) return;

  pPg = pager_lookup(pPager, pgno);
  assert( pPg!=0 );  /* We never call _dont_write unless the page is in mem */
  pPg->alwaysRollback = 1;
  if( pPg->dirty && !pPager->stmtInUse ){
    assert( pPager->state>=PAGER_SHARED );
    if( pPager->dbSize==(int)pPg->pgno && pPager->origDbSize<pPager->dbSize ){
      /* If this pages is the last page in the file and the file has grown
      ** during the current transaction, then do NOT mark the page as clean.
      ** When the database file grows, we must make sure that the last page
      ** gets written at least once so that the disk file will be the correct
      ** size. If you do not write this page and the size of the file
      ** on the disk ends up being too small, that can lead to database
      ** corruption during the next transaction.
      */
    }else{
      PAGERTRACE3("DONT_WRITE page %d of %d\n", pgno, PAGERID(pPager));
      IOTRACE(("CLEAN %p %d\n", pPager, pgno))
      makeClean(pPg);
#ifdef SQLITE_CHECK_PAGES
      pPg->pageHash = pager_pagehash(pPg);
#endif
    }
  }
}

/*
** A call to this routine tells the pager that if a rollback occurs,
** it is not necessary to restore the data on the given page.  This
** means that the pager does not have to record the given page in the
** rollback journal.





*/
void sqlite3PagerDontRollback(DbPage *pPg){
  Pager *pPager = pPg->pPager;

  assert( pPager->state>=PAGER_RESERVED );
  if( pPager->journalOpen==0 ) return;
  if( pPg->alwaysRollback || pPager->alwaysRollback || MEMDB ) return;
  if( !pPg->inJournal && (int)pPg->pgno <= pPager->origDbSize ){
    assert( pPager->aInJournal!=0 );
    pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7);
    pPg->inJournal = 1;

    if( pPager->stmtInUse ){
      pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
    }
    PAGERTRACE3("DONT_ROLLBACK page %d of %d\n", pPg->pgno, PAGERID(pPager));
    IOTRACE(("GARBAGE %p %d\n", pPager, pPg->pgno))
  }
  if( pPager->stmtInUse 







|







 







>







 







|
|
>


<
>
>
>
>
>

|

>
>







 







>
>







 







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







>
>







 







|

>
|







 







>
>







 







>
>
|







 







>







 







|
|
|
|
>
>
>

|
>
>
>
>
>







 







|







 







|

>
>






<
<

>
>
>
>







 







|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|
>







 







|







 







|
|
>


<
<
<













|
|













>
>
>
>
>











>







14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
...
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
...
294
295
296
297
298
299
300
301
302
303
304
305

306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
...
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
....
1225
1226
1227
1228
1229
1230
1231













































1232
1233
1234
1235
1236
1237
1238
....
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
....
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
....
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
....
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
....
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
....
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
....
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
....
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029


3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
....
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
....
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
....
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
....
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
....
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640



3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.327 2007/04/13 02:14:30 drh Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>
................................................................................
  PgHdr *pNextHash, *pPrevHash;  /* Hash collision chain for PgHdr.pgno */
  PgHdr *pNextFree, *pPrevFree;  /* Freelist of pages where nRef==0 */
  PgHdr *pNextAll;               /* A list of all pages */
  u8 inJournal;                  /* TRUE if has been written to journal */
  u8 dirty;                      /* TRUE if we need to write back changes */
  u8 needSync;                   /* Sync journal before writing this page */
  u8 alwaysRollback;             /* Disable DontRollback() for this page */
  u8 needRead;                   /* Read content if PagerWrite() is called */
  short int nRef;                /* Number of users of this page */
  PgHdr *pDirty, *pPrevDirty;    /* Dirty pages */
  u32 notUsed;                   /* Buffer space */
#ifdef SQLITE_CHECK_PAGES
  u32 pageHash;
#endif
  /* pPager->pageSize bytes of page data follow this header */
................................................................................
  Pager *pNext;               /* Linked list of pagers in this thread */
#endif
  char *pTmpSpace;            /* Pager.pageSize bytes of space for tmp use */
  u32 iChangeCount;           /* Db change-counter for which cache is valid */
};

/*
** The following global variables hold counters used for
** testing purposes only.  These variables do not exist in
** a non-testing build.  These variables are not thread-safe.
*/
#ifdef SQLITE_TEST

int sqlite3_pager_readdb_count = 0;    /* Number of full pages read from DB */
int sqlite3_pager_writedb_count = 0;   /* Number of full pages written to DB */
int sqlite3_pager_writej_count = 0;    /* Number of pages written to journal */
int sqlite3_pager_pgfree_count = 0;    /* Number of cache pages freed */
# define PAGER_INCR(v)  v++
#else
# define PAGER_INCR(v)
#endif



/*
** Journal files begin with the following magic string.  The data
** was obtained from /dev/random.  It is used only as a sanity check.
**
** Since version 2.8.0, the journal format contains additional sanity
** checking information.  If the power fails while the journal is begin
................................................................................
** opened.  Any outstanding pages are invalidated and subsequent attempts
** to access those pages will likely result in a coredump.
*/
static void pager_reset(Pager *pPager){
  PgHdr *pPg, *pNext;
  if( pPager->errCode ) return;
  for(pPg=pPager->pAll; pPg; pPg=pNext){
    IOTRACE(("PGFREE %p %d\n", pPager, pPg->pgno));
    PAGER_INCR(sqlite3_pager_pgfree_count);
    pNext = pPg->pNextAll;
    sqliteFree(pPg);
  }
  pPager->pStmt = 0;
  pPager->pFirst = 0;
  pPager->pFirstSynced = 0;
  pPager->pLast = 0;
................................................................................
  }  
  if( master_open ){
    sqlite3OsClose(&master);
  }
  return rc;
}















































static void pager_truncate_cache(Pager *pPager);

/*
** Truncate the main file of the given pager to the number of pages
** indicated. Also truncate the cached representation of the file.
*/
................................................................................
    if( pPg->pgno<=dbSize ){
      ppPg = &pPg->pNextAll;
    }else if( pPg->nRef>0 ){
      memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize);
      ppPg = &pPg->pNextAll;
    }else{
      *ppPg = pPg->pNextAll;
      IOTRACE(("PGFREE %p %d\n", pPager, pPg->pgno));
      PAGER_INCR(sqlite3_pager_pgfree_count);
      unlinkPage(pPg);
      makeClean(pPg);
      sqliteFree(pPg);
      pPager->nPage--;
    }
  }
}
................................................................................
    ** than Pager.dbSize, this means sqlite3PagerTruncate() was called to
    ** make the file smaller (presumably by auto-vacuum code). Do not write
    ** any such pages to the file.
    */
    if( pList->pgno<=pPager->dbSize ){
      char *pData = CODEC2(pPager, PGHDR_TO_DATA(pList), pList->pgno, 6);
      PAGERTRACE3("STORE %d page %d\n", PAGERID(pPager), pList->pgno);
      IOTRACE(("PGOUT %p %d\n", pPager, pList->pgno));
      rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize);
      PAGER_INCR(sqlite3_pager_writedb_count);
      PAGER_INCR(pPager->nWrite);
    }
#ifndef NDEBUG
    else{
      PAGERTRACE3("NOSTORE %d page %d\n", PAGERID(pPager), pList->pgno);
    }
#endif
    if( rc ) return rc;
................................................................................
        if( pPg==pPager->pAll ){
           pPager->pAll = pPg->pNextAll;
        }else{
          for( pTmp=pPager->pAll; pTmp->pNextAll!=pPg; pTmp=pTmp->pNextAll ){}
          pTmp->pNextAll = pPg->pNextAll;
        }
        nReleased += sqliteAllocSize(pPg);
        IOTRACE(("PGFREE %p %d\n", pPager, pPg->pgno));
        PAGER_INCR(sqlite3_pager_pgfree_count);
        sqliteFree(pPg);
      }

      if( rc!=SQLITE_OK ){
        /* An error occured whilst writing to the database file or 
        ** journal in pager_recycle(). The error is not returned to the 
        ** caller of this function. Instead, set the Pager.errCode variable.
................................................................................
  int rc;
  assert( MEMDB==0 );
  rc = sqlite3OsSeek(pPager->fd, (pgno-1)*(i64)pPager->pageSize);
  if( rc==SQLITE_OK ){
    rc = sqlite3OsRead(pPager->fd, PGHDR_TO_DATA(pPg),
                          pPager->pageSize);
  }
  PAGER_INCR(sqlite3_pager_readdb_count);
  PAGER_INCR(pPager->nRead);
  IOTRACE(("PGIN %p %d\n", pPager, pgno));
  PAGERTRACE3("FETCH %d page %d\n", PAGERID(pPager), pPg->pgno);
  CODEC1(pPager, PGHDR_TO_DATA(pPg), pPg->pgno, 3);
  return rc;
}


/*
................................................................................
            return rc;
          }
          iChangeCounter = (zC[0]<<24) + (zC[1]<<16) + (zC[2]<<8) + zC[3];
        }

        if( iChangeCounter!=pPager->iChangeCount ){
          pager_reset(pPager);
          pPager->iChangeCount = iChangeCounter;
        }
      }
    }
    assert( pPager->exclusiveMode || pPager->state<=PAGER_SHARED );
    if( pPager->state==PAGER_UNLOCK ){
      pPager->state = PAGER_SHARED;
    }
................................................................................
** to find a page in the in-memory cache first.  If the page is not already
** in memory, this routine goes to disk to read it in whereas _lookup()
** just returns 0.  This routine acquires a read-lock the first time it
** has to go to disk, and could also playback an old journal if necessary.
** Since _lookup() never goes to disk, it never has to deal with locks
** or journal files.
**
** If noContent is false, the page contents are actually read from disk.
** If noContent is true, it means that we do not care about the contents
** of the page at this time, so do not do a disk read.  Just fill in the
** page content with zeros.  But mark the fact that we have not read the
** content by setting the PgHdr.needRead flag.  Later on, if 
** sqlite3PagerWrite() is called on this page, that means that the
** content is needed and the disk read should occur at that point.
*/
int sqlite3PagerAcquire(
  Pager *pPager,      /* The pager open on the database file */
  Pgno pgno,          /* Page number to fetch */
  DbPage **ppPage,    /* Write a pointer to the page here */
  int noContent       /* Do not bother reading content from disk if true */
){
  PgHdr *pPg;
  int rc;

  assert( pPager->state==PAGER_UNLOCK || pPager->nRef>0 || pgno==1 );

  /* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page
  ** number greater than this, or zero, is requested.
................................................................................
  assert( pPager->state!=PAGER_UNLOCK );

  pPg = pager_lookup(pPager, pgno);
  if( pPg==0 ){
    /* The requested page is not in the page cache. */
    int nMax;
    int h;
    PAGER_INCR(pPager->nMiss);
    rc = pagerAllocatePage(pPager, &pPg);
    if( rc!=SQLITE_OK ){
      return rc;
    }

    pPg->pgno = pgno;
    assert( !MEMDB || pgno>pPager->stmtSize );
................................................................................
      rc = pPager->errCode;
      return rc;
    }

    /* Populate the page with data, either by reading from the database
    ** file, or by setting the entire page to zero.
    */
    if( nMax<(int)pgno || MEMDB || noContent ){
      memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize);
      pPg->needRead = noContent;
      IOTRACE(("ZERO %p %d\n", pPager, pgno));
    }else{
      rc = readDbPage(pPager, pPg, pgno);
      if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){
        pPg->pgno = 0;
        sqlite3PagerUnref(pPg);
        return rc;


      }
    }
    /* If this was page 1, then restore the value of Pager.iChangeCount */
    if( pgno==1 ){
      pPager->iChangeCount = retrieve32bits(pPg, 24);
    }

    /* Link the page into the page hash table */
    h = pgno & (pPager->nHash-1);
    assert( pgno!=0 );
    pPg->pNextHash = pPager->aHash[h];
    pPager->aHash[h] = pPg;
................................................................................

#ifdef SQLITE_CHECK_PAGES
    pPg->pageHash = pager_pagehash(pPg);
#endif
  }else{
    /* The requested page is in the page cache. */
    assert(pPager->nRef>0 || pgno==1);
    PAGER_INCR(pPager->nHit);
    page_ref(pPg);
  }
  *ppPage = pPg;
  return SQLITE_OK;
}

/*
................................................................................
  if( pPager->readOnly ){
    return SQLITE_PERM;
  }

  assert( !pPager->setMaster );

  CHECK_PAGE(pPg);

  /* If this page was previously acquired with noContent==1, that means
  ** we didn't really read in the content of the page.  This can happen
  ** (for example) when the page is being moved to the freelist.  But
  ** now we are (perhaps) moving the page off of the freelist for
  ** reuse and we need to know its original content so that content
  ** can be stored in the rollback journal.  So do the read at this
  ** time.
  */
  if( pPg->needRead ){
    rc = readDbPage(pPager, pPg, pPg->pgno);
    if( rc==SQLITE_OK ){
      pPg->needRead = 0;
    }else{
      return rc;
    }
  }

  /* Mark the page as dirty.  If the page has already been written
  ** to the journal then we can return right away.
  */
  makeDirty(pPg);
  if( pPg->inJournal && (pageInStatement(pPg) || pPager->stmtInUse==0) ){
    pPager->dirtyCache = 1;
................................................................................
          pData2 -= 4;
          saved = *(u32*)pEnd;
          put32bits(pEnd, cksum);
          szPg = pPager->pageSize+8;
          put32bits(pData2, pPg->pgno);
          rc = sqlite3OsWrite(pPager->jfd, pData2, szPg);
          IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno,
                   pPager->journalOff, szPg));
          PAGER_INCR(sqlite3_pager_writej_count);
          pPager->journalOff += szPg;
          PAGERTRACE4("JOURNAL %d page %d needSync=%d\n",
                  PAGERID(pPager), pPg->pgno, pPg->needSync);
          *(u32*)pEnd = saved;

	  /* An error has occured writing to the journal file. The 
          ** transaction will be rolled back by the layer above.
................................................................................
  }
  return rc;
}
#endif

/*
** A call to this routine tells the pager that it is not necessary to
** write the information on page pPg back to the disk, even though
** that page might be marked as dirty.
**
** The overlying software layer calls this routine when all of the data
** on the given page is unused.  The pager marks the page as clean so
** that it does not get written to disk.
**
** Tests show that this optimization, together with the
................................................................................
** a transaction then removed from the freelist during a later part
** of the same transaction and reused for some other purpose.  When it
** is first added to the freelist, this routine is called.  When reused,
** the sqlite3PagerDontRollback() routine is called.  But because the
** page contains critical data, we still need to be sure it gets
** rolled back in spite of the sqlite3PagerDontRollback() call.
*/
void sqlite3PagerDontWrite(DbPage *pDbPage){
  PgHdr *pPg = pDbPage;
  Pager *pPager = pPg->pPager;

  if( MEMDB ) return;



  pPg->alwaysRollback = 1;
  if( pPg->dirty && !pPager->stmtInUse ){
    assert( pPager->state>=PAGER_SHARED );
    if( pPager->dbSize==(int)pPg->pgno && pPager->origDbSize<pPager->dbSize ){
      /* If this pages is the last page in the file and the file has grown
      ** during the current transaction, then do NOT mark the page as clean.
      ** When the database file grows, we must make sure that the last page
      ** gets written at least once so that the disk file will be the correct
      ** size. If you do not write this page and the size of the file
      ** on the disk ends up being too small, that can lead to database
      ** corruption during the next transaction.
      */
    }else{
      PAGERTRACE3("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager));
      IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno))
      makeClean(pPg);
#ifdef SQLITE_CHECK_PAGES
      pPg->pageHash = pager_pagehash(pPg);
#endif
    }
  }
}

/*
** A call to this routine tells the pager that if a rollback occurs,
** it is not necessary to restore the data on the given page.  This
** means that the pager does not have to record the given page in the
** rollback journal.
**
** If we have not yet actually read the content of this page (if
** the PgHdr.needRead flag is set) then this routine acts as a promise
** that we will never need to read the page content in the future.
** so the needRead flag can be cleared at this point.
*/
void sqlite3PagerDontRollback(DbPage *pPg){
  Pager *pPager = pPg->pPager;

  assert( pPager->state>=PAGER_RESERVED );
  if( pPager->journalOpen==0 ) return;
  if( pPg->alwaysRollback || pPager->alwaysRollback || MEMDB ) return;
  if( !pPg->inJournal && (int)pPg->pgno <= pPager->origDbSize ){
    assert( pPager->aInJournal!=0 );
    pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7);
    pPg->inJournal = 1;
    pPg->needRead = 0;
    if( pPager->stmtInUse ){
      pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
    }
    PAGERTRACE3("DONT_ROLLBACK page %d of %d\n", pPg->pgno, PAGERID(pPager));
    IOTRACE(("GARBAGE %p %d\n", pPager, pPg->pgno))
  }
  if( pPager->stmtInUse 

Changes to src/pager.h.

9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
...
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
**    May you share freely, never taking more than you give.
**
*************************************************************************
** This header file defines the interface that the sqlite page cache
** subsystem.  The page cache subsystem reads and writes a file a page
** at a time and provides a journal for rollback.
**
** @(#) $Id: pager.h,v 1.57 2007/03/30 14:06:34 drh Exp $
*/

#ifndef _PAGER_H_
#define _PAGER_H_

/*
** The default size of a database page.
................................................................................
int sqlite3PagerCommitPhaseTwo(Pager*);
int sqlite3PagerRollback(Pager*);
int sqlite3PagerIsreadonly(Pager*);
int sqlite3PagerStmtBegin(Pager*);
int sqlite3PagerStmtCommit(Pager*);
int sqlite3PagerStmtRollback(Pager*);
void sqlite3PagerDontRollback(DbPage*);
void sqlite3PagerDontWrite(Pager*, Pgno);
int sqlite3PagerRefcount(Pager*);
int *sqlite3PagerStats(Pager*);
void sqlite3PagerSetSafetyLevel(Pager*,int,int);
const char *sqlite3PagerFilename(Pager*);
const char *sqlite3PagerDirname(Pager*);
const char *sqlite3PagerJournalname(Pager*);
int sqlite3PagerNosync(Pager*);







|







 







|







9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
...
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
**    May you share freely, never taking more than you give.
**
*************************************************************************
** This header file defines the interface that the sqlite page cache
** subsystem.  The page cache subsystem reads and writes a file a page
** at a time and provides a journal for rollback.
**
** @(#) $Id: pager.h,v 1.58 2007/04/13 02:14:30 drh Exp $
*/

#ifndef _PAGER_H_
#define _PAGER_H_

/*
** The default size of a database page.
................................................................................
int sqlite3PagerCommitPhaseTwo(Pager*);
int sqlite3PagerRollback(Pager*);
int sqlite3PagerIsreadonly(Pager*);
int sqlite3PagerStmtBegin(Pager*);
int sqlite3PagerStmtCommit(Pager*);
int sqlite3PagerStmtRollback(Pager*);
void sqlite3PagerDontRollback(DbPage*);
void sqlite3PagerDontWrite(DbPage*);
int sqlite3PagerRefcount(Pager*);
int *sqlite3PagerStats(Pager*);
void sqlite3PagerSetSafetyLevel(Pager*,int,int);
const char *sqlite3PagerFilename(Pager*);
const char *sqlite3PagerDirname(Pager*);
const char *sqlite3PagerJournalname(Pager*);
int sqlite3PagerNosync(Pager*);

Changes to src/test1.c.

9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
...
192
193
194
195
196
197
198



















































199
200
201
202
203
204
205
....
4212
4213
4214
4215
4216
4217
4218

4219
4220
4221
4222
4223
4224
4225
....
4334
4335
4336
4337
4338
4339
4340




4341
4342
4343
4344
4345
4346
4347
....
4373
4374
4375
4376
4377
4378
4379








4380
4381
4382
4383
4384
4385
4386
**    May you share freely, never taking more than you give.
**
*************************************************************************
** Code for testing all sorts of SQLite interfaces.  This code
** is not included in the SQLite library.  It is used for automated
** testing of the SQLite library.
**
** $Id: test1.c,v 1.236 2007/04/09 12:45:03 drh Exp $
*/
#include "sqliteInt.h"
#include "tcl.h"
#include "os.h"
#include <stdlib.h>
#include <string.h>

................................................................................
    }
  }
  for(i=0; i<argc; i++){
    Tcl_DStringAppendElement(str, argv[i] ? argv[i] : "NULL");
  }
  return 0;
}




















































/*
** Usage:  sqlite3_exec_printf  DB  FORMAT  STRING
**
** Invoke the sqlite3_exec_printf() interface using the open database
** DB.  The SQL is the string FORMAT.  The format string should contain
** one %s or %q.  STRING is the value inserted into %s or %q.
................................................................................
     { "sqlite3_interrupt",             (Tcl_CmdProc*)test_interrupt        },
     { "sqlite_delete_function",        (Tcl_CmdProc*)delete_function       },
     { "sqlite_delete_collation",       (Tcl_CmdProc*)delete_collation      },
     { "sqlite3_get_autocommit",        (Tcl_CmdProc*)get_autocommit        },
     { "sqlite3_stack_used",            (Tcl_CmdProc*)test_stack_used       },
     { "sqlite3_busy_timeout",          (Tcl_CmdProc*)test_busy_timeout     },
     { "printf",                        (Tcl_CmdProc*)test_printf           },

  };
  static struct {
     char *zName;
     Tcl_ObjCmdProc *xProc;
     void *clientData;
  } aObjCmd[] = {
     { "sqlite3_connection_pointer",    get_sqlite_pointer, 0 },
................................................................................
  extern int sqlite3_opentemp_count;
  extern int sqlite3_memUsed;
  extern char *sqlite3_malloc_id;
  extern int sqlite3_memMax;
  extern int sqlite3_like_count;
  extern int sqlite3_tsd_count;
  extern int sqlite3_xferopt_count;




#if OS_UNIX && defined(SQLITE_TEST) && defined(THREADSAFE) && THREADSAFE
  extern int threadsOverrideEachOthersLocks;
#endif
#if OS_WIN
  extern int sqlite3_os_type;
#endif
#ifdef SQLITE_DEBUG
................................................................................
      (char*)&sqlite3_current_time, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite_os_trace",
      (char*)&sqlite3_os_trace, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite3_tsd_count",
      (char*)&sqlite3_tsd_count, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite3_xferopt_count",
      (char*)&sqlite3_xferopt_count, TCL_LINK_INT);








#ifndef SQLITE_OMIT_UTF16
  Tcl_LinkVar(interp, "unaligned_string_counter",
      (char*)&unaligned_string_counter, TCL_LINK_INT);
#endif
#if OS_UNIX && defined(SQLITE_TEST) && defined(THREADSAFE) && THREADSAFE
  Tcl_LinkVar(interp, "threadsOverrideEachOthersLocks",
      (char*)&threadsOverrideEachOthersLocks, TCL_LINK_INT);







|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>







 







>
>
>
>







 







>
>
>
>
>
>
>
>







9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
...
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
....
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
....
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
....
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
**    May you share freely, never taking more than you give.
**
*************************************************************************
** Code for testing all sorts of SQLite interfaces.  This code
** is not included in the SQLite library.  It is used for automated
** testing of the SQLite library.
**
** $Id: test1.c,v 1.237 2007/04/13 02:14:30 drh Exp $
*/
#include "sqliteInt.h"
#include "tcl.h"
#include "os.h"
#include <stdlib.h>
#include <string.h>

................................................................................
    }
  }
  for(i=0; i<argc; i++){
    Tcl_DStringAppendElement(str, argv[i] ? argv[i] : "NULL");
  }
  return 0;
}

/*
** The I/O tracing callback.
*/
static FILE *iotrace_file = 0;
static void io_trace_callback(const char *zFormat, ...){
  va_list ap;
  va_start(ap, zFormat);
  vfprintf(iotrace_file, zFormat, ap);
  va_end(ap);
  fflush(iotrace_file);
}

/*
** Usage:  io_trace FILENAME
**
** Turn I/O tracing on or off.  If FILENAME is not an empty string,
** I/O tracing begins going into FILENAME. If FILENAME is an empty
** string, I/O tracing is turned off.
*/
static int test_io_trace(
  void *NotUsed,
  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
  int argc,              /* Number of arguments */
  char **argv            /* Text of each argument */
){
  if( argc!=2 ){
    Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
          " FILENAME\"", 0);
    return TCL_ERROR;
  }
  if( iotrace_file ){
    if( iotrace_file!=stdout && iotrace_file!=stderr ){
      fclose(iotrace_file);
    }
    iotrace_file = 0;
    sqlite3_io_trace = 0;
  }
  if( argv[1][0] ){
    if( strcmp(argv[1],"stdout")==0 ){
      iotrace_file = stdout;
    }else if( strcmp(argv[1],"stderr")==0 ){
      iotrace_file = stderr;
    }else{
      iotrace_file = fopen(argv[1], "w");
    }
    sqlite3_io_trace = io_trace_callback;
  }
  return SQLITE_OK;
}


/*
** Usage:  sqlite3_exec_printf  DB  FORMAT  STRING
**
** Invoke the sqlite3_exec_printf() interface using the open database
** DB.  The SQL is the string FORMAT.  The format string should contain
** one %s or %q.  STRING is the value inserted into %s or %q.
................................................................................
     { "sqlite3_interrupt",             (Tcl_CmdProc*)test_interrupt        },
     { "sqlite_delete_function",        (Tcl_CmdProc*)delete_function       },
     { "sqlite_delete_collation",       (Tcl_CmdProc*)delete_collation      },
     { "sqlite3_get_autocommit",        (Tcl_CmdProc*)get_autocommit        },
     { "sqlite3_stack_used",            (Tcl_CmdProc*)test_stack_used       },
     { "sqlite3_busy_timeout",          (Tcl_CmdProc*)test_busy_timeout     },
     { "printf",                        (Tcl_CmdProc*)test_printf           },
     { "sqlite3_io_trace",              (Tcl_CmdProc*)test_io_trace         },
  };
  static struct {
     char *zName;
     Tcl_ObjCmdProc *xProc;
     void *clientData;
  } aObjCmd[] = {
     { "sqlite3_connection_pointer",    get_sqlite_pointer, 0 },
................................................................................
  extern int sqlite3_opentemp_count;
  extern int sqlite3_memUsed;
  extern char *sqlite3_malloc_id;
  extern int sqlite3_memMax;
  extern int sqlite3_like_count;
  extern int sqlite3_tsd_count;
  extern int sqlite3_xferopt_count;
  extern int sqlite3_pager_readdb_count;
  extern int sqlite3_pager_writedb_count;
  extern int sqlite3_pager_writej_count;
  extern int sqlite3_pager_pgfree_count;
#if OS_UNIX && defined(SQLITE_TEST) && defined(THREADSAFE) && THREADSAFE
  extern int threadsOverrideEachOthersLocks;
#endif
#if OS_WIN
  extern int sqlite3_os_type;
#endif
#ifdef SQLITE_DEBUG
................................................................................
      (char*)&sqlite3_current_time, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite_os_trace",
      (char*)&sqlite3_os_trace, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite3_tsd_count",
      (char*)&sqlite3_tsd_count, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite3_xferopt_count",
      (char*)&sqlite3_xferopt_count, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite3_pager_readdb_count",
      (char*)&sqlite3_pager_readdb_count, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite3_pager_writedb_count",
      (char*)&sqlite3_pager_writedb_count, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite3_pager_writej_count",
      (char*)&sqlite3_pager_writej_count, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite3_pager_pgfree_count",
      (char*)&sqlite3_pager_pgfree_count, TCL_LINK_INT);
#ifndef SQLITE_OMIT_UTF16
  Tcl_LinkVar(interp, "unaligned_string_counter",
      (char*)&unaligned_string_counter, TCL_LINK_INT);
#endif
#if OS_UNIX && defined(SQLITE_TEST) && defined(THREADSAFE) && THREADSAFE
  Tcl_LinkVar(interp, "threadsOverrideEachOthersLocks",
      (char*)&threadsOverrideEachOthersLocks, TCL_LINK_INT);

Changes to test/ioerr2.test.

11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
..
78
79
80
81
82
83
84
85












86
87
88
89
90
91
92
93
94
95
96
97
98
# This file implements regression tests for SQLite library.  The
# focus of this file is testing for correct handling of I/O errors
# such as writes failing because the disk is full.
# 
# The tests in this file use special facilities that are only
# available in the SQLite test fixture.
#
# $Id: ioerr2.test,v 1.3 2007/04/09 12:45:03 drh Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

do_test ioerr2-1.1 {
  execsql {
    PRAGMA cache_size = 10;
................................................................................
  INSERT INTO t1 SELECT randstr(400,400), randstr(400,400) 
    WHERE (random()%7)==0;
  UPDATE t1 SET a = randstr(400,400), b = randstr(400,400) 
    WHERE (random()%7)==0;
  ROLLBACK;
}

breakpoint












foreach bPersist [list 0 1] {
  set ::go 1
  for {set ::N 1} {$::go} {incr ::N} {
    set ::sqlite_io_error_hit 0
    set ::sqlite_io_error_persist $bPersist
    set ::sqlite_io_error_pending $::N

    foreach {::go res} [catchsql $sql] {}
    check_db ioerr2-3.$bPersist.$::N
  }
}

finish_test







|







 







|
>
>
>
>
>
>
>
>
>
>
>
>








|




11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
..
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
# This file implements regression tests for SQLite library.  The
# focus of this file is testing for correct handling of I/O errors
# such as writes failing because the disk is full.
# 
# The tests in this file use special facilities that are only
# available in the SQLite test fixture.
#
# $Id: ioerr2.test,v 1.4 2007/04/13 02:14:30 drh Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

do_test ioerr2-1.1 {
  execsql {
    PRAGMA cache_size = 10;
................................................................................
  INSERT INTO t1 SELECT randstr(400,400), randstr(400,400) 
    WHERE (random()%7)==0;
  UPDATE t1 SET a = randstr(400,400), b = randstr(400,400) 
    WHERE (random()%7)==0;
  ROLLBACK;
}

foreach bPersist [list 0 1] {
  set ::go 1
  for {set ::N 1} {$::go} {incr ::N} {
    db close
    sqlite3 db test.db
    set ::sqlite_io_error_hit 0
    set ::sqlite_io_error_persist $bPersist
    set ::sqlite_io_error_pending $::N

    foreach {::go res} [catchsql $sql] {}
    check_db ioerr2-3.$bPersist.$::N
  }
}
foreach bPersist [list 0 1] {
  set ::go 1
  for {set ::N 1} {$::go} {incr ::N} {
    set ::sqlite_io_error_hit 0
    set ::sqlite_io_error_persist $bPersist
    set ::sqlite_io_error_pending $::N

    foreach {::go res} [catchsql $sql] {}
    check_db ioerr2-3.[expr {$bPersist+2}].$::N
  }
}

finish_test

Added test/pageropt.test.





















































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
# 2007 April 12
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.
# The focus of the tests in this file are to verify that the
# pager optimizations implemented in version 3.3.14 work.
#
# $Id: pageropt.test,v 1.1 2007/04/13 02:14:30 drh Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

ifcapable {!pager_pragmas} {
  finish_test
  return
}

# Run the SQL statement supplied by the argument and return
# the results.  Prepend four integers to the beginning of the
# result which are
#
#     (1)  The number of page reads from the database
#     (2)  The number of page writes to the database
#     (3)  The number of page writes to the journal
#     (4)  The number of cache pages freed
#
proc pagercount_sql {sql {db db}} {
  global sqlite3_pager_readdb_count
  global sqlite3_pager_writedb_count
  global sqlite3_pager_writej_count
  global sqlite3_pager_pgfree_count
  set sqlite3_pager_readdb_count 0
  set sqlite3_pager_writedb_count 0
  set sqlite3_pager_writej_count 0
  set sqlite3_pager_pgfree_count 0
  set r [$db eval $sql]
  set cnt [list $sqlite3_pager_readdb_count \
                $sqlite3_pager_writedb_count \
                $sqlite3_pager_writej_count \
                $sqlite3_pager_pgfree_count]
  return [concat $cnt $r]
}

# Setup the test database
#
do_test pageropt-1.1 {
  execsql {
    PRAGMA auto_vacuum = OFF;
    PRAGMA page_size = 1024;
  }
  pagercount_sql {
    CREATE TABLE t1(x);
  }
} {0 2 0 0}
do_test pageropt-1.2 {
  pagercount_sql {
    INSERT INTO t1 VALUES(randomblob(5000));
  }
} {0 6 2 0}

# Verify that values remain in cache on for subsequent reads.
# We should not have to go back to disk.
#
do_test pageropt-1.3 {
  pagercount_sql {
    SELECT length(x) FROM t1
  }
} {0 0 0 0 5000}

# If another thread reads the database, the original cache
# remains valid.
#
sqlite3 db2 test.db
set blobcontent [db2 one {SELECT hex(x) FROM t1}]
do_test pageropt-1.4 {
  pagercount_sql {
    SELECT hex(x) FROM t1
  }
} [list 0 0 0 0 $blobcontent]

# But if the other thread modifies the database, then the cache
# must refill.
#
do_test pageropt-1.5 {
  db2 eval {CREATE TABLE t2(y)}
  pagercount_sql {
    SELECT hex(x) FROM t1
  }
} [list 6 0 0 6 $blobcontent]
do_test pageropt-1.6 {
  pagercount_sql {
    SELECT hex(x) FROM t1
  }
} [list 0 0 0 0 $blobcontent]

# Verify that the last page of an overflow chain is not read from
# disk when deleting a row.  The one row of t1(x) has four pages
# of overflow.  So deleting that row from t1 should involve reading
# the sqlite_master table (1 page) the main page of t1 (1 page) and
# the three overflow pages of t1 for a total of 5 pages.
#
# Pages written are page 1 (for the freelist pointer), the root page
# of the table, and one of the overflow chain pointers because it
# becomes the trunk of the freelist.  Total 3.
#
do_test pageropt-2.1 {
  db close
  sqlite3 db test.db
  pagercount_sql {
    DELETE FROM t1 WHERE rowid=1
  }
} {5 3 3 0}

# When pulling pages off of the freelist, there is no reason
# to actually bring in the old content.
#
do_test pageropt-2.2 {
  db close
  sqlite3 db test.db
  pagercount_sql {
    INSERT INTO t1 VALUES(randomblob(1500));
  }
} {3 4 3 0}
do_test pageropt-2.3 {
  pagercount_sql {
    INSERT INTO t1 VALUES(randomblob(1500));
  }
} {0 4 3 0}

catch {db2 close}
finish_test