SQLite

Check-in [f1f94a971e]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Change the page cache so that a new sqlite3_pcache object is allocated as soon as the page cache is opened, not delayed until the first fetch request. This give a noticable performance boost. The interface between pager and the page cache has changed slightly, which might break ZIPVFS.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: f1f94a971e031e784f8c30a6faf829df58709329
User & Date: drh 2014-08-26 15:06:49.829
Context
2014-08-27
00:50
Performance enhancement in sqlite3VdbeMemNulTerminate(). (check-in: f94cacc393 user: drh tags: trunk)
2014-08-26
15:06
Change the page cache so that a new sqlite3_pcache object is allocated as soon as the page cache is opened, not delayed until the first fetch request. This give a noticable performance boost. The interface between pager and the page cache has changed slightly, which might break ZIPVFS. (check-in: f1f94a971e user: drh tags: trunk)
2014-08-25
22:37
Add an assert() and five testcase() macros to the OP_Cast opcode implementation to help verify that it is fully tested. (check-in: af364cce9d user: drh tags: trunk)
Changes
Side-by-Side Diff Ignore Whitespace Patch
Changes to src/pager.c.
3618
3619
3620
3621
3622
3623
3624
3625

3626
3627
3628
3629
3630
3631
3632
3618
3619
3620
3621
3622
3623
3624

3625
3626
3627
3628
3629
3630
3631
3632







-
+








    if( rc==SQLITE_OK ){
      pager_reset(pPager);
      pPager->dbSize = (Pgno)((nByte+pageSize-1)/pageSize);
      pPager->pageSize = pageSize;
      sqlite3PageFree(pPager->pTmpSpace);
      pPager->pTmpSpace = pNew;
      sqlite3PcacheSetPageSize(pPager->pPCache, pageSize);
      rc = sqlite3PcacheSetPageSize(pPager->pPCache, pageSize);
    }
  }

  *pPageSize = pPager->pageSize;
  if( rc==SQLITE_OK ){
    if( nReserve<0 ) nReserve = pPager->nReserve;
    assert( nReserve>=0 && nReserve<1000 );
4381
4382
4383
4384
4385
4386
4387
4388

4389
4390
4391
4392
4393
4394
4395
4381
4382
4383
4384
4385
4386
4387

4388
4389
4390
4391
4392
4393
4394
4395







-
+







  **
  ** The doNotSpill ROLLBACK and OFF bits inhibits all cache spilling
  ** regardless of whether or not a sync is required.  This is set during
  ** a rollback or by user request, respectively.
  **
  ** Spilling is also prohibited when in an error state since that could
  ** lead to database corruption.   In the current implementaton it 
  ** is impossible for sqlite3PcacheFetch() to be called with createFlag==1
  ** is impossible for sqlite3PcacheFetch() to be called with createFlag==3
  ** while in the error state, hence it is impossible for this routine to
  ** be called in the error state.  Nevertheless, we include a NEVER()
  ** test for the error state as a safeguard against future changes.
  */
  if( NEVER(pPager->errCode) ) return SQLITE_OK;
  testcase( pPager->doNotSpill & SPILLFLAG_ROLLBACK );
  testcase( pPager->doNotSpill & SPILLFLAG_OFF );
4717
4718
4719
4720
4721
4722
4723







4724
4725


4726
4727
4728
4729

4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730


4731
4732
4733
4734

4735
4736
4737
4738
4739
4740






4741
4742
4743
4744
4745
4746
4747







+
+
+
+
+
+
+
-
-
+
+


-

+




-
-
-
-
-
-







  */
  if( rc==SQLITE_OK ){
    assert( pPager->memDb==0 );
    rc = sqlite3PagerSetPagesize(pPager, &szPageDflt, -1);
    testcase( rc!=SQLITE_OK );
  }

  /* Initialize the PCache object. */
  if( rc==SQLITE_OK ){
    assert( nExtra<1000 );
    nExtra = ROUND8(nExtra);
    rc = sqlite3PcacheOpen(szPageDflt, nExtra, !memDb,
                           !memDb?pagerStress:0, (void *)pPager, pPager->pPCache);
  }
  /* If an error occurred in either of the blocks above, free the 
  ** Pager structure and close the file.

  /* If an error occurred above, free the  Pager structure and close the file.
  */
  if( rc!=SQLITE_OK ){
    assert( !pPager->pTmpSpace );
    sqlite3OsClose(pPager->fd);
    sqlite3PageFree(pPager->pTmpSpace);
    sqlite3_free(pPager);
    return rc;
  }

  /* Initialize the PCache object. */
  assert( nExtra<1000 );
  nExtra = ROUND8(nExtra);
  sqlite3PcacheOpen(szPageDflt, nExtra, !memDb,
                    !memDb?pagerStress:0, (void *)pPager, pPager->pPCache);

  PAGERTRACE(("OPEN %d %s\n", FILEHANDLEID(pPager->fd), pPager->zFilename));
  IOTRACE(("OPEN %p %s\n", pPager, pPager->zFilename))

  pPager->useJournal = (u8)useJournal;
  /* pPager->stmtOpen = 0; */
  /* pPager->stmtInUse = 0; */
  /* pPager->nRef = 0; */
5314
5315
5316
5317
5318
5319
5320
5321

5322
5323
5324
5325
5326
5327
5328
5315
5316
5317
5318
5319
5320
5321

5322
5323
5324
5325
5326
5327
5328
5329







-
+







        }
      }
      if( rc!=SQLITE_OK ){
        goto pager_acquire_err;
      }
    }

    rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage);
    rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 3, ppPage);
  }

  if( rc!=SQLITE_OK ){
    /* Either the call to sqlite3PcacheFetch() returned an error or the
    ** pager was already in the error-state when this function was called.
    ** Set pPg to 0 and jump to the exception handler.  */
    pPg = 0;
Changes to src/pcache.c.
139
140
141
142
143
144
145











146
147
148
149
150
151
152
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163







+
+
+
+
+
+
+
+
+
+
+







  if( p->pCache->bPurgeable ){
    if( p->pgno==1 ){
      p->pCache->pPage1 = 0;
    }
    sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 0);
  }
}

/*
** Compute the number of pages of cache requested.
*/
static int numberOfCachePages(PCache *p){
  if( p->szCache>=0 ){
    return p->szCache;
  }else{
    return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra));
  }
}

/*************************************************** General Interfaces ******
**
** Initialize and shutdown the page cache subsystem. Neither of these 
** functions are threadsafe.
*/
int sqlite3PcacheInitialize(void){
172
173
174
175
176
177
178
179

180
181
182
183
184
185
186
187
188

189
190
191
192
193
194

195
196
197
198
199
200
201

202







203
204
205




206
207
208
209


210
211
212
213
214
215
216

217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235

236

237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266




267
268
269
270
271
272
273
183
184
185
186
187
188
189

190
191
192
193
194
195
196
197
198

199
200
201
202
203
204
205
206
207
208
209
210
211
212

213
214
215
216
217
218
219
220
221



222
223
224
225
226



227
228







229



230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246

247
248
249



















250
251
252
253
254
255
256


257
258
259
260
261
262
263
264
265
266
267







-
+








-
+






+






-
+

+
+
+
+
+
+
+
-
-
-
+
+
+
+

-
-
-
+
+
-
-
-
-
-
-
-
+
-
-
-
















+
-
+


-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-







-
-
+
+
+
+








/*
** Create a new PCache object. Storage space to hold the object
** has already been allocated and is passed in as the p pointer. 
** The caller discovers how much space needs to be allocated by 
** calling sqlite3PcacheSize().
*/
void sqlite3PcacheOpen(
int sqlite3PcacheOpen(
  int szPage,                  /* Size of every page */
  int szExtra,                 /* Extra space associated with each page */
  int bPurgeable,              /* True if pages are on backing store */
  int (*xStress)(void*,PgHdr*),/* Call to try to make pages clean */
  void *pStress,               /* Argument to xStress */
  PCache *p                    /* Preallocated space for the PCache */
){
  memset(p, 0, sizeof(PCache));
  p->szPage = szPage;
  p->szPage = 1;
  p->szExtra = szExtra;
  p->bPurgeable = bPurgeable;
  p->eCreate = 2;
  p->xStress = xStress;
  p->pStress = pStress;
  p->szCache = 100;
  return sqlite3PcacheSetPageSize(p, szPage);
}

/*
** Change the page size for PCache object. The caller must ensure that there
** are no outstanding page references when this function is called.
*/
void sqlite3PcacheSetPageSize(PCache *pCache, int szPage){
int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){
  assert( pCache->nRef==0 && pCache->pDirty==0 );
  if( pCache->szPage ){
    sqlite3_pcache *pNew;
    pNew = sqlite3GlobalConfig.pcache2.xCreate(
                szPage, pCache->szExtra + sizeof(PgHdr), pCache->bPurgeable
    );
    if( pNew==0 ) return SQLITE_NOMEM;
    sqlite3GlobalConfig.pcache2.xCachesize(pNew, numberOfCachePages(pCache));
  if( pCache->pCache ){
    sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache);
    pCache->pCache = 0;
    if( pCache->pCache ){
      sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache);
    }
    pCache->pCache = pNew;
    pCache->pPage1 = 0;
  }
  pCache->szPage = szPage;
}
    pCache->szPage = szPage;
  }

/*
** Compute the number of pages of cache requested.
*/
static int numberOfCachePages(PCache *p){
  if( p->szCache>=0 ){
    return p->szCache;
  return SQLITE_OK;
  }else{
    return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra));
  }
}

/*
** Try to obtain a page from the cache.
*/
int sqlite3PcacheFetch(
  PCache *pCache,       /* Obtain the page from this cache */
  Pgno pgno,            /* Page number to obtain */
  int createFlag,       /* If true, create page if it does not exist already */
  PgHdr **ppPage        /* Write the page here */
){
  sqlite3_pcache_page *pPage;
  PgHdr *pPgHdr = 0;
  int eCreate;

  assert( pCache!=0 );
  assert( pCache->pCache!=0 );
  assert( createFlag==1 || createFlag==0 );
  assert( createFlag==3 || createFlag==0 );
  assert( pgno>0 );

  /* If the pluggable cache (sqlite3_pcache*) has not been allocated,
  ** allocate it now.
  */
  if( !pCache->pCache ){
    sqlite3_pcache *p;
    if( !createFlag ){
      *ppPage = 0;
      return SQLITE_OK;
    }
    p = sqlite3GlobalConfig.pcache2.xCreate(
        pCache->szPage, pCache->szExtra + sizeof(PgHdr), pCache->bPurgeable
    );
    if( !p ){
      return SQLITE_NOMEM;
    }
    sqlite3GlobalConfig.pcache2.xCachesize(p, numberOfCachePages(pCache));
    pCache->pCache = p;
  }

  /* eCreate defines what to do if the page does not exist.
  **    0     Do not allocate a new page.  (createFlag==0)
  **    1     Allocate a new page if doing so is inexpensive.
  **          (createFlag==1 AND bPurgeable AND pDirty)
  **    2     Allocate a new page even it doing so is difficult.
  **          (createFlag==1 AND !(bPurgeable AND pDirty)
  */
  eCreate = createFlag==0 ? 0 : pCache->eCreate;
  assert( (createFlag*(1+(!pCache->bPurgeable||!pCache->pDirty)))==eCreate );
  eCreate = createFlag & pCache->eCreate;
  assert( eCreate==0 || eCreate==1 || eCreate==2 );
  assert( createFlag==0 || pCache->eCreate==eCreate );
  assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) );
  pPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate);
  if( !pPage && eCreate==1 ){
    PgHdr *pPg;

    /* Find a dirty page to write-out and recycle. First try to find a 
    ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC
    ** cleared), but if that is not possible settle for any other 
467
468
469
470
471
472
473
474
475


476
477
478
479
480
481
482
483
461
462
463
464
465
466
467


468
469

470
471
472
473
474
475
476







-
-
+
+
-







  }
}

/*
** Close a cache.
*/
void sqlite3PcacheClose(PCache *pCache){
  if( pCache->pCache ){
    sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache);
  assert( pCache->pCache!=0 );
  sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache);
  }
}

/* 
** Discard the contents of the cache.
*/
void sqlite3PcacheClear(PCache *pCache){
  sqlite3PcacheTruncate(pCache, 0);
578
579
580
581
582
583
584
585
586
587


588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604

605
606
607
608


609
610
611
612
613
614
615
616
617


618
619
620
621
622
623
624
625
571
572
573
574
575
576
577



578
579


580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596



597
598

599
600
601
602
603
604


605
606

607
608
609
610
611
612
613







-
-
-
+
+
-
-















+

-
-
-
+
+
-






-
-
+
+
-







  return p->nRef;
}

/* 
** Return the total number of pages in the cache.
*/
int sqlite3PcachePagecount(PCache *pCache){
  int nPage = 0;
  if( pCache->pCache ){
    nPage = sqlite3GlobalConfig.pcache2.xPagecount(pCache->pCache);
  assert( pCache->pCache!=0 );
  return sqlite3GlobalConfig.pcache2.xPagecount(pCache->pCache);
  }
  return nPage;
}

#ifdef SQLITE_TEST
/*
** Get the suggested cache-size value.
*/
int sqlite3PcacheGetCachesize(PCache *pCache){
  return numberOfCachePages(pCache);
}
#endif

/*
** Set the suggested cache-size value.
*/
void sqlite3PcacheSetCachesize(PCache *pCache, int mxPage){
  assert( pCache->pCache!=0 );
  pCache->szCache = mxPage;
  if( pCache->pCache ){
    sqlite3GlobalConfig.pcache2.xCachesize(pCache->pCache,
                                           numberOfCachePages(pCache));
  sqlite3GlobalConfig.pcache2.xCachesize(pCache->pCache,
                                         numberOfCachePages(pCache));
  }
}

/*
** Free up as much memory as possible from the page cache.
*/
void sqlite3PcacheShrink(PCache *pCache){
  if( pCache->pCache ){
    sqlite3GlobalConfig.pcache2.xShrink(pCache->pCache);
  assert( pCache->pCache!=0 );
  sqlite3GlobalConfig.pcache2.xShrink(pCache->pCache);
  }
}

#if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG)
/*
** For all dirty pages currently in the cache, invoke the specified
** callback. This is only used if the SQLITE_CHECK_PAGES macro is
** defined.
Changes to src/pcache.h.
64
65
66
67
68
69
70
71

72
73
74
75
76
77
78
79
80
81

82
83
84
85
86
87
88
64
65
66
67
68
69
70

71
72
73
74
75
76
77
78
79
80

81
82
83
84
85
86
87
88







-
+









-
+







*/
void sqlite3PCacheBufferSetup(void *, int sz, int n);

/* Create a new pager cache.
** Under memory stress, invoke xStress to try to make pages clean.
** Only clean and unpinned pages can be reclaimed.
*/
void sqlite3PcacheOpen(
int sqlite3PcacheOpen(
  int szPage,                    /* Size of every page */
  int szExtra,                   /* Extra space associated with each page */
  int bPurgeable,                /* True if pages are on backing store */
  int (*xStress)(void*, PgHdr*), /* Call to try to make pages clean */
  void *pStress,                 /* Argument to xStress */
  PCache *pToInit                /* Preallocated space for the PCache */
);

/* Modify the page-size after the cache has been created. */
void sqlite3PcacheSetPageSize(PCache *, int);
int sqlite3PcacheSetPageSize(PCache *, int);

/* Return the size in bytes of a PCache object.  Used to preallocate
** storage space.
*/
int sqlite3PcacheSize(void);

/* One release per successful fetch.  Page is pinned until released.