/ Check-in [81629ba9]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem reading from temp databases in SQLITE_DIRECT_OVERFLOW_READ builds.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 81629ba91475938b6ad528e7b1dbef4ad22239782bb2e9c1bb59413aba11da87
User & Date: dan 2018-11-22 19:10:14
References
2018-11-27
14:41
Remove the sqlite3PagerUseWal() routine which was made obsolete by the [81629ba91475938b6ad] change. check-in: 4331b499 user: drh tags: trunk
Context
2018-11-24
16:07
Remove the unused mmapSizeActual field from the Windows sqlite3_file implementation. check-in: 0e7aa622 user: drh tags: trunk
2018-11-22
19:10
Fix a problem reading from temp databases in SQLITE_DIRECT_OVERFLOW_READ builds. check-in: 81629ba9 user: dan tags: trunk
2018-11-21
14:27
Improvements to the ossfuzz.c fuzz-testing module so that it works with -DSQLITE_OMIT_PROGRESS_CALLBACK and with -DSQLITE_OMIT_INIT. check-in: d343f7d6 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/btree.c.

  4754   4754             rc = getOverflowPage(pBt, nextPage, 0, &nextPage);
  4755   4755           }
  4756   4756           offset -= ovflSize;
  4757   4757         }else{
  4758   4758           /* Need to read this page properly. It contains some of the
  4759   4759           ** range of data that is being read (eOp==0) or written (eOp!=0).
  4760   4760           */
  4761         -#ifdef SQLITE_DIRECT_OVERFLOW_READ
  4762         -        sqlite3_file *fd;      /* File from which to do direct overflow read */
  4763         -#endif
  4764   4761           int a = amt;
  4765   4762           if( a + offset > ovflSize ){
  4766   4763             a = ovflSize - offset;
  4767   4764           }
  4768   4765   
  4769   4766   #ifdef SQLITE_DIRECT_OVERFLOW_READ
  4770   4767           /* If all the following are true:
  4771   4768           **
  4772   4769           **   1) this is a read operation, and 
  4773   4770           **   2) data is required from the start of this overflow page, and
  4774         -        **   3) there is no open write-transaction, and
         4771  +        **   3) there are no dirty pages in the page-cache
  4775   4772           **   4) the database is file-backed, and
  4776   4773           **   5) the page is not in the WAL file
  4777   4774           **   6) at least 4 bytes have already been read into the output buffer 
  4778   4775           **
  4779   4776           ** then data can be read directly from the database file into the
  4780   4777           ** output buffer, bypassing the page-cache altogether. This speeds
  4781   4778           ** up loading large records that span many overflow pages.
  4782   4779           */
  4783   4780           if( eOp==0                                             /* (1) */
  4784   4781            && offset==0                                          /* (2) */
  4785         -         && pBt->inTransaction==TRANS_READ                     /* (3) */
  4786         -         && (fd = sqlite3PagerFile(pBt->pPager))->pMethods     /* (4) */
  4787         -         && 0==sqlite3PagerUseWal(pBt->pPager, nextPage)       /* (5) */
         4782  +         && sqlite3PagerDirectReadOk(pBt->pPager, nextPage)    /* (3,4,5) */
  4788   4783            && &pBuf[-4]>=pBufStart                               /* (6) */
  4789   4784           ){
         4785  +          sqlite3_file *fd = sqlite3PagerFile(pBt->pPager);
  4790   4786             u8 aSave[4];
  4791   4787             u8 *aWrite = &pBuf[-4];
  4792   4788             assert( aWrite>=pBufStart );                         /* due to (6) */
  4793   4789             memcpy(aSave, aWrite, 4);
  4794   4790             rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1));
  4795   4791             nextPage = get4byte(aWrite);
  4796   4792             memcpy(aWrite, aSave, 4);

Changes to src/pager.c.

   820    820   **   if( isOpen(pPager->jfd) ){ ...
   821    821   **
   822    822   ** instead of
   823    823   **
   824    824   **   if( pPager->jfd->pMethods ){ ...
   825    825   */
   826    826   #define isOpen(pFd) ((pFd)->pMethods!=0)
          827  +
          828  +#ifdef SQLITE_DIRECT_OVERFLOW_READ
          829  +/*
          830  +** Return true if page pgno can be read directly from the database file
          831  +** by the b-tree layer. This is the case if:
          832  +**
          833  +**   * the database file is open,
          834  +**   * there are no dirty pages in the cache, and
          835  +**   * the desired page is not currently in the wal file.
          836  +*/
          837  +int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){
          838  +  if( pPager->fd->pMethods==0 ) return 0;
          839  +  if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0;
          840  +#ifndef SQLITE_OMIT_WAL
          841  +  if( pPager->pWal ){
          842  +    u32 iRead = 0;
          843  +    int rc;
          844  +    rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iRead);
          845  +    return (rc==SQLITE_OK && iRead==0);
          846  +  }
          847  +#endif
          848  +  return 1;
          849  +}
          850  +#endif
   827    851   
   828    852   /*
   829    853   ** Return true if this pager uses a write-ahead log to read page pgno.
   830    854   ** Return false if the pager reads pgno directly from the database.
   831    855   */
   832    856   #if !defined(SQLITE_OMIT_WAL) && defined(SQLITE_DIRECT_OVERFLOW_READ)
   833    857   int sqlite3PagerUseWal(Pager *pPager, Pgno pgno){

Changes to src/pager.h.

   188    188     int sqlite3PagerSnapshotRecover(Pager *pPager);
   189    189     int sqlite3PagerSnapshotCheck(Pager *pPager, sqlite3_snapshot *pSnapshot);
   190    190     void sqlite3PagerSnapshotUnlock(Pager *pPager);
   191    191   # endif
   192    192   #else
   193    193   # define sqlite3PagerUseWal(x,y) 0
   194    194   #endif
          195  +
          196  +#ifdef SQLITE_DIRECT_OVERFLOW_READ
          197  +  int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno);
          198  +#endif
   195    199   
   196    200   #ifdef SQLITE_ENABLE_ZIPVFS
   197    201     int sqlite3PagerWalFramesize(Pager *pPager);
   198    202   #endif
   199    203   
   200    204   /* Functions used to query pager state and configuration. */
   201    205   u8 sqlite3PagerIsreadonly(Pager*);

Changes to src/pcache.c.

   851    851   int sqlite3PCachePercentDirty(PCache *pCache){
   852    852     PgHdr *pDirty;
   853    853     int nDirty = 0;
   854    854     int nCache = numberOfCachePages(pCache);
   855    855     for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext) nDirty++;
   856    856     return nCache ? (int)(((i64)nDirty * 100) / nCache) : 0;
   857    857   }
          858  +
          859  +#ifdef SQLITE_DIRECT_OVERFLOW_READ
          860  +/* 
          861  +** Return true if there are one or more dirty pages in the cache. Else false.
          862  +*/
          863  +int sqlite3PCacheIsDirty(PCache *pCache){
          864  +  return (pCache->pDirty!=0);
          865  +}
          866  +#endif
   858    867   
   859    868   #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG)
   860    869   /*
   861    870   ** For all dirty pages currently in the cache, invoke the specified
   862    871   ** callback. This is only used if the SQLITE_CHECK_PAGES macro is
   863    872   ** defined.
   864    873   */

Changes to src/pcache.h.

   178    178   
   179    179   /* Return the header size */
   180    180   int sqlite3HeaderSizePcache(void);
   181    181   int sqlite3HeaderSizePcache1(void);
   182    182   
   183    183   /* Number of dirty pages as a percentage of the configured cache size */
   184    184   int sqlite3PCachePercentDirty(PCache*);
          185  +
          186  +#ifdef SQLITE_DIRECT_OVERFLOW_READ
          187  +int sqlite3PCacheIsDirty(PCache *pCache);
          188  +#endif
   185    189   
   186    190   #endif /* _PCACHE_H_ */

Changes to test/tempdb2.test.

    72     72     ROLLBACK;
    73     73   }
    74     74   
    75     75   do_execsql_test 1.4 {
    76     76     SELECT b=int2str(2) FROM t1
    77     77   } {1 1 1}
    78     78   
           79  +#-------------------------------------------------------------------------
           80  +db close
           81  +sqlite3 db ""
           82  +db func int2str int2str
           83  +
           84  +do_execsql_test 2.0 {
           85  +  PRAGMA cache_size = -100;
           86  +  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
           87  +  WITH c(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100 ) 
           88  +    INSERT INTO t1 SELECT x, int2str(x) FROM c;
           89  +}
           90  +
           91  +do_execsql_test 2.1 {
           92  +  INSERT INTO t1 VALUES(10001, int2str(1001) || int2str(1001) || int2str(1001));
           93  +}
           94  +
           95  +do_execsql_test 2.2 {
           96  +  SELECT b FROM t1 WHERE a = 10001;
           97  +} "[int2str 1001][int2str 1001][int2str 1001]"
           98  +
    79     99   finish_test
          100  +