SQLite

Check-in [81629ba914]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem reading from temp databases in SQLITE_DIRECT_OVERFLOW_READ builds.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 81629ba91475938b6ad528e7b1dbef4ad22239782bb2e9c1bb59413aba11da87
User & Date: dan 2018-11-22 19:10:14.702
References
2018-11-27
14:41
Remove the sqlite3PagerUseWal() routine which was made obsolete by the [81629ba91475938b6ad] change. (check-in: 4331b4990c user: drh tags: trunk)
Context
2018-11-24
16:07
Remove the unused mmapSizeActual field from the Windows sqlite3_file implementation. (check-in: 0e7aa62227 user: drh tags: trunk)
2018-11-22
19:10
Fix a problem reading from temp databases in SQLITE_DIRECT_OVERFLOW_READ builds. (check-in: 81629ba914 user: dan tags: trunk)
2018-11-21
14:27
Improvements to the ossfuzz.c fuzz-testing module so that it works with -DSQLITE_OMIT_PROGRESS_CALLBACK and with -DSQLITE_OMIT_INIT. (check-in: d343f7d6b0 user: drh tags: trunk)
Changes
Unified Diff Show Whitespace Changes Patch
Changes to src/btree.c.
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789

4790
4791
4792
4793
4794
4795
4796
          rc = getOverflowPage(pBt, nextPage, 0, &nextPage);
        }
        offset -= ovflSize;
      }else{
        /* Need to read this page properly. It contains some of the
        ** range of data that is being read (eOp==0) or written (eOp!=0).
        */
#ifdef SQLITE_DIRECT_OVERFLOW_READ
        sqlite3_file *fd;      /* File from which to do direct overflow read */
#endif
        int a = amt;
        if( a + offset > ovflSize ){
          a = ovflSize - offset;
        }

#ifdef SQLITE_DIRECT_OVERFLOW_READ
        /* If all the following are true:
        **
        **   1) this is a read operation, and 
        **   2) data is required from the start of this overflow page, and
        **   3) there is no open write-transaction, and
        **   4) the database is file-backed, and
        **   5) the page is not in the WAL file
        **   6) at least 4 bytes have already been read into the output buffer 
        **
        ** then data can be read directly from the database file into the
        ** output buffer, bypassing the page-cache altogether. This speeds
        ** up loading large records that span many overflow pages.
        */
        if( eOp==0                                             /* (1) */
         && offset==0                                          /* (2) */
         && pBt->inTransaction==TRANS_READ                     /* (3) */
         && (fd = sqlite3PagerFile(pBt->pPager))->pMethods     /* (4) */
         && 0==sqlite3PagerUseWal(pBt->pPager, nextPage)       /* (5) */
         && &pBuf[-4]>=pBufStart                               /* (6) */
        ){

          u8 aSave[4];
          u8 *aWrite = &pBuf[-4];
          assert( aWrite>=pBufStart );                         /* due to (6) */
          memcpy(aSave, aWrite, 4);
          rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1));
          nextPage = get4byte(aWrite);
          memcpy(aWrite, aSave, 4);







<
<
<










|










<
<
|


>







4754
4755
4756
4757
4758
4759
4760



4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781


4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
          rc = getOverflowPage(pBt, nextPage, 0, &nextPage);
        }
        offset -= ovflSize;
      }else{
        /* Need to read this page properly. It contains some of the
        ** range of data that is being read (eOp==0) or written (eOp!=0).
        */



        int a = amt;
        if( a + offset > ovflSize ){
          a = ovflSize - offset;
        }

#ifdef SQLITE_DIRECT_OVERFLOW_READ
        /* If all the following are true:
        **
        **   1) this is a read operation, and 
        **   2) data is required from the start of this overflow page, and
        **   3) there are no dirty pages in the page-cache
        **   4) the database is file-backed, and
        **   5) the page is not in the WAL file
        **   6) at least 4 bytes have already been read into the output buffer 
        **
        ** then data can be read directly from the database file into the
        ** output buffer, bypassing the page-cache altogether. This speeds
        ** up loading large records that span many overflow pages.
        */
        if( eOp==0                                             /* (1) */
         && offset==0                                          /* (2) */


         && sqlite3PagerDirectReadOk(pBt->pPager, nextPage)    /* (3,4,5) */
         && &pBuf[-4]>=pBufStart                               /* (6) */
        ){
          sqlite3_file *fd = sqlite3PagerFile(pBt->pPager);
          u8 aSave[4];
          u8 *aWrite = &pBuf[-4];
          assert( aWrite>=pBufStart );                         /* due to (6) */
          memcpy(aSave, aWrite, 4);
          rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1));
          nextPage = get4byte(aWrite);
          memcpy(aWrite, aSave, 4);
Changes to src/pager.c.
820
821
822
823
824
825
826
























827
828
829
830
831
832
833
**   if( isOpen(pPager->jfd) ){ ...
**
** instead of
**
**   if( pPager->jfd->pMethods ){ ...
*/
#define isOpen(pFd) ((pFd)->pMethods!=0)

























/*
** Return true if this pager uses a write-ahead log to read page pgno.
** Return false if the pager reads pgno directly from the database.
*/
#if !defined(SQLITE_OMIT_WAL) && defined(SQLITE_DIRECT_OVERFLOW_READ)
int sqlite3PagerUseWal(Pager *pPager, Pgno pgno){







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
**   if( isOpen(pPager->jfd) ){ ...
**
** instead of
**
**   if( pPager->jfd->pMethods ){ ...
*/
#define isOpen(pFd) ((pFd)->pMethods!=0)

#ifdef SQLITE_DIRECT_OVERFLOW_READ
/*
** Return true if page pgno can be read directly from the database file
** by the b-tree layer. This is the case if:
**
**   * the database file is open,
**   * there are no dirty pages in the cache, and
**   * the desired page is not currently in the wal file.
*/
int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){
  if( pPager->fd->pMethods==0 ) return 0;
  if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0;
#ifndef SQLITE_OMIT_WAL
  if( pPager->pWal ){
    u32 iRead = 0;
    int rc;
    rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iRead);
    return (rc==SQLITE_OK && iRead==0);
  }
#endif
  return 1;
}
#endif

/*
** Return true if this pager uses a write-ahead log to read page pgno.
** Return false if the pager reads pgno directly from the database.
*/
#if !defined(SQLITE_OMIT_WAL) && defined(SQLITE_DIRECT_OVERFLOW_READ)
int sqlite3PagerUseWal(Pager *pPager, Pgno pgno){
Changes to src/pager.h.
188
189
190
191
192
193
194




195
196
197
198
199
200
201
  int sqlite3PagerSnapshotRecover(Pager *pPager);
  int sqlite3PagerSnapshotCheck(Pager *pPager, sqlite3_snapshot *pSnapshot);
  void sqlite3PagerSnapshotUnlock(Pager *pPager);
# endif
#else
# define sqlite3PagerUseWal(x,y) 0
#endif





#ifdef SQLITE_ENABLE_ZIPVFS
  int sqlite3PagerWalFramesize(Pager *pPager);
#endif

/* Functions used to query pager state and configuration. */
u8 sqlite3PagerIsreadonly(Pager*);







>
>
>
>







188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
  int sqlite3PagerSnapshotRecover(Pager *pPager);
  int sqlite3PagerSnapshotCheck(Pager *pPager, sqlite3_snapshot *pSnapshot);
  void sqlite3PagerSnapshotUnlock(Pager *pPager);
# endif
#else
# define sqlite3PagerUseWal(x,y) 0
#endif

#ifdef SQLITE_DIRECT_OVERFLOW_READ
  int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno);
#endif

#ifdef SQLITE_ENABLE_ZIPVFS
  int sqlite3PagerWalFramesize(Pager *pPager);
#endif

/* Functions used to query pager state and configuration. */
u8 sqlite3PagerIsreadonly(Pager*);
Changes to src/pcache.c.
851
852
853
854
855
856
857









858
859
860
861
862
863
864
int sqlite3PCachePercentDirty(PCache *pCache){
  PgHdr *pDirty;
  int nDirty = 0;
  int nCache = numberOfCachePages(pCache);
  for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext) nDirty++;
  return nCache ? (int)(((i64)nDirty * 100) / nCache) : 0;
}










#if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG)
/*
** For all dirty pages currently in the cache, invoke the specified
** callback. This is only used if the SQLITE_CHECK_PAGES macro is
** defined.
*/







>
>
>
>
>
>
>
>
>







851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
int sqlite3PCachePercentDirty(PCache *pCache){
  PgHdr *pDirty;
  int nDirty = 0;
  int nCache = numberOfCachePages(pCache);
  for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext) nDirty++;
  return nCache ? (int)(((i64)nDirty * 100) / nCache) : 0;
}

#ifdef SQLITE_DIRECT_OVERFLOW_READ
/* 
** Return true if there are one or more dirty pages in the cache. Else false.
*/
int sqlite3PCacheIsDirty(PCache *pCache){
  return (pCache->pDirty!=0);
}
#endif

#if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG)
/*
** For all dirty pages currently in the cache, invoke the specified
** callback. This is only used if the SQLITE_CHECK_PAGES macro is
** defined.
*/
Changes to src/pcache.h.
178
179
180
181
182
183
184




185
186

/* Return the header size */
int sqlite3HeaderSizePcache(void);
int sqlite3HeaderSizePcache1(void);

/* Number of dirty pages as a percentage of the configured cache size */
int sqlite3PCachePercentDirty(PCache*);





#endif /* _PCACHE_H_ */







>
>
>
>


178
179
180
181
182
183
184
185
186
187
188
189
190

/* Return the header size */
int sqlite3HeaderSizePcache(void);
int sqlite3HeaderSizePcache1(void);

/* Number of dirty pages as a percentage of the configured cache size */
int sqlite3PCachePercentDirty(PCache*);

#ifdef SQLITE_DIRECT_OVERFLOW_READ
int sqlite3PCacheIsDirty(PCache *pCache);
#endif

#endif /* _PCACHE_H_ */
Changes to test/tempdb2.test.
72
73
74
75
76
77
78




















79

  ROLLBACK;
}

do_execsql_test 1.4 {
  SELECT b=int2str(2) FROM t1
} {1 1 1}





















finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

>
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
  ROLLBACK;
}

do_execsql_test 1.4 {
  SELECT b=int2str(2) FROM t1
} {1 1 1}

#-------------------------------------------------------------------------
db close
sqlite3 db ""
db func int2str int2str

do_execsql_test 2.0 {
  PRAGMA cache_size = -100;
  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
  WITH c(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100 ) 
    INSERT INTO t1 SELECT x, int2str(x) FROM c;
}

do_execsql_test 2.1 {
  INSERT INTO t1 VALUES(10001, int2str(1001) || int2str(1001) || int2str(1001));
}

do_execsql_test 2.2 {
  SELECT b FROM t1 WHERE a = 10001;
} "[int2str 1001][int2str 1001][int2str 1001]"

finish_test