Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Always truncate the pager cache when truncating the database file. Also reorganize the code to check the change-counter after first obtaining a shared lock. (CVS 3814) |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
9dc4100eff71be579480ce7939c7da71 |
User & Date: | danielk1977 2007-04-05 17:15:53.000 |
Context
2007-04-05
| ||
17:36 | New testfixture command: sqlite3_pager_refcounts. Returns a list of integers which is the pager refcount for each pager in the database. (CVS 3815) (check-in: 7338e68e0f user: drh tags: trunk) | |
17:15 | Always truncate the pager cache when truncating the database file. Also reorganize the code to check the change-counter after first obtaining a shared lock. (CVS 3814) (check-in: 9dc4100eff user: danielk1977 tags: trunk) | |
14:29 | Use the MEMDB macro instead of OMIT_MEMORYDB in pager_recycle(). (CVS 3813) (check-in: 97c5159816 user: danielk1977 tags: trunk) | |
Changes
Changes to src/pager.c.
︙ | ︙ | |||
14 15 16 17 18 19 20 | ** The pager is used to access a database disk file. It implements ** atomic commit and rollback through the use of a journal file that ** is separate from the database file. The pager also implements file ** locking to prevent two processes from writing the same database ** file simultaneously, or one process from reading the database while ** another is writing. ** | | | 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | ** The pager is used to access a database disk file. It implements ** atomic commit and rollback through the use of a journal file that ** is separate from the database file. The pager also implements file ** locking to prevent two processes from writing the same database ** file simultaneously, or one process from reading the database while ** another is writing. ** ** @(#) $Id: pager.c,v 1.323 2007/04/05 17:15:53 danielk1977 Exp $ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" #include "os.h" #include "pager.h" #include <assert.h> #include <string.h> |
︙ | ︙ | |||
885 886 887 888 889 890 891 | pager_unlock(p); assert( p->errCode || !p->journalOpen || (p->exclusiveMode&&!p->journalOff) ); assert( p->errCode || !p->stmtOpen || p->exclusiveMode ); } /* | | | 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 | pager_unlock(p); assert( p->errCode || !p->journalOpen || (p->exclusiveMode&&!p->journalOff) ); assert( p->errCode || !p->stmtOpen || p->exclusiveMode ); } /* ** Clear the in-memory cache. This routine ** sets the state of the pager back to what it was when it was first ** opened. Any outstanding pages are invalidated and subsequent attempts ** to access those pages will likely result in a coredump. */ static void pager_reset(Pager *pPager){ PgHdr *pPg, *pNext; if( pPager->errCode ) return; |
︙ | ︙ | |||
1261 1262 1263 1264 1265 1266 1267 1268 1269 | #endif } pPager->pDirty = 0; return rc; } #endif /* ** Truncate the main file of the given pager to the number of pages | > > | > | | > > > > > > | 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 | #endif } pPager->pDirty = 0; return rc; } #endif static void pager_truncate_cache(Pager *pPager); /* ** Truncate the main file of the given pager to the number of pages ** indicated. Also truncate the cached representation of the file. */ static int pager_truncate(Pager *pPager, int nPage){ int rc = SQLITE_OK; if( pPager->state>=PAGER_EXCLUSIVE ){ rc = sqlite3OsTruncate(pPager->fd, pPager->pageSize*(i64)nPage); } if( rc==SQLITE_OK ){ pPager->dbSize = nPage; pager_truncate_cache(pPager); } return rc; } /* ** Playback the journal and thus restore the database file to ** the state it was in before we started making changes. ** ** The journal file format is as follows: |
︙ | ︙ | |||
1394 1395 1396 1397 1398 1399 1400 | if( nRec==0 && !isHot ){ nRec = (szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager); } /* If this is the first header read from the journal, truncate the ** database file back to it's original size. */ | < | < < | 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 | if( nRec==0 && !isHot ){ nRec = (szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager); } /* If this is the first header read from the journal, truncate the ** database file back to it's original size. */ if( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ){ rc = pager_truncate(pPager, mxPg); if( rc!=SQLITE_OK ){ goto end_playback; } } /* Copy original pages out of the journal and back into the database file. */ for(i=0; i<nRec; i++){ rc = pager_playback_one_page(pPager, pPager->jfd, 1); if( rc!=SQLITE_OK ){ |
︙ | ︙ | |||
1488 1489 1490 1491 1492 1493 1494 | assert( pPager->fullSync || !hdrOff ); if( !hdrOff ){ hdrOff = szJ; } /* Truncate the database back to its original size. */ | < < < | 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 | assert( pPager->fullSync || !hdrOff ); if( !hdrOff ){ hdrOff = szJ; } /* Truncate the database back to its original size. */ rc = pager_truncate(pPager, pPager->stmtSize); assert( pPager->state>=PAGER_SHARED ); /* Figure out how many records are in the statement journal. */ assert( pPager->stmtInUse && pPager->journalOpen ); sqlite3OsSeek(pPager->stfd, 0); nRec = pPager->stmtNRec; |
︙ | ︙ | |||
2016 2017 2018 2019 2020 2021 2022 | } pPg->pNextFree = pPg->pPrevFree = 0; /* Unlink from the pgno hash table */ unlinkHashChain(pPager, pPg); } | < | > | > > > > > > | < < < | 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 | } pPg->pNextFree = pPg->pPrevFree = 0; /* Unlink from the pgno hash table */ unlinkHashChain(pPager, pPg); } /* ** This routine is used to truncate the cache when a database ** is truncated. Drop from the cache all pages whose pgno is ** larger than pPager->dbSize and is unreferenced. ** ** Referenced pages larger than pPager->dbSize are zeroed. ** ** Actually, at the point this routine is called, it would be ** an error to have a referenced page. But rather than delete ** that page and guarantee a subsequent segfault, it seems better ** to zero it and hope that we error out sanely. */ static void pager_truncate_cache(Pager *pPager){ PgHdr *pPg; PgHdr **ppPg; int dbSize = pPager->dbSize; ppPg = &pPager->pAll; while( (pPg = *ppPg)!=0 ){ if( pPg->pgno<=dbSize ){ ppPg = &pPg->pNextAll; }else if( pPg->nRef>0 ){ memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize); ppPg = &pPg->pNextAll; }else{ *ppPg = pPg->pNextAll; unlinkPage(pPg); makeClean(pPg); sqliteFree(pPg); pPager->nPage--; } } } /* ** Try to obtain a lock on a file. Invoke the busy callback if the lock ** is currently not available. Repeat until the busy callback returns ** false or until the lock succeeds. ** ** Return SQLITE_OK on success and an error code if we cannot obtain |
︙ | ︙ | |||
2096 2097 2098 2099 2100 2101 2102 | return rc; } if( nPage>=(unsigned)pPager->dbSize ){ return SQLITE_OK; } if( MEMDB ){ pPager->dbSize = nPage; | | < < < | 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 | return rc; } if( nPage>=(unsigned)pPager->dbSize ){ return SQLITE_OK; } if( MEMDB ){ pPager->dbSize = nPage; pager_truncate_cache(pPager); return SQLITE_OK; } rc = syncJournal(pPager); if( rc!=SQLITE_OK ){ return rc; } /* Get an exclusive lock on the database before truncating. */ rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ return rc; } rc = pager_truncate(pPager, nPage); return rc; } /* ** Shutdown the page cache. Free all memory and close all files. ** ** If a transaction was in progress when this routine is called, that |
︙ | ︙ | |||
2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 | } } } return nReleased; } #endif /* SQLITE_ENABLE_MEMORY_MANAGEMENT */ /* ** This function is called to obtain the shared lock required before ** data may be read from the pager cache. If the shared lock has already ** been obtained, this function is a no-op. ** ** Immediately after obtaining the shared lock (if required), this function | > > > > > > > > > > > > > > > > > > | 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 | } } } return nReleased; } #endif /* SQLITE_ENABLE_MEMORY_MANAGEMENT */ /* ** Read the content of page pPg out of the database file. */ static int readDbPage(Pager *pPager, PgHdr *pPg, Pgno pgno){ int rc; assert( MEMDB==0 ); rc = sqlite3OsSeek(pPager->fd, (pgno-1)*(i64)pPager->pageSize); if( rc==SQLITE_OK ){ rc = sqlite3OsRead(pPager->fd, PGHDR_TO_DATA(pPg), pPager->pageSize); } IOTRACE(("PGIN %p %d\n", pPager, pgno)) PAGERTRACE3("FETCH %d page %d\n", PAGERID(pPager), pPg->pgno); CODEC1(pPager, PGHDR_TO_DATA(pPg), pPg->pgno, 3); return rc; } /* ** This function is called to obtain the shared lock required before ** data may be read from the pager cache. If the shared lock has already ** been obtained, this function is a no-op. ** ** Immediately after obtaining the shared lock (if required), this function |
︙ | ︙ | |||
2788 2789 2790 2791 2792 2793 2794 2795 | ** stored in Pager.iChangeCount matches that found on page 1 of ** the database file, then no database changes have occured since ** the cache was last valid and it is safe to retain the cached ** pages. Otherwise, if Pager.iChangeCount does not match the ** change-counter on page 1 of the file, the current cache contents ** must be discarded. */ | > > > < | | | | < < < < < < | > | > | < < | < > > > | | < < < < < < < < < | < < | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | > | | < | 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 | ** stored in Pager.iChangeCount matches that found on page 1 of ** the database file, then no database changes have occured since ** the cache was last valid and it is safe to retain the cached ** pages. Otherwise, if Pager.iChangeCount does not match the ** change-counter on page 1 of the file, the current cache contents ** must be discarded. */ u8 zC[4]; u32 iChangeCounter = 0; sqlite3PagerPagecount(pPager); if( pPager->errCode ){ return pPager->errCode; } if( pPager->dbSize>0 ){ /* Read the 4-byte change counter directly from the file. */ rc = sqlite3OsSeek(pPager->fd, 24); if( rc!=SQLITE_OK ){ return rc; } rc = sqlite3OsRead(pPager->fd, zC, 4); if( rc!=SQLITE_OK ){ return rc; } iChangeCounter = (zC[0]<<24) + (zC[1]<<16) + (zC[2]<<8) + zC[3]; } if( iChangeCounter!=pPager->iChangeCount ){ pager_reset(pPager); } } } assert( pPager->exclusiveMode || pPager->state<=PAGER_SHARED ); if( pPager->state==PAGER_UNLOCK ){ pPager->state = PAGER_SHARED; } } return rc; } /* ** Allocate a PgHdr object. Either create a new one or reuse ** an existing one that is not otherwise in use. ** ** A new PgHdr structure is created if any of the following are ** true: ** ** (1) We have not exceeded our maximum allocated cache size ** as set by the "PRAGMA cache_size" command. ** ** (2) There are no unused PgHdr objects available at this time. ** ** (3) This is an in-memory database. ** ** (4) There are no PgHdr objects that do not require a journal ** file sync and a sync of the journal file is currently ** prohibited. ** ** Otherwise, reuse an existing PgHdr. In other words, reuse an ** existing PgHdr if all of the following are true: ** ** (1) We have reached or exceeded the maximum cache size ** allowed by "PRAGMA cache_size". ** ** (2) There is a PgHdr available with PgHdr->nRef==0 ** ** (3) We are not in an in-memory database ** ** (4) Either there is an available PgHdr that does not need ** to be synced to disk or else disk syncing is currently ** allowed. */ static int pagerAllocatePage(Pager *pPager, PgHdr **ppPg){ int rc = SQLITE_OK; PgHdr *pPg; /* Create a new PgHdr if any of the four conditions defined ** above is met: */ if( pPager->nPage<pPager->mxPage || pPager->pFirst==0 || MEMDB || (pPager->pFirstSynced==0 && pPager->doNotSync) ){ if( pPager->nPage>=pPager->nHash ){ pager_resize_hash_table(pPager, pPager->nHash<256 ? 256 : pPager->nHash*2); if( pPager->nHash==0 ){ rc = SQLITE_NOMEM; goto pager_allocate_out; } |
︙ | ︙ | |||
2997 2998 2999 3000 3001 3002 3003 | /* Populate the page with data, either by reading from the database ** file, or by setting the entire page to zero. */ if( nMax<(int)pgno || MEMDB || (clrFlag && !pPager->alwaysRollback) ){ memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize); }else{ | < < < < < < | < < | 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 | /* Populate the page with data, either by reading from the database ** file, or by setting the entire page to zero. */ if( nMax<(int)pgno || MEMDB || (clrFlag && !pPager->alwaysRollback) ){ memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize); }else{ rc = readDbPage(pPager, pPg, pgno); if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ pPg->pgno = 0; sqlite3PagerUnref(pPg); return rc; }else{ TEST_INCR(pPager->nRead); } |
︙ | ︙ | |||
3031 3032 3033 3034 3035 3036 3037 | #ifdef SQLITE_CHECK_PAGES pPg->pageHash = pager_pagehash(pPg); #endif }else{ /* The requested page is in the page cache. */ assert(pPager->nRef>0 || pgno==1); | < < < < < < < < < < < < < < | 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 | #ifdef SQLITE_CHECK_PAGES pPg->pageHash = pager_pagehash(pPg); #endif }else{ /* The requested page is in the page cache. */ assert(pPager->nRef>0 || pgno==1); TEST_INCR(pPager->nHit); page_ref(pPg); } *ppPage = pPg; return SQLITE_OK; } |
︙ | ︙ | |||
3887 3888 3889 3890 3891 3892 3893 | if( pPager->xReiniter ){ pPager->xReiniter(p, pPager->pageSize); } } pPager->pDirty = 0; pPager->pStmt = 0; pPager->dbSize = pPager->origDbSize; | | | 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 | if( pPager->xReiniter ){ pPager->xReiniter(p, pPager->pageSize); } } pPager->pDirty = 0; pPager->pStmt = 0; pPager->dbSize = pPager->origDbSize; pager_truncate_cache(pPager); pPager->stmtInUse = 0; pPager->state = PAGER_SHARED; return SQLITE_OK; } if( !pPager->dirtyCache || !pPager->journalOpen ){ rc = pager_end_transaction(pPager); |
︙ | ︙ | |||
3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 | rc2 = pager_end_transaction(pPager); if( rc==SQLITE_OK ){ rc = rc2; } }else{ rc = pager_playback(pPager, 0); } pPager->dbSize = -1; /* If an error occurs during a ROLLBACK, we can no longer trust the pager ** cache. So call pager_error() on the way out to make any error ** persistent. */ return pager_error(pPager, rc); | > | 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 | rc2 = pager_end_transaction(pPager); if( rc==SQLITE_OK ){ rc = rc2; } }else{ rc = pager_playback(pPager, 0); } /* pager_reset(pPager); */ pPager->dbSize = -1; /* If an error occurs during a ROLLBACK, we can no longer trust the pager ** cache. So call pager_error() on the way out to make any error ** persistent. */ return pager_error(pPager, rc); |
︙ | ︙ | |||
4063 4064 4065 4066 4067 4068 4069 | if( pHist->pStmt ){ memcpy(PGHDR_TO_DATA(pPg), pHist->pStmt, pPager->pageSize); sqliteFree(pHist->pStmt); pHist->pStmt = 0; } } pPager->dbSize = pPager->stmtSize; | | | 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 | if( pHist->pStmt ){ memcpy(PGHDR_TO_DATA(pPg), pHist->pStmt, pPager->pageSize); sqliteFree(pHist->pStmt); pHist->pStmt = 0; } } pPager->dbSize = pPager->stmtSize; pager_truncate_cache(pPager); rc = SQLITE_OK; }else{ rc = pager_stmt_playback(pPager); } sqlite3PagerStmtCommit(pPager); }else{ rc = SQLITE_OK; |
︙ | ︙ |
Changes to test/diskfull.test.
︙ | ︙ | |||
8 9 10 11 12 13 14 | # May you share freely, never taking more than you give. # #*********************************************************************** # This file implements regression tests for SQLite library. The # focus of this file is testing for correct handling of disk full # errors. # | | | 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | # May you share freely, never taking more than you give. # #*********************************************************************** # This file implements regression tests for SQLite library. The # focus of this file is testing for correct handling of disk full # errors. # # $Id: diskfull.test,v 1.6 2007/04/05 17:15:53 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl set sqlite_io_error_persist 0 set sqlite_io_error_hit 0 set sqlite_io_error_pending 0 |
︙ | ︙ | |||
52 53 54 55 56 57 58 | set sqlite_diskfull_pending 0 set sqlite_io_error_hit 0 integrity_check diskfull-1.6 proc do_diskfull_test {prefix sql} { set ::go 1 set ::sql $sql | | | 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 | set sqlite_diskfull_pending 0 set sqlite_io_error_hit 0 integrity_check diskfull-1.6 proc do_diskfull_test {prefix sql} { set ::go 1 set ::sql $sql set ::i 1 while {$::go} { incr ::i do_test ${prefix}.$::i.1 { set ::sqlite_diskfull_pending $::i set ::sqlite_diskfull 0 set r [catchsql $::sql] if {!$::sqlite_diskfull} { |
︙ | ︙ |
Changes to test/pager.test.
1 2 3 4 5 6 7 8 9 10 11 12 13 | # 2001 September 15 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # This file implements regression tests for SQLite library. The # focus of this script is page cache subsystem. # | | | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | # 2001 September 15 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # This file implements regression tests for SQLite library. The # focus of this script is page cache subsystem. # # $Id: pager.test,v 1.28 2007/04/05 17:15:53 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl if {[info commands pager_open]!=""} { db close |
︙ | ︙ | |||
115 116 117 118 119 120 121 | expr {$::g1!=0} } {1} do_test pager-2.12 { page_number $::g1 } {1} do_test pager-2.13 { pager_stats $::p1 | | | | | | | 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 | expr {$::g1!=0} } {1} do_test pager-2.12 { page_number $::g1 } {1} do_test pager-2.13 { pager_stats $::p1 } {ref 1 page 1 max 10 size 0 state 1 err 0 hit 1 miss 1 ovfl 0} do_test pager-2.14 { set v [catch { page_write $::g1 "Page-One" } msg] lappend v $msg } {0 {}} do_test pager-2.15 { pager_stats $::p1 } {ref 1 page 1 max 10 size 1 state 2 err 0 hit 1 miss 1 ovfl 0} do_test pager-2.16 { page_read $::g1 } {Page-One} do_test pager-2.17 { set v [catch { pager_commit $::p1 } msg] lappend v $msg } {0 {}} do_test pager-2.20 { pager_stats $::p1 } {ref 1 page 1 max 10 size -1 state 1 err 0 hit 2 miss 1 ovfl 0} do_test pager-2.19 { pager_pagecount $::p1 } {1} do_test pager-2.21 { pager_stats $::p1 } {ref 1 page 1 max 10 size 1 state 1 err 0 hit 2 miss 1 ovfl 0} do_test pager-2.22 { page_unref $::g1 } {} do_test pager-2.23 { pager_stats $::p1 } {ref 0 page 1 max 10 size -1 state 0 err 0 hit 2 miss 1 ovfl 0} do_test pager-2.24 { set v [catch { page_get $::p1 1 } ::g1] if {$v} {lappend v $::g1} set v } {0} |
︙ | ︙ |