Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch wal-sync-refactor Excluding Merge-Ins
This is equivalent to a diff from 25292b9a to bf65dae8
2017-08-25
| ||
11:44 | Improve the internal mechanism used to keep track of what kind of syncing to do for WAL transaction commits and checkpoint operations. Use the checkpoint-style of syncing to sync the header of a new or restarted WAL file. (check-in: e8d23afe user: drh tags: trunk) | |
09:17 | Avoid returning duplicate rows in experimental pragmas "pragma_list", "module_list" and "function_list". (check-in: b79cc8dc user: dan tags: trunk) | |
01:14 | Update the mechanism used to keep track of what kind of syncing to do for WAL transaction commits and checkpoint operations. Use the checkpoint-style of syncing to sync the header of a new or restarted WAL file. (Closed-Leaf check-in: bf65dae8 user: drh tags: wal-sync-refactor) | |
2017-08-24
| ||
20:54 | Fix an incorrect hyperlink in a comment. (check-in: 25292b9a user: drh tags: trunk) | |
15:43 | Fixes to documentation about SQLITE_OPEN_URI. (check-in: 7ec72314 user: drh tags: trunk) | |
Changes to src/os.c.
︙ | ︙ | |||
94 95 96 97 98 99 100 | return id->pMethods->xWrite(id, pBuf, amt, offset); } int sqlite3OsTruncate(sqlite3_file *id, i64 size){ return id->pMethods->xTruncate(id, size); } int sqlite3OsSync(sqlite3_file *id, int flags){ DO_OS_MALLOC_TEST(id); | | | 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | return id->pMethods->xWrite(id, pBuf, amt, offset); } int sqlite3OsTruncate(sqlite3_file *id, i64 size){ return id->pMethods->xTruncate(id, size); } int sqlite3OsSync(sqlite3_file *id, int flags){ DO_OS_MALLOC_TEST(id); return flags ? id->pMethods->xSync(id, flags) : SQLITE_OK; } int sqlite3OsFileSize(sqlite3_file *id, i64 *pSize){ DO_OS_MALLOC_TEST(id); return id->pMethods->xFileSize(id, pSize); } int sqlite3OsLock(sqlite3_file *id, int lockType){ DO_OS_MALLOC_TEST(id); |
︙ | ︙ |
Changes to src/pager.c.
︙ | ︙ | |||
612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 | ** ** errCode ** ** The Pager.errCode variable is only ever used in PAGER_ERROR state. It ** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode ** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX ** sub-codes. */ struct Pager { sqlite3_vfs *pVfs; /* OS functions to use for IO */ u8 exclusiveMode; /* Boolean. True if locking_mode==EXCLUSIVE */ u8 journalMode; /* One of the PAGER_JOURNALMODE_* values */ u8 useJournal; /* Use a rollback journal on this file */ u8 noSync; /* Do not sync the journal if true */ u8 fullSync; /* Do extra syncs of the journal for robustness */ u8 extraSync; /* sync directory after journal delete */ | > > > > > > > > > > > > | | < | 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 | ** ** errCode ** ** The Pager.errCode variable is only ever used in PAGER_ERROR state. It ** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode ** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX ** sub-codes. ** ** syncFlags, walSyncFlags ** ** syncFlags is either SQLITE_SYNC_NORMAL (0x02) or SQLITE_SYNC_FULL (0x03). ** syncFlags is used for rollback mode. walSyncFlags is used for WAL mode ** and contains the flags used to sync the checkpoint operations in the ** lower two bits, and sync flags used for transaction commits in the WAL ** file in bits 0x04 and 0x08. In other words, to get the correct sync flags ** for checkpoint operations, use (walSyncFlags&0x03) and to get the correct ** sync flags for transaction commit, use ((walSyncFlags>>2)&0x03). Note ** that with synchronous=NORMAL in WAL mode, transaction commit is not synced ** meaning that the 0x04 and 0x08 bits are both zero. */ struct Pager { sqlite3_vfs *pVfs; /* OS functions to use for IO */ u8 exclusiveMode; /* Boolean. True if locking_mode==EXCLUSIVE */ u8 journalMode; /* One of the PAGER_JOURNALMODE_* values */ u8 useJournal; /* Use a rollback journal on this file */ u8 noSync; /* Do not sync the journal if true */ u8 fullSync; /* Do extra syncs of the journal for robustness */ u8 extraSync; /* sync directory after journal delete */ u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ u8 walSyncFlags; /* See description above */ u8 tempFile; /* zFilename is a temporary or immutable file */ u8 noLock; /* Do not lock (except in WAL mode) */ u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ /************************************************************************** ** The following block contains those class members that change during |
︙ | ︙ | |||
3594 3595 3596 3597 3598 3599 3600 | }else{ pPager->noSync = level==PAGER_SYNCHRONOUS_OFF ?1:0; pPager->fullSync = level>=PAGER_SYNCHRONOUS_FULL ?1:0; pPager->extraSync = level==PAGER_SYNCHRONOUS_EXTRA ?1:0; } if( pPager->noSync ){ pPager->syncFlags = 0; | < < < < < < | | > > > | 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 | }else{ pPager->noSync = level==PAGER_SYNCHRONOUS_OFF ?1:0; pPager->fullSync = level>=PAGER_SYNCHRONOUS_FULL ?1:0; pPager->extraSync = level==PAGER_SYNCHRONOUS_EXTRA ?1:0; } if( pPager->noSync ){ pPager->syncFlags = 0; }else if( pgFlags & PAGER_FULLFSYNC ){ pPager->syncFlags = SQLITE_SYNC_FULL; }else{ pPager->syncFlags = SQLITE_SYNC_NORMAL; } pPager->walSyncFlags = (pPager->syncFlags<<2); if( pPager->fullSync ){ pPager->walSyncFlags |= pPager->syncFlags; } if( (pgFlags & PAGER_CKPT_FULLFSYNC) && !pPager->noSync ){ pPager->walSyncFlags |= (SQLITE_SYNC_FULL<<2); } if( pgFlags & PAGER_CACHESPILL ){ pPager->doNotSpill &= ~SPILLFLAG_OFF; }else{ pPager->doNotSpill |= SPILLFLAG_OFF; } } |
︙ | ︙ | |||
4106 4107 4108 4109 4110 4111 4112 | disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); pagerFreeMapHdrs(pPager); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL assert( db || pPager->pWal==0 ); | | | 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 | disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); pagerFreeMapHdrs(pPager); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL assert( db || pPager->pWal==0 ); sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize, (db && (db->flags & SQLITE_NoCkptOnClose) ? 0 : pTmp) ); pPager->pWal = 0; #endif pager_reset(pPager); if( MEMDB ){ pager_unlock(pPager); |
︙ | ︙ | |||
4914 4915 4916 4917 4918 4919 4920 | assert( useJournal || pPager->tempFile ); pPager->noSync = pPager->tempFile; if( pPager->noSync ){ assert( pPager->fullSync==0 ); assert( pPager->extraSync==0 ); assert( pPager->syncFlags==0 ); assert( pPager->walSyncFlags==0 ); | < | < | 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 | assert( useJournal || pPager->tempFile ); pPager->noSync = pPager->tempFile; if( pPager->noSync ){ assert( pPager->fullSync==0 ); assert( pPager->extraSync==0 ); assert( pPager->syncFlags==0 ); assert( pPager->walSyncFlags==0 ); }else{ pPager->fullSync = 1; pPager->extraSync = 0; pPager->syncFlags = SQLITE_SYNC_NORMAL; pPager->walSyncFlags = SQLITE_SYNC_NORMAL | (SQLITE_SYNC_NORMAL<<2); } /* pPager->pFirst = 0; */ /* pPager->pFirstSynced = 0; */ /* pPager->pLast = 0; */ pPager->nExtra = (u16)nExtra; pPager->journalSizeLimit = SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT; assert( isOpen(pPager->fd) || tempFile ); |
︙ | ︙ | |||
7368 7369 7370 7371 7372 7373 7374 | int *pnCkpt /* OUT: Final number of checkpointed frames */ ){ int rc = SQLITE_OK; if( pPager->pWal ){ rc = sqlite3WalCheckpoint(pPager->pWal, db, eMode, (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), pPager->pBusyHandlerArg, | | | 7374 7375 7376 7377 7378 7379 7380 7381 7382 7383 7384 7385 7386 7387 7388 | int *pnCkpt /* OUT: Final number of checkpointed frames */ ){ int rc = SQLITE_OK; if( pPager->pWal ){ rc = sqlite3WalCheckpoint(pPager->pWal, db, eMode, (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), pPager->pBusyHandlerArg, pPager->walSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, pnLog, pnCkpt ); } return rc; } int sqlite3PagerWalCallback(Pager *pPager){ |
︙ | ︙ | |||
7525 7526 7527 7528 7529 7530 7531 | /* Checkpoint and close the log. Because an EXCLUSIVE lock is held on ** the database file, the log and log-summary files will be deleted. */ if( rc==SQLITE_OK && pPager->pWal ){ rc = pagerExclusiveLock(pPager); if( rc==SQLITE_OK ){ | | | 7531 7532 7533 7534 7535 7536 7537 7538 7539 7540 7541 7542 7543 7544 7545 | /* Checkpoint and close the log. Because an EXCLUSIVE lock is held on ** the database file, the log and log-summary files will be deleted. */ if( rc==SQLITE_OK && pPager->pWal ){ rc = pagerExclusiveLock(pPager); if( rc==SQLITE_OK ){ rc = sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; pagerFixMaplimit(pPager); if( rc && !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); } } return rc; |
︙ | ︙ |
Changes to src/wal.c.
︙ | ︙ | |||
1794 1795 1796 1797 1798 1799 1800 | ){ i64 nSize; /* Current size of database file */ u32 nBackfill = pInfo->nBackfill; pInfo->nBackfillAttempted = mxSafeFrame; /* Sync the WAL to disk */ | < | < | 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 | ){ i64 nSize; /* Current size of database file */ u32 nBackfill = pInfo->nBackfill; pInfo->nBackfillAttempted = mxSafeFrame; /* Sync the WAL to disk */ rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); /* If the database may grow as a result of this checkpoint, hint ** about the eventual size of the db file to the VFS layer. */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); |
︙ | ︙ | |||
1837 1838 1839 1840 1841 1842 1843 | /* If work was actually accomplished... */ if( rc==SQLITE_OK ){ if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ i64 szDb = pWal->hdr.nPage*(i64)szPage; testcase( IS_BIG_INT(szDb) ); rc = sqlite3OsTruncate(pWal->pDbFd, szDb); | | | | 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 | /* If work was actually accomplished... */ if( rc==SQLITE_OK ){ if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ i64 szDb = pWal->hdr.nPage*(i64)szPage; testcase( IS_BIG_INT(szDb) ); rc = sqlite3OsTruncate(pWal->pDbFd, szDb); if( rc==SQLITE_OK ){ rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags)); } } if( rc==SQLITE_OK ){ pInfo->nBackfill = mxSafeFrame; } } |
︙ | ︙ | |||
2944 2945 2946 2947 2948 2949 2950 | if( iOffset<p->iSyncPoint && iOffset+iAmt>=p->iSyncPoint ){ int iFirstAmt = (int)(p->iSyncPoint - iOffset); rc = sqlite3OsWrite(p->pFd, pContent, iFirstAmt, iOffset); if( rc ) return rc; iOffset += iFirstAmt; iAmt -= iFirstAmt; pContent = (void*)(iFirstAmt + (char*)pContent); | | | | 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 | if( iOffset<p->iSyncPoint && iOffset+iAmt>=p->iSyncPoint ){ int iFirstAmt = (int)(p->iSyncPoint - iOffset); rc = sqlite3OsWrite(p->pFd, pContent, iFirstAmt, iOffset); if( rc ) return rc; iOffset += iFirstAmt; iAmt -= iFirstAmt; pContent = (void*)(iFirstAmt + (char*)pContent); assert( WAL_SYNC_FLAGS(p->syncFlags)!=0 ); rc = sqlite3OsSync(p->pFd, WAL_SYNC_FLAGS(p->syncFlags)); if( iAmt==0 || rc ) return rc; } rc = sqlite3OsWrite(p->pFd, pContent, iAmt, iOffset); return rc; } /* |
︙ | ︙ | |||
3117 3118 3119 3120 3121 3122 3123 | /* Sync the header (unless SQLITE_IOCAP_SEQUENTIAL is true or unless ** all syncing is turned off by PRAGMA synchronous=OFF). Otherwise ** an out-of-order write following a WAL restart could result in ** database corruption. See the ticket: ** ** https://sqlite.org/src/info/ff5be73dee */ | | | | 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 | /* Sync the header (unless SQLITE_IOCAP_SEQUENTIAL is true or unless ** all syncing is turned off by PRAGMA synchronous=OFF). Otherwise ** an out-of-order write following a WAL restart could result in ** database corruption. See the ticket: ** ** https://sqlite.org/src/info/ff5be73dee */ if( pWal->syncHeader ){ rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); if( rc ) return rc; } } assert( (int)pWal->szPage==szPage ); /* Setup information needed to write frames into the WAL */ w.pWal = pWal; |
︙ | ︙ | |||
3193 3194 3195 3196 3197 3198 3199 | ** If SQLITE_IOCAP_POWERSAFE_OVERWRITE is defined, then padding is not ** needed and only the sync is done. If padding is needed, then the ** final frame is repeated (with its commit mark) until the next sector ** boundary is crossed. Only the part of the WAL prior to the last ** sector boundary is synced; the part of the last frame that extends ** past the sector boundary is written after the sync. */ | | | | 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 | ** If SQLITE_IOCAP_POWERSAFE_OVERWRITE is defined, then padding is not ** needed and only the sync is done. If padding is needed, then the ** final frame is repeated (with its commit mark) until the next sector ** boundary is crossed. Only the part of the WAL prior to the last ** sector boundary is synced; the part of the last frame that extends ** past the sector boundary is written after the sync. */ if( isCommit && WAL_SYNC_FLAGS(sync_flags)!=0 ){ int bSync = 1; if( pWal->padToSectorBoundary ){ int sectorSize = sqlite3SectorSize(pWal->pWalFd); w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize; bSync = (w.iSyncPoint==iOffset); testcase( bSync ); while( iOffset<w.iSyncPoint ){ rc = walWriteOneFrame(&w, pLast, nTruncate, iOffset); if( rc ) return rc; iOffset += szFrame; nExtra++; } } if( bSync ){ assert( rc==SQLITE_OK ); rc = sqlite3OsSync(w.pFd, WAL_SYNC_FLAGS(sync_flags)); } } /* If this frame set completes the first transaction in the WAL and ** if PRAGMA journal_size_limit is set, then truncate the WAL to the ** journal size limit, if possible. */ |
︙ | ︙ |
Changes to src/wal.h.
︙ | ︙ | |||
15 16 17 18 19 20 21 | */ #ifndef SQLITE_WAL_H #define SQLITE_WAL_H #include "sqliteInt.h" | | | | | | 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | */ #ifndef SQLITE_WAL_H #define SQLITE_WAL_H #include "sqliteInt.h" /* Macros for extracting appropriate sync flags for either transaction ** commits (WAL_SYNC_FLAGS(X)) or for checkpoint ops (CKPT_SYNC_FLAGS(X)): */ #define WAL_SYNC_FLAGS(X) ((X)&0x03) #define CKPT_SYNC_FLAGS(X) (((X)>>2)&0x03) #ifdef SQLITE_OMIT_WAL # define sqlite3WalOpen(x,y,z) 0 # define sqlite3WalLimit(x,y) # define sqlite3WalClose(v,w,x,y,z) 0 # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) |
︙ | ︙ |
Changes to test/wal2.test.
︙ | ︙ | |||
1187 1188 1189 1190 1191 1192 1193 | } #------------------------------------------------------------------------- # Test that "PRAGMA checkpoint_fullsync" appears to be working. # foreach {tn sql reslist} { 1 { } {10 0 4 0 6 0} | | | 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 | } #------------------------------------------------------------------------- # Test that "PRAGMA checkpoint_fullsync" appears to be working. # foreach {tn sql reslist} { 1 { } {10 0 4 0 6 0} 2 { PRAGMA checkpoint_fullfsync = 1 } {10 6 4 3 6 3} 3 { PRAGMA checkpoint_fullfsync = 0 } {10 0 4 0 6 0} } { ifcapable default_ckptfullfsync { if {[string trim $sql]==""} continue } faultsim_delete_and_reopen |
︙ | ︙ | |||
1257 1258 1259 1260 1261 1262 1263 | 3 {0 0 full} {2 0} {1 0} {2 0} 4 {0 1 off} {0 0} {0 0} {0 0} 5 {0 1 normal} {0 1} {0 0} {0 2} 6 {0 1 full} {0 2} {0 1} {0 2} 7 {1 0 off} {0 0} {0 0} {0 0} | | | | 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 | 3 {0 0 full} {2 0} {1 0} {2 0} 4 {0 1 off} {0 0} {0 0} {0 0} 5 {0 1 normal} {0 1} {0 0} {0 2} 6 {0 1 full} {0 2} {0 1} {0 2} 7 {1 0 off} {0 0} {0 0} {0 0} 8 {1 0 normal} {0 1} {0 0} {0 2} 9 {1 0 full} {1 1} {1 0} {0 2} 10 {1 1 off} {0 0} {0 0} {0 0} 11 {1 1 normal} {0 1} {0 0} {0 2} 12 {1 1 full} {0 2} {0 1} {0 2} } { forcedelete test.db |
︙ | ︙ |