Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Ensure that "PRAGMA wal_checkpoint = TRUNCATE|FULL|RESTART" block on other connections and truncate the wal file as required even if the entire wal file has already been checkpointed. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
53429689d4fcf472edbc89cc50b5e69b |
User & Date: | dan 2015-01-29 19:12:12.112 |
Original Comment: | Ensure that "PRAGMA wal_checkpoint = TRUNCATE|FULL|RESTART" block on other connections and truncate the database file as required even if the entire wal file has already been checkpointed. |
Context
2015-01-29
| ||
19:59 | Ensure that "PRAGMA wal_checkpoint = TRUNCATE|FULL|RESTART" block on other connections and truncate the wal file as required even if the entire wal file has already been checkpointed. Cherry-pick from trunk. (check-in: 18610617af user: drh tags: branch-3.8.8) | |
19:27 | Fix some duplicated test names. (check-in: 1797158db2 user: mistachkin tags: trunk) | |
19:12 | Ensure that "PRAGMA wal_checkpoint = TRUNCATE|FULL|RESTART" block on other connections and truncate the wal file as required even if the entire wal file has already been checkpointed. (check-in: 53429689d4 user: dan tags: trunk) | |
18:38 | Split up the SRC variable in Makefile.msc to avoid over-long cmd.exe commands when TOP is set to a long pathname. (check-in: 7d70ac65c1 user: drh tags: trunk) | |
Changes
Changes to src/main.c.
︙ | ︙ | |||
1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 | if( zDb && zDb[0] ){ iDb = sqlite3FindDbName(db, zDb); } if( iDb<0 ){ rc = SQLITE_ERROR; sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb); }else{ rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); sqlite3Error(db, rc); } rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; #endif | > | 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 | if( zDb && zDb[0] ){ iDb = sqlite3FindDbName(db, zDb); } if( iDb<0 ){ rc = SQLITE_ERROR; sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb); }else{ db->busyHandler.nBusy = 0; rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); sqlite3Error(db, rc); } rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; #endif |
︙ | ︙ |
Changes to src/wal.c.
︙ | ︙ | |||
1690 1691 1692 1693 1694 1695 1696 | Wal *pWal, /* Wal connection */ int eMode, /* One of PASSIVE, FULL or RESTART */ int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags for OsSync() (or 0) */ u8 *zBuf /* Temporary buffer to use */ ){ | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | > > | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | > | | 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 | Wal *pWal, /* Wal connection */ int eMode, /* One of PASSIVE, FULL or RESTART */ int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags for OsSync() (or 0) */ u8 *zBuf /* Temporary buffer to use */ ){ int rc = SQLITE_OK; /* Return code */ int szPage; /* Database page-size */ WalIterator *pIter = 0; /* Wal iterator context */ u32 iDbpage = 0; /* Next database page to write */ u32 iFrame = 0; /* Wal frame containing data for iDbpage */ u32 mxSafeFrame; /* Max frame that can be backfilled */ u32 mxPage; /* Max database page to write */ int i; /* Loop counter */ volatile WalCkptInfo *pInfo; /* The checkpoint status information */ szPage = walPagesize(pWal); testcase( szPage<=32768 ); testcase( szPage>=65536 ); pInfo = walCkptInfo(pWal); if( pInfo->nBackfill<pWal->hdr.mxFrame ){ /* Allocate the iterator */ rc = walIteratorInit(pWal, &pIter); if( rc!=SQLITE_OK ){ return rc; } assert( pIter ); /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); /* Compute in mxSafeFrame the index of the last frame of the WAL that is ** safe to write into the database. Frames beyond mxSafeFrame might ** overwrite database pages that are in use by active readers and thus ** cannot be backfilled from the WAL. */ mxSafeFrame = pWal->hdr.mxFrame; mxPage = pWal->hdr.nPage; for(i=1; i<WAL_NREADER; i++){ u32 y = pInfo->aReadMark[i]; if( mxSafeFrame>y ){ assert( y<=pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); if( rc==SQLITE_OK ){ pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED); walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); }else if( rc==SQLITE_BUSY ){ mxSafeFrame = y; xBusy = 0; }else{ goto walcheckpoint_out; } } } if( pInfo->nBackfill<mxSafeFrame && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0),1))==SQLITE_OK ){ i64 nSize; /* Current size of database file */ u32 nBackfill = pInfo->nBackfill; /* Sync the WAL to disk */ if( sync_flags ){ rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } /* If the database may grow as a result of this checkpoint, hint ** about the eventual size of the db file to the VFS layer. */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); if( rc==SQLITE_OK && nSize<nReq ){ sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); } } /* Iterate through the contents of the WAL, copying data to the db file */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; assert( walFramePgno(pWal, iFrame)==iDbpage ); if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ){ continue; } iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; iOffset = (iDbpage-1)*(i64)szPage; testcase( IS_BIG_INT(iOffset) ); rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; } /* If work was actually accomplished... */ if( rc==SQLITE_OK ){ if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ i64 szDb = pWal->hdr.nPage*(i64)szPage; testcase( IS_BIG_INT(szDb) ); rc = sqlite3OsTruncate(pWal->pDbFd, szDb); if( rc==SQLITE_OK && sync_flags ){ rc = sqlite3OsSync(pWal->pDbFd, sync_flags); } } if( rc==SQLITE_OK ){ pInfo->nBackfill = mxSafeFrame; } } /* Release the reader lock held while backfilling */ walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); } if( rc==SQLITE_BUSY ){ /* Reset the return code so as not to report a checkpoint failure ** just because there are active readers. */ rc = SQLITE_OK; } } /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the ** entire wal file has been copied into the database file, then block ** until all readers have finished using the wal file. This ensures that ** the next process to write to the database restarts the wal file. */ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); if( pInfo->nBackfill<pWal->hdr.mxFrame ){ rc = SQLITE_BUSY; }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ u32 salt1; sqlite3_randomness(4, &salt1); assert( pInfo->nBackfill==pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){ /* IMPLEMENTATION-OF: R-44699-57140 This mode works the same way as ** SQLITE_CHECKPOINT_RESTART with the addition that it also ** truncates the log file to zero bytes just prior to a ** successful return. |
︙ | ︙ |
Changes to test/wal5.test.
︙ | ︙ | |||
386 387 388 389 390 391 392 393 394 395 396 | do_test 4.$tn.5 { sql2 { INSERT INTO t1 VALUES('a', 'b') } file size test.db-wal } [wal_file_size 2 1024] } } finish_test | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 | do_test 4.$tn.5 { sql2 { INSERT INTO t1 VALUES('a', 'b') } file size test.db-wal } [wal_file_size 2 1024] } # Test that FULL, RESTART and TRUNCATE callbacks block on other clients # and truncate the wal file as required even if the entire wal file has # already been checkpointed when they are invoked. # do_multiclient_test tn { code1 $do_wal_checkpoint code2 $do_wal_checkpoint code3 $do_wal_checkpoint do_test 5.$tn.1 { sql1 { PRAGMA page_size = 1024; PRAGMA auto_vacuum = 0; PRAGMA journal_mode = WAL; PRAGMA synchronous = normal; CREATE TABLE t1(x, y); CREATE INDEX i1 ON t1(x, y); INSERT INTO t1 VALUES(1, 2); INSERT INTO t1 VALUES(3, 4); INSERT INTO t1 VALUES(5, 6); } file size test.db-wal } [wal_file_size 10 1024] do_test 5.$tn.2 { sql2 { BEGIN; SELECT * FROM t1 } } {1 2 3 4 5 6} do_test 5.$tn.3 { do_wal_checkpoint db -mode passive } {0 10 10} do_test 5.$tn.4 { sql3 { BEGIN; INSERT INTO t1 VALUES(7, 8); } } {} do_test 5.$tn.5 { do_wal_checkpoint db -mode passive } {0 10 10} do_test 5.$tn.6 { do_wal_checkpoint db -mode full } {1 10 10} do_test 5.$tn.7 { sql3 { ROLLBACK } } {} do_test 5.$tn.8 { do_wal_checkpoint db -mode full } {0 10 10} do_test 5.$tn.9 { do_wal_checkpoint db -mode truncate } {1 10 10} do_test 5.$tn.10 { file size test.db-wal } [wal_file_size 10 1024] proc xBusyHandler {n} { sql2 { COMMIT } ; return 0 } db busy xBusyHandler do_test 5.$tn.11 { do_wal_checkpoint db -mode truncate } {0 0 0} do_test 5.$tn.12 { file size test.db-wal } 0 do_test 5.$tn.13 { sql1 { INSERT INTO t1 VALUES(7, 8); INSERT INTO t1 VALUES(9, 10); SELECT * FROM t1; } } {1 2 3 4 5 6 7 8 9 10} do_test 5.$tn.14 { sql2 { BEGIN; SELECT * FROM t1 } } {1 2 3 4 5 6 7 8 9 10} proc xBusyHandler {n} { return 1 } do_test 5.$tn.14 { do_wal_checkpoint db -mode truncate } {1 4 4} do_test 5.$tn.15 { file size test.db-wal } [wal_file_size 4 1024] do_test 5.$tn.16 { do_wal_checkpoint db -mode restart } {1 4 4} proc xBusyHandler {n} { sql2 { COMMIT } ; return 0 } db busy xBusyHandler do_test 5.$tn.17 { do_wal_checkpoint db -mode restart } {0 4 4} do_test 5.$tn.18 { file size test.db-wal } [wal_file_size 4 1024] do_test 5.$tn.19 { do_wal_checkpoint db -mode truncate } {0 0 0} do_test 5.$tn.20 { file size test.db-wal } 0 } } finish_test |