/ Check-in [09ccc4a1]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge the fix for [a1fa75cbdd02] from the experimental branch. Also fix the persistent-wal mode feature of truncating the WAL on close so that it always truncates the WAL to zero bytes.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:09ccc4a1be7ba81890f10aac6623dd90dab4f990
User & Date: drh 2011-12-16 15:38:52
References
2011-12-16
17:01
Add code for a test that was failing before the persistent-wal related changes of [09ccc4a1be]. check-in: 49d21ce5 user: dan tags: trunk
Context
2011-12-16
19:34
Proposed changes that ensure that the WAL header is written prior to the first commit mark. check-in: 91d0437c user: drh tags: wal-header-sync
17:01
Add code for a test that was failing before the persistent-wal related changes of [09ccc4a1be]. check-in: 49d21ce5 user: dan tags: trunk
15:38
Merge the fix for [a1fa75cbdd02] from the experimental branch. Also fix the persistent-wal mode feature of truncating the WAL on close so that it always truncates the WAL to zero bytes. check-in: 09ccc4a1 user: drh tags: trunk
15:11
Improved logging of master-journal name conflicts. check-in: b1005ef4 user: drh tags: trunk
13:24
Experimental fix for [a1fa75cbdd]. Closed-Leaf check-in: 6492af76 user: dan tags: experimental
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/wal.c.

   417    417     volatile u32 **apWiData;   /* Pointer to wal-index content in memory */
   418    418     u32 szPage;                /* Database page size */
   419    419     i16 readLock;              /* Which read lock is being held.  -1 for none */
   420    420     u8 exclusiveMode;          /* Non-zero if connection is in exclusive mode */
   421    421     u8 writeLock;              /* True if in a write transaction */
   422    422     u8 ckptLock;               /* True if holding a checkpoint lock */
   423    423     u8 readOnly;               /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */
          424  +  u8 truncateOnCommit;       /* True to truncate WAL file on commit */
   424    425     WalIndexHdr hdr;           /* Wal-index header for current transaction */
   425    426     const char *zWalName;      /* Name of WAL file */
   426    427     u32 nCkpt;                 /* Checkpoint sequence counter in the wal-header */
   427    428   #ifdef SQLITE_DEBUG
   428    429     u8 lockError;              /* True if a locking error has occurred */
   429    430   #endif
   430    431   };
................................................................................
  1778   1779   
  1779   1780    walcheckpoint_out:
  1780   1781     walIteratorFree(pIter);
  1781   1782     return rc;
  1782   1783   }
  1783   1784   
  1784   1785   /*
  1785         -** Attempt to limit the WAL size to the size limit defined by
  1786         -** PRAGMA journal_size_limit.
         1786  +** If the WAL file is currently larger than nMax bytes in size, truncate
         1787  +** it to exactly nMax bytes. If an error occurs while doing so, ignore it.
  1787   1788   */
  1788         -static void walLimitSize(Wal *pWal){
  1789         -  if( pWal->mxWalSize>=0 ){
  1790         -    i64 sz;
  1791         -    int rx;
  1792         -    sqlite3BeginBenignMalloc();
  1793         -    rx = sqlite3OsFileSize(pWal->pWalFd, &sz);
  1794         -    if( rx==SQLITE_OK && (sz > pWal->mxWalSize) ){
  1795         -      rx = sqlite3OsTruncate(pWal->pWalFd, pWal->mxWalSize);
  1796         -    }
  1797         -    sqlite3EndBenignMalloc();
  1798         -    if( rx ){
  1799         -      sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName);
  1800         -    }
         1789  +static void walLimitSize(Wal *pWal, i64 nMax){
         1790  +  i64 sz;
         1791  +  int rx;
         1792  +  sqlite3BeginBenignMalloc();
         1793  +  rx = sqlite3OsFileSize(pWal->pWalFd, &sz);
         1794  +  if( rx==SQLITE_OK && (sz > nMax ) ){
         1795  +    rx = sqlite3OsTruncate(pWal->pWalFd, nMax);
         1796  +  }
         1797  +  sqlite3EndBenignMalloc();
         1798  +  if( rx ){
         1799  +    sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName);
  1801   1800     }
  1802   1801   }
  1803   1802   
  1804   1803   /*
  1805   1804   ** Close a connection to a log file.
  1806   1805   */
  1807   1806   int sqlite3WalClose(
................................................................................
  1820   1819       ** the database. In this case checkpoint the database and unlink both
  1821   1820       ** the wal and wal-index files.
  1822   1821       **
  1823   1822       ** The EXCLUSIVE lock is not released before returning.
  1824   1823       */
  1825   1824       rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE);
  1826   1825       if( rc==SQLITE_OK ){
  1827         -      int bPersistWal = -1;
  1828   1826         if( pWal->exclusiveMode==WAL_NORMAL_MODE ){
  1829   1827           pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
  1830   1828         }
  1831   1829         rc = sqlite3WalCheckpoint(
  1832   1830             pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0
  1833   1831         );
  1834         -      sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersistWal);
  1835         -      if( rc==SQLITE_OK && bPersistWal!=1 ){
  1836         -        isDelete = 1;
  1837         -      }else{
  1838         -        walLimitSize(pWal);
         1832  +      if( rc==SQLITE_OK ){
         1833  +        int bPersist = -1;
         1834  +        sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersist);
         1835  +        if( bPersist!=1 ){
         1836  +          /* Try to delete the WAL file if the checkpoint completed and
         1837  +          ** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal
         1838  +          ** mode (!bPersist) */
         1839  +          isDelete = 1;
         1840  +        }else if( pWal->mxWalSize>=0 ){
         1841  +          /* Try to truncate the WAL file to zero bytes if the checkpoint
         1842  +          ** completed and fsynced (rc==SQLITE_OK) and we are in persistent
         1843  +          ** WAL mode (bPersist) and if the PRAGMA journal_size_limit is a
         1844  +          ** non-negative value (pWal->mxWalSize>=0).  Note that we truncate
         1845  +          ** to zero bytes as truncating to the journal_size_limit might
         1846  +          ** leave a corrupt WAL file on disk. */
         1847  +          walLimitSize(pWal, 0);
         1848  +        }
  1839   1849         }
  1840   1850       }
  1841   1851   
  1842   1852       walIndexClose(pWal, isDelete);
  1843   1853       sqlite3OsClose(pWal->pWalFd);
  1844   1854       if( isDelete ){
  1845   1855         sqlite3OsDelete(pWal->pVfs, pWal->zWalName, 0);
................................................................................
  2440   2450   ** End a write transaction.  The commit has already been done.  This
  2441   2451   ** routine merely releases the lock.
  2442   2452   */
  2443   2453   int sqlite3WalEndWriteTransaction(Wal *pWal){
  2444   2454     if( pWal->writeLock ){
  2445   2455       walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1);
  2446   2456       pWal->writeLock = 0;
         2457  +    pWal->truncateOnCommit = 0;
  2447   2458     }
  2448   2459     return SQLITE_OK;
  2449   2460   }
  2450   2461   
  2451   2462   /*
  2452   2463   ** If any data has been written (but not committed) to the log file, this
  2453   2464   ** function moves the write-pointer back to the start of the transaction.
................................................................................
  2574   2585           ** at this point. But updating the actual wal-index header is also
  2575   2586           ** safe and means there is no special case for sqlite3WalUndo()
  2576   2587           ** to handle if this transaction is rolled back.
  2577   2588           */
  2578   2589           int i;                    /* Loop counter */
  2579   2590           u32 *aSalt = pWal->hdr.aSalt;       /* Big-endian salt values */
  2580   2591   
  2581         -        walLimitSize(pWal);
  2582   2592           pWal->nCkpt++;
  2583   2593           pWal->hdr.mxFrame = 0;
  2584   2594           sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0]));
  2585   2595           aSalt[1] = salt1;
  2586   2596           walIndexWriteHdr(pWal);
  2587   2597           pInfo->nBackfill = 0;
  2588   2598           for(i=1; i<WAL_NREADER; i++) pInfo->aReadMark[i] = READMARK_NOT_USED;
................................................................................
  2662   2672       sqlite3Put4byte(&aWalHdr[24], aCksum[0]);
  2663   2673       sqlite3Put4byte(&aWalHdr[28], aCksum[1]);
  2664   2674       
  2665   2675       pWal->szPage = szPage;
  2666   2676       pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN;
  2667   2677       pWal->hdr.aFrameCksum[0] = aCksum[0];
  2668   2678       pWal->hdr.aFrameCksum[1] = aCksum[1];
         2679  +    pWal->truncateOnCommit = 1;
  2669   2680   
  2670   2681       rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0);
  2671   2682       WALTRACE(("WAL%p: wal-header write %s\n", pWal, rc ? "failed" : "ok"));
  2672   2683       if( rc!=SQLITE_OK ){
  2673   2684         return rc;
  2674   2685       }
  2675   2686     }
................................................................................
  2734   2745         }
  2735   2746         nLast++;
  2736   2747         iOffset += szPage;
  2737   2748       }
  2738   2749   
  2739   2750       rc = sqlite3OsSync(pWal->pWalFd, sync_flags);
  2740   2751     }
         2752  +
         2753  +  if( isCommit && pWal->truncateOnCommit && pWal->mxWalSize>=0 ){
         2754  +    i64 sz = pWal->mxWalSize;
         2755  +    if( walFrameOffset(iFrame+nLast+1, szPage)>pWal->mxWalSize ){
         2756  +      sz = walFrameOffset(iFrame+nLast+1, szPage);
         2757  +    }
         2758  +    walLimitSize(pWal, sz);
         2759  +    pWal->truncateOnCommit = 0;
         2760  +  }
  2741   2761   
  2742   2762     /* Append data to the wal-index. It is not necessary to lock the 
  2743   2763     ** wal-index to do this as the SQLITE_SHM_WRITE lock held on the wal-index
  2744   2764     ** guarantees that there are no other writers, and no data that may
  2745   2765     ** be in use by existing readers is being overwritten.
  2746   2766     */
  2747   2767     iFrame = pWal->hdr.mxFrame;

Added test/walcrash3.test.

            1  +# 2011 December 16
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# This test simulates an application crash immediately following a
           13  +# system call to truncate a file. Specifically, the system call that
           14  +# truncates the WAL file if "PRAGMA journal_size_limit" is configured.
           15  +#
           16  +
           17  +set testdir [file dirname $argv0]
           18  +source $testdir/tester.tcl
           19  +
           20  +ifcapable !wal {finish_test ; return }
           21  +set testprefix walcrash3
           22  +
           23  +db close
           24  +testvfs tvfs
           25  +tvfs filter {xTruncate xWrite}
           26  +tvfs script tvfs_callback
           27  +proc tvfs_callback {args} {}
           28  +
           29  +sqlite3 db test.db -vfs tvfs
           30  +do_execsql_test 1.1 {
           31  +  PRAGMA page_size = 1024;
           32  +  PRAGMA journal_mode = WAL;
           33  +  PRAGMA wal_autocheckpoint = 128;
           34  +  PRAGMA journal_size_limit = 16384;
           35  +
           36  +  CREATE TABLE t1(a BLOB, b BLOB, UNIQUE(a, b));
           37  +  INSERT INTO t1 VALUES(randomblob(10), randomblob(1000));
           38  +} {wal 128 16384}
           39  +
           40  +proc tvfs_callback {method file arglist} {
           41  +  if {$::state==1} {
           42  +    foreach f [glob -nocomplain xx_test.*] { forcedelete $f }
           43  +    foreach f [glob -nocomplain test.*]    { forcecopy $f "xx_$f" }
           44  +    set ::state 2
           45  +  }
           46  +  if {$::state==0 && $method=="xTruncate" && [file tail $file]=="test.db-wal"} {
           47  +    set ::state 1
           48  +  }
           49  +}
           50  +
           51  +for {set i 2} {$i<1000} {incr i} {
           52  +
           53  +  # If the WAL file is truncated within the following, within the following
           54  +  # xWrite call the [tvfs_callback] makes a copy of the database and WAL 
           55  +  # files set sets $::state to 2. So that the copied files are in the same
           56  +  # state as the real database and WAL files would be if an application crash 
           57  +  # occurred immediately following the xTruncate().
           58  +  # 
           59  +  set ::state 0
           60  +  do_execsql_test 1.$i.1 {
           61  +    INSERT INTO t1 VALUES(randomblob(10), randomblob(1000));
           62  +  }
           63  +
           64  +  # If a copy was made, open it and run the integrity-check.
           65  +  #
           66  +  if {$::state==2} {
           67  +    sqlite3 db2 xx_test.db
           68  +    do_test 1.$i.2 { execsql { PRAGMA integrity_check  } db2 } "ok"
           69  +    do_test 1.$i.3 { execsql { SELECT count(*) FROM t1 } db2 } [expr $i-1]
           70  +    db2 close
           71  +  }
           72  +}
           73  +
           74  +finish_test
           75  +

Changes to test/walpersist.test.

    64     64   } {0 1}
    65     65   do_test walpersist-1.11 {
    66     66     db close
    67     67     list [file exists test.db] [file exists test.db-wal] [file exists test.db-shm]
    68     68   } {1 1 1}
    69     69   
    70     70   # Make sure the journal_size_limit works to limit the size of the
    71         -# persisted wal file.
           71  +# persisted wal file.  In persistent-wal mode, any non-negative
           72  +# journal_size_limit causes the WAL file to be truncated to zero bytes
           73  +# when closing.
           74  +#
    72     75   forcedelete test.db test.db-shm test.db-wal
    73     76   do_test walpersist-2.1 {
    74     77     sqlite3 db test.db
    75     78     db eval {
    76     79       PRAGMA journal_mode=WAL;
    77     80       PRAGMA wal_autocheckpoint=OFF;
    78     81       PRAGMA journal_size_limit=12000;
................................................................................
    81     84       UPDATE t1 SET x=randomblob(50000);
    82     85     }
    83     86     expr {[file size test.db-wal]>100000}
    84     87   } {1}
    85     88   do_test walpersist-2.2 {
    86     89     file_control_persist_wal db 1
    87     90     db close
    88         -  file size test.db-wal
    89         -} {12000}
           91  +  concat [file exists test.db-wal] [file size test.db-wal]
           92  +} {1 0}
    90     93   
    91     94   finish_test