/ Check-in [6492af76]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Experimental fix for [a1fa75cbdd].
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | experimental
Files: files | file ages | folders
SHA1: 6492af76ea6585a1b377d69751af930c0ccfe688
User & Date: dan 2011-12-16 13:24:27
Context
2011-12-16
15:38
Merge the fix for [a1fa75cbdd02] from the experimental branch. Also fix the persistent-wal mode feature of truncating the WAL on close so that it always truncates the WAL to zero bytes. check-in: 09ccc4a1 user: drh tags: trunk
13:24
Experimental fix for [a1fa75cbdd]. Closed-Leaf check-in: 6492af76 user: dan tags: experimental
2011-12-15
17:44
Add stdio support to the quota VFS. check-in: 322bd15f user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/wal.c.

417
418
419
420
421
422
423

424
425
426
427
428
429
430
....
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
....
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
....
2440
2441
2442
2443
2444
2445
2446

2447
2448
2449
2450
2451
2452
2453
....
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
....
2662
2663
2664
2665
2666
2667
2668

2669
2670
2671
2672
2673
2674
2675
....
2734
2735
2736
2737
2738
2739
2740









2741
2742
2743
2744
2745
2746
2747
  volatile u32 **apWiData;   /* Pointer to wal-index content in memory */
  u32 szPage;                /* Database page size */
  i16 readLock;              /* Which read lock is being held.  -1 for none */
  u8 exclusiveMode;          /* Non-zero if connection is in exclusive mode */
  u8 writeLock;              /* True if in a write transaction */
  u8 ckptLock;               /* True if holding a checkpoint lock */
  u8 readOnly;               /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */

  WalIndexHdr hdr;           /* Wal-index header for current transaction */
  const char *zWalName;      /* Name of WAL file */
  u32 nCkpt;                 /* Checkpoint sequence counter in the wal-header */
#ifdef SQLITE_DEBUG
  u8 lockError;              /* True if a locking error has occurred */
#endif
};
................................................................................

 walcheckpoint_out:
  walIteratorFree(pIter);
  return rc;
}

/*
** Attempt to limit the WAL size to the size limit defined by
** PRAGMA journal_size_limit.
*/
static void walLimitSize(Wal *pWal){
  if( pWal->mxWalSize>=0 ){
    i64 sz;
    int rx;
    sqlite3BeginBenignMalloc();
    rx = sqlite3OsFileSize(pWal->pWalFd, &sz);
    if( rx==SQLITE_OK && (sz > pWal->mxWalSize) ){
      rx = sqlite3OsTruncate(pWal->pWalFd, pWal->mxWalSize);
    }
    sqlite3EndBenignMalloc();
    if( rx ){
      sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName);
    }
  }
}

/*
** Close a connection to a log file.
*/
int sqlite3WalClose(
................................................................................
      }
      rc = sqlite3WalCheckpoint(
          pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0
      );
      sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersistWal);
      if( rc==SQLITE_OK && bPersistWal!=1 ){
        isDelete = 1;
      }else{
        walLimitSize(pWal);
      }
    }

    walIndexClose(pWal, isDelete);
    sqlite3OsClose(pWal->pWalFd);
    if( isDelete ){
      sqlite3OsDelete(pWal->pVfs, pWal->zWalName, 0);
................................................................................
** End a write transaction.  The commit has already been done.  This
** routine merely releases the lock.
*/
int sqlite3WalEndWriteTransaction(Wal *pWal){
  if( pWal->writeLock ){
    walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1);
    pWal->writeLock = 0;

  }
  return SQLITE_OK;
}

/*
** If any data has been written (but not committed) to the log file, this
** function moves the write-pointer back to the start of the transaction.
................................................................................
        ** at this point. But updating the actual wal-index header is also
        ** safe and means there is no special case for sqlite3WalUndo()
        ** to handle if this transaction is rolled back.
        */
        int i;                    /* Loop counter */
        u32 *aSalt = pWal->hdr.aSalt;       /* Big-endian salt values */

        walLimitSize(pWal);
        pWal->nCkpt++;
        pWal->hdr.mxFrame = 0;
        sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0]));
        aSalt[1] = salt1;
        walIndexWriteHdr(pWal);
        pInfo->nBackfill = 0;
        for(i=1; i<WAL_NREADER; i++) pInfo->aReadMark[i] = READMARK_NOT_USED;
................................................................................
    sqlite3Put4byte(&aWalHdr[24], aCksum[0]);
    sqlite3Put4byte(&aWalHdr[28], aCksum[1]);
    
    pWal->szPage = szPage;
    pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN;
    pWal->hdr.aFrameCksum[0] = aCksum[0];
    pWal->hdr.aFrameCksum[1] = aCksum[1];


    rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0);
    WALTRACE(("WAL%p: wal-header write %s\n", pWal, rc ? "failed" : "ok"));
    if( rc!=SQLITE_OK ){
      return rc;
    }
  }
................................................................................
      }
      nLast++;
      iOffset += szPage;
    }

    rc = sqlite3OsSync(pWal->pWalFd, sync_flags);
  }










  /* Append data to the wal-index. It is not necessary to lock the 
  ** wal-index to do this as the SQLITE_SHM_WRITE lock held on the wal-index
  ** guarantees that there are no other writers, and no data that may
  ** be in use by existing readers is being overwritten.
  */
  iFrame = pWal->hdr.mxFrame;







>







 







|
|

|
<
|
|
|
|
|
|
|
|
|
|
<







 







|
|







 







>







 







<







 







>







 







>
>
>
>
>
>
>
>
>







417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
....
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789

1790
1791
1792
1793
1794
1795
1796
1797
1798
1799

1800
1801
1802
1803
1804
1805
1806
....
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
....
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
....
2574
2575
2576
2577
2578
2579
2580

2581
2582
2583
2584
2585
2586
2587
....
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
....
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
  volatile u32 **apWiData;   /* Pointer to wal-index content in memory */
  u32 szPage;                /* Database page size */
  i16 readLock;              /* Which read lock is being held.  -1 for none */
  u8 exclusiveMode;          /* Non-zero if connection is in exclusive mode */
  u8 writeLock;              /* True if in a write transaction */
  u8 ckptLock;               /* True if holding a checkpoint lock */
  u8 readOnly;               /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */
  u8 truncateOnCommit;       /* True to truncate WAL file on commit */
  WalIndexHdr hdr;           /* Wal-index header for current transaction */
  const char *zWalName;      /* Name of WAL file */
  u32 nCkpt;                 /* Checkpoint sequence counter in the wal-header */
#ifdef SQLITE_DEBUG
  u8 lockError;              /* True if a locking error has occurred */
#endif
};
................................................................................

 walcheckpoint_out:
  walIteratorFree(pIter);
  return rc;
}

/*
** If the WAL file is currently larger than nMax bytes in size, truncate
** it to exactly nMax bytes. If an error occurs while doing so, ignore it.
*/
static void walLimitSize(Wal *pWal, i64 nMax){

  i64 sz;
  int rx;
  sqlite3BeginBenignMalloc();
  rx = sqlite3OsFileSize(pWal->pWalFd, &sz);
  if( rx==SQLITE_OK && (sz > nMax ) ){
    rx = sqlite3OsTruncate(pWal->pWalFd, nMax);
  }
  sqlite3EndBenignMalloc();
  if( rx ){
    sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName);

  }
}

/*
** Close a connection to a log file.
*/
int sqlite3WalClose(
................................................................................
      }
      rc = sqlite3WalCheckpoint(
          pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0
      );
      sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersistWal);
      if( rc==SQLITE_OK && bPersistWal!=1 ){
        isDelete = 1;
      }else if( pWal->mxWalSize>=0 ){
        walLimitSize(pWal, pWal->mxWalSize);
      }
    }

    walIndexClose(pWal, isDelete);
    sqlite3OsClose(pWal->pWalFd);
    if( isDelete ){
      sqlite3OsDelete(pWal->pVfs, pWal->zWalName, 0);
................................................................................
** End a write transaction.  The commit has already been done.  This
** routine merely releases the lock.
*/
int sqlite3WalEndWriteTransaction(Wal *pWal){
  if( pWal->writeLock ){
    walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1);
    pWal->writeLock = 0;
    pWal->truncateOnCommit = 0;
  }
  return SQLITE_OK;
}

/*
** If any data has been written (but not committed) to the log file, this
** function moves the write-pointer back to the start of the transaction.
................................................................................
        ** at this point. But updating the actual wal-index header is also
        ** safe and means there is no special case for sqlite3WalUndo()
        ** to handle if this transaction is rolled back.
        */
        int i;                    /* Loop counter */
        u32 *aSalt = pWal->hdr.aSalt;       /* Big-endian salt values */


        pWal->nCkpt++;
        pWal->hdr.mxFrame = 0;
        sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0]));
        aSalt[1] = salt1;
        walIndexWriteHdr(pWal);
        pInfo->nBackfill = 0;
        for(i=1; i<WAL_NREADER; i++) pInfo->aReadMark[i] = READMARK_NOT_USED;
................................................................................
    sqlite3Put4byte(&aWalHdr[24], aCksum[0]);
    sqlite3Put4byte(&aWalHdr[28], aCksum[1]);
    
    pWal->szPage = szPage;
    pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN;
    pWal->hdr.aFrameCksum[0] = aCksum[0];
    pWal->hdr.aFrameCksum[1] = aCksum[1];
    pWal->truncateOnCommit = 1;

    rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0);
    WALTRACE(("WAL%p: wal-header write %s\n", pWal, rc ? "failed" : "ok"));
    if( rc!=SQLITE_OK ){
      return rc;
    }
  }
................................................................................
      }
      nLast++;
      iOffset += szPage;
    }

    rc = sqlite3OsSync(pWal->pWalFd, sync_flags);
  }

  if( isCommit && pWal->truncateOnCommit && pWal->mxWalSize>=0 ){
    i64 sz = pWal->mxWalSize;
    if( walFrameOffset(iFrame+nLast+1, szPage)>pWal->mxWalSize ){
      sz = walFrameOffset(iFrame+nLast+1, szPage);
    }
    walLimitSize(pWal, sz);
    pWal->truncateOnCommit = 0;
  }

  /* Append data to the wal-index. It is not necessary to lock the 
  ** wal-index to do this as the SQLITE_SHM_WRITE lock held on the wal-index
  ** guarantees that there are no other writers, and no data that may
  ** be in use by existing readers is being overwritten.
  */
  iFrame = pWal->hdr.mxFrame;

Added test/walcrash3.test.























































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
# 2011 December 16
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This test simulates an application crash immediately following a
# system call to truncate a file. Specifically, the system call that
# truncates the WAL file if "PRAGMA journal_size_limit" is configured.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

ifcapable !wal {finish_test ; return }
set testprefix walcrash3

db close
testvfs tvfs
tvfs filter {xTruncate xWrite}
tvfs script tvfs_callback
proc tvfs_callback {args} {}

sqlite3 db test.db -vfs tvfs
do_execsql_test 1.1 {
  PRAGMA page_size = 1024;
  PRAGMA journal_mode = WAL;
  PRAGMA wal_autocheckpoint = 128;
  PRAGMA journal_size_limit = 16384;

  CREATE TABLE t1(a BLOB, b BLOB, UNIQUE(a, b));
  INSERT INTO t1 VALUES(randomblob(10), randomblob(1000));
} {wal 128 16384}

proc tvfs_callback {method file arglist} {
  if {$::state==1} {
    foreach f [glob -nocomplain xx_test.*] { forcedelete $f }
    foreach f [glob -nocomplain test.*]    { forcecopy $f "xx_$f" }
    set ::state 2
  }
  if {$::state==0 && $method=="xTruncate" && [file tail $file]=="test.db-wal"} {
    set ::state 1
  }
}

for {set i 2} {$i<1000} {incr i} {

  # If the WAL file is truncated within the following, within the following
  # xWrite call the [tvfs_callback] makes a copy of the database and WAL 
  # files set sets $::state to 2. So that the copied files are in the same
  # state as the real database and WAL files would be if an application crash 
  # occurred immediately following the xTruncate().
  # 
  set ::state 0
  do_execsql_test 1.$i.1 {
    INSERT INTO t1 VALUES(randomblob(10), randomblob(1000));
  }

  # If a copy was made, open it and run the integrity-check.
  #
  if {$::state==2} {
    sqlite3 db2 xx_test.db
    do_test 1.$i.2 { execsql { PRAGMA integrity_check  } db2 } "ok"
    do_test 1.$i.3 { execsql { SELECT count(*) FROM t1 } db2 } [expr $i-1]
    db2 close
  }
}

finish_test