/ Check-in [8a53f12c]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Change the checksum used in WAL files so that each frames checksum depends on the content of the WAL header and all frame headers and content up to and including the frame to which the checksum is attached.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 8a53f12c83a107684b99f4a9de371b5ea3ca810a
User & Date: dan 2010-05-24 13:57:43
Context
2010-05-24
17:00
Fix the wal2.test script so that it works in auto_vacuum mode. check-in: 6a818afb user: drh tags: trunk
13:57
Change the checksum used in WAL files so that each frames checksum depends on the content of the WAL header and all frame headers and content up to and including the frame to which the checksum is attached. check-in: 8a53f12c user: dan tags: trunk
13:28
Make sure a WAL frame of all zeros is detected as an invalid frame. check-in: 02d99ad4 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/pager.c.

217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
....
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
....
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
typedef struct PagerSavepoint PagerSavepoint;
struct PagerSavepoint {
  i64 iOffset;                 /* Starting offset in main journal */
  i64 iHdrOffset;              /* See above */
  Bitvec *pInSavepoint;        /* Set of pages in this savepoint */
  Pgno nOrig;                  /* Original number of pages in file */
  Pgno iSubRec;                /* Index of first record in sub-journal */
  u32 iFrame;                  /* Last frame in WAL when savepoint opened */
};

/*
** A open page cache is an instance of the following structure.
**
** errCode
**
................................................................................
  ** will be skipped.  Out-of-range pages are also skipped.
  */
  if( pSavepoint ){
    u32 ii;            /* Loop counter */
    i64 offset = pSavepoint->iSubRec*(4+pPager->pageSize);

    if( pagerUseWal(pPager) ){
      rc = sqlite3WalSavepointUndo(pPager->pWal, pSavepoint->iFrame);
    }
    for(ii=pSavepoint->iSubRec; rc==SQLITE_OK && ii<pPager->nSubRec; ii++){
      assert( offset==ii*(4+pPager->pageSize) );
      rc = pager_playback_one_page(pPager, &offset, pDone, 0, 1);
    }
    assert( rc!=SQLITE_DONE );
  }
................................................................................
      }
      aNew[ii].iSubRec = pPager->nSubRec;
      aNew[ii].pInSavepoint = sqlite3BitvecCreate(nPage);
      if( !aNew[ii].pInSavepoint ){
        return SQLITE_NOMEM;
      }
      if( pagerUseWal(pPager) ){
        aNew[ii].iFrame = sqlite3WalSavepoint(pPager->pWal);
      }
    }

    /* Open the sub-journal, if it is not already opened. */
    rc = openSubJournal(pPager);
    assertTruncateConstraint(pPager);
  }







|







 







|







 







|







217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
....
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
....
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
typedef struct PagerSavepoint PagerSavepoint;
struct PagerSavepoint {
  i64 iOffset;                 /* Starting offset in main journal */
  i64 iHdrOffset;              /* See above */
  Bitvec *pInSavepoint;        /* Set of pages in this savepoint */
  Pgno nOrig;                  /* Original number of pages in file */
  Pgno iSubRec;                /* Index of first record in sub-journal */
  u32 aWalData[WAL_SAVEPOINT_NDATA];        /* WAL savepoint context */
};

/*
** A open page cache is an instance of the following structure.
**
** errCode
**
................................................................................
  ** will be skipped.  Out-of-range pages are also skipped.
  */
  if( pSavepoint ){
    u32 ii;            /* Loop counter */
    i64 offset = pSavepoint->iSubRec*(4+pPager->pageSize);

    if( pagerUseWal(pPager) ){
      rc = sqlite3WalSavepointUndo(pPager->pWal, pSavepoint->aWalData);
    }
    for(ii=pSavepoint->iSubRec; rc==SQLITE_OK && ii<pPager->nSubRec; ii++){
      assert( offset==ii*(4+pPager->pageSize) );
      rc = pager_playback_one_page(pPager, &offset, pDone, 0, 1);
    }
    assert( rc!=SQLITE_DONE );
  }
................................................................................
      }
      aNew[ii].iSubRec = pPager->nSubRec;
      aNew[ii].pInSavepoint = sqlite3BitvecCreate(nPage);
      if( !aNew[ii].pInSavepoint ){
        return SQLITE_NOMEM;
      }
      if( pagerUseWal(pPager) ){
        sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData);
      }
    }

    /* Open the sub-journal, if it is not already opened. */
    rc = openSubJournal(pPager);
    assertTruncateConstraint(pPager);
  }

Changes to src/wal.c.

210
211
212
213
214
215
216
217
218
219
220
221

222
223
224
225
226
227
228
229
230
...
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
...
447
448
449
450
451
452
453

454
455
456
457
458
459
460
461
462
...
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
...
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
...
775
776
777
778
779
780
781
782
783
784
785



786
787
788
789
790
791
792
...
805
806
807
808
809
810
811
812
813
814


815
816
817
818
819
820
821
822
823
824
825
826
827
828

829
830
831
832
833
834
835
....
1622
1623
1624
1625
1626
1627
1628
1629
1630



1631
1632
1633
1634


1635
1636

1637
1638



1639
1640
1641
1642
1643
1644
1645
1646
1647


1648
1649
1650
1651
1652
1653
1654
....
1690
1691
1692
1693
1694
1695
1696

1697
1698
1699
1700
1701
1702
1703
/*
** The following object holds a copy of the wal-index header content.
**
** The actual header in the wal-index consists of two copies of this
** object.
*/
struct WalIndexHdr {
  u32 iChange;      /* Counter incremented each transaction */
  u16 bigEndCksum;  /* True if checksums in WAL are big-endian */
  u16 szPage;       /* Database page size in bytes */
  u32 mxFrame;      /* Index of last valid frame in the WAL */
  u32 nPage;        /* Size of database in pages */

  u32 aSalt[2];     /* Salt-1 and salt-2 values copied from WAL header */
  u32 aCksum[2];    /* Checksum over all prior fields */
};

/* A block of WALINDEX_LOCK_RESERVED bytes beginning at
** WALINDEX_LOCK_OFFSET is reserved for locks. Since some systems
** only support mandatory file-locks, we do not read or write data
** from the region of the file on which locks are applied.
*/
................................................................................
  Wal *pWal,                      /* The write-ahead log */
  u32 iPage,                      /* Database page number for frame */
  u32 nTruncate,                  /* New db size (or 0 for non-commit frames) */
  u8 *aData,                      /* Pointer to page data */
  u8 *aFrame                      /* OUT: Write encoded frame here */
){
  int nativeCksum;                /* True for native byte-order checksums */
  u32 aCksum[2];
  assert( WAL_FRAME_HDRSIZE==24 );
  sqlite3Put4byte(&aFrame[0], iPage);
  sqlite3Put4byte(&aFrame[4], nTruncate);
  memcpy(&aFrame[8], pWal->hdr.aSalt, 8);

  nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN);
  walChecksumBytes(nativeCksum, aFrame, 16, 0, aCksum);
  walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum);

  sqlite3Put4byte(&aFrame[16], aCksum[0]);
  sqlite3Put4byte(&aFrame[20], aCksum[1]);
}

/*
................................................................................
  Wal *pWal,                      /* The write-ahead log */
  u32 *piPage,                    /* OUT: Database page number for frame */
  u32 *pnTruncate,                /* OUT: New db size (or 0 if not commit) */
  u8 *aData,                      /* Pointer to page data (for checksum) */
  u8 *aFrame                      /* Frame data */
){
  int nativeCksum;                /* True for native byte-order checksums */

  u32 pgno;                       /* Page number of the frame */
  u32 aCksum[2];
  assert( WAL_FRAME_HDRSIZE==24 );

  /* A frame is only valid if the salt values in the frame-header
  ** match the salt values in the wal-header. 
  */
  if( memcmp(&pWal->hdr.aSalt, &aFrame[8], 8)!=0 ){
    return 0;
................................................................................
  }

  /* A frame is only valid if a checksum of the first 16 bytes
  ** of the frame-header, and the frame-data matches
  ** the checksum in the last 8 bytes of the frame-header.
  */
  nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN);
  walChecksumBytes(nativeCksum, aFrame, 16, 0, aCksum);
  walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum);
  if( aCksum[0]!=sqlite3Get4byte(&aFrame[16]) 
   || aCksum[1]!=sqlite3Get4byte(&aFrame[20]) 
  ){
    /* Checksum failed. */
    return 0;
  }
................................................................................
/*
** Recover the wal-index by reading the write-ahead log file. 
** The caller must hold RECOVER lock on the wal-index file.
*/
static int walIndexRecover(Wal *pWal){
  int rc;                         /* Return Code */
  i64 nSize;                      /* Size of log file */
  WalIndexHdr hdr;                /* Recovered wal-index header */

  assert( pWal->lockState>SQLITE_SHM_READ );
  memset(&hdr, 0, sizeof(hdr));

  rc = sqlite3OsFileSize(pWal->pWalFd, &nSize);
  if( rc!=SQLITE_OK ){
    return rc;
  }

  if( nSize>WAL_HDRSIZE ){
................................................................................
    if( (magic&0xFFFFFFFE)!=WAL_MAGIC 
     || szPage&(szPage-1) 
     || szPage>SQLITE_MAX_PAGE_SIZE 
     || szPage<512 
    ){
      goto finished;
    }
    hdr.bigEndCksum = pWal->hdr.bigEndCksum = (magic&0x00000001);
    pWal->szPage = szPage;
    pWal->nCkpt = sqlite3Get4byte(&aBuf[12]);
    memcpy(&pWal->hdr.aSalt, &aBuf[16], 8);




    /* Malloc a buffer to read frames into. */
    szFrame = szPage + WAL_FRAME_HDRSIZE;
    aFrame = (u8 *)sqlite3_malloc(szFrame);
    if( !aFrame ){
      return SQLITE_NOMEM;
    }
................................................................................
      isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame);
      if( !isValid ) break;
      rc = walIndexAppend(pWal, ++iFrame, pgno);
      if( rc!=SQLITE_OK ) break;

      /* If nTruncate is non-zero, this is a commit record. */
      if( nTruncate ){
        hdr.mxFrame = iFrame;
        hdr.nPage = nTruncate;
        hdr.szPage = szPage;


      }
    }

    sqlite3_free(aFrame);
  }else{
    memset(&hdr, 0, sizeof(hdr));
  }

finished:
  if( rc==SQLITE_OK && hdr.mxFrame==0 ){
    rc = walIndexRemap(pWal, WALINDEX_MMAP_INCREMENT);
  }
  if( rc==SQLITE_OK ){
    memcpy(&pWal->hdr, &hdr, sizeof(hdr));

    walIndexWriteHdr(pWal);
  }
  return rc;
}

/*
** Close an open wal-index.
................................................................................
      }
    }
    walIndexUnmap(pWal);
  }
  return rc;
}

/* Return an integer that records the current (uncommitted) write
** position in the WAL



*/
u32 sqlite3WalSavepoint(Wal *pWal){
  assert( pWal->lockState==SQLITE_SHM_WRITE );
  return pWal->hdr.mxFrame;


}


/* Move the write position of the WAL back to iFrame.  Called in
** response to a ROLLBACK TO command.



*/
int sqlite3WalSavepointUndo(Wal *pWal, u32 iFrame){
  int rc = SQLITE_OK;
  assert( pWal->lockState==SQLITE_SHM_WRITE );

  assert( iFrame<=pWal->hdr.mxFrame );
  if( iFrame<pWal->hdr.mxFrame ){
    rc = walIndexMap(pWal, walMappingSize(pWal->hdr.mxFrame));
    pWal->hdr.mxFrame = iFrame;


    if( rc==SQLITE_OK ){
      walCleanupHash(pWal);
      walIndexUnmap(pWal);
    }
  }
  return rc;
}
................................................................................
    pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN;
    sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt);
    memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8);
    rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0);
    if( rc!=SQLITE_OK ){
      return rc;
    }

  }
  assert( pWal->szPage==szPage );

    /* Write the log file. */
  for(p=pList; p; p=p->pDirty){
    u32 nDbsize;                  /* Db-size field for frame header */
    i64 iOffset;                  /* Write offset in log file */







|
|
|
|
|
>
|
|







 







|






|







 







>

<







 







|







 







|


|







 







|



>
>
>







 







|
|
|
>
>




<
<



|



|
>







 







|
|
>
>
>

|

|
>
>


>
|
<
>
>
>

|



|
|

|
>
>







 







>







210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
...
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
...
448
449
450
451
452
453
454
455
456

457
458
459
460
461
462
463
...
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
...
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
...
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
...
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824


825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
....
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648

1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
....
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
/*
** The following object holds a copy of the wal-index header content.
**
** The actual header in the wal-index consists of two copies of this
** object.
*/
struct WalIndexHdr {
  u32 iChange;                    /* Counter incremented each transaction */
  u16 bigEndCksum;                /* True if checksums in WAL are big-endian */
  u16 szPage;                     /* Database page size in bytes */
  u32 mxFrame;                    /* Index of last valid frame in the WAL */
  u32 nPage;                      /* Size of database in pages */
  u32 aFrameCksum[2];             /* Checksum of last frame in log */
  u32 aSalt[2];                   /* Two salt values copied from WAL header */
  u32 aCksum[2];                  /* Checksum over all prior fields */
};

/* A block of WALINDEX_LOCK_RESERVED bytes beginning at
** WALINDEX_LOCK_OFFSET is reserved for locks. Since some systems
** only support mandatory file-locks, we do not read or write data
** from the region of the file on which locks are applied.
*/
................................................................................
  Wal *pWal,                      /* The write-ahead log */
  u32 iPage,                      /* Database page number for frame */
  u32 nTruncate,                  /* New db size (or 0 for non-commit frames) */
  u8 *aData,                      /* Pointer to page data */
  u8 *aFrame                      /* OUT: Write encoded frame here */
){
  int nativeCksum;                /* True for native byte-order checksums */
  u32 *aCksum = pWal->hdr.aFrameCksum;
  assert( WAL_FRAME_HDRSIZE==24 );
  sqlite3Put4byte(&aFrame[0], iPage);
  sqlite3Put4byte(&aFrame[4], nTruncate);
  memcpy(&aFrame[8], pWal->hdr.aSalt, 8);

  nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN);
  walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum);
  walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum);

  sqlite3Put4byte(&aFrame[16], aCksum[0]);
  sqlite3Put4byte(&aFrame[20], aCksum[1]);
}

/*
................................................................................
  Wal *pWal,                      /* The write-ahead log */
  u32 *piPage,                    /* OUT: Database page number for frame */
  u32 *pnTruncate,                /* OUT: New db size (or 0 if not commit) */
  u8 *aData,                      /* Pointer to page data (for checksum) */
  u8 *aFrame                      /* Frame data */
){
  int nativeCksum;                /* True for native byte-order checksums */
  u32 *aCksum = pWal->hdr.aFrameCksum;
  u32 pgno;                       /* Page number of the frame */

  assert( WAL_FRAME_HDRSIZE==24 );

  /* A frame is only valid if the salt values in the frame-header
  ** match the salt values in the wal-header. 
  */
  if( memcmp(&pWal->hdr.aSalt, &aFrame[8], 8)!=0 ){
    return 0;
................................................................................
  }

  /* A frame is only valid if a checksum of the first 16 bytes
  ** of the frame-header, and the frame-data matches
  ** the checksum in the last 8 bytes of the frame-header.
  */
  nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN);
  walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum);
  walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum);
  if( aCksum[0]!=sqlite3Get4byte(&aFrame[16]) 
   || aCksum[1]!=sqlite3Get4byte(&aFrame[20]) 
  ){
    /* Checksum failed. */
    return 0;
  }
................................................................................
/*
** Recover the wal-index by reading the write-ahead log file. 
** The caller must hold RECOVER lock on the wal-index file.
*/
static int walIndexRecover(Wal *pWal){
  int rc;                         /* Return Code */
  i64 nSize;                      /* Size of log file */
  u32 aFrameCksum[2] = {0, 0};

  assert( pWal->lockState>SQLITE_SHM_READ );
  memset(&pWal->hdr, 0, sizeof(WalIndexHdr));

  rc = sqlite3OsFileSize(pWal->pWalFd, &nSize);
  if( rc!=SQLITE_OK ){
    return rc;
  }

  if( nSize>WAL_HDRSIZE ){
................................................................................
    if( (magic&0xFFFFFFFE)!=WAL_MAGIC 
     || szPage&(szPage-1) 
     || szPage>SQLITE_MAX_PAGE_SIZE 
     || szPage<512 
    ){
      goto finished;
    }
    pWal->hdr.bigEndCksum = (magic&0x00000001);
    pWal->szPage = szPage;
    pWal->nCkpt = sqlite3Get4byte(&aBuf[12]);
    memcpy(&pWal->hdr.aSalt, &aBuf[16], 8);
    walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, 
        aBuf, WAL_HDRSIZE, 0, pWal->hdr.aFrameCksum
    );

    /* Malloc a buffer to read frames into. */
    szFrame = szPage + WAL_FRAME_HDRSIZE;
    aFrame = (u8 *)sqlite3_malloc(szFrame);
    if( !aFrame ){
      return SQLITE_NOMEM;
    }
................................................................................
      isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame);
      if( !isValid ) break;
      rc = walIndexAppend(pWal, ++iFrame, pgno);
      if( rc!=SQLITE_OK ) break;

      /* If nTruncate is non-zero, this is a commit record. */
      if( nTruncate ){
        pWal->hdr.mxFrame = iFrame;
        pWal->hdr.nPage = nTruncate;
        pWal->hdr.szPage = szPage;
        aFrameCksum[0] = pWal->hdr.aFrameCksum[0];
        aFrameCksum[1] = pWal->hdr.aFrameCksum[1];
      }
    }

    sqlite3_free(aFrame);


  }

finished:
  if( rc==SQLITE_OK && pWal->hdr.mxFrame==0 ){
    rc = walIndexRemap(pWal, WALINDEX_MMAP_INCREMENT);
  }
  if( rc==SQLITE_OK ){
    pWal->hdr.aFrameCksum[0] = aFrameCksum[0];
    pWal->hdr.aFrameCksum[1] = aFrameCksum[1];
    walIndexWriteHdr(pWal);
  }
  return rc;
}

/*
** Close an open wal-index.
................................................................................
      }
    }
    walIndexUnmap(pWal);
  }
  return rc;
}

/* 
** Argument aWalData must point to an array of WAL_SAVEPOINT_NDATA u32 
** values. This function populates the array with values required to 
** "rollback" the write position of the WAL handle back to the current 
** point in the event of a savepoint rollback (via WalSavepointUndo()).
*/
void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){
  assert( pWal->lockState==SQLITE_SHM_WRITE );
  aWalData[0] = pWal->hdr.mxFrame;
  aWalData[1] = pWal->hdr.aFrameCksum[0];
  aWalData[2] = pWal->hdr.aFrameCksum[1];
}

/* 
** Move the write position of the WAL back to the point identified by

** the values in the aWalData[] array. aWalData must point to an array
** of WAL_SAVEPOINT_NDATA u32 values that has been previously populated
** by a call to WalSavepoint().
*/
int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){
  int rc = SQLITE_OK;
  assert( pWal->lockState==SQLITE_SHM_WRITE );

  assert( aWalData[0]<=pWal->hdr.mxFrame );
  if( aWalData[0]<pWal->hdr.mxFrame ){
    rc = walIndexMap(pWal, walMappingSize(pWal->hdr.mxFrame));
    pWal->hdr.mxFrame = aWalData[0];
    pWal->hdr.aFrameCksum[0] = aWalData[1];
    pWal->hdr.aFrameCksum[1] = aWalData[2];
    if( rc==SQLITE_OK ){
      walCleanupHash(pWal);
      walIndexUnmap(pWal);
    }
  }
  return rc;
}
................................................................................
    pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN;
    sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt);
    memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8);
    rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0);
    if( rc!=SQLITE_OK ){
      return rc;
    }
    walChecksumBytes(1, aWalHdr, sizeof(aWalHdr), 0, pWal->hdr.aFrameCksum);
  }
  assert( pWal->szPage==szPage );

    /* Write the log file. */
  for(p=pList; p; p=p->pDirty){
    u32 nDbsize;                  /* Db-size field for frame header */
    i64 iOffset;                  /* Write offset in log file */

Changes to src/wal.h.

24
25
26
27
28
29
30
31
32
33
34
35
36


37
38
39
40
41
42
43
..
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
# define sqlite3WalClose(w,x,y,z)          0
# define sqlite3WalOpenSnapshot(y,z)       0
# define sqlite3WalCloseSnapshot(z) 
# define sqlite3WalRead(v,w,x,y,z)         0
# define sqlite3WalDbsize(y,z)
# define sqlite3WalWriteLock(y,z)          0
# define sqlite3WalUndo(x,y,z)             0
# define sqlite3WalSavepoint(z)            0
# define sqlite3WalSavepointUndo(y,z)      0
# define sqlite3WalFrames(u,v,w,x,y,z)     0
# define sqlite3WalCheckpoint(u,v,w,x,y,z) 0
# define sqlite3WalCallback(z)             0
#else



/* Connection to a write-ahead log (WAL) file. 
** There is one object of this type for each pager. 
*/
typedef struct Wal Wal;

/* Open and close a connection to a write-ahead log. */
................................................................................
int sqlite3WalWriteLock(Wal *pWal, int op);

/* Undo any frames written (but not committed) to the log */
int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx);

/* Return an integer that records the current (uncommitted) write
** position in the WAL */
u32 sqlite3WalSavepoint(Wal *pWal);

/* Move the write position of the WAL back to iFrame.  Called in
** response to a ROLLBACK TO command. */
int sqlite3WalSavepointUndo(Wal *pWal, u32 iFrame);

/* Write a frame or frames to the log. */
int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int);

/* Copy pages from the log to the database file */ 
int sqlite3WalCheckpoint(
  Wal *pWal,                      /* Write-ahead log connection */







|





>
>







 







|



|







24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
..
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
# define sqlite3WalClose(w,x,y,z)          0
# define sqlite3WalOpenSnapshot(y,z)       0
# define sqlite3WalCloseSnapshot(z) 
# define sqlite3WalRead(v,w,x,y,z)         0
# define sqlite3WalDbsize(y,z)
# define sqlite3WalWriteLock(y,z)          0
# define sqlite3WalUndo(x,y,z)             0
# define sqlite3WalSavepoint(y,z)
# define sqlite3WalSavepointUndo(y,z)      0
# define sqlite3WalFrames(u,v,w,x,y,z)     0
# define sqlite3WalCheckpoint(u,v,w,x,y,z) 0
# define sqlite3WalCallback(z)             0
#else

#define WAL_SAVEPOINT_NDATA 3

/* Connection to a write-ahead log (WAL) file. 
** There is one object of this type for each pager. 
*/
typedef struct Wal Wal;

/* Open and close a connection to a write-ahead log. */
................................................................................
int sqlite3WalWriteLock(Wal *pWal, int op);

/* Undo any frames written (but not committed) to the log */
int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx);

/* Return an integer that records the current (uncommitted) write
** position in the WAL */
void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData);

/* Move the write position of the WAL back to iFrame.  Called in
** response to a ROLLBACK TO command. */
int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData);

/* Write a frame or frames to the log. */
int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int);

/* Copy pages from the log to the database file */ 
int sqlite3WalCheckpoint(
  Wal *pWal,                      /* Write-ahead log connection */

Changes to test/wal.test.

1344
1345
1346
1347
1348
1349
1350
1351

1352
1353
1354
1355
1356
1357
1358
      # be ignored.
      #
      set walhdr [binary format IIIIII 931071618 3007000 $pgsz 1234 22 23]
      set framebody [randomblob $pgsz]
      set framehdr  [binary format IIII $pg 5 22 23]
      set c1 0
      set c2 0
      logcksum c1 c2 $framehdr

      logcksum c1 c2 $framebody
      set framehdr [binary format IIIIII $pg 5 22 23 $c1 $c2]

      set fd [open test.db-wal w]
      fconfigure $fd -encoding binary -translation binary
      puts -nonewline $fd $walhdr
      puts -nonewline $fd $framehdr







|
>







1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
      # be ignored.
      #
      set walhdr [binary format IIIIII 931071618 3007000 $pgsz 1234 22 23]
      set framebody [randomblob $pgsz]
      set framehdr  [binary format IIII $pg 5 22 23]
      set c1 0
      set c2 0
      logcksum c1 c2 $walhdr
      logcksum c1 c2 [string range $framehdr 0 7]
      logcksum c1 c2 $framebody
      set framehdr [binary format IIIIII $pg 5 22 23 $c1 $c2]

      set fd [open test.db-wal w]
      fconfigure $fd -encoding binary -translation binary
      puts -nonewline $fd $walhdr
      puts -nonewline $fd $framehdr

Changes to test/wal2.test.

15
16
17
18
19
20
21





22
23
24
25
26
27

28

29
30
31
32
33
34
35
36
37
38
39

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
ifcapable !wal {finish_test ; return }

proc set_tvfs_hdr {file args} {





  if {[llength $args]>1} {
    return -code error {wrong # args: should be "set_tvfs_hdr fileName ?val?"}
  }

  set blob [tvfs shm $file]
  if {[llength $args]} {

    set blob [binary format i16a* [lindex $args 0] [string range $blob 64 end]]

    tvfs shm $file $blob
  }

  binary scan $blob i16 ints
  return $ints
}

proc incr_tvfs_hdr {file idx incrval} {
  set ints [set_tvfs_hdr $file]
  set v [lindex $ints $idx]
  incr v $incrval







>
>
>
>
>






>
|
>



|







15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
ifcapable !wal {finish_test ; return }

proc set_tvfs_hdr {file args} {

  # Set $nHdr to the number of bytes in the wal-index header:
  set nHdr 80
  set nInt [expr {$nHdr/4}]

  if {[llength $args]>1} {
    return -code error {wrong # args: should be "set_tvfs_hdr fileName ?val?"}
  }

  set blob [tvfs shm $file]
  if {[llength $args]} {
    set blob [
      binary format i${nInt}a* [lindex $args 0] [string range $blob $nHdr end]
    ]
    tvfs shm $file $blob
  }

  binary scan $blob i${nInt} ints
  return $ints
}

proc incr_tvfs_hdr {file idx incrval} {
  set ints [set_tvfs_hdr $file]
  set v [lindex $ints $idx]
  incr v $incrval

Changes to test/walcksum.test.

58
59
60
61
62
63
64
65
66
67
68

69
70
71
72
73
74
75
76

77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

96
97
98
99
100
101
102
103
104



















105
106
107
108
109
110
111
...
176
177
178
179
180
181
182

183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
...
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278









































279
# File $filename must be a WAL file on disk. Check that the checksum of frame
# $iFrame in the file is correct when interpreting data as $endian-endian
# integers ($endian must be either "big" or "little"). If the checksum looks
# correct, return 1. Otherwise 0.
#
proc log_checksum_verify {filename iFrame endian} {
  set data [readfile $filename]
  set c1 0
  set c2 0
  
  binary scan [string range $data 8 11] I pgsz


  set n [log_file_size [expr $iFrame-1] $pgsz]
  binary scan [string range $data [expr $n+16] [expr $n+23]] II expect1 expect2
  log_cksum $endian c1 c2 [string range $data $n [expr $n+15]]
  log_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]]

  set expect1 [expr $expect1&0xFFFFFFFF]
  set expect2 [expr $expect2&0xFFFFFFFF]

  expr {$c1==$expect1 && $c2==$expect2}
}

#
# File $filename must be a WAL file on disk. Compute the checksum for frame
# $iFrame in the file by interpreting data as $endian-endian integers 
# ($endian must be either "big" or "little"). Then write the computed 
# checksum into the file.
#
proc log_checksum_write {filename iFrame endian} {
  set data [readfile $filename]
  set c1 0
  set c2 0
  
  binary scan [string range $data 8 11] I pgsz

  set n [log_file_size [expr $iFrame-1] $pgsz]
  log_cksum $endian c1 c2 [string range $data $n [expr $n+15]]
  log_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]]


  set bin [binary format II $c1 $c2]
  set fd [open $filename r+]
  fconfigure $fd -encoding binary
  fconfigure $fd -translation binary
  seek $fd [expr $n+16]
  puts -nonewline $fd $bin
  close $fd
}




















#
# File $filename must be a WAL file on disk. Set the 'magic' field of the
# WAL header to indicate that checksums are $endian-endian ($endian must be
# either "big" or "little").
#
proc log_checksum_writemagic {filename endian} {
................................................................................
      log_checksum_verify test2.db-wal $f $native
    } 1
  }

  # Replace all checksums in the current WAL file with $endian versions.
  # Then check that it is still possible to recover and read the database.
  #

  for {set f 1} {$f <= 6} {incr f} {
    do_test walcksum-1.$endian.3.$f {
      log_checksum_write test2.db-wal $f $endian
      log_checksum_verify test2.db-wal $f $endian
    } {1}
  }
  do_test walcksum-1.$endian.4.1 {
    log_checksum_writemagic test2.db-wal $endian
    file copy -force test2.db test.db
    file copy -force test2.db-wal test.db-wal
    sqlite3 db test.db
    execsql { SELECT a FROM t1 }
  } {1 2 3 5 8 13 21}

  # Following recovery, any frames written to the log should use the same 
................................................................................
    log_checksum_verify test.db-wal 1 $native
  } {1}
  do_test walcksum-1.$endian.8.2 {
    log_checksum_verify test.db-wal 2 $native
  } {1}
  do_test walcksum-1.$endian.8.3 {
    log_checksum_verify test.db-wal 3 $native
  } [expr {$native == $endian}]

  do_test walcksum-1.$endian.9 {
    execsql { 
      PRAGMA integrity_check;
      SELECT a FROM t1;
    } db2
  } {ok 1 2 3 5 8 13 21 34 55 89}

  catch { db close }
  catch { db2 close }
}










































finish_test







<
<
|
<
>

<
|
<
<
<


>











<
<
|
<
<
<
<
<
>





|



>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>







<







 







|












>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

58
59
60
61
62
63
64


65

66
67

68



69
70
71
72
73
74
75
76
77
78
79
80
81
82


83





84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
...
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198

199
200
201
202
203
204
205
...
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
# File $filename must be a WAL file on disk. Check that the checksum of frame
# $iFrame in the file is correct when interpreting data as $endian-endian
# integers ($endian must be either "big" or "little"). If the checksum looks
# correct, return 1. Otherwise 0.
#
proc log_checksum_verify {filename iFrame endian} {
  set data [readfile $filename]




  foreach {offset c1 c2} [log_checksum_calc $data $iFrame $endian] {}


  binary scan [string range $data $offset [expr $offset+7]] II expect1 expect2



  set expect1 [expr $expect1&0xFFFFFFFF]
  set expect2 [expr $expect2&0xFFFFFFFF]

  expr {$c1==$expect1 && $c2==$expect2}
}

#
# File $filename must be a WAL file on disk. Compute the checksum for frame
# $iFrame in the file by interpreting data as $endian-endian integers 
# ($endian must be either "big" or "little"). Then write the computed 
# checksum into the file.
#
proc log_checksum_write {filename iFrame endian} {
  set data [readfile $filename]








  foreach {offset c1 c2} [log_checksum_calc $data $iFrame $endian] {}

  set bin [binary format II $c1 $c2]
  set fd [open $filename r+]
  fconfigure $fd -encoding binary
  fconfigure $fd -translation binary
  seek $fd $offset
  puts -nonewline $fd $bin
  close $fd
}

proc log_checksum_calc {data iFrame endian} {
  
  binary scan [string range $data 8 11] I pgsz
  if {$iFrame > 1} {
    set n [log_file_size [expr $iFrame-2] $pgsz]
    binary scan [string range $data [expr $n+16] [expr $n+23]] II c1 c2
  } else {
    set c1 0
    set c2 0
    log_cksum $endian c1 c2 [string range $data 0 23]
  }

  set n [log_file_size [expr $iFrame-1] $pgsz]
  log_cksum $endian c1 c2 [string range $data $n [expr $n+7]]
  log_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]]

  list [expr $n+16] $c1 $c2
}

#
# File $filename must be a WAL file on disk. Set the 'magic' field of the
# WAL header to indicate that checksums are $endian-endian ($endian must be
# either "big" or "little").
#
proc log_checksum_writemagic {filename endian} {
................................................................................
      log_checksum_verify test2.db-wal $f $native
    } 1
  }

  # Replace all checksums in the current WAL file with $endian versions.
  # Then check that it is still possible to recover and read the database.
  #
  log_checksum_writemagic test2.db-wal $endian
  for {set f 1} {$f <= 6} {incr f} {
    do_test walcksum-1.$endian.3.$f {
      log_checksum_write test2.db-wal $f $endian
      log_checksum_verify test2.db-wal $f $endian
    } {1}
  }
  do_test walcksum-1.$endian.4.1 {

    file copy -force test2.db test.db
    file copy -force test2.db-wal test.db-wal
    sqlite3 db test.db
    execsql { SELECT a FROM t1 }
  } {1 2 3 5 8 13 21}

  # Following recovery, any frames written to the log should use the same 
................................................................................
    log_checksum_verify test.db-wal 1 $native
  } {1}
  do_test walcksum-1.$endian.8.2 {
    log_checksum_verify test.db-wal 2 $native
  } {1}
  do_test walcksum-1.$endian.8.3 {
    log_checksum_verify test.db-wal 3 $native
  } {0}

  do_test walcksum-1.$endian.9 {
    execsql { 
      PRAGMA integrity_check;
      SELECT a FROM t1;
    } db2
  } {ok 1 2 3 5 8 13 21 34 55 89}

  catch { db close }
  catch { db2 close }
}

do_test walcksum-2.1 {
  file delete -force test.db test.db-wal test.db-journal
  sqlite3 db test.db
  execsql {
    PRAGMA synchronous = NORMAL;
    PRAGMA page_size = 1024;
    PRAGMA journal_mode = WAL;
    PRAGMA cache_size = 10;
    CREATE TABLE t1(x PRIMARY KEY);
    PRAGMA wal_checkpoint;
    INSERT INTO t1 VALUES(randomblob(800));
    BEGIN;
      INSERT INTO t1 SELECT randomblob(800) FROM t1;   /*   2 */
      INSERT INTO t1 SELECT randomblob(800) FROM t1;   /*   4 */
      INSERT INTO t1 SELECT randomblob(800) FROM t1;   /*   8 */
      INSERT INTO t1 SELECT randomblob(800) FROM t1;   /*  16 */
      SAVEPOINT one;
        INSERT INTO t1 SELECT randomblob(800) FROM t1;   /*  32 */
        INSERT INTO t1 SELECT randomblob(800) FROM t1;   /*  64 */
        INSERT INTO t1 SELECT randomblob(800) FROM t1;   /* 128 */
        INSERT INTO t1 SELECT randomblob(800) FROM t1;   /* 256 */
      ROLLBACK TO one;
      INSERT INTO t1 SELECT randomblob(800) FROM t1;   /*  32 */
      INSERT INTO t1 SELECT randomblob(800) FROM t1;   /*  64 */
      INSERT INTO t1 SELECT randomblob(800) FROM t1;   /* 128 */
      INSERT INTO t1 SELECT randomblob(800) FROM t1;   /* 256 */
    COMMIT;
  }

  file copy -force test.db test2.db
  file copy -force test.db-wal test2.db-wal

  sqlite3 db2 test2.db
  execsql {
    PRAGMA integrity_check;
    SELECT count(*) FROM t1;
  } db2
} {ok 256}
catch { db close }
catch { db2 close }

finish_test