SQLite

Check-in [a031aa1fa0]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Reserve some space at the start of the log-summary file to apply locks to.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | wal
Files: files | file ages | folders
SHA1: a031aa1fa01f062b087154833738f011f7bc4eb5
User & Date: dan 2010-04-24 04:49:15.000
Context
2010-04-24
04:53
Merge with [e79dac3c2f]. (check-in: 1e793d3a6d user: dan tags: wal)
04:49
Reserve some space at the start of the log-summary file to apply locks to. (check-in: a031aa1fa0 user: dan tags: wal)
2010-04-23
19:15
Fixes and tests for backup of a WAL database. (check-in: 480d12db4c user: dan tags: wal)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/log.c.
24
25
26
27
28
29
30
31
32



33





















34
35
36
37
38
39
40
**     4: For commit records, the size of the database image in pages 
**        after the commit. For all other records, zero.
**     8: Checksum value 1.
**    12: Checksum value 2.
*/

/* 
** LOG SUMMARY FORMAT
**



** TODO.





















*/

#include "log.h"

#include <unistd.h>
#include <fcntl.h>
#include <sys/mman.h>







|

>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
**     4: For commit records, the size of the database image in pages 
**        after the commit. For all other records, zero.
**     8: Checksum value 1.
**    12: Checksum value 2.
*/

/* 
** LOG SUMMARY FILE FORMAT
**
** The log-summary file consists of a header region, followed by an 
** region that contains no useful data (used to apply byte-range locks
** to), followed by the data region. 
**
** The contents of both the header and data region are specified in terms
** of 1, 2 and 4 byte unsigned integers. All integers are stored in 
** machine-endian order.
**
** A log-summary file is essentially a shadow-pager map. It contains a
** mapping from database page number to the set of locations in the log 
** file that contain versions of the database page. When a database 
** client needs to read a page of data, it first queries the log-summary
** file to determine if the required version of the page is stored in
** the log. If so, it is read from the log file. If not, it is read from
** the database file.
**
** Whenever a transaction is appended to the log or a checkpoint transfers
** data from the log file into the database file, the log-summary is 
** updated accordingly.
**
** The fields in the log-summary file header are described in the comment 
** directly above the definition of struct LogSummaryHdr (see below). 
** Immediately following the fields in the LogSummaryHdr structure is
** an 8 byte checksum based on the contents of the header. This field is
** not the same as the iCheck1 and iCheck2 fields of the LogSummaryHdr.
*/

#include "log.h"

#include <unistd.h>
#include <fcntl.h>
#include <sys/mman.h>
61
62
63
64
65
66
67






68
69
70
71

72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
  u32 iCheck1;                    /* Checkpoint value 1 */
  u32 iCheck2;                    /* Checkpoint value 2 */
};

/* Size of serialized LogSummaryHdr object. */
#define LOGSUMMARY_HDR_NFIELD (sizeof(LogSummaryHdr) / sizeof(u32))







#define LOGSUMMARY_FRAME_OFFSET \
  (LOGSUMMARY_HDR_NFIELD + LOG_CKSM_BYTES/sizeof(u32))




/* Size of frame header */
#define LOG_FRAME_HDRSIZE 16
#define LOG_HDRSIZE       12

/*
** Return the offset of frame iFrame in the log file, assuming a database
** page size of pgsz bytes. The offset returned is to the start of the
** log frame-header.
*/
#define logFrameOffset(iFrame, pgsz) (                               \
  LOG_HDRSIZE + ((iFrame)-1)*((pgsz)+LOG_FRAME_HDRSIZE)              \
)

/*
** If using mmap() to access a shared (or otherwise) log-summary file, then
** the mapping size is incremented in units of the following size.
**
** A 64 KB log-summary mapping corresponds to a log file containing over







>
>
>
>
>
>
|
<

|
>

|
<
|







|







85
86
87
88
89
90
91
92
93
94
95
96
97
98

99
100
101
102
103

104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
  u32 iCheck1;                    /* Checkpoint value 1 */
  u32 iCheck2;                    /* Checkpoint value 2 */
};

/* Size of serialized LogSummaryHdr object. */
#define LOGSUMMARY_HDR_NFIELD (sizeof(LogSummaryHdr) / sizeof(u32))

/* A block of 16 bytes beginning at LOGSUMMARY_LOCK_OFFSET is reserved
** for locks. Since some systems only feature mandatory file-locks, we
** do not read or write data from the region of the file on which locks
** are applied.
*/
#define LOGSUMMARY_LOCK_OFFSET   ((sizeof(LogSummaryHdr))+2*sizeof(u32))
#define LOGSUMMARY_LOCK_RESERVED 16


/* Size of header before each frame in log file */
#define LOG_FRAME_HDRSIZE 16

/* Size of log header */

#define LOG_HDRSIZE 12

/*
** Return the offset of frame iFrame in the log file, assuming a database
** page size of pgsz bytes. The offset returned is to the start of the
** log frame-header.
*/
#define logFrameOffset(iFrame, pgsz) (                               \
  LOG_HDRSIZE + ((iFrame)-1)*((pgsz)+LOG_FRAME_HDRSIZE)        \
)

/*
** If using mmap() to access a shared (or otherwise) log-summary file, then
** the mapping size is incremented in units of the following size.
**
** A 64 KB log-summary mapping corresponds to a log file containing over
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
**         However, there may still exist region D readers using data in 
**         the body of the log file, so the log file itself cannot be 
**         truncated or overwritten until all region D readers have finished.
**         That requirement is satisfied, because writers (the clients that
**         write to the log file) require an exclusive lock on region D.
**         Which they cannot get until all region D readers have finished.
*/
#define LOG_LOCK_MUTEX  12
#define LOG_LOCK_DMH    13
#define LOG_LOCK_REGION 14

/*
** The four lockable regions associated with each log-summary. A connection
** may take either a SHARED or EXCLUSIVE lock on each. An ORed combination
** of the following bitmasks is passed as the second argument to the
** logLockRegion() function.
*/







|
|
|







222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
**         However, there may still exist region D readers using data in 
**         the body of the log file, so the log file itself cannot be 
**         truncated or overwritten until all region D readers have finished.
**         That requirement is satisfied, because writers (the clients that
**         write to the log file) require an exclusive lock on region D.
**         Which they cannot get until all region D readers have finished.
*/
#define LOG_LOCK_MUTEX  (LOGSUMMARY_LOCK_OFFSET)
#define LOG_LOCK_DMH    (LOG_LOCK_MUTEX+1)
#define LOG_LOCK_REGION (LOG_LOCK_DMH+1)

/*
** The four lockable regions associated with each log-summary. A connection
** may take either a SHARED or EXCLUSIVE lock on each. An ORed combination
** of the following bitmasks is passed as the second argument to the
** logLockRegion() function.
*/
382
383
384
385
386
387
388
389

390
391

392
393
394
395
396
397
398
399
400
401
402
    close(pSummary->fd);
    pSummary->fd = -1;
  }
  return rc;
}

static void logSummaryWriteHdr(LogSummary *pSummary, LogSummaryHdr *pHdr){
  u32 *aData = pSummary->aData;

  memcpy(aData, pHdr, sizeof(LogSummaryHdr));
  aData[LOGSUMMARY_HDR_NFIELD] = 1;

  aData[LOGSUMMARY_HDR_NFIELD+1] = 1;
  logChecksumBytes(
    (u8 *)aData, sizeof(LogSummaryHdr), &aData[LOGSUMMARY_HDR_NFIELD]
  );
}

/*
** This function encodes a single frame header and writes it to a buffer
** supplied by the caller. A log frame-header is made up of a series of 
** 4-byte big-endian integers, as follows:
**







|
>
|
|
>
|
|
<
<







411
412
413
414
415
416
417
418
419
420
421
422
423
424


425
426
427
428
429
430
431
    close(pSummary->fd);
    pSummary->fd = -1;
  }
  return rc;
}

static void logSummaryWriteHdr(LogSummary *pSummary, LogSummaryHdr *pHdr){
  u32 *aHdr = pSummary->aData;                   /* Write header here */
  u32 *aCksum = &aHdr[LOGSUMMARY_HDR_NFIELD];    /* Write header cksum here */

  assert( LOGSUMMARY_HDR_NFIELD==sizeof(LogSummaryHdr)/4 );
  memcpy(aHdr, pHdr, sizeof(LogSummaryHdr));
  aCksum[0] = aCksum[1] = 1;
  logChecksumBytes((u8 *)aHdr, sizeof(LogSummaryHdr), aCksum);


}

/*
** This function encodes a single frame header and writes it to a buffer
** supplied by the caller. A log frame-header is made up of a series of 
** 4-byte big-endian integers, as follows:
**
549
550
551
552
553
554
555
556




557
558
559
560
561
562
563

/*
** Return the index in the LogSummary.aData array that corresponds to 
** frame iFrame. The log-summary file consists of a header, followed by
** alternating "map" and "index" blocks.
*/
static int logSummaryEntry(u32 iFrame){
  return ((((iFrame-1)>>8)<<6) + iFrame-1 + 2 + LOGSUMMARY_HDR_NFIELD);




}


/*
** Set an entry in the log-summary map to map log frame iFrame to db 
** page iPage. Values are always appended to the log-summary (i.e. the
** value of iFrame is always exactly one more than the value passed to







|
>
>
>
>







578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596

/*
** Return the index in the LogSummary.aData array that corresponds to 
** frame iFrame. The log-summary file consists of a header, followed by
** alternating "map" and "index" blocks.
*/
static int logSummaryEntry(u32 iFrame){
  return (
      (LOGSUMMARY_LOCK_OFFSET+LOGSUMMARY_LOCK_RESERVED)/sizeof(u32)
    + (((iFrame-1)>>8)<<6)        /* Indexes that occur before iFrame */
    + iFrame-1                    /* Db page numbers that occur before iFrame */
  );
}


/*
** Set an entry in the log-summary map to map log frame iFrame to db 
** page iPage. Values are always appended to the log-summary (i.e. the
** value of iFrame is always exactly one more than the value passed to
688
689
690
691
692
693
694



695
696
697
698
699
700
701
  logSummaryWriteHdr(pSummary, &hdr);
  return rc;
}

/*
** Place, modify or remove a lock on the log-summary file associated 
** with pSummary.



*/
static int logLockFd(
  LogSummary *pSummary,           /* The log-summary object to lock */
  int iStart,                     /* First byte to lock */
  int nByte,                      /* Number of bytes to lock */
  int op                          /* LOG_UNLOCK, RDLOCK, WRLOCK or WRLOCKW */
){







>
>
>







721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
  logSummaryWriteHdr(pSummary, &hdr);
  return rc;
}

/*
** Place, modify or remove a lock on the log-summary file associated 
** with pSummary.
**
** The locked byte-range should be inside the region dedicated to 
** locking. This region of the log-summary file is never read or written.
*/
static int logLockFd(
  LogSummary *pSummary,           /* The log-summary object to lock */
  int iStart,                     /* First byte to lock */
  int nByte,                      /* Number of bytes to lock */
  int op                          /* LOG_UNLOCK, RDLOCK, WRLOCK or WRLOCKW */
){
712
713
714
715
716
717
718







719
720
721
722
723
724
725
    F_SETLKW                      /* LOG_WRLOCKW */
  };
  struct flock f;                 /* Locking operation */
  int rc;                         /* Value returned by fcntl() */

  assert( ArraySize(aType)==ArraySize(aOp) );
  assert( op>=0 && op<ArraySize(aType) );








  memset(&f, 0, sizeof(f));
  f.l_type = aType[op];
  f.l_whence = SEEK_SET;
  f.l_start = iStart;
  f.l_len = nByte;
  rc = fcntl(pSummary->fd, aOp[op], &f);







>
>
>
>
>
>
>







748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
    F_SETLKW                      /* LOG_WRLOCKW */
  };
  struct flock f;                 /* Locking operation */
  int rc;                         /* Value returned by fcntl() */

  assert( ArraySize(aType)==ArraySize(aOp) );
  assert( op>=0 && op<ArraySize(aType) );
  assert( nByte>0 );
  assert( iStart>=LOGSUMMARY_LOCK_OFFSET 
       && iStart+nByte<=LOGSUMMARY_LOCK_OFFSET+LOGSUMMARY_LOCK_RESERVED
  );
#if defined(SQLITE_DEBUG) && defined(SQLITE_OS_UNIX)
  if( pSummary->aData ) memset(&((u8*)pSummary->aData)[iStart], op, nByte);
#endif

  memset(&f, 0, sizeof(f));
  f.l_type = aType[op];
  f.l_whence = SEEK_SET;
  f.l_start = iStart;
  f.l_len = nByte;
  rc = fcntl(pSummary->fd, aOp[op], &f);
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
   || (op==LOG_UNLOCK && mRegion) 
   || (op==LOG_RDLOCK && (mOther&mRegion)!=mRegion)
  ){
    struct LockMap {
      int iStart;                 /* Byte offset to start locking operation */
      int iLen;                   /* Length field for locking operation */
    } aMap[] = {
      /* 0000 */ {0, 0},                    /* 0001 */ {4+LOG_LOCK_REGION, 1}, 
      /* 0010 */ {3+LOG_LOCK_REGION, 1},    /* 0011 */ {3+LOG_LOCK_REGION, 2},
      /* 0100 */ {2+LOG_LOCK_REGION, 1},    /* 0101 */ {0, 0}, 
      /* 0110 */ {2+LOG_LOCK_REGION, 2},    /* 0111 */ {2+LOG_LOCK_REGION, 3},
      /* 1000 */ {1+LOG_LOCK_REGION, 1},    /* 1001 */ {0, 0}, 
      /* 1010 */ {0, 0},                    /* 1011 */ {0, 0},
      /* 1100 */ {1+LOG_LOCK_REGION, 2},    /* 1101 */ {0, 0}, 
      /* 1110 */ {0, 0},                    /* 1111 */ {0, 0}
    };
    int rc;                       /* Return code of logLockFd() */

    assert( mRegion<ArraySize(aMap) && aMap[mRegion].iStart!=0 );

    rc = logLockFd(pSummary, aMap[mRegion].iStart, aMap[mRegion].iLen, op);







|
|
|
|
|

|







873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
   || (op==LOG_UNLOCK && mRegion) 
   || (op==LOG_RDLOCK && (mOther&mRegion)!=mRegion)
  ){
    struct LockMap {
      int iStart;                 /* Byte offset to start locking operation */
      int iLen;                   /* Length field for locking operation */
    } aMap[] = {
      /* 0000 */ {0, 0},                    /* 0001 */ {3+LOG_LOCK_REGION, 1}, 
      /* 0010 */ {2+LOG_LOCK_REGION, 1},    /* 0011 */ {2+LOG_LOCK_REGION, 2},
      /* 0100 */ {1+LOG_LOCK_REGION, 1},    /* 0101 */ {0, 0}, 
      /* 0110 */ {1+LOG_LOCK_REGION, 2},    /* 0111 */ {1+LOG_LOCK_REGION, 3},
      /* 1000 */ {0+LOG_LOCK_REGION, 1},    /* 1001 */ {0, 0}, 
      /* 1010 */ {0, 0},                    /* 1011 */ {0, 0},
      /* 1100 */ {0+LOG_LOCK_REGION, 2},    /* 1101 */ {0, 0}, 
      /* 1110 */ {0, 0},                    /* 1111 */ {0, 0}
    };
    int rc;                       /* Return code of logLockFd() */

    assert( mRegion<ArraySize(aMap) && aMap[mRegion].iStart!=0 );

    rc = logLockFd(pSummary, aMap[mRegion].iStart, aMap[mRegion].iLen, op);