SQLite

Check-in [1bd011c9fe]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add a new xShmBarrier method to the VFS - a shared-memory fence operation. Implement the same in both unix and win32. Use it to make the WAL subsystem more robust.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 1bd011c9fed5ef29fb616b4d0a52df3b82221b1f
User & Date: drh 2010-05-20 23:51:07.000
Context
2010-05-21
13:16
Fix the wal-index header read routine so that it correctly detects a zero header as being malformed and in need of a wal-index rebuild. (check-in: 1a4eb3a3ef user: drh tags: trunk)
2010-05-20
23:51
Add a new xShmBarrier method to the VFS - a shared-memory fence operation. Implement the same in both unix and win32. Use it to make the WAL subsystem more robust. (check-in: 1bd011c9fe user: drh tags: trunk)
21:21
Make use of the extra information in the WAL header and frame header to enhance robustness. (check-in: 9580ecb7e3 user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/os.c.
108
109
110
111
112
113
114



115
116
117
118
119
120
121
  return id->pMethods->xShmGet(id, reqSize, pSize, pp);
}
int sqlite3OsShmRelease(sqlite3_file *id){
  return id->pMethods->xShmRelease(id);
}
int sqlite3OsShmLock(sqlite3_file *id, int desiredLock, int *pGotLock){
  return id->pMethods->xShmLock(id, desiredLock, pGotLock);



}
int sqlite3OsShmClose(sqlite3_file *id, int deleteFlag){
  return id->pMethods->xShmClose(id, deleteFlag);
}

/*
** The next group of routines are convenience wrappers around the







>
>
>







108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
  return id->pMethods->xShmGet(id, reqSize, pSize, pp);
}
int sqlite3OsShmRelease(sqlite3_file *id){
  return id->pMethods->xShmRelease(id);
}
int sqlite3OsShmLock(sqlite3_file *id, int desiredLock, int *pGotLock){
  return id->pMethods->xShmLock(id, desiredLock, pGotLock);
}
void sqlite3OsShmBarrier(sqlite3_file *id){
  id->pMethods->xShmBarrier(id);
}
int sqlite3OsShmClose(sqlite3_file *id, int deleteFlag){
  return id->pMethods->xShmClose(id, deleteFlag);
}

/*
** The next group of routines are convenience wrappers around the
Changes to src/os.h.
244
245
246
247
248
249
250

251
252
253
254
255
256
257
int sqlite3OsSectorSize(sqlite3_file *id);
int sqlite3OsDeviceCharacteristics(sqlite3_file *id);
int sqlite3OsShmOpen(sqlite3_file *id);
int sqlite3OsShmSize(sqlite3_file *id, int, int*);
int sqlite3OsShmGet(sqlite3_file *id, int, int*, void volatile**);
int sqlite3OsShmRelease(sqlite3_file *id);
int sqlite3OsShmLock(sqlite3_file *id, int, int*);

int sqlite3OsShmClose(sqlite3_file *id, int);

/* 
** Functions for accessing sqlite3_vfs methods 
*/
int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *);
int sqlite3OsDelete(sqlite3_vfs *, const char *, int);







>







244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
int sqlite3OsSectorSize(sqlite3_file *id);
int sqlite3OsDeviceCharacteristics(sqlite3_file *id);
int sqlite3OsShmOpen(sqlite3_file *id);
int sqlite3OsShmSize(sqlite3_file *id, int, int*);
int sqlite3OsShmGet(sqlite3_file *id, int, int*, void volatile**);
int sqlite3OsShmRelease(sqlite3_file *id);
int sqlite3OsShmLock(sqlite3_file *id, int, int*);
void sqlite3OsShmBarrier(sqlite3_file *id);
int sqlite3OsShmClose(sqlite3_file *id, int);

/* 
** Functions for accessing sqlite3_vfs methods 
*/
int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *);
int sqlite3OsDelete(sqlite3_vfs *, const char *, int);
Changes to src/os_unix.c.
3878
3879
3880
3881
3882
3883
3884


















3885
3886
3887
3888
3889
3890
3891

3892
3893
3894
3895
3896
3897
3898
  }
  sqlite3_mutex_leave(pShmNode->mutex);
  OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %s\n",
           p->id, getpid(), azLkName[p->lockState]));
  if( pGotLock ) *pGotLock = p->lockState;
  return rc;
}



















#else
# define unixShmOpen    0
# define unixShmSize    0
# define unixShmGet     0
# define unixShmRelease 0
# define unixShmLock    0

# define unixShmClose   0
#endif /* #ifndef SQLITE_OMIT_WAL */

/*
** Here ends the implementation of all sqlite3_file methods.
**
********************** End sqlite3_file Methods *******************************







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







>







3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
  }
  sqlite3_mutex_leave(pShmNode->mutex);
  OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %s\n",
           p->id, getpid(), azLkName[p->lockState]));
  if( pGotLock ) *pGotLock = p->lockState;
  return rc;
}

/*
** Implement a memory barrier or memory fence on shared memory.  
**
** All loads and stores begun before the barrier must complete before
** any load or store begun after the barrier.
*/
static void unixShmBarrier(
  sqlite3_file *fd           /* Database file holding the shared memory */
){
#ifdef __GNUC__
  __sync_synchronize();
#else
  unixMutexEnter();
  unixMutexLeave();
#endif
}


#else
# define unixShmOpen    0
# define unixShmSize    0
# define unixShmGet     0
# define unixShmRelease 0
# define unixShmLock    0
# define unixShmBarrier 0
# define unixShmClose   0
#endif /* #ifndef SQLITE_OMIT_WAL */

/*
** Here ends the implementation of all sqlite3_file methods.
**
********************** End sqlite3_file Methods *******************************
3948
3949
3950
3951
3952
3953
3954

3955
3956
3957
3958
3959
3960
3961
   unixSectorSize,             /* xSectorSize */                             \
   unixDeviceCharacteristics,  /* xDeviceCapabilities */                     \
   unixShmOpen,                /* xShmOpen */                                \
   unixShmSize,                /* xShmSize */                                \
   unixShmGet,                 /* xShmGet */                                 \
   unixShmRelease,             /* xShmRelease */                             \
   unixShmLock,                /* xShmLock */                                \

   unixShmClose                /* xShmClose */                               \
};                                                                           \
static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){   \
  UNUSED_PARAMETER(z); UNUSED_PARAMETER(p);                                  \
  return &METHOD;                                                            \
}                                                                            \
static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p)    \







>







3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
   unixSectorSize,             /* xSectorSize */                             \
   unixDeviceCharacteristics,  /* xDeviceCapabilities */                     \
   unixShmOpen,                /* xShmOpen */                                \
   unixShmSize,                /* xShmSize */                                \
   unixShmGet,                 /* xShmGet */                                 \
   unixShmRelease,             /* xShmRelease */                             \
   unixShmLock,                /* xShmLock */                                \
   unixShmBarrier,             /* xShmBarrier */                             \
   unixShmClose                /* xShmClose */                               \
};                                                                           \
static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){   \
  UNUSED_PARAMETER(z); UNUSED_PARAMETER(p);                                  \
  return &METHOD;                                                            \
}                                                                            \
static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p)    \
Changes to src/os_win.c.
2073
2074
2075
2076
2077
2078
2079














2080
2081
2082
2083
2084
2085
2086

2087
2088
2089
2090
2091
2092
2093
  sqlite3_mutex_leave(pShmNode->mutex);
  OSTRACE(("SHM-LOCK %d shmid-%d, pid-%d got %s\n",
           pShmNode->hFile.h, 
           p->id, (int)GetCurrentProcessId(), azLkName[p->lockState]));
  if( pGotLock ) *pGotLock = p->lockState;
  return rc;
}















#else
# define winShmOpen    0
# define winShmSize    0
# define winShmGet     0
# define winShmRelease 0
# define winShmLock    0

# define winShmClose   0
#endif /* #ifndef SQLITE_OMIT_WAL */
/*
***************************** End Shared Memory *****************************
****************************************************************************/

/*







>
>
>
>
>
>
>
>
>
>
>
>
>
>







>







2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
  sqlite3_mutex_leave(pShmNode->mutex);
  OSTRACE(("SHM-LOCK %d shmid-%d, pid-%d got %s\n",
           pShmNode->hFile.h, 
           p->id, (int)GetCurrentProcessId(), azLkName[p->lockState]));
  if( pGotLock ) *pGotLock = p->lockState;
  return rc;
}

/*
** Implement a memory barrier or memory fence on shared memory.  
**
** All loads and stores begun before the barrier must complete before
** any load or store begun after the barrier.
*/
static void winShmBarrier(
  sqlite3_file *fd          /* Database holding the shared memory */
){
  /* MemoryBarrier(); // does not work -- do not know why not */
  winShmEnterMutex();
  winShmLeaveMutex();
}

#else
# define winShmOpen    0
# define winShmSize    0
# define winShmGet     0
# define winShmRelease 0
# define winShmLock    0
# define winShmBarrier 0
# define winShmClose   0
#endif /* #ifndef SQLITE_OMIT_WAL */
/*
***************************** End Shared Memory *****************************
****************************************************************************/

/*
2109
2110
2111
2112
2113
2114
2115

2116
2117
2118
2119
2120
2121
2122
  winSectorSize,
  winDeviceCharacteristics,
  winShmOpen,              /* xShmOpen */
  winShmSize,              /* xShmSize */
  winShmGet,               /* xShmGet */
  winShmRelease,           /* xShmRelease */
  winShmLock,              /* xShmLock */

  winShmClose              /* xShmClose */
};

/***************************************************************************
** Here ends the I/O methods that form the sqlite3_io_methods object.
**
** The next block of code implements the VFS methods.







>







2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
  winSectorSize,
  winDeviceCharacteristics,
  winShmOpen,              /* xShmOpen */
  winShmSize,              /* xShmSize */
  winShmGet,               /* xShmGet */
  winShmRelease,           /* xShmRelease */
  winShmLock,              /* xShmLock */
  winShmBarrier,           /* xShmBarrier */
  winShmClose              /* xShmClose */
};

/***************************************************************************
** Here ends the I/O methods that form the sqlite3_io_methods object.
**
** The next block of code implements the VFS methods.
Changes to src/sqlite.h.in.
655
656
657
658
659
660
661

662
663
664
665
666
667
668
  int (*xDeviceCharacteristics)(sqlite3_file*);
  /* Methods above are valid for version 1 */
  int (*xShmOpen)(sqlite3_file*);
  int (*xShmSize)(sqlite3_file*, int reqSize, int *pNewSize);
  int (*xShmGet)(sqlite3_file*, int reqSize, int *pSize, void volatile**);
  int (*xShmRelease)(sqlite3_file*);
  int (*xShmLock)(sqlite3_file*, int desiredLock, int *gotLock);

  int (*xShmClose)(sqlite3_file*, int deleteFlag);
  /* Methods above are valid for version 2 */
  /* Additional methods may be added in future releases */
};

/*
** CAPI3REF: Standard File Control Opcodes







>







655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
  int (*xDeviceCharacteristics)(sqlite3_file*);
  /* Methods above are valid for version 1 */
  int (*xShmOpen)(sqlite3_file*);
  int (*xShmSize)(sqlite3_file*, int reqSize, int *pNewSize);
  int (*xShmGet)(sqlite3_file*, int reqSize, int *pSize, void volatile**);
  int (*xShmRelease)(sqlite3_file*);
  int (*xShmLock)(sqlite3_file*, int desiredLock, int *gotLock);
  void (*xShmBarrier)(sqlite3_file*);
  int (*xShmClose)(sqlite3_file*, int deleteFlag);
  /* Methods above are valid for version 2 */
  /* Additional methods may be added in future releases */
};

/*
** CAPI3REF: Standard File Control Opcodes
Changes to src/test6.c.
539
540
541
542
543
544
545



546
547
548
549
550
551
552
}
static int cfShmRelease(sqlite3_file *pFile){
  return sqlite3OsShmRelease(((CrashFile*)pFile)->pRealFile);
}
static int cfShmLock(sqlite3_file *pFile, int desired, int *pGot){
  return sqlite3OsShmLock(((CrashFile*)pFile)->pRealFile, desired, pGot);
}



static int cfShmClose(sqlite3_file *pFile, int delFlag){
  return sqlite3OsShmClose(((CrashFile*)pFile)->pRealFile, delFlag);
}


static const sqlite3_io_methods CrashFileVtab = {
  2,                            /* iVersion */







>
>
>







539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
}
static int cfShmRelease(sqlite3_file *pFile){
  return sqlite3OsShmRelease(((CrashFile*)pFile)->pRealFile);
}
static int cfShmLock(sqlite3_file *pFile, int desired, int *pGot){
  return sqlite3OsShmLock(((CrashFile*)pFile)->pRealFile, desired, pGot);
}
static void cfShmBarrier(sqlite3_file *pFile){
  sqlite3OsShmBarrier(((CrashFile*)pFile)->pRealFile);
}
static int cfShmClose(sqlite3_file *pFile, int delFlag){
  return sqlite3OsShmClose(((CrashFile*)pFile)->pRealFile, delFlag);
}


static const sqlite3_io_methods CrashFileVtab = {
  2,                            /* iVersion */
563
564
565
566
567
568
569

570
571
572
573
574
575
576
  cfSectorSize,                 /* xSectorSize */
  cfDeviceCharacteristics,      /* xDeviceCharacteristics */
  cfShmOpen,                    /* xShmOpen */
  cfShmSize,                    /* xShmSize */
  cfShmGet,                     /* xShmGet */
  cfShmRelease,                 /* xShmRelease */
  cfShmLock,                    /* xShmLock */

  cfShmClose                    /* xShmClose */
};

/*
** Application data for the crash VFS
*/
struct crashAppData {







>







566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
  cfSectorSize,                 /* xSectorSize */
  cfDeviceCharacteristics,      /* xDeviceCharacteristics */
  cfShmOpen,                    /* xShmOpen */
  cfShmSize,                    /* xShmSize */
  cfShmGet,                     /* xShmGet */
  cfShmRelease,                 /* xShmRelease */
  cfShmLock,                    /* xShmLock */
  cfShmBarrier,                 /* xShmBarrier */
  cfShmClose                    /* xShmClose */
};

/*
** Application data for the crash VFS
*/
struct crashAppData {
Changes to src/test_devsym.c.
51
52
53
54
55
56
57

58
59
60
61
62
63
64
static int devsymSectorSize(sqlite3_file*);
static int devsymDeviceCharacteristics(sqlite3_file*);
static int devsymShmOpen(sqlite3_file*);
static int devsymShmSize(sqlite3_file*,int,int*);
static int devsymShmGet(sqlite3_file*,int,int*,volatile void**);
static int devsymShmRelease(sqlite3_file*);
static int devsymShmLock(sqlite3_file*,int,int*);

static int devsymShmClose(sqlite3_file*,int);

/*
** Method declarations for devsym_vfs.
*/
static int devsymOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
static int devsymDelete(sqlite3_vfs*, const char *zName, int syncDir);







>







51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
static int devsymSectorSize(sqlite3_file*);
static int devsymDeviceCharacteristics(sqlite3_file*);
static int devsymShmOpen(sqlite3_file*);
static int devsymShmSize(sqlite3_file*,int,int*);
static int devsymShmGet(sqlite3_file*,int,int*,volatile void**);
static int devsymShmRelease(sqlite3_file*);
static int devsymShmLock(sqlite3_file*,int,int*);
static void devsymShmBarrier(sqlite3_file*);
static int devsymShmClose(sqlite3_file*,int);

/*
** Method declarations for devsym_vfs.
*/
static int devsymOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
static int devsymDelete(sqlite3_vfs*, const char *zName, int syncDir);
119
120
121
122
123
124
125

126
127
128
129
130
131
132
  devsymSectorSize,                 /* xSectorSize */
  devsymDeviceCharacteristics,      /* xDeviceCharacteristics */
  devsymShmOpen,                    /* xShmOpen */
  devsymShmSize,                    /* xShmSize */
  devsymShmGet,                     /* xShmGet */
  devsymShmRelease,                 /* xShmRelease */
  devsymShmLock,                    /* xShmLock */

  devsymShmClose                    /* xShmClose */
};

struct DevsymGlobal {
  sqlite3_vfs *pVfs;
  int iDeviceChar;
  int iSectorSize;







>







120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
  devsymSectorSize,                 /* xSectorSize */
  devsymDeviceCharacteristics,      /* xDeviceCharacteristics */
  devsymShmOpen,                    /* xShmOpen */
  devsymShmSize,                    /* xShmSize */
  devsymShmGet,                     /* xShmGet */
  devsymShmRelease,                 /* xShmRelease */
  devsymShmLock,                    /* xShmLock */
  devsymShmBarrier,                 /* xShmBarrier */
  devsymShmClose                    /* xShmClose */
};

struct DevsymGlobal {
  sqlite3_vfs *pVfs;
  int iDeviceChar;
  int iSectorSize;
260
261
262
263
264
265
266




267
268
269
270
271
272
273
static int devsymShmRelease(sqlite3_file *pFile){
  devsym_file *p = (devsym_file *)pFile;
  return sqlite3OsShmRelease(p->pReal);
}
static int devsymShmLock(sqlite3_file *pFile, int desired, int *pGot){
  devsym_file *p = (devsym_file *)pFile;
  return sqlite3OsShmLock(p->pReal, desired, pGot);




}
static int devsymShmClose(sqlite3_file *pFile, int delFlag){
  devsym_file *p = (devsym_file *)pFile;
  return sqlite3OsShmClose(p->pReal, delFlag);
}









>
>
>
>







262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
static int devsymShmRelease(sqlite3_file *pFile){
  devsym_file *p = (devsym_file *)pFile;
  return sqlite3OsShmRelease(p->pReal);
}
static int devsymShmLock(sqlite3_file *pFile, int desired, int *pGot){
  devsym_file *p = (devsym_file *)pFile;
  return sqlite3OsShmLock(p->pReal, desired, pGot);
}
static void devsymShmBarrier(sqlite3_file *pFile){
  devsym_file *p = (devsym_file *)pFile;
  sqlite3OsShmBarrier(p->pReal);
}
static int devsymShmClose(sqlite3_file *pFile, int delFlag){
  devsym_file *p = (devsym_file *)pFile;
  return sqlite3OsShmClose(p->pReal, delFlag);
}


Changes to src/test_onefile.c.
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
  fsCheckReservedLock,          /* xCheckReservedLock */
  fsFileControl,                /* xFileControl */
  fsSectorSize,                 /* xSectorSize */
  fsDeviceCharacteristics,      /* xDeviceCharacteristics */
  0,                            /* xShmOpen */
  0,                            /* xShmSize */
  0,                            /* xShmLock */
  0,                            /* xShmClose */
  0,                            /* xShmDelete */
};


static sqlite3_io_methods tmp_io_methods = {
  1,                            /* iVersion */
  tmpClose,                     /* xClose */
  tmpRead,                      /* xRead */
  tmpWrite,                     /* xWrite */
  tmpTruncate,                  /* xTruncate */
  tmpSync,                      /* xSync */
  tmpFileSize,                  /* xFileSize */
  tmpLock,                      /* xLock */
  tmpUnlock,                    /* xUnlock */
  tmpCheckReservedLock,         /* xCheckReservedLock */
  tmpFileControl,               /* xFileControl */
  tmpSectorSize,                /* xSectorSize */
  tmpDeviceCharacteristics,     /* xDeviceCharacteristics */
  0,                            /* xShmOpen */
  0,                            /* xShmSize */
  0,                            /* xShmLock */
  0,                            /* xShmClose */
  0,                            /* xShmDelete */
};

/* Useful macros used in several places */
#define MIN(x,y) ((x)<(y)?(x):(y))
#define MAX(x,y) ((x)>(y)?(x):(y))









|
|




















|
|







219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
  fsCheckReservedLock,          /* xCheckReservedLock */
  fsFileControl,                /* xFileControl */
  fsSectorSize,                 /* xSectorSize */
  fsDeviceCharacteristics,      /* xDeviceCharacteristics */
  0,                            /* xShmOpen */
  0,                            /* xShmSize */
  0,                            /* xShmLock */
  0,                            /* xShmBarrier */
  0                             /* xShmClose */
};


static sqlite3_io_methods tmp_io_methods = {
  1,                            /* iVersion */
  tmpClose,                     /* xClose */
  tmpRead,                      /* xRead */
  tmpWrite,                     /* xWrite */
  tmpTruncate,                  /* xTruncate */
  tmpSync,                      /* xSync */
  tmpFileSize,                  /* xFileSize */
  tmpLock,                      /* xLock */
  tmpUnlock,                    /* xUnlock */
  tmpCheckReservedLock,         /* xCheckReservedLock */
  tmpFileControl,               /* xFileControl */
  tmpSectorSize,                /* xSectorSize */
  tmpDeviceCharacteristics,     /* xDeviceCharacteristics */
  0,                            /* xShmOpen */
  0,                            /* xShmSize */
  0,                            /* xShmLock */
  0,                            /* xShmBarrier */
  0                             /* xShmClose */
};

/* Useful macros used in several places */
#define MIN(x,y) ((x)<(y)?(x):(y))
#define MAX(x,y) ((x)>(y)?(x):(y))


Changes to src/test_osinst.c.
94
95
96
97
98
99
100

101
102
103
104
105
106
107
108
109
110
111
#define OS_UNLOCK            19
#define OS_WRITE             20
#define OS_SHMOPEN           21
#define OS_SHMCLOSE          22
#define OS_SHMGET            23
#define OS_SHMRELEASE        24
#define OS_SHMLOCK           25

#define OS_SHMSIZE           26
#define OS_ANNOTATE          27

#define OS_NUMEVENTS         28

#define VFSLOG_BUFFERSIZE 8192

typedef struct VfslogVfs VfslogVfs;
typedef struct VfslogFile VfslogFile;

struct VfslogVfs {







>
|
|

|







94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
#define OS_UNLOCK            19
#define OS_WRITE             20
#define OS_SHMOPEN           21
#define OS_SHMCLOSE          22
#define OS_SHMGET            23
#define OS_SHMRELEASE        24
#define OS_SHMLOCK           25
#define OS_SHMBARRIER        26
#define OS_SHMSIZE           27
#define OS_ANNOTATE          28

#define OS_NUMEVENTS         29

#define VFSLOG_BUFFERSIZE 8192

typedef struct VfslogVfs VfslogVfs;
typedef struct VfslogFile VfslogFile;

struct VfslogVfs {
146
147
148
149
150
151
152

153
154
155
156
157
158
159
static int vfslogDeviceCharacteristics(sqlite3_file*);

static int vfslogShmOpen(sqlite3_file *pFile);
static int vfslogShmSize(sqlite3_file *pFile, int reqSize, int *pNewSize);
static int vfslogShmGet(sqlite3_file *pFile, int,int*,volatile void **);
static int vfslogShmRelease(sqlite3_file *pFile);
static int vfslogShmLock(sqlite3_file *pFile, int desiredLock, int *gotLock);

static int vfslogShmClose(sqlite3_file *pFile, int deleteFlag);

/*
** Method declarations for vfslog_vfs.
*/
static int vfslogOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
static int vfslogDelete(sqlite3_vfs*, const char *zName, int syncDir);







>







147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
static int vfslogDeviceCharacteristics(sqlite3_file*);

static int vfslogShmOpen(sqlite3_file *pFile);
static int vfslogShmSize(sqlite3_file *pFile, int reqSize, int *pNewSize);
static int vfslogShmGet(sqlite3_file *pFile, int,int*,volatile void **);
static int vfslogShmRelease(sqlite3_file *pFile);
static int vfslogShmLock(sqlite3_file *pFile, int desiredLock, int *gotLock);
static void vfslogShmBarrier(sqlite3_file*);
static int vfslogShmClose(sqlite3_file *pFile, int deleteFlag);

/*
** Method declarations for vfslog_vfs.
*/
static int vfslogOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
static int vfslogDelete(sqlite3_vfs*, const char *zName, int syncDir);
202
203
204
205
206
207
208

209
210
211
212
213
214
215
  vfslogSectorSize,               /* xSectorSize */
  vfslogDeviceCharacteristics,    /* xDeviceCharacteristics */
  vfslogShmOpen,                  /* xShmOpen */
  vfslogShmSize,                  /* xShmSize */
  vfslogShmGet,                   /* xShmGet */
  vfslogShmRelease,               /* xShmRelease */
  vfslogShmLock,                  /* xShmLock */

  vfslogShmClose                  /* xShmClose */
};

#if defined(SQLITE_OS_UNIX) && !defined(NO_GETTOD)
#include <sys/time.h>
static sqlite3_uint64 vfslog_time(){
  struct timeval sTime;







>







204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
  vfslogSectorSize,               /* xSectorSize */
  vfslogDeviceCharacteristics,    /* xDeviceCharacteristics */
  vfslogShmOpen,                  /* xShmOpen */
  vfslogShmSize,                  /* xShmSize */
  vfslogShmGet,                   /* xShmGet */
  vfslogShmRelease,               /* xShmRelease */
  vfslogShmLock,                  /* xShmLock */
  vfslogShmBarrier,               /* xShmBarrier */
  vfslogShmClose                  /* xShmClose */
};

#if defined(SQLITE_OS_UNIX) && !defined(NO_GETTOD)
#include <sys/time.h>
static sqlite3_uint64 vfslog_time(){
  struct timeval sTime;
451
452
453
454
455
456
457








458
459
460
461
462
463
464
  VfslogFile *p = (VfslogFile *)pFile;
  t = vfslog_time();
  rc = p->pReal->pMethods->xShmLock(p->pReal, desiredLock, gotLock);
  t = vfslog_time() - t;
  vfslog_call(p->pVfslog, OS_SHMLOCK, p->iFileId, t, rc, 0, 0);
  return rc;
}








static int vfslogShmClose(sqlite3_file *pFile, int deleteFlag){
  int rc;
  sqlite3_uint64 t;
  VfslogFile *p = (VfslogFile *)pFile;
  t = vfslog_time();
  rc = p->pReal->pMethods->xShmClose(p->pReal, deleteFlag);
  t = vfslog_time() - t;







>
>
>
>
>
>
>
>







454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
  VfslogFile *p = (VfslogFile *)pFile;
  t = vfslog_time();
  rc = p->pReal->pMethods->xShmLock(p->pReal, desiredLock, gotLock);
  t = vfslog_time() - t;
  vfslog_call(p->pVfslog, OS_SHMLOCK, p->iFileId, t, rc, 0, 0);
  return rc;
}
static void vfslogShmBarrier(sqlite3_file *pFile){
  sqlite3_uint64 t;
  VfslogFile *p = (VfslogFile *)pFile;
  t = vfslog_time();
  p->pReal->pMethods->xShmBarrier(p->pReal);
  t = vfslog_time() - t;
  vfslog_call(p->pVfslog, OS_SHMBARRIER, p->iFileId, t, SQLITE_OK, 0, 0);
}
static int vfslogShmClose(sqlite3_file *pFile, int deleteFlag){
  int rc;
  sqlite3_uint64 t;
  VfslogFile *p = (VfslogFile *)pFile;
  t = vfslog_time();
  rc = p->pReal->pMethods->xShmClose(p->pReal, deleteFlag);
  t = vfslog_time() - t;
777
778
779
780
781
782
783

784
785
786
787
788
789
790

    case OS_SHMCLOSE:          zEvent = "xShmClose"; break;
    case OS_SHMOPEN:           zEvent = "xShmOpen"; break;
    case OS_SHMGET:            zEvent = "xShmGet"; break;
    case OS_SHMSIZE:           zEvent = "xShmSize"; break;
    case OS_SHMRELEASE:        zEvent = "xShmRelease"; break;
    case OS_SHMLOCK:           zEvent = "xShmLock"; break;


    case OS_ANNOTATE:          zEvent = "annotation"; break;
  }

  return zEvent;
}








>







788
789
790
791
792
793
794
795
796
797
798
799
800
801
802

    case OS_SHMCLOSE:          zEvent = "xShmClose"; break;
    case OS_SHMOPEN:           zEvent = "xShmOpen"; break;
    case OS_SHMGET:            zEvent = "xShmGet"; break;
    case OS_SHMSIZE:           zEvent = "xShmSize"; break;
    case OS_SHMRELEASE:        zEvent = "xShmRelease"; break;
    case OS_SHMLOCK:           zEvent = "xShmLock"; break;
    case OS_SHMBARRIER:        zEvent = "xShmBarrier"; break;

    case OS_ANNOTATE:          zEvent = "annotation"; break;
  }

  return zEvent;
}

Changes to src/test_vfs.c.
99
100
101
102
103
104
105

106
107
108
109
110
111
112
static int tvfsCurrentTime(sqlite3_vfs*, double*);

static int tvfsShmOpen(sqlite3_file*);
static int tvfsShmSize(sqlite3_file*, int , int *);
static int tvfsShmGet(sqlite3_file*, int , int *, volatile void **);
static int tvfsShmRelease(sqlite3_file*);
static int tvfsShmLock(sqlite3_file*, int , int *);

static int tvfsShmClose(sqlite3_file*, int);

static sqlite3_io_methods tvfs_io_methods = {
  2,                            /* iVersion */
  tvfsClose,                      /* xClose */
  tvfsRead,                       /* xRead */
  tvfsWrite,                      /* xWrite */







>







99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
static int tvfsCurrentTime(sqlite3_vfs*, double*);

static int tvfsShmOpen(sqlite3_file*);
static int tvfsShmSize(sqlite3_file*, int , int *);
static int tvfsShmGet(sqlite3_file*, int , int *, volatile void **);
static int tvfsShmRelease(sqlite3_file*);
static int tvfsShmLock(sqlite3_file*, int , int *);
static void tvfsShmBarrier(sqlite3_file*);
static int tvfsShmClose(sqlite3_file*, int);

static sqlite3_io_methods tvfs_io_methods = {
  2,                            /* iVersion */
  tvfsClose,                      /* xClose */
  tvfsRead,                       /* xRead */
  tvfsWrite,                      /* xWrite */
120
121
122
123
124
125
126

127
128
129
130
131
132
133
  tvfsSectorSize,                 /* xSectorSize */
  tvfsDeviceCharacteristics,      /* xDeviceCharacteristics */
  tvfsShmOpen,                    /* xShmOpen */
  tvfsShmSize,                    /* xShmSize */
  tvfsShmGet,                     /* xShmGet */
  tvfsShmRelease,                 /* xShmRelease */
  tvfsShmLock,                    /* xShmLock */

  tvfsShmClose                    /* xShmClose */
};

/*
** Close an tvfs-file.
*/
static int tvfsClose(sqlite3_file *pFile){







>







121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
  tvfsSectorSize,                 /* xSectorSize */
  tvfsDeviceCharacteristics,      /* xDeviceCharacteristics */
  tvfsShmOpen,                    /* xShmOpen */
  tvfsShmSize,                    /* xShmSize */
  tvfsShmGet,                     /* xShmGet */
  tvfsShmRelease,                 /* xShmRelease */
  tvfsShmLock,                    /* xShmLock */
  tvfsShmBarrier,                 /* xShmBarrier */
  tvfsShmClose                    /* xShmClose */
};

/*
** Close an tvfs-file.
*/
static int tvfsClose(sqlite3_file *pFile){
265
266
267
268
269
270
271

272
273
274
275
276
277
278
    if( ((Testvfs *)pVfs->pAppData)->isNoshm ){
      pMethods->xShmOpen = 0;
      pMethods->xShmGet = 0;
      pMethods->xShmSize = 0;
      pMethods->xShmRelease = 0;
      pMethods->xShmClose = 0;
      pMethods->xShmLock = 0;

    }
    pFile->pMethods = pMethods;
  }

  return rc;
}








>







267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
    if( ((Testvfs *)pVfs->pAppData)->isNoshm ){
      pMethods->xShmOpen = 0;
      pMethods->xShmGet = 0;
      pMethods->xShmSize = 0;
      pMethods->xShmRelease = 0;
      pMethods->xShmClose = 0;
      pMethods->xShmLock = 0;
      pMethods->xShmBarrier = 0;
    }
    pFile->pMethods = pMethods;
  }

  return rc;
}

564
565
566
567
568
569
570











571
572
573
574
575
576
577
  tvfsResultCode(p, &rc);
  if( rc==SQLITE_OK ){
    *gotLock = desiredLock;
  }

  return rc;
}












static int tvfsShmClose(
  sqlite3_file *pFile,
  int deleteFlag
){
  int rc = SQLITE_OK;
  TestvfsFile *pFd = (TestvfsFile *)pFile;







>
>
>
>
>
>
>
>
>
>
>







567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
  tvfsResultCode(p, &rc);
  if( rc==SQLITE_OK ){
    *gotLock = desiredLock;
  }

  return rc;
}

static void tvfsShmBarrier(sqlite3_file *pFile){
  int rc = SQLITE_OK;
  TestvfsFile *pFd = (TestvfsFile *)pFile;
  Testvfs *p = (Testvfs *)(pFd->pVfs->pAppData);

  tvfsExecTcl(p, "xShmBarrier", 
      Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0
  );
  tvfsResultCode(p, &rc);
}

static int tvfsShmClose(
  sqlite3_file *pFile,
  int deleteFlag
){
  int rc = SQLITE_OK;
  TestvfsFile *pFd = (TestvfsFile *)pFile;
Changes to src/wal.c.
203
204
205
206
207
208
209
210



211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233

/* Object declarations */
typedef struct WalIndexHdr WalIndexHdr;
typedef struct WalIterator WalIterator;


/*
** The following object holds an exact copy of the wal-index header.



*/
struct WalIndexHdr {
  u32 iChange;      /* Counter incremented each transaction */
  u32 szPage;       /* Database page size in bytes */
  u32 mxFrame;      /* Index of last valid frame in the WAL */
  u32 nPage;        /* Size of database in pages */
  u32 aSalt[2];     /* Salt-1 and salt-2 values copied from WAL header */
  u32 aCksum[2];    /* Checksum over all prior fields */
};

/* A block of WALINDEX_LOCK_RESERVED bytes beginning at
** WALINDEX_LOCK_OFFSET is reserved for locks. Since some systems
** only support mandatory file-locks, we do not read or write data
** from the region of the file on which locks are applied.
*/
#define WALINDEX_LOCK_OFFSET   (sizeof(WalIndexHdr))
#define WALINDEX_LOCK_RESERVED 8

/* Size of header before each frame in wal */
#define WAL_FRAME_HDRSIZE 24

/* Size of write ahead log header */
#define WAL_HDRSIZE 24







|
>
>
>















|







203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236

/* Object declarations */
typedef struct WalIndexHdr WalIndexHdr;
typedef struct WalIterator WalIterator;


/*
** The following object holds a copy of the wal-index header content.
**
** The actual header in the wal-index consists of two copies of this
** object.
*/
struct WalIndexHdr {
  u32 iChange;      /* Counter incremented each transaction */
  u32 szPage;       /* Database page size in bytes */
  u32 mxFrame;      /* Index of last valid frame in the WAL */
  u32 nPage;        /* Size of database in pages */
  u32 aSalt[2];     /* Salt-1 and salt-2 values copied from WAL header */
  u32 aCksum[2];    /* Checksum over all prior fields */
};

/* A block of WALINDEX_LOCK_RESERVED bytes beginning at
** WALINDEX_LOCK_OFFSET is reserved for locks. Since some systems
** only support mandatory file-locks, we do not read or write data
** from the region of the file on which locks are applied.
*/
#define WALINDEX_LOCK_OFFSET   (sizeof(WalIndexHdr)*2)
#define WALINDEX_LOCK_RESERVED 8

/* Size of header before each frame in wal */
#define WAL_FRAME_HDRSIZE 24

/* Size of write ahead log header */
#define WAL_HDRSIZE 24
302
303
304
305
306
307
308

309
310
311
312
313
314
315
316
317
318
319
320
321
322
static void walChecksumBytes(
  u8 *a,           /* Content to be checksummed */
  int nByte,       /* Bytes of content in a[].  Must be a multiple of 8. */
  const u32 *aIn,  /* Initial checksum value input */
  u32 *aOut        /* OUT: Final checksum value output */
){
  u32 s1, s2;

  if( aIn ){
    s1 = aIn[0];
    s2 = aIn[1];
  }else{
    s1 = s2 = 0;
  }
  u8 *aEnd = (u8*)&a[nByte];

  assert( nByte>=8 );
  assert( (nByte&0x00000003)==0 );

  do {
    s1 += (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3] + s2;
    s2 += (a[4]<<24) + (a[5]<<16) + (a[6]<<8) + a[7] + s1;







>






<







305
306
307
308
309
310
311
312
313
314
315
316
317
318

319
320
321
322
323
324
325
static void walChecksumBytes(
  u8 *a,           /* Content to be checksummed */
  int nByte,       /* Bytes of content in a[].  Must be a multiple of 8. */
  const u32 *aIn,  /* Initial checksum value input */
  u32 *aOut        /* OUT: Final checksum value output */
){
  u32 s1, s2;
  u8 *aEnd = (u8*)&a[nByte];
  if( aIn ){
    s1 = aIn[0];
    s2 = aIn[1];
  }else{
    s1 = s2 = 0;
  }


  assert( nByte>=8 );
  assert( (nByte&0x00000003)==0 );

  do {
    s1 += (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3] + s2;
    s2 += (a[4]<<24) + (a[5]<<16) + (a[6]<<8) + a[7] + s1;
351
352
353
354
355
356
357

358
359
360

361


362
363
364
365
366
367
368

/*
** Write the header information in pWal->hdr into the wal-index.
**
** The checksum on pWal->hdr is updated before it is written.
*/
static void walIndexWriteHdr(Wal *pWal){

  walChecksumBytes((u8*)&pWal->hdr,
                   sizeof(pWal->hdr) - sizeof(pWal->hdr.aCksum),
                   0, pWal->hdr.aCksum);

  memcpy((void*)pWal->pWiData, &pWal->hdr, sizeof(pWal->hdr));


}

/*
** This function encodes a single frame header and writes it to a buffer
** supplied by the caller. A frame-header is made up of a series of 
** 4-byte big-endian integers, as follows:
**







>



>
|
>
>







354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375

/*
** Write the header information in pWal->hdr into the wal-index.
**
** The checksum on pWal->hdr is updated before it is written.
*/
static void walIndexWriteHdr(Wal *pWal){
  WalIndexHdr *aHdr;
  walChecksumBytes((u8*)&pWal->hdr,
                   sizeof(pWal->hdr) - sizeof(pWal->hdr.aCksum),
                   0, pWal->hdr.aCksum);
  aHdr = (WalIndexHdr*)pWal->pWiData;
  memcpy(&aHdr[1], &pWal->hdr, sizeof(pWal->hdr));
  sqlite3OsShmBarrier(pWal->pDbFd);
  memcpy(&aHdr[0], &pWal->hdr, sizeof(pWal->hdr));
}

/*
** This function encodes a single frame header and writes it to a buffer
** supplied by the caller. A frame-header is made up of a series of 
** 4-byte big-endian integers, as follows:
**
1114
1115
1116
1117
1118
1119
1120
1121
1122

1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138








1139
1140

1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
** pWal->hdr, then pWal->hdr is updated to the content of the new header
** and *pChanged is set to 1.
**
** If the checksum cannot be verified return non-zero. If the header
** is read successfully and the checksum verified, return zero.
*/
int walIndexTryHdr(Wal *pWal, int *pChanged){
  u32 aCksum[2];
  WalIndexHdr hdr;


  assert( pWal->pWiData );
  if( pWal->szWIndex==0 ){
    /* The wal-index is of size 0 bytes. This is handled in the same way
    ** as an invalid header. The caller will run recovery to construct
    ** a valid wal-index file before accessing the database.
    */
    return 1;
  }

  /* Read the header. The caller may or may not have an exclusive 
  ** (WRITE, PENDING, CHECKPOINT or RECOVER) lock on the wal-index
  ** file, meaning it is possible that an inconsistent snapshot is read
  ** from the file. If this happens, return non-zero.
  */
  memcpy(&hdr, (void*)pWal->pWiData, sizeof(hdr));








  walChecksumBytes((u8*)&hdr, sizeof(hdr)-sizeof(hdr.aCksum), 0, aCksum);
  if( aCksum[0]!=hdr.aCksum[0] || aCksum[1]!=hdr.aCksum[1] ){

    return 1;
  }

  if( memcmp(&pWal->hdr, &hdr, sizeof(WalIndexHdr)) ){
    *pChanged = 1;
    memcpy(&pWal->hdr, &hdr, sizeof(WalIndexHdr));
    pWal->szPage = pWal->hdr.szPage;
  }

  /* The header was successfully read. Return zero. */
  return 0;
}








|
|
>















|
>
>
>
>
>
>
>
>
|
|
>



|

|







1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
** pWal->hdr, then pWal->hdr is updated to the content of the new header
** and *pChanged is set to 1.
**
** If the checksum cannot be verified return non-zero. If the header
** is read successfully and the checksum verified, return zero.
*/
int walIndexTryHdr(Wal *pWal, int *pChanged){
  u32 aCksum[2];               /* Checksum on the header content */
  WalIndexHdr hdr1, hdr2;      /* Two copies of the header content */
  WalIndexHdr *aHdr;           /* Header in shared memory */

  assert( pWal->pWiData );
  if( pWal->szWIndex==0 ){
    /* The wal-index is of size 0 bytes. This is handled in the same way
    ** as an invalid header. The caller will run recovery to construct
    ** a valid wal-index file before accessing the database.
    */
    return 1;
  }

  /* Read the header. The caller may or may not have an exclusive 
  ** (WRITE, PENDING, CHECKPOINT or RECOVER) lock on the wal-index
  ** file, meaning it is possible that an inconsistent snapshot is read
  ** from the file. If this happens, return non-zero.
  */
  aHdr = (WalIndexHdr*)pWal->pWiData;
  memcpy(&hdr1, &aHdr[0], sizeof(hdr1));
  sqlite3OsShmBarrier(pWal->pDbFd);
  memcpy(&hdr2, &aHdr[1], sizeof(hdr2));

  if( memcmp(&hdr1, &hdr2, sizeof(hdr1))!=0 ){
    /* Dirty read */
    return 1;
  }  
  walChecksumBytes((u8*)&hdr1, sizeof(hdr1)-sizeof(hdr1.aCksum), 0, aCksum);
  if( aCksum[0]!=hdr1.aCksum[0] || aCksum[1]!=hdr1.aCksum[1] ){
    /* Malformed header */
    return 1;
  }

  if( memcmp(&pWal->hdr, &hdr1, sizeof(WalIndexHdr)) ){
    *pChanged = 1;
    memcpy(&pWal->hdr, &hdr1, sizeof(WalIndexHdr));
    pWal->szPage = pWal->hdr.szPage;
  }

  /* The header was successfully read. Return zero. */
  return 0;
}

Changes to test/wal2.test.
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
proc set_tvfs_hdr {file args} {
  if {[llength $args]>1} {
    return -code error {wrong # args: should be "set_tvfs_hdr fileName ?val?"}
  }

  set blob [tvfs shm $file]
  if {[llength $args]} {
    set blob [binary format i8a* [lindex $args 0] [string range $blob 32 end]]
    tvfs shm $file $blob
  }

  binary scan $blob i8 ints
  return $ints
}

proc incr_tvfs_hdr {file idx incrval} {
  set ints [set_tvfs_hdr $file]
  set v [lindex $ints $idx]
  incr v $incrval







|



|







21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
proc set_tvfs_hdr {file args} {
  if {[llength $args]>1} {
    return -code error {wrong # args: should be "set_tvfs_hdr fileName ?val?"}
  }

  set blob [tvfs shm $file]
  if {[llength $args]} {
    set blob [binary format i16a* [lindex $args 0] [string range $blob 64 end]]
    tvfs shm $file $blob
  }

  binary scan $blob i16 ints
  return $ints
}

proc incr_tvfs_hdr {file idx incrval} {
  set ints [set_tvfs_hdr $file]
  set v [lindex $ints $idx]
  incr v $incrval