SQLite

Check-in [8d2d1c4ff9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add a debugging memory allocator. (CVS 4227)
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 8d2d1c4ff9dca61f75e3048107ee9712d346a28c
User & Date: drh 2007-08-15 17:07:57.000
Context
2007-08-15
17:08
Modify the crash-recovery test code in test6.c for 3.5. Also change some other code to use the new sqlite3_io_methods interface. Lots of things are broken now. (CVS 4228) (check-in: af9503daf3 user: danielk1977 tags: trunk)
17:07
Add a debugging memory allocator. (CVS 4227) (check-in: 8d2d1c4ff9 user: drh tags: trunk)
13:04
Add initial implementations of mutex and memory subsystem modules. (CVS 4226) (check-in: c0fa376979 user: drh tags: trunk)
Changes
Unified Diff Show Whitespace Changes Patch
Changes to main.mk.
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
TCCX = $(TCC) $(OPTS) $(THREADSAFE) $(USLEEP) -I. -I$(TOP)/src

# Object files for the SQLite library.
#
LIBOBJ+= alter.o analyze.o attach.o auth.o btree.o build.o \
         callback.o complete.o date.o delete.o \
         expr.o func.o hash.o insert.o loadext.o \
         main.o malloc.o mem1.o mutex.o \
         opcodes.o os.o os_os2.o os_unix.o os_win.o \
         pager.o parse.o pragma.o prepare.o printf.o random.o \
         select.o table.o tclsqlite.o tokenize.o trigger.o \
         update.o util.o vacuum.o \
         vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbefifo.o vdbemem.o \
         where.o utf.o legacy.o vtab.o








|







54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
TCCX = $(TCC) $(OPTS) $(THREADSAFE) $(USLEEP) -I. -I$(TOP)/src

# Object files for the SQLite library.
#
LIBOBJ+= alter.o analyze.o attach.o auth.o btree.o build.o \
         callback.o complete.o date.o delete.o \
         expr.o func.o hash.o insert.o loadext.o \
         main.o malloc.o mem1.o mem2.o mutex.o \
         opcodes.o os.o os_os2.o os_unix.o os_win.o \
         pager.o parse.o pragma.o prepare.o printf.o random.o \
         select.o table.o tclsqlite.o tokenize.o trigger.o \
         update.o util.o vacuum.o \
         vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbefifo.o vdbemem.o \
         where.o utf.o legacy.o vtab.o

88
89
90
91
92
93
94
95

96
97
98
99
100
101
102
  $(TOP)/src/hash.c \
  $(TOP)/src/hash.h \
  $(TOP)/src/insert.c \
  $(TOP)/src/legacy.c \
  $(TOP)/src/loadext.c \
  $(TOP)/src/main.c \
  $(TOP)/src/malloc.c \
  $(TOP)/src/mem.c \

  $(TOP)/src/mutex.c \
  $(TOP)/src/os.c \
  $(TOP)/src/os_os2.c \
  $(TOP)/src/os_unix.c \
  $(TOP)/src/os_win.c \
  $(TOP)/src/pager.c \
  $(TOP)/src/pager.h \







|
>







88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
  $(TOP)/src/hash.c \
  $(TOP)/src/hash.h \
  $(TOP)/src/insert.c \
  $(TOP)/src/legacy.c \
  $(TOP)/src/loadext.c \
  $(TOP)/src/main.c \
  $(TOP)/src/malloc.c \
  $(TOP)/src/mem1.c \
  $(TOP)/src/mem2.c \
  $(TOP)/src/mutex.c \
  $(TOP)/src/os.c \
  $(TOP)/src/os_os2.c \
  $(TOP)/src/os_unix.c \
  $(TOP)/src/os_win.c \
  $(TOP)/src/pager.c \
  $(TOP)/src/pager.h \
342
343
344
345
346
347
348



349
350
351
352
353
354
355
	$(TCCX) -c $(TOP)/src/main.c

malloc.o:	$(TOP)/src/malloc.c $(HDR)
	$(TCCX) -c $(TOP)/src/malloc.c

mem1.o:	$(TOP)/src/mem1.c $(HDR)
	$(TCCX) -c $(TOP)/src/mem1.c




mutex.o:	$(TOP)/src/mutex.c $(HDR)
	$(TCCX) -c $(TOP)/src/mutex.c

pager.o:	$(TOP)/src/pager.c $(HDR) $(TOP)/src/pager.h
	$(TCCX) -c $(TOP)/src/pager.c








>
>
>







343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
	$(TCCX) -c $(TOP)/src/main.c

malloc.o:	$(TOP)/src/malloc.c $(HDR)
	$(TCCX) -c $(TOP)/src/malloc.c

mem1.o:	$(TOP)/src/mem1.c $(HDR)
	$(TCCX) -c $(TOP)/src/mem1.c

mem2.o:	$(TOP)/src/mem2.c $(HDR)
	$(TCCX) -c $(TOP)/src/mem2.c

mutex.o:	$(TOP)/src/mutex.c $(HDR)
	$(TCCX) -c $(TOP)/src/mutex.c

pager.o:	$(TOP)/src/pager.c $(HDR) $(TOP)/src/pager.h
	$(TCCX) -c $(TOP)/src/pager.c

Changes to src/mem1.c.
8
9
10
11
12
13
14
15
16







17
18
19
20
21
22
23
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains the C functions that implement a memory
** allocation subsystem for use by SQLite.  
**
** $Id: mem1.c,v 1.1 2007/08/15 13:04:54 drh Exp $
*/








/*
** We will eventually construct multiple memory allocation subsystems
** suitable for use in various contexts:
**
**    *  Normal multi-threaded builds
**    *  Normal single-threaded builds







|

>
>
>
>
>
>
>







8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains the C functions that implement a memory
** allocation subsystem for use by SQLite.  
**
** $Id: mem1.c,v 1.2 2007/08/15 17:07:57 drh Exp $
*/

/*
** This version of the memory allocator is the default.  It is
** used when no other memory allocator is specified using compile-time
** macros.
*/
#if !defined(SQLITE_MEMDEBUG) && !defined(SQLITE_OMIT_MEMORY_ALLOCATION)

/*
** We will eventually construct multiple memory allocation subsystems
** suitable for use in various contexts:
**
**    *  Normal multi-threaded builds
**    *  Normal single-threaded builds
198
199
200
201
202
203
204


    if( nowUsed>mxUsed ){
      mxUsed = nowUsed;
    }
  }
  sqlite3_mutex_leave(memMutex);
  return (void*)p;
}









>
>
205
206
207
208
209
210
211
212
213
    if( nowUsed>mxUsed ){
      mxUsed = nowUsed;
    }
  }
  sqlite3_mutex_leave(memMutex);
  return (void*)p;
}

#endif /* !SQLITE_MEMDEBUG && !SQLITE_OMIT_MEMORY_ALLOCATION */
Added src/mem2.c.




























































































































































































































































































































































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
/*
** 2007 August 15
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains the C functions that implement a memory
** allocation subsystem for use by SQLite.  
**
** $Id: mem2.c,v 1.1 2007/08/15 17:07:57 drh Exp $
*/

/*
** This version of the memory allocator is used only if the
** SQLITE_MEMDEBUG macro is defined and SQLITE_OMIT_MEMORY_ALLOCATION
** is not defined.
*/
#if defined(SQLITE_MEMDEBUG) && !defined(SQLITE_OMIT_MEMORY_ALLOCATION)

/*
** We will eventually construct multiple memory allocation subsystems
** suitable for use in various contexts:
**
**    *  Normal multi-threaded builds
**    *  Normal single-threaded builds
**    *  Debugging builds
**
** This version is suitable for use in debugging builds.
**
** Features:
**
**    * Every allocate has guards at both ends.
**    * New allocations are initialized with randomness
**    * Allocations are overwritten with randomness when freed
**    * Optional logs of malloc activity generated
**    * Summary of outstanding allocations with backtraces to the
**      point of allocation.
**    * The ability to simulate memory allocation failure
*/
#include "sqliteInt.h"
#include <stdio.h>

/*
** The backtrace functionality is only available with GLIBC
*/
#ifdef __GLIBC__
  extern int backtrace(void**,int);
  extern void backtrace_symbols_fd(void*const*,int,int);
#else
# define backtrace(A,B) 0
# define backtrace_symbols_fd(A,B,C)
#endif


/*
** Mutex to control access to the memory allocation subsystem.
*/
static sqlite3_mutex *memMutex = 0;

/*
** Current allocation and high-water mark.
*/
static sqlite3_uint64 nowUsed = 0;
static sqlite3_uint64 mxUsed = 0;

/*
** The alarm callback and its arguments.  The memMutex lock will
** be held while the callback is running.  Recursive calls into
** the memory subsystem are allowed, but no new callbacks will be
** issued.  The alarmBusy variable is set to prevent recursive
** callbacks.
*/
static void (*alarmCallback)(void*, sqlite3_uint64, unsigned) = 0;
static void *alarmArg = 0;
static sqlite3_uint64 alarmThreshold = (((sqlite3_uint64)1)<<63);
static int alarmBusy = 0;


/*
** Return the amount of memory currently checked out.
*/
sqlite3_uint64 sqlite3_memory_used(void){
  sqlite3_uint64 n;
  if( memMutex==0 ){
    memMutex = sqlite3_mutex_alloc(1);
  }
  sqlite3_mutex_enter(memMutex, 1);
  n = nowUsed;
  sqlite3_mutex_leave(memMutex);  
  return n;
}

/*
** Return the maximum amount of memory that has ever been
** checked out since either the beginning of this process
** or since the most recent reset.
*/
sqlite3_uint64 sqlite3_memory_highwater(int resetFlag){
  sqlite3_uint64 n;
  if( memMutex==0 ){
    memMutex = sqlite3_mutex_alloc(1);
  }
  sqlite3_mutex_enter(memMutex, 1);
  n = mxUsed;
  if( resetFlag ){
    mxUsed = nowUsed;
  }
  sqlite3_mutex_leave(memMutex);  
  return n;
}

/*
** Change the alarm callback
*/
int sqlite3_memory_alarm(
  void(*xCallback)(void *pArg, sqlite3_uint64 used, unsigned int N),
  void *pArg,
  sqlite3_uint64 iThreshold
){
  if( memMutex==0 ){
    memMutex = sqlite3_mutex_alloc(1);
  }
  sqlite3_mutex_enter(memMutex, 1);
  alarmCallback = xCallback;
  alarmArg = pArg;
  alarmThreshold = iThreshold;
  sqlite3_mutex_leave(memMutex);
  return SQLITE_OK;
}

/*
** Trigger the alarm 
*/
static void sqlite3MemsysAlarm(unsigned nByte){
  if( alarmCallback==0 || alarmBusy  ) return;
  alarmBusy = 1;
  alarmCallback(alarmArg, nowUsed, nByte);
  alarmBusy = 0;
}

/*
** Each memory allocation looks like this:
**
**    ----------------------------------------------------------------
**    |  backtrace pointers |  MemBlockHdr |  allocation |  EndGuard |
**    ----------------------------------------------------------------
**
** The application code sees only a pointer to the allocation.  We have
** to back up from the allocation pointer to find the MemBlockHdr.  The
** MemBlockHdr tells us the size of the allocation and the number of
** backtrace pointers.  There is also a guard word at the end of the
** MemBlockHdr.
*/
struct MemBlockHdr {
  struct MemBlockHdr *pNext, *pPrev;  /* Linked list of all unfreed memory */
  unsigned int iSize;                 /* Size of this allocation */
  unsigned short nBacktrace;          /* Number of backtraces on this alloc */
  unsigned short nBacktraceSlots;     /* Available backtrace slots */
  unsigned int iForeGuard;            /* Guard word for sanity */
};

/*
** Guard words
*/
#define FOREGUARD 0x80F5E153
#define REARGUARD 0xE4676B53

/*
** Head and tail of a linked list of all outstanding allocations
*/
static struct MemBlockHdr *pFirst = 0;
static struct MemBlockHdr *pLast = 0;

/*
** The number of levels of backtrace to save in new allocations.
*/
static int backtraceLevels = 0;

/*
** Given an allocation, find the MemBlockHdr for that allocation.
**
** This routine checks the guards at either end of the allocation and
** if they are incorrect it asserts.
*/
static struct MemBlockHdr *sqlite3MemsysGetHeader(void *pAllocation){
  struct MemBlockHdr *p;
  unsigned int *pInt;

  p = (struct MemBlockHdr*)pAllocation;
  p--;
  assert( p->iForeGuard==FOREGUARD );
  assert( (p->iSize & 3)==0 );
  pInt = (unsigned int*)pAllocation;
  assert( pInt[p->iSize/sizeof(unsigned int)]==REARGUARD );
  return p;
}

/*
** Allocate nByte of memory
*/
void *sqlite3_malloc(unsigned int nByte){
  struct MemBlockHdr *pHdr;
  void **pBt;
  unsigned int *pInt;
  void *p;
  unsigned int totalSize;

  if( memMutex==0 ){
    memMutex = sqlite3_mutex_alloc(1);
  }
  sqlite3_mutex_enter(memMutex, 1);
  if( nowUsed+nByte>=alarmThreshold ){
    sqlite3MemsysAlarm(nByte);
  }
  nByte = (nByte+3)&~3;
  totalSize = nByte + sizeof(*pHdr) + sizeof(unsigned int) +
               backtraceLevels*sizeof(void*);
  p = malloc(totalSize);
  if( p==0 ){
    sqlite3MemsysAlarm(nByte);
    p = malloc(totalSize);
  }
  if( p ){
    pBt = p;
    pHdr = (struct MemBlockHdr*)&pBt[backtraceLevels];
    pHdr->pNext = 0;
    pHdr->pPrev = pLast;
    if( pLast ){
      pLast->pNext = pHdr;
    }else{
      pFirst = pHdr;
    }
    pLast = pHdr;
    pHdr->iForeGuard = FOREGUARD;
    pHdr->nBacktraceSlots = backtraceLevels;
    if( backtraceLevels ){
      pHdr->nBacktrace = backtrace(pBt, backtraceLevels);
    }else{
      pHdr->nBacktrace = 0;
    }
    pHdr->iSize = nByte;
    pInt = (unsigned int *)&pHdr[1];
    pInt[nByte/sizeof(unsigned int)] = REARGUARD;
    memset(pInt, 0x65, nByte);
    nowUsed += nByte;
    if( nowUsed>mxUsed ){
      mxUsed = nowUsed;
    }
    p = (void*)pInt;
  }
  sqlite3_mutex_leave(memMutex);
  return p; 
}

/*
** Free memory.
*/
void sqlite3_free(void *pPrior){
  struct MemBlockHdr *pHdr;
  void **pBt;
  if( pPrior==0 ){
    return;
  }
  assert( memMutex!=0 );
  pHdr = sqlite3MemsysGetHeader(pPrior);
  pBt = (void**)pHdr;
  pBt -= pHdr->nBacktraceSlots;
  sqlite3_mutex_enter(memMutex, 1);
  nowUsed -= pHdr->iSize;
  if( pHdr->pPrev ){
    assert( pHdr->pPrev->pNext==pHdr );
    pHdr->pPrev->pNext = pHdr->pNext;
  }else{
    assert( pFirst==pHdr );
    pFirst = pHdr->pNext;
  }
  if( pHdr->pNext ){
    assert( pHdr->pNext->pPrev==pHdr );
    pHdr->pNext->pPrev = pHdr->pPrev;
  }else{
    assert( pLast==pHdr );
    pLast = pHdr->pPrev;
  }
  memset(pBt, 0x2b, sizeof(void*)*pHdr->nBacktrace + sizeof(*pHdr) +
                    pHdr->iSize + sizeof(unsigned int));
  free(pBt);
  sqlite3_mutex_leave(memMutex);  
}

/*
** Change the size of an existing memory allocation.
**
** For this debugging implementation, we *always* make a copy of the
** allocation into a new place in memory.  In this way, if the 
** higher level code is using pointer to the old allocation, it is 
** much more likely to break and we are much more liking to find
** the error.
*/
void *sqlite3_realloc(void *pPrior, unsigned int nByte){
  struct MemBlockHdr *pOldHdr;
  void *pNew;
  unsigned nOld;
  if( pPrior==0 ){
    return sqlite3_malloc(nByte);
  }
  if( nByte==0 ){
    sqlite3_free(pPrior);
    return;
  }
  pOldHdr = sqlite3MemsysGetHeader(pPrior);
  pNew = sqlite3_malloc(nByte);
  if( pNew ){
    memcpy(pNew, pPrior, nByte<pOldHdr->iSize ? nByte : pOldHdr->iSize);
    if( nByte>pOldHdr->iSize ){
      memset(&((char*)pNew)[pOldHdr->iSize], 0x2b, nByte - pOldHdr->iSize);
    }
    sqlite3_free(pPrior);
  }
  return pNew;
}

/*
** Set the number of backtrace levels kept for each allocation.
** A value of zero turns of backtracing.  The number is always rounded
** up to a multiple of 2.
*/
void sqlite3_memdebug_backtrace_depth(int depth){
  if( depth<0 ){ depth = 0; }
  if( depth>20 ){ depth = 20; }
  depth = (depth+1)&~1;
  backtraceLevels = depth;
}

/*
** Open the file indicated and write a log of all unfreed memory 
** allocations into that log.
*/
void sqlite3_memdebug_dump(const char *zFilename){
  FILE *out;
  struct MemBlockHdr *pHdr;
  void **pBt;
  out = fopen(zFilename, "w");
  if( out==0 ){
    fprintf(stderr, "** Unable to output memory debug output log: %s **\n",
                    zFilename);
    return;
  }
  for(pHdr=pFirst; pHdr; pHdr=pHdr->pNext){
    fprintf(out, "**** %d bytes at %p ****\n", pHdr->iSize, &pHdr[1]);
    if( pHdr->nBacktrace ){
      fflush(out);
      pBt = (void**)pHdr;
      pBt -= pHdr->nBacktraceSlots;
      backtrace_symbols_fd(pBt, pHdr->nBacktrace, fileno(out));
      fprintf(out, "\n");
    }
  }
  fclose(out);
}

#endif /* SQLITE_MEMDEBUG && !SQLITE_OMIT_MEMORY_ALLOCATION */