/ Check-in [8d2d1c4f]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add a debugging memory allocator. (CVS 4227)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 8d2d1c4ff9dca61f75e3048107ee9712d346a28c
User & Date: drh 2007-08-15 17:07:57
Context
2007-08-15
17:08
Modify the crash-recovery test code in test6.c for 3.5. Also change some other code to use the new sqlite3_io_methods interface. Lots of things are broken now. (CVS 4228) check-in: af9503da user: danielk1977 tags: trunk
17:07
Add a debugging memory allocator. (CVS 4227) check-in: 8d2d1c4f user: drh tags: trunk
13:04
Add initial implementations of mutex and memory subsystem modules. (CVS 4226) check-in: c0fa3769 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to main.mk.

    54     54   TCCX = $(TCC) $(OPTS) $(THREADSAFE) $(USLEEP) -I. -I$(TOP)/src
    55     55   
    56     56   # Object files for the SQLite library.
    57     57   #
    58     58   LIBOBJ+= alter.o analyze.o attach.o auth.o btree.o build.o \
    59     59            callback.o complete.o date.o delete.o \
    60     60            expr.o func.o hash.o insert.o loadext.o \
    61         -         main.o malloc.o mem1.o mutex.o \
           61  +         main.o malloc.o mem1.o mem2.o mutex.o \
    62     62            opcodes.o os.o os_os2.o os_unix.o os_win.o \
    63     63            pager.o parse.o pragma.o prepare.o printf.o random.o \
    64     64            select.o table.o tclsqlite.o tokenize.o trigger.o \
    65     65            update.o util.o vacuum.o \
    66     66            vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbefifo.o vdbemem.o \
    67     67            where.o utf.o legacy.o vtab.o
    68     68   
................................................................................
    88     88     $(TOP)/src/hash.c \
    89     89     $(TOP)/src/hash.h \
    90     90     $(TOP)/src/insert.c \
    91     91     $(TOP)/src/legacy.c \
    92     92     $(TOP)/src/loadext.c \
    93     93     $(TOP)/src/main.c \
    94     94     $(TOP)/src/malloc.c \
    95         -  $(TOP)/src/mem.c \
           95  +  $(TOP)/src/mem1.c \
           96  +  $(TOP)/src/mem2.c \
    96     97     $(TOP)/src/mutex.c \
    97     98     $(TOP)/src/os.c \
    98     99     $(TOP)/src/os_os2.c \
    99    100     $(TOP)/src/os_unix.c \
   100    101     $(TOP)/src/os_win.c \
   101    102     $(TOP)/src/pager.c \
   102    103     $(TOP)/src/pager.h \
................................................................................
   342    343   	$(TCCX) -c $(TOP)/src/main.c
   343    344   
   344    345   malloc.o:	$(TOP)/src/malloc.c $(HDR)
   345    346   	$(TCCX) -c $(TOP)/src/malloc.c
   346    347   
   347    348   mem1.o:	$(TOP)/src/mem1.c $(HDR)
   348    349   	$(TCCX) -c $(TOP)/src/mem1.c
          350  +
          351  +mem2.o:	$(TOP)/src/mem2.c $(HDR)
          352  +	$(TCCX) -c $(TOP)/src/mem2.c
   349    353   
   350    354   mutex.o:	$(TOP)/src/mutex.c $(HDR)
   351    355   	$(TCCX) -c $(TOP)/src/mutex.c
   352    356   
   353    357   pager.o:	$(TOP)/src/pager.c $(HDR) $(TOP)/src/pager.h
   354    358   	$(TCCX) -c $(TOP)/src/pager.c
   355    359   

Changes to src/mem1.c.

     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   *************************************************************************
    12     12   ** This file contains the C functions that implement a memory
    13     13   ** allocation subsystem for use by SQLite.  
    14     14   **
    15         -** $Id: mem1.c,v 1.1 2007/08/15 13:04:54 drh Exp $
           15  +** $Id: mem1.c,v 1.2 2007/08/15 17:07:57 drh Exp $
           16  +*/
           17  +
           18  +/*
           19  +** This version of the memory allocator is the default.  It is
           20  +** used when no other memory allocator is specified using compile-time
           21  +** macros.
    16     22   */
           23  +#if !defined(SQLITE_MEMDEBUG) && !defined(SQLITE_OMIT_MEMORY_ALLOCATION)
    17     24   
    18     25   /*
    19     26   ** We will eventually construct multiple memory allocation subsystems
    20     27   ** suitable for use in various contexts:
    21     28   **
    22     29   **    *  Normal multi-threaded builds
    23     30   **    *  Normal single-threaded builds
................................................................................
   198    205       if( nowUsed>mxUsed ){
   199    206         mxUsed = nowUsed;
   200    207       }
   201    208     }
   202    209     sqlite3_mutex_leave(memMutex);
   203    210     return (void*)p;
   204    211   }
          212  +
          213  +#endif /* !SQLITE_MEMDEBUG && !SQLITE_OMIT_MEMORY_ALLOCATION */

Added src/mem2.c.

            1  +/*
            2  +** 2007 August 15
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +** This file contains the C functions that implement a memory
           13  +** allocation subsystem for use by SQLite.  
           14  +**
           15  +** $Id: mem2.c,v 1.1 2007/08/15 17:07:57 drh Exp $
           16  +*/
           17  +
           18  +/*
           19  +** This version of the memory allocator is used only if the
           20  +** SQLITE_MEMDEBUG macro is defined and SQLITE_OMIT_MEMORY_ALLOCATION
           21  +** is not defined.
           22  +*/
           23  +#if defined(SQLITE_MEMDEBUG) && !defined(SQLITE_OMIT_MEMORY_ALLOCATION)
           24  +
           25  +/*
           26  +** We will eventually construct multiple memory allocation subsystems
           27  +** suitable for use in various contexts:
           28  +**
           29  +**    *  Normal multi-threaded builds
           30  +**    *  Normal single-threaded builds
           31  +**    *  Debugging builds
           32  +**
           33  +** This version is suitable for use in debugging builds.
           34  +**
           35  +** Features:
           36  +**
           37  +**    * Every allocate has guards at both ends.
           38  +**    * New allocations are initialized with randomness
           39  +**    * Allocations are overwritten with randomness when freed
           40  +**    * Optional logs of malloc activity generated
           41  +**    * Summary of outstanding allocations with backtraces to the
           42  +**      point of allocation.
           43  +**    * The ability to simulate memory allocation failure
           44  +*/
           45  +#include "sqliteInt.h"
           46  +#include <stdio.h>
           47  +
           48  +/*
           49  +** The backtrace functionality is only available with GLIBC
           50  +*/
           51  +#ifdef __GLIBC__
           52  +  extern int backtrace(void**,int);
           53  +  extern void backtrace_symbols_fd(void*const*,int,int);
           54  +#else
           55  +# define backtrace(A,B) 0
           56  +# define backtrace_symbols_fd(A,B,C)
           57  +#endif
           58  +
           59  +
           60  +/*
           61  +** Mutex to control access to the memory allocation subsystem.
           62  +*/
           63  +static sqlite3_mutex *memMutex = 0;
           64  +
           65  +/*
           66  +** Current allocation and high-water mark.
           67  +*/
           68  +static sqlite3_uint64 nowUsed = 0;
           69  +static sqlite3_uint64 mxUsed = 0;
           70  +
           71  +/*
           72  +** The alarm callback and its arguments.  The memMutex lock will
           73  +** be held while the callback is running.  Recursive calls into
           74  +** the memory subsystem are allowed, but no new callbacks will be
           75  +** issued.  The alarmBusy variable is set to prevent recursive
           76  +** callbacks.
           77  +*/
           78  +static void (*alarmCallback)(void*, sqlite3_uint64, unsigned) = 0;
           79  +static void *alarmArg = 0;
           80  +static sqlite3_uint64 alarmThreshold = (((sqlite3_uint64)1)<<63);
           81  +static int alarmBusy = 0;
           82  +
           83  +
           84  +/*
           85  +** Return the amount of memory currently checked out.
           86  +*/
           87  +sqlite3_uint64 sqlite3_memory_used(void){
           88  +  sqlite3_uint64 n;
           89  +  if( memMutex==0 ){
           90  +    memMutex = sqlite3_mutex_alloc(1);
           91  +  }
           92  +  sqlite3_mutex_enter(memMutex, 1);
           93  +  n = nowUsed;
           94  +  sqlite3_mutex_leave(memMutex);  
           95  +  return n;
           96  +}
           97  +
           98  +/*
           99  +** Return the maximum amount of memory that has ever been
          100  +** checked out since either the beginning of this process
          101  +** or since the most recent reset.
          102  +*/
          103  +sqlite3_uint64 sqlite3_memory_highwater(int resetFlag){
          104  +  sqlite3_uint64 n;
          105  +  if( memMutex==0 ){
          106  +    memMutex = sqlite3_mutex_alloc(1);
          107  +  }
          108  +  sqlite3_mutex_enter(memMutex, 1);
          109  +  n = mxUsed;
          110  +  if( resetFlag ){
          111  +    mxUsed = nowUsed;
          112  +  }
          113  +  sqlite3_mutex_leave(memMutex);  
          114  +  return n;
          115  +}
          116  +
          117  +/*
          118  +** Change the alarm callback
          119  +*/
          120  +int sqlite3_memory_alarm(
          121  +  void(*xCallback)(void *pArg, sqlite3_uint64 used, unsigned int N),
          122  +  void *pArg,
          123  +  sqlite3_uint64 iThreshold
          124  +){
          125  +  if( memMutex==0 ){
          126  +    memMutex = sqlite3_mutex_alloc(1);
          127  +  }
          128  +  sqlite3_mutex_enter(memMutex, 1);
          129  +  alarmCallback = xCallback;
          130  +  alarmArg = pArg;
          131  +  alarmThreshold = iThreshold;
          132  +  sqlite3_mutex_leave(memMutex);
          133  +  return SQLITE_OK;
          134  +}
          135  +
          136  +/*
          137  +** Trigger the alarm 
          138  +*/
          139  +static void sqlite3MemsysAlarm(unsigned nByte){
          140  +  if( alarmCallback==0 || alarmBusy  ) return;
          141  +  alarmBusy = 1;
          142  +  alarmCallback(alarmArg, nowUsed, nByte);
          143  +  alarmBusy = 0;
          144  +}
          145  +
          146  +/*
          147  +** Each memory allocation looks like this:
          148  +**
          149  +**    ----------------------------------------------------------------
          150  +**    |  backtrace pointers |  MemBlockHdr |  allocation |  EndGuard |
          151  +**    ----------------------------------------------------------------
          152  +**
          153  +** The application code sees only a pointer to the allocation.  We have
          154  +** to back up from the allocation pointer to find the MemBlockHdr.  The
          155  +** MemBlockHdr tells us the size of the allocation and the number of
          156  +** backtrace pointers.  There is also a guard word at the end of the
          157  +** MemBlockHdr.
          158  +*/
          159  +struct MemBlockHdr {
          160  +  struct MemBlockHdr *pNext, *pPrev;  /* Linked list of all unfreed memory */
          161  +  unsigned int iSize;                 /* Size of this allocation */
          162  +  unsigned short nBacktrace;          /* Number of backtraces on this alloc */
          163  +  unsigned short nBacktraceSlots;     /* Available backtrace slots */
          164  +  unsigned int iForeGuard;            /* Guard word for sanity */
          165  +};
          166  +
          167  +/*
          168  +** Guard words
          169  +*/
          170  +#define FOREGUARD 0x80F5E153
          171  +#define REARGUARD 0xE4676B53
          172  +
          173  +/*
          174  +** Head and tail of a linked list of all outstanding allocations
          175  +*/
          176  +static struct MemBlockHdr *pFirst = 0;
          177  +static struct MemBlockHdr *pLast = 0;
          178  +
          179  +/*
          180  +** The number of levels of backtrace to save in new allocations.
          181  +*/
          182  +static int backtraceLevels = 0;
          183  +
          184  +/*
          185  +** Given an allocation, find the MemBlockHdr for that allocation.
          186  +**
          187  +** This routine checks the guards at either end of the allocation and
          188  +** if they are incorrect it asserts.
          189  +*/
          190  +static struct MemBlockHdr *sqlite3MemsysGetHeader(void *pAllocation){
          191  +  struct MemBlockHdr *p;
          192  +  unsigned int *pInt;
          193  +
          194  +  p = (struct MemBlockHdr*)pAllocation;
          195  +  p--;
          196  +  assert( p->iForeGuard==FOREGUARD );
          197  +  assert( (p->iSize & 3)==0 );
          198  +  pInt = (unsigned int*)pAllocation;
          199  +  assert( pInt[p->iSize/sizeof(unsigned int)]==REARGUARD );
          200  +  return p;
          201  +}
          202  +
          203  +/*
          204  +** Allocate nByte of memory
          205  +*/
          206  +void *sqlite3_malloc(unsigned int nByte){
          207  +  struct MemBlockHdr *pHdr;
          208  +  void **pBt;
          209  +  unsigned int *pInt;
          210  +  void *p;
          211  +  unsigned int totalSize;
          212  +
          213  +  if( memMutex==0 ){
          214  +    memMutex = sqlite3_mutex_alloc(1);
          215  +  }
          216  +  sqlite3_mutex_enter(memMutex, 1);
          217  +  if( nowUsed+nByte>=alarmThreshold ){
          218  +    sqlite3MemsysAlarm(nByte);
          219  +  }
          220  +  nByte = (nByte+3)&~3;
          221  +  totalSize = nByte + sizeof(*pHdr) + sizeof(unsigned int) +
          222  +               backtraceLevels*sizeof(void*);
          223  +  p = malloc(totalSize);
          224  +  if( p==0 ){
          225  +    sqlite3MemsysAlarm(nByte);
          226  +    p = malloc(totalSize);
          227  +  }
          228  +  if( p ){
          229  +    pBt = p;
          230  +    pHdr = (struct MemBlockHdr*)&pBt[backtraceLevels];
          231  +    pHdr->pNext = 0;
          232  +    pHdr->pPrev = pLast;
          233  +    if( pLast ){
          234  +      pLast->pNext = pHdr;
          235  +    }else{
          236  +      pFirst = pHdr;
          237  +    }
          238  +    pLast = pHdr;
          239  +    pHdr->iForeGuard = FOREGUARD;
          240  +    pHdr->nBacktraceSlots = backtraceLevels;
          241  +    if( backtraceLevels ){
          242  +      pHdr->nBacktrace = backtrace(pBt, backtraceLevels);
          243  +    }else{
          244  +      pHdr->nBacktrace = 0;
          245  +    }
          246  +    pHdr->iSize = nByte;
          247  +    pInt = (unsigned int *)&pHdr[1];
          248  +    pInt[nByte/sizeof(unsigned int)] = REARGUARD;
          249  +    memset(pInt, 0x65, nByte);
          250  +    nowUsed += nByte;
          251  +    if( nowUsed>mxUsed ){
          252  +      mxUsed = nowUsed;
          253  +    }
          254  +    p = (void*)pInt;
          255  +  }
          256  +  sqlite3_mutex_leave(memMutex);
          257  +  return p; 
          258  +}
          259  +
          260  +/*
          261  +** Free memory.
          262  +*/
          263  +void sqlite3_free(void *pPrior){
          264  +  struct MemBlockHdr *pHdr;
          265  +  void **pBt;
          266  +  if( pPrior==0 ){
          267  +    return;
          268  +  }
          269  +  assert( memMutex!=0 );
          270  +  pHdr = sqlite3MemsysGetHeader(pPrior);
          271  +  pBt = (void**)pHdr;
          272  +  pBt -= pHdr->nBacktraceSlots;
          273  +  sqlite3_mutex_enter(memMutex, 1);
          274  +  nowUsed -= pHdr->iSize;
          275  +  if( pHdr->pPrev ){
          276  +    assert( pHdr->pPrev->pNext==pHdr );
          277  +    pHdr->pPrev->pNext = pHdr->pNext;
          278  +  }else{
          279  +    assert( pFirst==pHdr );
          280  +    pFirst = pHdr->pNext;
          281  +  }
          282  +  if( pHdr->pNext ){
          283  +    assert( pHdr->pNext->pPrev==pHdr );
          284  +    pHdr->pNext->pPrev = pHdr->pPrev;
          285  +  }else{
          286  +    assert( pLast==pHdr );
          287  +    pLast = pHdr->pPrev;
          288  +  }
          289  +  memset(pBt, 0x2b, sizeof(void*)*pHdr->nBacktrace + sizeof(*pHdr) +
          290  +                    pHdr->iSize + sizeof(unsigned int));
          291  +  free(pBt);
          292  +  sqlite3_mutex_leave(memMutex);  
          293  +}
          294  +
          295  +/*
          296  +** Change the size of an existing memory allocation.
          297  +**
          298  +** For this debugging implementation, we *always* make a copy of the
          299  +** allocation into a new place in memory.  In this way, if the 
          300  +** higher level code is using pointer to the old allocation, it is 
          301  +** much more likely to break and we are much more liking to find
          302  +** the error.
          303  +*/
          304  +void *sqlite3_realloc(void *pPrior, unsigned int nByte){
          305  +  struct MemBlockHdr *pOldHdr;
          306  +  void *pNew;
          307  +  unsigned nOld;
          308  +  if( pPrior==0 ){
          309  +    return sqlite3_malloc(nByte);
          310  +  }
          311  +  if( nByte==0 ){
          312  +    sqlite3_free(pPrior);
          313  +    return;
          314  +  }
          315  +  pOldHdr = sqlite3MemsysGetHeader(pPrior);
          316  +  pNew = sqlite3_malloc(nByte);
          317  +  if( pNew ){
          318  +    memcpy(pNew, pPrior, nByte<pOldHdr->iSize ? nByte : pOldHdr->iSize);
          319  +    if( nByte>pOldHdr->iSize ){
          320  +      memset(&((char*)pNew)[pOldHdr->iSize], 0x2b, nByte - pOldHdr->iSize);
          321  +    }
          322  +    sqlite3_free(pPrior);
          323  +  }
          324  +  return pNew;
          325  +}
          326  +
          327  +/*
          328  +** Set the number of backtrace levels kept for each allocation.
          329  +** A value of zero turns of backtracing.  The number is always rounded
          330  +** up to a multiple of 2.
          331  +*/
          332  +void sqlite3_memdebug_backtrace_depth(int depth){
          333  +  if( depth<0 ){ depth = 0; }
          334  +  if( depth>20 ){ depth = 20; }
          335  +  depth = (depth+1)&~1;
          336  +  backtraceLevels = depth;
          337  +}
          338  +
          339  +/*
          340  +** Open the file indicated and write a log of all unfreed memory 
          341  +** allocations into that log.
          342  +*/
          343  +void sqlite3_memdebug_dump(const char *zFilename){
          344  +  FILE *out;
          345  +  struct MemBlockHdr *pHdr;
          346  +  void **pBt;
          347  +  out = fopen(zFilename, "w");
          348  +  if( out==0 ){
          349  +    fprintf(stderr, "** Unable to output memory debug output log: %s **\n",
          350  +                    zFilename);
          351  +    return;
          352  +  }
          353  +  for(pHdr=pFirst; pHdr; pHdr=pHdr->pNext){
          354  +    fprintf(out, "**** %d bytes at %p ****\n", pHdr->iSize, &pHdr[1]);
          355  +    if( pHdr->nBacktrace ){
          356  +      fflush(out);
          357  +      pBt = (void**)pHdr;
          358  +      pBt -= pHdr->nBacktraceSlots;
          359  +      backtrace_symbols_fd(pBt, pHdr->nBacktrace, fileno(out));
          360  +      fprintf(out, "\n");
          361  +    }
          362  +  }
          363  +  fclose(out);
          364  +}
          365  +
          366  +#endif /* SQLITE_MEMDEBUG && !SQLITE_OMIT_MEMORY_ALLOCATION */