/ Check-in [fd97f876]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Change the allocator in mem2.c (used when SQLITE_MEMDEBUG is defined) so that allocations are not rounded up to the nearest 4 byte boundary. Fix a couple of errors in malloc.test related to sqlite3OsAccess() returning -1. (CVS 4956)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: fd97f8762cb1e4653c932402940f74d7c0ebf71f
User & Date: danielk1977 2008-04-03 10:13:01
Context
2008-04-03
14:36
Avoid an out-of-bounds read in sqlite3_prepare() and fix a case where the output variable *pzTail was being set incorrectly. Fix for #3027. (CVS 4957) check-in: c287a7b2 user: danielk1977 tags: trunk
10:13
Change the allocator in mem2.c (used when SQLITE_MEMDEBUG is defined) so that allocations are not rounded up to the nearest 4 byte boundary. Fix a couple of errors in malloc.test related to sqlite3OsAccess() returning -1. (CVS 4956) check-in: fd97f876 user: danielk1977 tags: trunk
2008-04-02
18:33
Minor optimizations. (CVS 4955) check-in: e8529455 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/mem2.c.

8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
...
208
209
210
211
212
213
214


215
216
217
218
219
220

221



222
223
224
225
226
227
228
...
242
243
244
245
246
247
248

249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
...
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains the C functions that implement a memory
** allocation subsystem for use by SQLite.  
**
** $Id: mem2.c,v 1.24 2008/03/28 07:42:54 danielk1977 Exp $
*/
#include "sqliteInt.h"

/*
** This version of the memory allocator is used only if the
** SQLITE_MEMDEBUG macro is defined
*/
................................................................................
**
** This routine checks the guards at either end of the allocation and
** if they are incorrect it asserts.
*/
static struct MemBlockHdr *sqlite3MemsysGetHeader(void *pAllocation){
  struct MemBlockHdr *p;
  int *pInt;



  p = (struct MemBlockHdr*)pAllocation;
  p--;
  assert( p->iForeGuard==FOREGUARD );
  assert( (p->iSize & 3)==0 );
  pInt = (int*)pAllocation;

  assert( pInt[p->iSize/sizeof(int)]==REARGUARD );



  return p;
}

/*
** Return the number of bytes currently allocated at address p.
*/
int sqlite3MallocSize(void *p){
................................................................................
  void **pBt;
  char *z;
  int *pInt;
  void *p = 0;
  int totalSize;

  if( nByte>0 ){

    enterMem();
    assert( mem.disallow==0 );
    if( mem.alarmCallback!=0 && mem.nowUsed+nByte>=mem.alarmThreshold ){
      sqlite3MemsysAlarm(nByte);
    }
    nByte = (nByte+3)&~3;
    if( nByte/8>NCSIZE-1 ){
      mem.sizeCnt[NCSIZE-1]++;
    }else{
      mem.sizeCnt[nByte/8]++;
    }
    totalSize = nByte + sizeof(*pHdr) + sizeof(int) +
                 mem.nBacktrace*sizeof(void*) + mem.nTitle;
    if( sqlite3FaultStep(SQLITE_FAULTINJECTOR_MALLOC) ){
      p = 0;
    }else{
      p = malloc(totalSize);
      if( p==0 ){
        sqlite3MemsysAlarm(nByte);
................................................................................
        pHdr->nBacktrace = 0;
      }
      if( mem.nTitle ){
        memcpy(z, mem.zTitle, mem.nTitle);
      }
      pHdr->iSize = nByte;
      pInt = (int*)&pHdr[1];
      pInt[nByte/sizeof(int)] = REARGUARD;
      memset(pInt, 0x65, nByte);
      mem.nowUsed += nByte;
      if( mem.nowUsed>mem.mxUsed ){
        mem.mxUsed = mem.nowUsed;
      }
      p = (void*)pInt;
    }
    sqlite3_mutex_leave(mem.mutex);







|







 







>
>




|

>
|
>
>
>







 







>





|
|


|

|







 







|
|







8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
...
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
...
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
...
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains the C functions that implement a memory
** allocation subsystem for use by SQLite.  
**
** $Id: mem2.c,v 1.25 2008/04/03 10:13:01 danielk1977 Exp $
*/
#include "sqliteInt.h"

/*
** This version of the memory allocator is used only if the
** SQLITE_MEMDEBUG macro is defined
*/
................................................................................
**
** This routine checks the guards at either end of the allocation and
** if they are incorrect it asserts.
*/
static struct MemBlockHdr *sqlite3MemsysGetHeader(void *pAllocation){
  struct MemBlockHdr *p;
  int *pInt;
  u8 *pU8;
  int nReserve;

  p = (struct MemBlockHdr*)pAllocation;
  p--;
  assert( p->iForeGuard==FOREGUARD );
  nReserve = (p->iSize+3)&~3;
  pInt = (int*)pAllocation;
  pU8 = (u8*)pAllocation;
  assert( pInt[nReserve/sizeof(int)]==REARGUARD );
  assert( (nReserve-0)<=p->iSize || pU8[nReserve-1]==0x65 );
  assert( (nReserve-1)<=p->iSize || pU8[nReserve-2]==0x65 );
  assert( (nReserve-2)<=p->iSize || pU8[nReserve-3]==0x65 );
  return p;
}

/*
** Return the number of bytes currently allocated at address p.
*/
int sqlite3MallocSize(void *p){
................................................................................
  void **pBt;
  char *z;
  int *pInt;
  void *p = 0;
  int totalSize;

  if( nByte>0 ){
    int nReserve;
    enterMem();
    assert( mem.disallow==0 );
    if( mem.alarmCallback!=0 && mem.nowUsed+nByte>=mem.alarmThreshold ){
      sqlite3MemsysAlarm(nByte);
    }
    nReserve = (nByte+3)&~3;
    if( nReserve/8>NCSIZE-1 ){
      mem.sizeCnt[NCSIZE-1]++;
    }else{
      mem.sizeCnt[nReserve/8]++;
    }
    totalSize = nReserve + sizeof(*pHdr) + sizeof(int) +
                 mem.nBacktrace*sizeof(void*) + mem.nTitle;
    if( sqlite3FaultStep(SQLITE_FAULTINJECTOR_MALLOC) ){
      p = 0;
    }else{
      p = malloc(totalSize);
      if( p==0 ){
        sqlite3MemsysAlarm(nByte);
................................................................................
        pHdr->nBacktrace = 0;
      }
      if( mem.nTitle ){
        memcpy(z, mem.zTitle, mem.nTitle);
      }
      pHdr->iSize = nByte;
      pInt = (int*)&pHdr[1];
      pInt[nReserve/sizeof(int)] = REARGUARD;
      memset(pInt, 0x65, nReserve);
      mem.nowUsed += nByte;
      if( mem.nowUsed>mem.mxUsed ){
        mem.mxUsed = mem.nowUsed;
      }
      p = (void*)pInt;
    }
    sqlite3_mutex_leave(mem.mutex);

Changes to src/pager.c.

14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
....
1759
1760
1761
1762
1763
1764
1765

1766
1767
1768
1769
1770
1771
1772
....
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791





1792
1793
1794
1795
1796
1797
1798
....
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428


3429
3430
3431
3432
3433
3434
3435
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.422 2008/03/28 17:41:14 drh Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include <assert.h>
#include <string.h>

/*
................................................................................
static int pager_playback(Pager *pPager, int isHot){
  sqlite3_vfs *pVfs = pPager->pVfs;
  i64 szJ;                 /* Size of the journal file in bytes */
  u32 nRec;                /* Number of Records in the journal */
  int i;                   /* Loop counter */
  Pgno mxPg = 0;           /* Size of the original file in pages */
  int rc;                  /* Result code of a subroutine */

  char *zMaster = 0;       /* Name of master journal file if any */

  /* Figure out how many records are in the journal.  Abort early if
  ** the journal is empty.
  */
  assert( pPager->journalOpen );
  rc = sqlite3OsFileSize(pPager->jfd, &szJ);
................................................................................
  /* Read the master journal name from the journal, if it is present.
  ** If a master journal file name is specified, but the file is not
  ** present on disk, then the journal is not hot and does not need to be
  ** played back.
  */
  zMaster = pPager->pTmpSpace;
  rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1);
  if( rc!=SQLITE_OK 
   || (zMaster[0] && sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS)==0 ) 
  ){
    zMaster = 0;
    goto end_playback;
  }
  pPager->journalOff = 0;
  zMaster = 0;






  /* This loop terminates either when the readJournalHdr() call returns
  ** SQLITE_DONE or an IO error occurs. */
  while( 1 ){

    /* Read the next journal header from the journal file.  If there are
    ** not enough bytes left in the journal file for a complete header, or
................................................................................
        ** Open the journal for read/write access. This is because in 
        ** exclusive-access mode the file descriptor will be kept open and
        ** possibly used for a transaction later on. On some systems, the
        ** OsTruncate() call used in exclusive-access mode also requires
        ** a read/write file handle.
        */
        if( !isHot ){
          rc = SQLITE_BUSY;
          if( sqlite3OsAccess(pVfs, pPager->zJournal,SQLITE_ACCESS_EXISTS)==1 ){
            int fout = 0;
            int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL;
            assert( !pPager->tempFile );
            rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout);
            assert( rc!=SQLITE_OK || pPager->jfd->pMethods );
            if( fout&SQLITE_OPEN_READONLY ){
              rc = SQLITE_BUSY;
              sqlite3OsClose(pPager->jfd);
            }


          }
        }
        if( rc!=SQLITE_OK ){
          pager_unlock(pPager);
          switch( rc ){
            case SQLITE_NOMEM:
            case SQLITE_IOERR_UNLOCK:







|







 







>







 







|
|




<

>
>
>
>
>







 







|
|









>
>







14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
....
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
....
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790

1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
....
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.423 2008/04/03 10:13:01 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include <assert.h>
#include <string.h>

/*
................................................................................
static int pager_playback(Pager *pPager, int isHot){
  sqlite3_vfs *pVfs = pPager->pVfs;
  i64 szJ;                 /* Size of the journal file in bytes */
  u32 nRec;                /* Number of Records in the journal */
  int i;                   /* Loop counter */
  Pgno mxPg = 0;           /* Size of the original file in pages */
  int rc;                  /* Result code of a subroutine */
  int res = 0;             /* Value returned by sqlite3OsAccess() */
  char *zMaster = 0;       /* Name of master journal file if any */

  /* Figure out how many records are in the journal.  Abort early if
  ** the journal is empty.
  */
  assert( pPager->journalOpen );
  rc = sqlite3OsFileSize(pPager->jfd, &szJ);
................................................................................
  /* Read the master journal name from the journal, if it is present.
  ** If a master journal file name is specified, but the file is not
  ** present on disk, then the journal is not hot and does not need to be
  ** played back.
  */
  zMaster = pPager->pTmpSpace;
  rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1);
  if( rc!=SQLITE_OK || (zMaster[0] 
   && (res=sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS))==0 ) 
  ){
    zMaster = 0;
    goto end_playback;
  }

  zMaster = 0;
  if( res<0 ){
    rc = SQLITE_IOERR_NOMEM;
    goto end_playback;
  }
  pPager->journalOff = 0;

  /* This loop terminates either when the readJournalHdr() call returns
  ** SQLITE_DONE or an IO error occurs. */
  while( 1 ){

    /* Read the next journal header from the journal file.  If there are
    ** not enough bytes left in the journal file for a complete header, or
................................................................................
        ** Open the journal for read/write access. This is because in 
        ** exclusive-access mode the file descriptor will be kept open and
        ** possibly used for a transaction later on. On some systems, the
        ** OsTruncate() call used in exclusive-access mode also requires
        ** a read/write file handle.
        */
        if( !isHot ){
          int res = sqlite3OsAccess(pVfs,pPager->zJournal,SQLITE_ACCESS_EXISTS);
          if( res==1 ){
            int fout = 0;
            int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL;
            assert( !pPager->tempFile );
            rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout);
            assert( rc!=SQLITE_OK || pPager->jfd->pMethods );
            if( fout&SQLITE_OPEN_READONLY ){
              rc = SQLITE_BUSY;
              sqlite3OsClose(pPager->jfd);
            }
          }else{
            rc = (res==0?SQLITE_BUSY:SQLITE_IOERR_NOMEM);
          }
        }
        if( rc!=SQLITE_OK ){
          pager_unlock(pPager);
          switch( rc ){
            case SQLITE_NOMEM:
            case SQLITE_IOERR_UNLOCK:

Changes to src/vdbemem.c.

80
81
82
83
84
85
86
87
88

89
90
91
92
93
94
95
    ((pMem->flags&MEM_Static) ? 1 : 0)
  );

  if( !pMem->zMalloc || sqlite3MallocSize(pMem->zMalloc)<n ){
    n = (n>32?n:32);
    if( preserve && pMem->z==pMem->zMalloc ){
      pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n);
      if ( !pMem->z )
        pMem->flags = MEM_Null;

      preserve = 0;
    }else{
      sqlite3_free(pMem->zMalloc);
      pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n);
    }
  }








|

>







80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
    ((pMem->flags&MEM_Static) ? 1 : 0)
  );

  if( !pMem->zMalloc || sqlite3MallocSize(pMem->zMalloc)<n ){
    n = (n>32?n:32);
    if( preserve && pMem->z==pMem->zMalloc ){
      pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n);
      if( !pMem->z ){
        pMem->flags = MEM_Null;
      }
      preserve = 0;
    }else{
      sqlite3_free(pMem->zMalloc);
      pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n);
    }
  }

Changes to test/malloc.test.

12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
..
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
# This file attempts to check the behavior of the SQLite library in 
# an out-of-memory situation. When compiled with -DSQLITE_DEBUG=1, 
# the SQLite library accepts a special command (sqlite3_memdebug_fail N C)
# which causes the N-th malloc to fail.  This special feature is used
# to see what happens in the library if a malloc were to really fail
# due to an out-of-memory situation.
#
# $Id: malloc.test,v 1.59 2008/02/18 22:24:58 drh Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Only run these tests if memory debugging is turned on.
#
source $testdir/malloc_common.tcl
................................................................................
# Do a couple of memory dumps just to exercise the memory dump logic
# that that we can say that we have.
#
puts stderr "This is a test.  Ignore the error that follows:"
sqlite3_memdebug_dump $testdir
puts "Memory dump to file memdump.txt..."
sqlite3_memdebug_dump memdump.txt


ifcapable bloblit&&subquery {
  do_malloc_test 1 -tclprep {
    db close
  } -tclbody {
    if {[catch {sqlite3 db test.db}]} {
      error "out of memory"







|







 







<







12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
..
33
34
35
36
37
38
39

40
41
42
43
44
45
46
# This file attempts to check the behavior of the SQLite library in 
# an out-of-memory situation. When compiled with -DSQLITE_DEBUG=1, 
# the SQLite library accepts a special command (sqlite3_memdebug_fail N C)
# which causes the N-th malloc to fail.  This special feature is used
# to see what happens in the library if a malloc were to really fail
# due to an out-of-memory situation.
#
# $Id: malloc.test,v 1.60 2008/04/03 10:13:01 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Only run these tests if memory debugging is turned on.
#
source $testdir/malloc_common.tcl
................................................................................
# Do a couple of memory dumps just to exercise the memory dump logic
# that that we can say that we have.
#
puts stderr "This is a test.  Ignore the error that follows:"
sqlite3_memdebug_dump $testdir
puts "Memory dump to file memdump.txt..."
sqlite3_memdebug_dump memdump.txt


ifcapable bloblit&&subquery {
  do_malloc_test 1 -tclprep {
    db close
  } -tclbody {
    if {[catch {sqlite3 db test.db}]} {
      error "out of memory"