SQLite

Check-in [cb72ee0478]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the SQLITE_CONFIG_MEMDB_MAXSIZE configuration option for configuring the default maximum size of an in-memory database created using sqlite3_deserialize(). This is necessary to make the interface reasonably testable.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: cb72ee0478ce98c48aae059fd5de4e36caf2b8c953e08fcb799bfd119ad46b73
User & Date: drh 2019-01-31 15:38:53.697
Context
2019-01-31
16:09
Remove an incorrect assert() from the deserialize in-memory database VFS. (check-in: 80151d7e3b user: drh tags: trunk)
15:38
Add the SQLITE_CONFIG_MEMDB_MAXSIZE configuration option for configuring the default maximum size of an in-memory database created using sqlite3_deserialize(). This is necessary to make the interface reasonably testable. (check-in: cb72ee0478 user: drh tags: trunk)
14:37
Fix another buffer overrun that could occur when quering a corrupt database using an fts5vocab table. (check-in: b80119bd60 user: dan tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/global.c.
185
186
187
188
189
190
191







192
193
194
195
196
197
198
** sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE);
*/
#ifndef SQLITE_DEFAULT_LOOKASIDE
# define SQLITE_DEFAULT_LOOKASIDE 1200,100
#endif









/*
** The following singleton contains the global configuration for
** the SQLite library.
*/
SQLITE_WSD struct Sqlite3Config sqlite3Config = {
   SQLITE_DEFAULT_MEMSTATUS,  /* bMemstat */
   1,                         /* bCoreMutex */







>
>
>
>
>
>
>







185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
** sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE);
*/
#ifndef SQLITE_DEFAULT_LOOKASIDE
# define SQLITE_DEFAULT_LOOKASIDE 1200,100
#endif


/* The default maximum size of an in-memory database created using
** sqlite3_deserialize()
*/
#ifndef SQLITE_MEMDB_DEFAULT_MAXSIZE
# define SQLITE_MEMDB_DEFAULT_MAXSIZE 1073741824
#endif

/*
** The following singleton contains the global configuration for
** the SQLite library.
*/
SQLITE_WSD struct Sqlite3Config sqlite3Config = {
   SQLITE_DEFAULT_MEMSTATUS,  /* bMemstat */
   1,                         /* bCoreMutex */
232
233
234
235
236
237
238



239
240
241
242
243
244
245
246
247
248
249
250
251
252
   0,                         /* xSqllog */
   0,                         /* pSqllogArg */
#endif
#ifdef SQLITE_VDBE_COVERAGE
   0,                         /* xVdbeBranch */
   0,                         /* pVbeBranchArg */
#endif



#ifndef SQLITE_UNTESTABLE
   0,                         /* xTestCallback */
#endif
   0,                         /* bLocaltimeFault */
   0,                         /* bInternalFunctions */
   0x7ffffffe,                /* iOnceResetThreshold */
   SQLITE_DEFAULT_SORTERREF_SIZE   /* szSorterRef */
};

/*
** Hash table for global functions - functions common to all
** database connections.  After initialization, this table is
** read-only.
*/







>
>
>






|







239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
   0,                         /* xSqllog */
   0,                         /* pSqllogArg */
#endif
#ifdef SQLITE_VDBE_COVERAGE
   0,                         /* xVdbeBranch */
   0,                         /* pVbeBranchArg */
#endif
#ifdef SQLITE_ENABLE_DESERIALIZE
   SQLITE_MEMDB_DEFAULT_MAXSIZE,   /* mxMemdbSize */
#endif
#ifndef SQLITE_UNTESTABLE
   0,                         /* xTestCallback */
#endif
   0,                         /* bLocaltimeFault */
   0,                         /* bInternalFunctions */
   0x7ffffffe,                /* iOnceResetThreshold */
   SQLITE_DEFAULT_SORTERREF_SIZE,   /* szSorterRef */
};

/*
** Hash table for global functions - functions common to all
** database connections.  After initialization, this table is
** read-only.
*/
Changes to src/main.c.
649
650
651
652
653
654
655







656
657
658
659
660
661
662
        iVal = SQLITE_DEFAULT_SORTERREF_SIZE;
      }
      sqlite3GlobalConfig.szSorterRef = (u32)iVal;
      break;
    }
#endif /* SQLITE_ENABLE_SORTER_REFERENCES */








    default: {
      rc = SQLITE_ERROR;
      break;
    }
  }
  va_end(ap);
  return rc;







>
>
>
>
>
>
>







649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
        iVal = SQLITE_DEFAULT_SORTERREF_SIZE;
      }
      sqlite3GlobalConfig.szSorterRef = (u32)iVal;
      break;
    }
#endif /* SQLITE_ENABLE_SORTER_REFERENCES */

#ifdef SQLITE_ENABLE_DESERIALIZE
    case SQLITE_CONFIG_MEMDB_MAXSIZE: {
      sqlite3GlobalConfig.mxMemdbSize = va_arg(ap, sqlite3_int64);
      break;
    }
#endif /* SQLITE_ENABLE_DESERIALIZE */

    default: {
      rc = SQLITE_ERROR;
      break;
    }
  }
  va_end(ap);
  return rc;
Changes to src/memdb.c.
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
  sqlite3_int64 szMax;            /* Maximum allowed size of the file */
  unsigned char *aData;           /* content of the file */
  int nMmap;                      /* Number of memory mapped pages */
  unsigned mFlags;                /* Flags */
  int eLock;                      /* Most recent lock against this file */
};

/* The default maximum size of an in-memory database */
#ifndef SQLITE_MEMDB_DEFAULT_MAXSIZE
# define SQLITE_MEMDB_DEFAULT_MAXSIZE 1073741824
#endif

/*
** Methods for MemFile
*/
static int memdbClose(sqlite3_file*);
static int memdbRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
static int memdbWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
static int memdbTruncate(sqlite3_file*, sqlite3_int64 size);







<
<
<
<
<







38
39
40
41
42
43
44





45
46
47
48
49
50
51
  sqlite3_int64 szMax;            /* Maximum allowed size of the file */
  unsigned char *aData;           /* content of the file */
  int nMmap;                      /* Number of memory mapped pages */
  unsigned mFlags;                /* Flags */
  int eLock;                      /* Most recent lock against this file */
};






/*
** Methods for MemFile
*/
static int memdbClose(sqlite3_file*);
static int memdbRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
static int memdbWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
static int memdbTruncate(sqlite3_file*, sqlite3_int64 size);
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
    return ORIGVFS(pVfs)->xOpen(ORIGVFS(pVfs), zName, pFile, flags, pOutFlags);
  }
  memset(p, 0, sizeof(*p));
  p->mFlags = SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE;
  assert( pOutFlags!=0 );  /* True because flags==SQLITE_OPEN_MAIN_DB */
  *pOutFlags = flags | SQLITE_OPEN_MEMORY;
  p->base.pMethods = &memdb_io_methods;
  p->szMax = SQLITE_MEMDB_DEFAULT_MAXSIZE;
  return SQLITE_OK;
}

#if 0 /* Only used to delete rollback journals, master journals, and WAL
      ** files, none of which exist in memdb.  So this routine is never used */
/*
** Delete the file located at zPath. If the dirSync argument is true,







|







337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
    return ORIGVFS(pVfs)->xOpen(ORIGVFS(pVfs), zName, pFile, flags, pOutFlags);
  }
  memset(p, 0, sizeof(*p));
  p->mFlags = SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE;
  assert( pOutFlags!=0 );  /* True because flags==SQLITE_OPEN_MAIN_DB */
  *pOutFlags = flags | SQLITE_OPEN_MEMORY;
  p->base.pMethods = &memdb_io_methods;
  p->szMax = sqlite3GlobalConfig.mxMemdbSize;
  return SQLITE_OK;
}

#if 0 /* Only used to delete rollback journals, master journals, and WAL
      ** files, none of which exist in memdb.  So this routine is never used */
/*
** Delete the file located at zPath. If the dirSync argument is true,
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
  if( p==0 ){
    rc = SQLITE_ERROR;
  }else{
    p->aData = pData;
    p->sz = szDb;
    p->szAlloc = szBuf;
    p->szMax = szBuf;
    if( p->szMax<SQLITE_MEMDB_DEFAULT_MAXSIZE ){
      p->szMax = SQLITE_MEMDB_DEFAULT_MAXSIZE;
    }
    p->mFlags = mFlags;
    rc = SQLITE_OK;
  }

end_deserialize:
  sqlite3_finalize(pStmt);







|
|







589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
  if( p==0 ){
    rc = SQLITE_ERROR;
  }else{
    p->aData = pData;
    p->sz = szDb;
    p->szAlloc = szBuf;
    p->szMax = szBuf;
    if( p->szMax<sqlite3GlobalConfig.mxMemdbSize ){
      p->szMax = sqlite3GlobalConfig.mxMemdbSize;
    }
    p->mFlags = mFlags;
    rc = SQLITE_OK;
  }

end_deserialize:
  sqlite3_finalize(pStmt);
Changes to src/sqlite.h.in.
1978
1979
1980
1981
1982
1983
1984











1985
1986
1987
1988
1989
1990
1991
** than the configured sorter-reference size threshold - then a reference
** is stored in each sorted record and the required column values loaded
** from the database as records are returned in sorted order. The default
** value for this option is to never use this optimization. Specifying a 
** negative value for this option restores the default behaviour.
** This option is only available if SQLite is compiled with the
** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option.











** </dl>
*/
#define SQLITE_CONFIG_SINGLETHREAD  1  /* nil */
#define SQLITE_CONFIG_MULTITHREAD   2  /* nil */
#define SQLITE_CONFIG_SERIALIZED    3  /* nil */
#define SQLITE_CONFIG_MALLOC        4  /* sqlite3_mem_methods* */
#define SQLITE_CONFIG_GETMALLOC     5  /* sqlite3_mem_methods* */







>
>
>
>
>
>
>
>
>
>
>







1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
** than the configured sorter-reference size threshold - then a reference
** is stored in each sorted record and the required column values loaded
** from the database as records are returned in sorted order. The default
** value for this option is to never use this optimization. Specifying a 
** negative value for this option restores the default behaviour.
** This option is only available if SQLite is compiled with the
** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option.
**
** [[SQLITE_CONFIG_MEMDB_MAXSIZE]]
** <dt>SQLITE_CONFIG_MEMDB_MAXSIZE
** <dd>The SQLITE_CONFIG_MEMDB_MAXSIZE option accepts a single parameter
** [sqlite3_int64] parameter which is the default maximum size for an in-memory
** database created using [sqlite3_deserialize()].  This default maximum
** size can be adjusted up or down for individual databases using the
** [SQLITE_FCNTL_SIZE_LIMIT] [sqlite3_file_control|file-control].  If this
** configuration setting is never used, then the default maximum is determined
** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option.  If that
** compile-time option is not set, then the default maximum is 1073741824.
** </dl>
*/
#define SQLITE_CONFIG_SINGLETHREAD  1  /* nil */
#define SQLITE_CONFIG_MULTITHREAD   2  /* nil */
#define SQLITE_CONFIG_SERIALIZED    3  /* nil */
#define SQLITE_CONFIG_MALLOC        4  /* sqlite3_mem_methods* */
#define SQLITE_CONFIG_GETMALLOC     5  /* sqlite3_mem_methods* */
2008
2009
2010
2011
2012
2013
2014

2015
2016
2017
2018
2019
2020
2021
#define SQLITE_CONFIG_MMAP_SIZE    22  /* sqlite3_int64, sqlite3_int64 */
#define SQLITE_CONFIG_WIN32_HEAPSIZE      23  /* int nByte */
#define SQLITE_CONFIG_PCACHE_HDRSZ        24  /* int *psz */
#define SQLITE_CONFIG_PMASZ               25  /* unsigned int szPma */
#define SQLITE_CONFIG_STMTJRNL_SPILL      26  /* int nByte */
#define SQLITE_CONFIG_SMALL_MALLOC        27  /* boolean */
#define SQLITE_CONFIG_SORTERREF_SIZE      28  /* int nByte */


/*
** CAPI3REF: Database Connection Configuration Options
**
** These constants are the available integer configuration options that
** can be passed as the second argument to the [sqlite3_db_config()] interface.
**







>







2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
#define SQLITE_CONFIG_MMAP_SIZE    22  /* sqlite3_int64, sqlite3_int64 */
#define SQLITE_CONFIG_WIN32_HEAPSIZE      23  /* int nByte */
#define SQLITE_CONFIG_PCACHE_HDRSZ        24  /* int *psz */
#define SQLITE_CONFIG_PMASZ               25  /* unsigned int szPma */
#define SQLITE_CONFIG_STMTJRNL_SPILL      26  /* int nByte */
#define SQLITE_CONFIG_SMALL_MALLOC        27  /* boolean */
#define SQLITE_CONFIG_SORTERREF_SIZE      28  /* int nByte */
#define SQLITE_CONFIG_MEMDB_MAXSIZE       29  /* sqlite3_int64 */

/*
** CAPI3REF: Database Connection Configuration Options
**
** These constants are the available integer configuration options that
** can be passed as the second argument to the [sqlite3_db_config()] interface.
**
Changes to src/sqliteInt.h.
3424
3425
3426
3427
3428
3429
3430



3431
3432
3433
3434
3435
3436
3437
#ifdef SQLITE_VDBE_COVERAGE
  /* The following callback (if not NULL) is invoked on every VDBE branch
  ** operation.  Set the callback using SQLITE_TESTCTRL_VDBE_COVERAGE.
  */
  void (*xVdbeBranch)(void*,unsigned iSrcLine,u8 eThis,u8 eMx);  /* Callback */
  void *pVdbeBranchArg;                                     /* 1st argument */
#endif



#ifndef SQLITE_UNTESTABLE
  int (*xTestCallback)(int);        /* Invoked by sqlite3FaultSim() */
#endif
  int bLocaltimeFault;              /* True to fail localtime() calls */
  int bInternalFunctions;           /* Internal SQL functions are visible */
  int iOnceResetThreshold;          /* When to reset OP_Once counters */
  u32 szSorterRef;                  /* Min size in bytes to use sorter-refs */







>
>
>







3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
#ifdef SQLITE_VDBE_COVERAGE
  /* The following callback (if not NULL) is invoked on every VDBE branch
  ** operation.  Set the callback using SQLITE_TESTCTRL_VDBE_COVERAGE.
  */
  void (*xVdbeBranch)(void*,unsigned iSrcLine,u8 eThis,u8 eMx);  /* Callback */
  void *pVdbeBranchArg;                                     /* 1st argument */
#endif
#ifdef SQLITE_ENABLE_DESERIALIZE
  sqlite3_int64 mxMemdbSize;        /* Default max memdb size */
#endif
#ifndef SQLITE_UNTESTABLE
  int (*xTestCallback)(int);        /* Invoked by sqlite3FaultSim() */
#endif
  int bLocaltimeFault;              /* True to fail localtime() calls */
  int bInternalFunctions;           /* Internal SQL functions are visible */
  int iOnceResetThreshold;          /* When to reset OP_Once counters */
  u32 szSorterRef;                  /* Min size in bytes to use sorter-refs */