/ Check-in [cb72ee04]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the SQLITE_CONFIG_MEMDB_MAXSIZE configuration option for configuring the default maximum size of an in-memory database created using sqlite3_deserialize(). This is necessary to make the interface reasonably testable.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: cb72ee0478ce98c48aae059fd5de4e36caf2b8c953e08fcb799bfd119ad46b73
User & Date: drh 2019-01-31 15:38:53
Context
2019-01-31
16:09
Remove an incorrect assert() from the deserialize in-memory database VFS. check-in: 80151d7e user: drh tags: trunk
15:38
Add the SQLITE_CONFIG_MEMDB_MAXSIZE configuration option for configuring the default maximum size of an in-memory database created using sqlite3_deserialize(). This is necessary to make the interface reasonably testable. check-in: cb72ee04 user: drh tags: trunk
14:37
Fix another buffer overrun that could occur when quering a corrupt database using an fts5vocab table. check-in: b80119bd user: dan tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/global.c.

   185    185   ** sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE);
   186    186   */
   187    187   #ifndef SQLITE_DEFAULT_LOOKASIDE
   188    188   # define SQLITE_DEFAULT_LOOKASIDE 1200,100
   189    189   #endif
   190    190   
   191    191   
          192  +/* The default maximum size of an in-memory database created using
          193  +** sqlite3_deserialize()
          194  +*/
          195  +#ifndef SQLITE_MEMDB_DEFAULT_MAXSIZE
          196  +# define SQLITE_MEMDB_DEFAULT_MAXSIZE 1073741824
          197  +#endif
          198  +
   192    199   /*
   193    200   ** The following singleton contains the global configuration for
   194    201   ** the SQLite library.
   195    202   */
   196    203   SQLITE_WSD struct Sqlite3Config sqlite3Config = {
   197    204      SQLITE_DEFAULT_MEMSTATUS,  /* bMemstat */
   198    205      1,                         /* bCoreMutex */
................................................................................
   232    239      0,                         /* xSqllog */
   233    240      0,                         /* pSqllogArg */
   234    241   #endif
   235    242   #ifdef SQLITE_VDBE_COVERAGE
   236    243      0,                         /* xVdbeBranch */
   237    244      0,                         /* pVbeBranchArg */
   238    245   #endif
          246  +#ifdef SQLITE_ENABLE_DESERIALIZE
          247  +   SQLITE_MEMDB_DEFAULT_MAXSIZE,   /* mxMemdbSize */
          248  +#endif
   239    249   #ifndef SQLITE_UNTESTABLE
   240    250      0,                         /* xTestCallback */
   241    251   #endif
   242    252      0,                         /* bLocaltimeFault */
   243    253      0,                         /* bInternalFunctions */
   244    254      0x7ffffffe,                /* iOnceResetThreshold */
   245         -   SQLITE_DEFAULT_SORTERREF_SIZE   /* szSorterRef */
          255  +   SQLITE_DEFAULT_SORTERREF_SIZE,   /* szSorterRef */
   246    256   };
   247    257   
   248    258   /*
   249    259   ** Hash table for global functions - functions common to all
   250    260   ** database connections.  After initialization, this table is
   251    261   ** read-only.
   252    262   */

Changes to src/main.c.

   649    649           iVal = SQLITE_DEFAULT_SORTERREF_SIZE;
   650    650         }
   651    651         sqlite3GlobalConfig.szSorterRef = (u32)iVal;
   652    652         break;
   653    653       }
   654    654   #endif /* SQLITE_ENABLE_SORTER_REFERENCES */
   655    655   
          656  +#ifdef SQLITE_ENABLE_DESERIALIZE
          657  +    case SQLITE_CONFIG_MEMDB_MAXSIZE: {
          658  +      sqlite3GlobalConfig.mxMemdbSize = va_arg(ap, sqlite3_int64);
          659  +      break;
          660  +    }
          661  +#endif /* SQLITE_ENABLE_DESERIALIZE */
          662  +
   656    663       default: {
   657    664         rc = SQLITE_ERROR;
   658    665         break;
   659    666       }
   660    667     }
   661    668     va_end(ap);
   662    669     return rc;

Changes to src/memdb.c.

    38     38     sqlite3_int64 szMax;            /* Maximum allowed size of the file */
    39     39     unsigned char *aData;           /* content of the file */
    40     40     int nMmap;                      /* Number of memory mapped pages */
    41     41     unsigned mFlags;                /* Flags */
    42     42     int eLock;                      /* Most recent lock against this file */
    43     43   };
    44     44   
    45         -/* The default maximum size of an in-memory database */
    46         -#ifndef SQLITE_MEMDB_DEFAULT_MAXSIZE
    47         -# define SQLITE_MEMDB_DEFAULT_MAXSIZE 1073741824
    48         -#endif
    49         -
    50     45   /*
    51     46   ** Methods for MemFile
    52     47   */
    53     48   static int memdbClose(sqlite3_file*);
    54     49   static int memdbRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
    55     50   static int memdbWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
    56     51   static int memdbTruncate(sqlite3_file*, sqlite3_int64 size);
................................................................................
   342    337       return ORIGVFS(pVfs)->xOpen(ORIGVFS(pVfs), zName, pFile, flags, pOutFlags);
   343    338     }
   344    339     memset(p, 0, sizeof(*p));
   345    340     p->mFlags = SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE;
   346    341     assert( pOutFlags!=0 );  /* True because flags==SQLITE_OPEN_MAIN_DB */
   347    342     *pOutFlags = flags | SQLITE_OPEN_MEMORY;
   348    343     p->base.pMethods = &memdb_io_methods;
   349         -  p->szMax = SQLITE_MEMDB_DEFAULT_MAXSIZE;
          344  +  p->szMax = sqlite3GlobalConfig.mxMemdbSize;
   350    345     return SQLITE_OK;
   351    346   }
   352    347   
   353    348   #if 0 /* Only used to delete rollback journals, master journals, and WAL
   354    349         ** files, none of which exist in memdb.  So this routine is never used */
   355    350   /*
   356    351   ** Delete the file located at zPath. If the dirSync argument is true,
................................................................................
   594    589     if( p==0 ){
   595    590       rc = SQLITE_ERROR;
   596    591     }else{
   597    592       p->aData = pData;
   598    593       p->sz = szDb;
   599    594       p->szAlloc = szBuf;
   600    595       p->szMax = szBuf;
   601         -    if( p->szMax<SQLITE_MEMDB_DEFAULT_MAXSIZE ){
   602         -      p->szMax = SQLITE_MEMDB_DEFAULT_MAXSIZE;
          596  +    if( p->szMax<sqlite3GlobalConfig.mxMemdbSize ){
          597  +      p->szMax = sqlite3GlobalConfig.mxMemdbSize;
   603    598       }
   604    599       p->mFlags = mFlags;
   605    600       rc = SQLITE_OK;
   606    601     }
   607    602   
   608    603   end_deserialize:
   609    604     sqlite3_finalize(pStmt);

Changes to src/sqlite.h.in.

  1978   1978   ** than the configured sorter-reference size threshold - then a reference
  1979   1979   ** is stored in each sorted record and the required column values loaded
  1980   1980   ** from the database as records are returned in sorted order. The default
  1981   1981   ** value for this option is to never use this optimization. Specifying a 
  1982   1982   ** negative value for this option restores the default behaviour.
  1983   1983   ** This option is only available if SQLite is compiled with the
  1984   1984   ** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option.
         1985  +**
         1986  +** [[SQLITE_CONFIG_MEMDB_MAXSIZE]]
         1987  +** <dt>SQLITE_CONFIG_MEMDB_MAXSIZE
         1988  +** <dd>The SQLITE_CONFIG_MEMDB_MAXSIZE option accepts a single parameter
         1989  +** [sqlite3_int64] parameter which is the default maximum size for an in-memory
         1990  +** database created using [sqlite3_deserialize()].  This default maximum
         1991  +** size can be adjusted up or down for individual databases using the
         1992  +** [SQLITE_FCNTL_SIZE_LIMIT] [sqlite3_file_control|file-control].  If this
         1993  +** configuration setting is never used, then the default maximum is determined
         1994  +** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option.  If that
         1995  +** compile-time option is not set, then the default maximum is 1073741824.
  1985   1996   ** </dl>
  1986   1997   */
  1987   1998   #define SQLITE_CONFIG_SINGLETHREAD  1  /* nil */
  1988   1999   #define SQLITE_CONFIG_MULTITHREAD   2  /* nil */
  1989   2000   #define SQLITE_CONFIG_SERIALIZED    3  /* nil */
  1990   2001   #define SQLITE_CONFIG_MALLOC        4  /* sqlite3_mem_methods* */
  1991   2002   #define SQLITE_CONFIG_GETMALLOC     5  /* sqlite3_mem_methods* */
................................................................................
  2008   2019   #define SQLITE_CONFIG_MMAP_SIZE    22  /* sqlite3_int64, sqlite3_int64 */
  2009   2020   #define SQLITE_CONFIG_WIN32_HEAPSIZE      23  /* int nByte */
  2010   2021   #define SQLITE_CONFIG_PCACHE_HDRSZ        24  /* int *psz */
  2011   2022   #define SQLITE_CONFIG_PMASZ               25  /* unsigned int szPma */
  2012   2023   #define SQLITE_CONFIG_STMTJRNL_SPILL      26  /* int nByte */
  2013   2024   #define SQLITE_CONFIG_SMALL_MALLOC        27  /* boolean */
  2014   2025   #define SQLITE_CONFIG_SORTERREF_SIZE      28  /* int nByte */
         2026  +#define SQLITE_CONFIG_MEMDB_MAXSIZE       29  /* sqlite3_int64 */
  2015   2027   
  2016   2028   /*
  2017   2029   ** CAPI3REF: Database Connection Configuration Options
  2018   2030   **
  2019   2031   ** These constants are the available integer configuration options that
  2020   2032   ** can be passed as the second argument to the [sqlite3_db_config()] interface.
  2021   2033   **

Changes to src/sqliteInt.h.

  3424   3424   #ifdef SQLITE_VDBE_COVERAGE
  3425   3425     /* The following callback (if not NULL) is invoked on every VDBE branch
  3426   3426     ** operation.  Set the callback using SQLITE_TESTCTRL_VDBE_COVERAGE.
  3427   3427     */
  3428   3428     void (*xVdbeBranch)(void*,unsigned iSrcLine,u8 eThis,u8 eMx);  /* Callback */
  3429   3429     void *pVdbeBranchArg;                                     /* 1st argument */
  3430   3430   #endif
         3431  +#ifdef SQLITE_ENABLE_DESERIALIZE
         3432  +  sqlite3_int64 mxMemdbSize;        /* Default max memdb size */
         3433  +#endif
  3431   3434   #ifndef SQLITE_UNTESTABLE
  3432   3435     int (*xTestCallback)(int);        /* Invoked by sqlite3FaultSim() */
  3433   3436   #endif
  3434   3437     int bLocaltimeFault;              /* True to fail localtime() calls */
  3435   3438     int bInternalFunctions;           /* Internal SQL functions are visible */
  3436   3439     int iOnceResetThreshold;          /* When to reset OP_Once counters */
  3437   3440     u32 szSorterRef;                  /* Min size in bytes to use sorter-refs */