/ Check-in [1b37c4ef]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Memory-mapped I/O is now on by default. The "PRAGMA mmap_limit(N)" can be used to issue a hint to the VFS to limit mmap space to N bytes. The VFS is free to ignore that hint if desired. However, if "PRAGMA mmap_limit(0)" is used, xFetch is never called.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | experimental-mmap
Files: files | file ages | folders
SHA1: 1b37c4effdd03aa2ea938a71b4f22ed27391689b
User & Date: drh 2013-03-25 23:09:28
Context
2013-03-26
01:07
Previous check-in accidently left mmap turned off by default. This checkin fixes that. Unfortunately, shared.test is now segfaulting. All other veryquick tests appear to work, however. check-in: a850c731 user: drh tags: experimental-mmap
2013-03-25
23:09
Memory-mapped I/O is now on by default. The "PRAGMA mmap_limit(N)" can be used to issue a hint to the VFS to limit mmap space to N bytes. The VFS is free to ignore that hint if desired. However, if "PRAGMA mmap_limit(0)" is used, xFetch is never called. check-in: 1b37c4ef user: drh tags: experimental-mmap
20:50
Add munmap and mremap to the set of os interfaces that can be overloaded in os_unix.c. check-in: 8776047b user: drh tags: experimental-mmap
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/btree.c.

2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
  return SQLITE_OK;
}

/*
** Change the limit on the amount of the database file that may be
** memory mapped.
*/
int sqlite3BtreeSetMmapSize(Btree *p, int nMap){
  BtShared *pBt = p->pBt;
  assert( sqlite3_mutex_held(p->db->mutex) );
  sqlite3BtreeEnter(p);
  sqlite3PagerSetMmapsize(pBt->pPager, nMap);
  sqlite3BtreeLeave(p);
  return SQLITE_OK;
}

/*
** Change the way data is synced to disk in order to increase or decrease
** how well the database resists damage due to OS crashes and power







|



|







2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
  return SQLITE_OK;
}

/*
** Change the limit on the amount of the database file that may be
** memory mapped.
*/
int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 mxMmap){
  BtShared *pBt = p->pBt;
  assert( sqlite3_mutex_held(p->db->mutex) );
  sqlite3BtreeEnter(p);
  sqlite3PagerSetMmapLimit(pBt->pPager, mxMmap);
  sqlite3BtreeLeave(p);
  return SQLITE_OK;
}

/*
** Change the way data is synced to disk in order to increase or decrease
** how well the database resists damage due to OS crashes and power

Changes to src/btree.h.

59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
#define BTREE_OMIT_JOURNAL  1  /* Do not create or use a rollback journal */
#define BTREE_MEMORY        2  /* This is an in-memory DB */
#define BTREE_SINGLE        4  /* The file contains at most 1 b-tree */
#define BTREE_UNORDERED     8  /* Use of a hash implementation is OK */

int sqlite3BtreeClose(Btree*);
int sqlite3BtreeSetCacheSize(Btree*,int);
int sqlite3BtreeSetMmapSize(Btree*, int);
int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int);
int sqlite3BtreeSyncDisabled(Btree*);
int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix);
int sqlite3BtreeGetPageSize(Btree*);
int sqlite3BtreeMaxPageCount(Btree*,int);
u32 sqlite3BtreeLastPage(Btree*);
int sqlite3BtreeSecureDelete(Btree*,int);







|







59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
#define BTREE_OMIT_JOURNAL  1  /* Do not create or use a rollback journal */
#define BTREE_MEMORY        2  /* This is an in-memory DB */
#define BTREE_SINGLE        4  /* The file contains at most 1 b-tree */
#define BTREE_UNORDERED     8  /* Use of a hash implementation is OK */

int sqlite3BtreeClose(Btree*);
int sqlite3BtreeSetCacheSize(Btree*,int);
int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64);
int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int);
int sqlite3BtreeSyncDisabled(Btree*);
int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix);
int sqlite3BtreeGetPageSize(Btree*);
int sqlite3BtreeMaxPageCount(Btree*,int);
u32 sqlite3BtreeLastPage(Btree*);
int sqlite3BtreeSecureDelete(Btree*,int);

Changes to src/os_unix.c.

221
222
223
224
225
226
227





228
229
230
231
232
233
234
...
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
....
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
....
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
....
4965
4966
4967
4968
4969
4970
4971

4972
4973
4974
4975
4976
4977
4978
  unsigned short int ctrlFlags;       /* Behavioral bits.  UNIXFILE_* flags */
  int lastErrno;                      /* The unix errno from last I/O error */
  void *lockingContext;               /* Locking style specific state */
  UnixUnusedFd *pUnused;              /* Pre-allocated UnixUnusedFd */
  const char *zPath;                  /* Name of the file */
  unixShm *pShm;                      /* Shared memory segment information */
  int szChunk;                        /* Configured by FCNTL_CHUNK_SIZE */





#ifdef __QNXNTO__
  int sectorSize;                     /* Device sector size */
  int deviceCharacteristics;          /* Precomputed device characteristics */
#endif
#if SQLITE_ENABLE_LOCKING_STYLE
  int openFlags;                      /* The flags specified at open() */
#endif
................................................................................
  ** one described by ticket #3584. 
  */
  unsigned char transCntrChng;   /* True if the transaction counter changed */
  unsigned char dbUpdate;        /* True if any part of database file changed */
  unsigned char inNormalWrite;   /* True if in a normal write operation */

#endif
  sqlite3_int64 mmapSize;         /* Usable size of mapping at pMapRegion */
  sqlite3_int64 mmapOrigsize;     /* Actual size of mapping at pMapRegion */
  sqlite3_int64 mmapLimit;        /* Configured FCNTL_MMAP_SIZE value */
  void *pMapRegion;               /* Memory mapped region */
  int nFetchOut;                  /* Number of outstanding xFetch refs */

#ifdef SQLITE_TEST
  /* In test mode, increase the size of this structure a bit so that 
  ** it is larger than the struct CrashFile defined in test6.c.
  */
  char aPadding[32];
#endif
................................................................................
      char *zTFile = sqlite3_malloc( pFile->pVfs->mxPathname );
      if( zTFile ){
        unixGetTempname(pFile->pVfs->mxPathname, zTFile);
        *(char**)pArg = zTFile;
      }
      return SQLITE_OK;
    }
    case SQLITE_FCNTL_MMAP_SIZE: {
      pFile->mmapLimit = *(i64*)pArg;
      return SQLITE_OK;
    }
#ifdef SQLITE_DEBUG
    /* The pager calls this method to signal that it has done
    ** a rollback and that the database is therefore unchanged and
    ** it hence it is OK for the transaction change counter to be
................................................................................
** there already exists a mapping for this file, and there are still 
** outstanding xFetch() references to it, this function is a no-op.
**
** If parameter nByte is non-negative, then it is the requested size of 
** the mapping to create. Otherwise, if nByte is less than zero, then the 
** requested size is the size of the file on disk. The actual size of the
** created mapping is either the requested size or the value configured 
** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller.
**
** SQLITE_OK is returned if no error occurs (even if the mapping is not
** recreated as a result of outstanding references) or an SQLite error
** code otherwise.
*/
static int unixMapfile(unixFile *pFd, i64 nByte){
  i64 nMap = nByte;
................................................................................
  assert( zFilename!=0 || (ctrlFlags & UNIXFILE_NOLOCK)!=0 );

  OSTRACE(("OPEN    %-3d %s\n", h, zFilename));
  pNew->h = h;
  pNew->pVfs = pVfs;
  pNew->zPath = zFilename;
  pNew->ctrlFlags = (u8)ctrlFlags;

  if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0),
                           "psow", SQLITE_POWERSAFE_OVERWRITE) ){
    pNew->ctrlFlags |= UNIXFILE_PSOW;
  }
  if( strcmp(pVfs->zName,"unix-excl")==0 ){
    pNew->ctrlFlags |= UNIXFILE_EXCL;
  }







>
>
>
>
>







 







<
<
<
<
<







 







|







 







|







 







>







221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
...
252
253
254
255
256
257
258





259
260
261
262
263
264
265
....
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
....
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
....
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
  unsigned short int ctrlFlags;       /* Behavioral bits.  UNIXFILE_* flags */
  int lastErrno;                      /* The unix errno from last I/O error */
  void *lockingContext;               /* Locking style specific state */
  UnixUnusedFd *pUnused;              /* Pre-allocated UnixUnusedFd */
  const char *zPath;                  /* Name of the file */
  unixShm *pShm;                      /* Shared memory segment information */
  int szChunk;                        /* Configured by FCNTL_CHUNK_SIZE */
  int nFetchOut;                      /* Number of outstanding xFetch refs */
  sqlite3_int64 mmapSize;             /* Usable size of mapping at pMapRegion */
  sqlite3_int64 mmapOrigsize;         /* Actual size of mapping at pMapRegion */
  sqlite3_int64 mmapLimit;            /* Configured FCNTL_MMAP_LIMIT value */
  void *pMapRegion;                   /* Memory mapped region */
#ifdef __QNXNTO__
  int sectorSize;                     /* Device sector size */
  int deviceCharacteristics;          /* Precomputed device characteristics */
#endif
#if SQLITE_ENABLE_LOCKING_STYLE
  int openFlags;                      /* The flags specified at open() */
#endif
................................................................................
  ** one described by ticket #3584. 
  */
  unsigned char transCntrChng;   /* True if the transaction counter changed */
  unsigned char dbUpdate;        /* True if any part of database file changed */
  unsigned char inNormalWrite;   /* True if in a normal write operation */

#endif






#ifdef SQLITE_TEST
  /* In test mode, increase the size of this structure a bit so that 
  ** it is larger than the struct CrashFile defined in test6.c.
  */
  char aPadding[32];
#endif
................................................................................
      char *zTFile = sqlite3_malloc( pFile->pVfs->mxPathname );
      if( zTFile ){
        unixGetTempname(pFile->pVfs->mxPathname, zTFile);
        *(char**)pArg = zTFile;
      }
      return SQLITE_OK;
    }
    case SQLITE_FCNTL_MMAP_LIMIT: {
      pFile->mmapLimit = *(i64*)pArg;
      return SQLITE_OK;
    }
#ifdef SQLITE_DEBUG
    /* The pager calls this method to signal that it has done
    ** a rollback and that the database is therefore unchanged and
    ** it hence it is OK for the transaction change counter to be
................................................................................
** there already exists a mapping for this file, and there are still 
** outstanding xFetch() references to it, this function is a no-op.
**
** If parameter nByte is non-negative, then it is the requested size of 
** the mapping to create. Otherwise, if nByte is less than zero, then the 
** requested size is the size of the file on disk. The actual size of the
** created mapping is either the requested size or the value configured 
** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller.
**
** SQLITE_OK is returned if no error occurs (even if the mapping is not
** recreated as a result of outstanding references) or an SQLite error
** code otherwise.
*/
static int unixMapfile(unixFile *pFd, i64 nByte){
  i64 nMap = nByte;
................................................................................
  assert( zFilename!=0 || (ctrlFlags & UNIXFILE_NOLOCK)!=0 );

  OSTRACE(("OPEN    %-3d %s\n", h, zFilename));
  pNew->h = h;
  pNew->pVfs = pVfs;
  pNew->zPath = zFilename;
  pNew->ctrlFlags = (u8)ctrlFlags;
  pNew->mmapLimit = SQLITE_DEFAULT_MMAP_LIMIT;
  if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0),
                           "psow", SQLITE_POWERSAFE_OVERWRITE) ){
    pNew->ctrlFlags |= UNIXFILE_PSOW;
  }
  if( strcmp(pVfs->zName,"unix-excl")==0 ){
    pNew->ctrlFlags |= UNIXFILE_EXCL;
  }

Changes to src/pager.c.

653
654
655
656
657
658
659
660
661

662
663
664
665
666
667
668
....
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373

3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
  i64 journalHdr;             /* Byte offset to previous journal header */
  sqlite3_backup *pBackup;    /* Pointer to list of ongoing backup processes */
  PagerSavepoint *aSavepoint; /* Array of active savepoints */
  int nSavepoint;             /* Number of elements in aSavepoint[] */
  char dbFileVers[16];        /* Changes whenever database file changes */

  u8 bUseFetch;               /* True to use xFetch() */
  int nMapCfgLimit;           /* Configured limit value */
  int nMmapOut;               /* Number of mmap pages currently outstanding */

  PgHdr *pFree;               /* List of free mmap page headers (pDirty) */
  /*
  ** End of the routinely-changing class members
  ***************************************************************************/

  u16 nExtra;                 /* Add this many bytes to each in-memory page */
  i16 nReserve;               /* Number of unused bytes at end of each page */
................................................................................
** Change the maximum number of in-memory pages that are allowed.
*/
void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){
  sqlite3PcacheSetCachesize(pPager->pPCache, mxPage);
}

/*
** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of nMapCfgLimit.
*/
static void pagerFixMaplimit(Pager *pPager){
  sqlite3_file *fd = pPager->fd;
  if( isOpen(fd) ){
    pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->nMapCfgLimit!=0;
    if( pPager->bUseFetch ){
      void *p;
      i64 nMapLimit;
      if( pPager->nMapCfgLimit<0 ){
        nMapLimit = (i64)pPager->nMapCfgLimit * -1024;
      }else{
        nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize;
      }

      p = (void *)&nMapLimit;
      sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, p);

    }
  }
}

/*
** Change the maximum size of any memory mapping made of the database file.
*/
void sqlite3PagerSetMmapsize(Pager *pPager, int nMap){
  pPager->nMapCfgLimit = nMap;
  pagerFixMaplimit(pPager);
}

/*
** Free as much memory as possible from the pager.
*/
void sqlite3PagerShrink(Pager *pPager){







<

>







 







|




|

<
<
<
<
<
<
<
<
<
|
>







|
|







653
654
655
656
657
658
659

660
661
662
663
664
665
666
667
668
....
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363









3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
  i64 journalHdr;             /* Byte offset to previous journal header */
  sqlite3_backup *pBackup;    /* Pointer to list of ongoing backup processes */
  PagerSavepoint *aSavepoint; /* Array of active savepoints */
  int nSavepoint;             /* Number of elements in aSavepoint[] */
  char dbFileVers[16];        /* Changes whenever database file changes */

  u8 bUseFetch;               /* True to use xFetch() */

  int nMmapOut;               /* Number of mmap pages currently outstanding */
  sqlite3_int64 mxMmap;       /* Desired maximum mmap size */
  PgHdr *pFree;               /* List of free mmap page headers (pDirty) */
  /*
  ** End of the routinely-changing class members
  ***************************************************************************/

  u16 nExtra;                 /* Add this many bytes to each in-memory page */
  i16 nReserve;               /* Number of unused bytes at end of each page */
................................................................................
** Change the maximum number of in-memory pages that are allowed.
*/
void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){
  sqlite3PcacheSetCachesize(pPager->pPCache, mxPage);
}

/*
** Invoke SQLITE_FCNTL_MMAP_LIMIT based on the current value of mxMmap.
*/
static void pagerFixMaplimit(Pager *pPager){
  sqlite3_file *fd = pPager->fd;
  if( isOpen(fd) ){
    pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->mxMmap>0;
    if( pPager->bUseFetch ){









      sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT,
                               (void*)&pPager->mxMmap);
    }
  }
}

/*
** Change the maximum size of any memory mapping made of the database file.
*/
void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 mxMmap){
  pPager->mxMmap = mxMmap;
  pagerFixMaplimit(pPager);
}

/*
** Free as much memory as possible from the pager.
*/
void sqlite3PagerShrink(Pager *pPager){

Changes to src/pager.h.

104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
int sqlite3PagerReadFileheader(Pager*, int, unsigned char*);

/* Functions used to configure a Pager object. */
void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *);
int sqlite3PagerSetPagesize(Pager*, u32*, int);
int sqlite3PagerMaxPageCount(Pager*, int);
void sqlite3PagerSetCachesize(Pager*, int);
void sqlite3PagerSetMmapsize(Pager *, int);
void sqlite3PagerShrink(Pager*);
void sqlite3PagerSetSafetyLevel(Pager*,int,int,int);
int sqlite3PagerLockingMode(Pager *, int);
int sqlite3PagerSetJournalMode(Pager *, int);
int sqlite3PagerGetJournalMode(Pager*);
int sqlite3PagerOkToChangeJournalMode(Pager*);
i64 sqlite3PagerJournalSizeLimit(Pager *, i64);







|







104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
int sqlite3PagerReadFileheader(Pager*, int, unsigned char*);

/* Functions used to configure a Pager object. */
void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *);
int sqlite3PagerSetPagesize(Pager*, u32*, int);
int sqlite3PagerMaxPageCount(Pager*, int);
void sqlite3PagerSetCachesize(Pager*, int);
void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64);
void sqlite3PagerShrink(Pager*);
void sqlite3PagerSetSafetyLevel(Pager*,int,int,int);
int sqlite3PagerLockingMode(Pager *, int);
int sqlite3PagerSetJournalMode(Pager *, int);
int sqlite3PagerGetJournalMode(Pager*);
int sqlite3PagerOkToChangeJournalMode(Pager*);
i64 sqlite3PagerJournalSizeLimit(Pager *, i64);

Changes to src/pragma.c.

741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758



759
760
761
762

763
764
765
766
767
768
769
770
771
772
773
774
      int size = sqlite3Atoi(zRight);
      pDb->pSchema->cache_size = size;
      sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size);
    }
  }else

  /*
  **  PRAGMA [database.]mmap_size
  **  PRAGMA [database.]mmap_size=N
  **
  ** Used to set or query the mapping size limit. The mapping size limit is
  ** used to limit the aggregate size of all memory mapped regions of the
  ** database file. If this parameter is set to zero, then memory mapping
  ** is not used at all. If it is set to a positive value, then it is
  ** interpreted as a maximum size in pages. If set to less than zero, then
  ** the absolute value is interpreted as a size limit in KB.
  **
  ** The default value is zero (do not use memory mapped IO).



  */
  if( sqlite3StrICmp(zLeft,"mmap_size")==0 ){
    assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
    if( !zRight ){

      returnSingleInt(pParse, "mmap_size", pDb->pSchema->mmap_size);
    }else{
      int size = sqlite3Atoi(zRight);
      pDb->pSchema->mmap_size = size;
      sqlite3BtreeSetMmapSize(pDb->pBt, pDb->pSchema->mmap_size);
    }
  }else

  /*
  **   PRAGMA temp_store
  **   PRAGMA temp_store = "default"|"memory"|"file"
  **







|
<

|


|
<
<

<
>
>
>

|

|
>
|
<
<
<
|







741
742
743
744
745
746
747
748

749
750
751
752
753


754

755
756
757
758
759
760
761
762
763



764
765
766
767
768
769
770
771
      int size = sqlite3Atoi(zRight);
      pDb->pSchema->cache_size = size;
      sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size);
    }
  }else

  /*
  **  PRAGMA [database.]mmap_limit(N)

  **
  ** Used to set mapping size limit. The mapping size limit is
  ** used to limit the aggregate size of all memory mapped regions of the
  ** database file. If this parameter is set to zero, then memory mapping
  ** is not used at all. The parameter N is measured in bytes.


  **

  ** This value is advisory.  The underlying VFS is free to memory map
  ** as little or as much as it wants.  Except, if N is set to 0 then the
  ** upper layers will never invoke the xFetch interfaces to the VFS.
  */
  if( sqlite3StrICmp(zLeft,"mmap_limit")==0 ){
    assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
    if( zRight ){
      sqlite3_int64 size;
      sqlite3Atoi64(zRight, &size, 1000, SQLITE_UTF8);



      sqlite3BtreeSetMmapLimit(pDb->pBt, size);
    }
  }else

  /*
  **   PRAGMA temp_store
  **   PRAGMA temp_store = "default"|"memory"|"file"
  **

Changes to src/sqlite.h.in.

882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
...
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
** ^Application can invoke this file-control to have SQLite generate a
** temporary filename using the same algorithm that is followed to generate
** temporary filenames for TEMP tables and other internal uses.  The
** argument should be a char** which will be filled with the filename
** written into memory obtained from [sqlite3_malloc()].  The caller should
** invoke [sqlite3_free()] on the result to avoid a memory leak.
**
** <li>[[SQLITE_FCNTL_MMAP_SIZE]]
** The argument is assumed to point to a value of type sqlite3_int64. An
** advisory maximum amount of this file to memory map in bytes.
**
** </ul>
*/
#define SQLITE_FCNTL_LOCKSTATE               1
#define SQLITE_GET_LOCKPROXYFILE             2
#define SQLITE_SET_LOCKPROXYFILE             3
#define SQLITE_LAST_ERRNO                    4
................................................................................
#define SQLITE_FCNTL_PERSIST_WAL            10
#define SQLITE_FCNTL_OVERWRITE              11
#define SQLITE_FCNTL_VFSNAME                12
#define SQLITE_FCNTL_POWERSAFE_OVERWRITE    13
#define SQLITE_FCNTL_PRAGMA                 14
#define SQLITE_FCNTL_BUSYHANDLER            15
#define SQLITE_FCNTL_TEMPFILENAME           16
#define SQLITE_FCNTL_MMAP_SIZE              18

/*
** CAPI3REF: Mutex Handle
**
** The mutex module within SQLite defines [sqlite3_mutex] to be an
** abstract type for a mutex object.  The SQLite core never looks
** at the internal representation of an [sqlite3_mutex].  It only







|
|
|







 







|







882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
...
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
** ^Application can invoke this file-control to have SQLite generate a
** temporary filename using the same algorithm that is followed to generate
** temporary filenames for TEMP tables and other internal uses.  The
** argument should be a char** which will be filled with the filename
** written into memory obtained from [sqlite3_malloc()].  The caller should
** invoke [sqlite3_free()] on the result to avoid a memory leak.
**
** <li>[[SQLITE_FCNTL_MMAP_LIMIT]]
** The argument is assumed to pointer to a value of type sqlite3_int64 that
** is an advisory maximum number of bytes in the file to memory map.
**
** </ul>
*/
#define SQLITE_FCNTL_LOCKSTATE               1
#define SQLITE_GET_LOCKPROXYFILE             2
#define SQLITE_SET_LOCKPROXYFILE             3
#define SQLITE_LAST_ERRNO                    4
................................................................................
#define SQLITE_FCNTL_PERSIST_WAL            10
#define SQLITE_FCNTL_OVERWRITE              11
#define SQLITE_FCNTL_VFSNAME                12
#define SQLITE_FCNTL_POWERSAFE_OVERWRITE    13
#define SQLITE_FCNTL_PRAGMA                 14
#define SQLITE_FCNTL_BUSYHANDLER            15
#define SQLITE_FCNTL_TEMPFILENAME           16
#define SQLITE_FCNTL_MMAP_LIMIT             18

/*
** CAPI3REF: Mutex Handle
**
** The mutex module within SQLite defines [sqlite3_mutex] to be an
** abstract type for a mutex object.  The SQLite core never looks
** at the internal representation of an [sqlite3_mutex].  It only

Changes to src/sqliteInt.h.

742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
  Hash trigHash;       /* All triggers indexed by name */
  Hash fkeyHash;       /* All foreign keys by referenced table name */
  Table *pSeqTab;      /* The sqlite_sequence table used by AUTOINCREMENT */
  u8 file_format;      /* Schema format version for this file */
  u8 enc;              /* Text encoding used by this database */
  u16 flags;           /* Flags associated with this schema */
  int cache_size;      /* Number of pages to use in the cache */
  int mmap_size;       /* Number of pages to memory map */
};

/*
** These macros can be used to test, set, or clear bits in the 
** Db.pSchema->flags field.
*/
#define DbHasProperty(D,I,P)     (((D)->aDb[I].pSchema->flags&(P))==(P))







<







742
743
744
745
746
747
748

749
750
751
752
753
754
755
  Hash trigHash;       /* All triggers indexed by name */
  Hash fkeyHash;       /* All foreign keys by referenced table name */
  Table *pSeqTab;      /* The sqlite_sequence table used by AUTOINCREMENT */
  u8 file_format;      /* Schema format version for this file */
  u8 enc;              /* Text encoding used by this database */
  u16 flags;           /* Flags associated with this schema */
  int cache_size;      /* Number of pages to use in the cache */

};

/*
** These macros can be used to test, set, or clear bits in the 
** Db.pSchema->flags field.
*/
#define DbHasProperty(D,I,P)     (((D)->aDb[I].pSchema->flags&(P))==(P))

Changes to src/sqliteLimit.h.

202
203
204
205
206
207
208







** A value of 1 means that a trigger program will not be able to itself
** fire any triggers. A value of 0 means that no trigger programs at all 
** may be executed.
*/
#ifndef SQLITE_MAX_TRIGGER_DEPTH
# define SQLITE_MAX_TRIGGER_DEPTH 1000
#endif














>
>
>
>
>
>
>
202
203
204
205
206
207
208
209
210
211
212
213
214
215
** A value of 1 means that a trigger program will not be able to itself
** fire any triggers. A value of 0 means that no trigger programs at all 
** may be executed.
*/
#ifndef SQLITE_MAX_TRIGGER_DEPTH
# define SQLITE_MAX_TRIGGER_DEPTH 1000
#endif

/*
** Default maximum size of memory used by xFetch in the VFS.
*/
#ifndef SQLITE_DEFAULT_MMAP_LIMIT
# define SQLITE_DEFAULT_MMAP_LIMIT (256*1024*1024)
#endif

Changes to test/dbstatus2.test.

36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
proc db_write {db {reset 0}} {
  sqlite3_db_status $db CACHE_WRITE $reset
}

do_test 1.1 {
  db close
  sqlite3 db test.db
  execsql { PRAGMA mmap_size = 0 }
  expr {[file size test.db] / 1024}
} 6

do_test 1.2 {
  execsql { SELECT b FROM t1 WHERE a=2 }
  db_hit_miss db
} {{0 2 0} {0 4 0}}







|







36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
proc db_write {db {reset 0}} {
  sqlite3_db_status $db CACHE_WRITE $reset
}

do_test 1.1 {
  db close
  sqlite3 db test.db
  execsql { PRAGMA mmap_limit = 0 }
  expr {[file size test.db] / 1024}
} 6

do_test 1.2 {
  execsql { SELECT b FROM t1 WHERE a=2 }
  db_hit_miss db
} {{0 2 0} {0 4 0}}

Changes to test/incrblob.test.

119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
...
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
...
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
...
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
    }
  }

  db close
  forcedelete test.db test.db-journal

  sqlite3 db test.db
  execsql "PRAGMA mmap_size = 0"
  execsql "PRAGMA auto_vacuum = $AutoVacuumMode"

  do_test incrblob-2.$AutoVacuumMode.1 {
    set ::str [string repeat abcdefghij 2900]
    execsql {
      BEGIN;
      CREATE TABLE blobs(k PRIMARY KEY, v BLOB, i INTEGER);
................................................................................
    } $AutoVacuumMode
  }

  do_test incrblob-2.$AutoVacuumMode.3 {
    # Open and close the db to make sure the page cache is empty.
    db close
    sqlite3 db test.db
    execsql "PRAGMA mmap_size = 0"
  
    # Read the last 20 bytes of the blob via a blob handle.
    set ::blob [db incrblob blobs v 1]
    seek $::blob -20 end
    set ::fragment [read $::blob]
    close $::blob
  
................................................................................
    string range [db one {SELECT v FROM blobs}] end-19 end
  } $::fragment

  do_test incrblob-2.$AutoVacuumMode.5 {
    # Open and close the db to make sure the page cache is empty.
    db close
    sqlite3 db test.db
    execsql "PRAGMA mmap_size = 0"
  
    # Write the second-to-last 20 bytes of the blob via a blob handle.
    #
    set ::blob [db incrblob blobs v 1]
    seek $::blob -40 end
    puts -nonewline $::blob "1234567890abcdefghij"
    flush $::blob
................................................................................
    string range [db one {SELECT v FROM blobs}] end-39 end-20
  } "1234567890abcdefghij"

  do_test incrblob-2.$AutoVacuumMode.8 {
    # Open and close the db to make sure the page cache is empty.
    db close
    sqlite3 db test.db
    execsql { PRAGMA mmap_size = 0 }

    execsql { SELECT i FROM blobs } 
  } {45}

  do_test incrblob-2.$AutoVacuumMode.9 {
    nRead db
  } [expr $AutoVacuumMode ? 4 : 30]







|







 







|







 







|







 







|







119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
...
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
...
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
...
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
    }
  }

  db close
  forcedelete test.db test.db-journal

  sqlite3 db test.db
  execsql "PRAGMA mmap_limit = 0"
  execsql "PRAGMA auto_vacuum = $AutoVacuumMode"

  do_test incrblob-2.$AutoVacuumMode.1 {
    set ::str [string repeat abcdefghij 2900]
    execsql {
      BEGIN;
      CREATE TABLE blobs(k PRIMARY KEY, v BLOB, i INTEGER);
................................................................................
    } $AutoVacuumMode
  }

  do_test incrblob-2.$AutoVacuumMode.3 {
    # Open and close the db to make sure the page cache is empty.
    db close
    sqlite3 db test.db
    execsql "PRAGMA mmap_limit = 0"
  
    # Read the last 20 bytes of the blob via a blob handle.
    set ::blob [db incrblob blobs v 1]
    seek $::blob -20 end
    set ::fragment [read $::blob]
    close $::blob
  
................................................................................
    string range [db one {SELECT v FROM blobs}] end-19 end
  } $::fragment

  do_test incrblob-2.$AutoVacuumMode.5 {
    # Open and close the db to make sure the page cache is empty.
    db close
    sqlite3 db test.db
    execsql "PRAGMA mmap_limit = 0"
  
    # Write the second-to-last 20 bytes of the blob via a blob handle.
    #
    set ::blob [db incrblob blobs v 1]
    seek $::blob -40 end
    puts -nonewline $::blob "1234567890abcdefghij"
    flush $::blob
................................................................................
    string range [db one {SELECT v FROM blobs}] end-39 end-20
  } "1234567890abcdefghij"

  do_test incrblob-2.$AutoVacuumMode.8 {
    # Open and close the db to make sure the page cache is empty.
    db close
    sqlite3 db test.db
    execsql { PRAGMA mmap_limit = 0 }

    execsql { SELECT i FROM blobs } 
  } {45}

  do_test incrblob-2.$AutoVacuumMode.9 {
    nRead db
  } [expr $AutoVacuumMode ? 4 : 30]

Changes to test/mmap1.test.

19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
..
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
..
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
...
126
127
128
129
130
131
132
133
  set bt [btree_from_db $db]
  db_enter $db
  array set stats [btree_pager_stats $bt]
  db_leave $db
  return $stats(read)
}

foreach {t mmap_size nRead c2init} {
  1.1 { PRAGMA mmap_size = -65536 }   4 {}
  1.2 { PRAGMA mmap_size = -50    } 156 {}
  1.3 { PRAGMA mmap_size = 0      } 344 {}
  1.4 { PRAGMA mmap_size = -65536 }   4 {PRAGMA mmap_size = -65536}
  1.5 { PRAGMA mmap_size = -50    } 156 {PRAGMA mmap_size = -65536}
  1.6 { PRAGMA mmap_size = 0      } 344 {PRAGMA mmap_size = -65536}
} {
  do_multiclient_test tn {
    sql1 $mmap_size
    sql2 $c2init

    code2 {
      set ::rcnt 0
      proc rblob {n} {
        set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF]
        set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] 
................................................................................
    # Have connection 2 grow the file again. Check connection 1 is still ok.
    sql2 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1 }
    do_test $t.$tn.4 {
      sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count"
    } {64 ok 149}

    # Check that the number of pages read by connection 1 indicates that the
    # "PRAGMA mmap_size" command worked.
    do_test $t.$tn.5 { nRead db } $nRead
  }
}

set ::rcnt 0
proc rblob {n} {
  set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF]
................................................................................
}

reset_db
db func rblob rblob

do_execsql_test 2.1 {
  PRAGMA auto_vacuum = 1;
  PRAGMA mmap_size = -65536;
  PRAGMA journal_mode = wal;
  CREATE TABLE t1(a, b, UNIQUE(a, b));
  INSERT INTO t1 VALUES(rblob(500), rblob(500));
  INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; --    2
  INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; --    4
  INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; --    8
  INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; --   16
................................................................................
} {16}

do_execsql_test 2.4 {
  PRAGMA wal_checkpoint;
} {0 24 24}

finish_test








|
|
|
|
|
|
|


|







 







|







 







|







 







<
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
..
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
..
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
...
126
127
128
129
130
131
132

  set bt [btree_from_db $db]
  db_enter $db
  array set stats [btree_pager_stats $bt]
  db_leave $db
  return $stats(read)
}

foreach {t mmap_limit nRead c2init} {
  1.1 { PRAGMA mmap_limit = 70000000 }   4 {}
  1.2 { PRAGMA mmap_limit =    51200 } 156 {}
  1.3 { PRAGMA mmap_limit =        0 } 344 {}
  1.4 { PRAGMA mmap_limit = 70000000 }   4 {PRAGMA mmap_limit = 70000000 }
  1.5 { PRAGMA mmap_limit =    51200 } 156 {PRAGMA mmap_limit = 70000000 }
  1.6 { PRAGMA mmap_limit =        0 } 344 {PRAGMA mmap_limit = 70000000 }
} {
  do_multiclient_test tn {
    sql1 $mmap_limit
    sql2 $c2init

    code2 {
      set ::rcnt 0
      proc rblob {n} {
        set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF]
        set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] 
................................................................................
    # Have connection 2 grow the file again. Check connection 1 is still ok.
    sql2 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1 }
    do_test $t.$tn.4 {
      sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count"
    } {64 ok 149}

    # Check that the number of pages read by connection 1 indicates that the
    # "PRAGMA mmap_limit" command worked.
    do_test $t.$tn.5 { nRead db } $nRead
  }
}

set ::rcnt 0
proc rblob {n} {
  set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF]
................................................................................
}

reset_db
db func rblob rblob

do_execsql_test 2.1 {
  PRAGMA auto_vacuum = 1;
  PRAGMA mmap_limit = 70000000;
  PRAGMA journal_mode = wal;
  CREATE TABLE t1(a, b, UNIQUE(a, b));
  INSERT INTO t1 VALUES(rblob(500), rblob(500));
  INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; --    2
  INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; --    4
  INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; --    8
  INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; --   16
................................................................................
} {16}

do_execsql_test 2.4 {
  PRAGMA wal_checkpoint;
} {0 24 24}

finish_test

Changes to test/permutations.test.

134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
  "Very" quick test suite. Runs in less than 5 minutes on a workstation. 
  This test suite is the same as the "quick" tests, except that some files
  that test malloc and IO errors are omitted.
} -files [
  test_set $allquicktests -exclude *malloc* *ioerr* *fault*
]

test_suite "mmap" -prefix "mm-" -description {
  Similar to veryquick. Except with memory mapping enabled.
} -presql {
  pragma mmap_size = -65536;
} -files [
  test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test
]

test_suite "valgrind" -prefix "" -description {
  Run the "veryquick" test suite with a couple of multi-process tests (that
  fail under valgrind) omitted.







|
|

|







134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
  "Very" quick test suite. Runs in less than 5 minutes on a workstation. 
  This test suite is the same as the "quick" tests, except that some files
  that test malloc and IO errors are omitted.
} -files [
  test_set $allquicktests -exclude *malloc* *ioerr* *fault*
]

test_suite "no-mmap" -prefix "nomm-" -description {
  Similar to veryquick. Except with memory mapping disabled.
} -presql {
  pragma mmap_size = 0;
} -files [
  test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test
]

test_suite "valgrind" -prefix "" -description {
  Run the "veryquick" test suite with a couple of multi-process tests (that
  fail under valgrind) omitted.

Changes to test/speed1p.test.

61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
  return $txt
}

# Create a database schema.
#
do_test speed1p-1.0 {
  execsql {
    PRAGMA mmap_size=1000000;
    PRAGMA page_size=1024;
    PRAGMA cache_size=500;
    PRAGMA locking_mode=EXCLUSIVE;
    CREATE TABLE t1(a INTEGER, b INTEGER, c TEXT);
    CREATE TABLE t2(a INTEGER, b INTEGER, c TEXT);
    CREATE INDEX i2a ON t2(a);
    CREATE INDEX i2b ON t2(b);







<







61
62
63
64
65
66
67

68
69
70
71
72
73
74
  return $txt
}

# Create a database schema.
#
do_test speed1p-1.0 {
  execsql {

    PRAGMA page_size=1024;
    PRAGMA cache_size=500;
    PRAGMA locking_mode=EXCLUSIVE;
    CREATE TABLE t1(a INTEGER, b INTEGER, c TEXT);
    CREATE TABLE t2(a INTEGER, b INTEGER, c TEXT);
    CREATE INDEX i2a ON t2(a);
    CREATE INDEX i2b ON t2(b);

Changes to test/sysfault.test.

258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
}
faultsim_save_and_close

do_faultsim_test 4 -faults vfsfault-* -prep {
  faultsim_restore_and_reopen
  file_control_chunksize_test db main 8192
  execsql { 
    PRAGMA mmap_size = 1000000;
  }
} -body {
  test_syscall errno mmap     EACCES

  execsql {
    SELECT * FROM t1;
  }
} -test {
  faultsim_test_result {0 {1 2}} {1 {disk I/O error}}
}

finish_test








|












<
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277

}
faultsim_save_and_close

do_faultsim_test 4 -faults vfsfault-* -prep {
  faultsim_restore_and_reopen
  file_control_chunksize_test db main 8192
  execsql { 
    PRAGMA mmap_limit = 1000000;
  }
} -body {
  test_syscall errno mmap     EACCES

  execsql {
    SELECT * FROM t1;
  }
} -test {
  faultsim_test_result {0 {1 2}} {1 {disk I/O error}}
}

finish_test

Changes to test/wal5.test.

237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
...
347
348
349
350
351
352
353
354
    do_test 2.3.$tn.7 { code1 { do_wal_checkpoint db -mode full } } {1 4 3}

    # The checkpoint above only writes page 1 of the db file. The other
    # page (page 2) is locked by the read-transaction opened by the
    # [sql2] commmand above. So normally, the db is 1 page in size here.
    # However, in mmap() mode, the db is pre-allocated to 2 pages at the
    # start of the checkpoint, even though page 2 cannot be written.
    set nDb 1
    if {[permutation]=="mmap"} {set nDb 2}
    do_test 2.3.$tn.8 { file_page_counts } [list $nDb 4 2 4]
  }

  # Check that checkpoints block on the correct locks. And respond correctly
  # if they cannot obtain those locks. There are three locks that a checkpoint
  # may block on (in the following order):
  #
................................................................................

    do_test 3.$tn.6 { code3 { do_wal_checkpoint db3 } } {0 0 0}
  }
}


finish_test








|
|







 







<
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
...
347
348
349
350
351
352
353

    do_test 2.3.$tn.7 { code1 { do_wal_checkpoint db -mode full } } {1 4 3}

    # The checkpoint above only writes page 1 of the db file. The other
    # page (page 2) is locked by the read-transaction opened by the
    # [sql2] commmand above. So normally, the db is 1 page in size here.
    # However, in mmap() mode, the db is pre-allocated to 2 pages at the
    # start of the checkpoint, even though page 2 cannot be written.
    set nDb 2
    if {[permutation]=="no-mmap"} {set nDb 1}
    do_test 2.3.$tn.8 { file_page_counts } [list $nDb 4 2 4]
  }

  # Check that checkpoints block on the correct locks. And respond correctly
  # if they cannot obtain those locks. There are three locks that a checkpoint
  # may block on (in the following order):
  #
................................................................................

    do_test 3.$tn.6 { code3 { do_wal_checkpoint db3 } } {0 0 0}
  }
}


finish_test