Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Avoid attempting to mmap memory from an offset that is not a multiple of the system page size on systems with page sizes larger than 32KB. |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
db7d62c8d58eb1e8654a762c9b199ae4 |
User & Date: | dan 2014-03-24 11:23:17 |
Context
2014-03-24
| ||
15:00 | Fix arithmetic operators so that they do not change the affinity of their input operands. Ticket [a8a0d2996a]. (check-in: 221f8f94 user: drh tags: trunk) | |
12:33 | Cancel column-cache entries that are involved in a comparison operator since the comparison might have forced an affinity change. Originally proposed as a fix for ticket [a8a0d2996a], but later determined to be incorrect. (Closed-Leaf check-in: 0b95b7a8 user: drh tags: tkt-a8a0d2996a) | |
11:23 | Avoid attempting to mmap memory from an offset that is not a multiple of the system page size on systems with page sizes larger than 32KB. (check-in: db7d62c8 user: dan tags: trunk) | |
2014-03-23
| ||
16:29 | Avoid a possible use of an uninitialized variable following an I/O or OOM error. (check-in: 641408a1 user: drh tags: trunk) | |
2014-03-20
| ||
09:42 | Add a test to ensure os_unix.c works with 64KiB OS pages. (Closed-Leaf check-in: e3d2be3b user: dan tags: shm-mapping-fix) | |
Changes
Changes to src/os_unix.c.
︙ | ︙ | |||
319 320 321 322 323 324 325 326 327 328 329 330 331 332 | */ static int posixFchown(int fd, uid_t uid, gid_t gid){ return geteuid() ? 0 : fchown(fd,uid,gid); } /* Forward reference */ static int openDirectory(const char*, int*); /* ** Many system calls are accessed through pointer-to-functions so that ** they may be overridden at runtime to facilitate fault injection during ** testing and sandboxing. The following array holds the names and pointers ** to all overrideable system calls. */ | > | 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 | */ static int posixFchown(int fd, uid_t uid, gid_t gid){ return geteuid() ? 0 : fchown(fd,uid,gid); } /* Forward reference */ static int openDirectory(const char*, int*); static int unixGetpagesize(void); /* ** Many system calls are accessed through pointer-to-functions so that ** they may be overridden at runtime to facilitate fault injection during ** testing and sandboxing. The following array holds the names and pointers ** to all overrideable system calls. */ |
︙ | ︙ | |||
441 442 443 444 445 446 447 448 449 450 451 452 453 454 | #if HAVE_MREMAP { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, #else { "mremap", (sqlite3_syscall_ptr)0, 0 }, #endif #define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) #endif }; /* End of the overrideable system calls */ /* ** This is the xSetSystemCall() method of sqlite3_vfs for all of the ** "unix" VFSes. Return SQLITE_OK opon successfully updating the ** system call pointer, or SQLITE_NOTFOUND if there is no configurable | > > > | 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 | #if HAVE_MREMAP { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, #else { "mremap", (sqlite3_syscall_ptr)0, 0 }, #endif #define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) #endif { "getpagesize", (sqlite3_syscall_ptr)unixGetpagesize, 0 }, #define osGetpagesize ((int(*)(void))aSyscall[24].pCurrent) }; /* End of the overrideable system calls */ /* ** This is the xSetSystemCall() method of sqlite3_vfs for all of the ** "unix" VFSes. Return SQLITE_OK opon successfully updating the ** system call pointer, or SQLITE_NOTFOUND if there is no configurable |
︙ | ︙ | |||
4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 | pShmNode->sharedMask, pShmNode->exclMask)); } #endif return rc; } /* ** Purge the unixShmNodeList list of all entries with unixShmNode.nRef==0. ** ** This is not a VFS shared-memory method; it is a utility function called ** by VFS shared-memory methods. */ static void unixShmPurge(unixFile *pFd){ unixShmNode *p = pFd->pInode->pShmNode; assert( unixMutexHeld() ); if( p && p->nRef==0 ){ int i; assert( p->pInode==pFd->pInode ); sqlite3_mutex_free(p->mutex); | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 | pShmNode->sharedMask, pShmNode->exclMask)); } #endif return rc; } /* ** Return the system page size. ** ** This function should not be called directly by other code in this file. ** Instead, it should be called via macro osGetpagesize(). */ static int unixGetpagesize(void){ #if defined(_BSD_SOURCE) return getpagesize(); #else return (int)sysconf(_SC_PAGESIZE); #endif } /* ** Return the minimum number of 32KB shm regions that should be mapped at ** a time, assuming that each mapping must be an integer multiple of the ** current system page-size. ** ** Usually, this is 1. The exception seems to be systems that are configured ** to use 64KB pages - in this case each mapping must cover at least two ** shm regions. */ static int unixShmRegionPerMap(void){ int shmsz = 32*1024; /* SHM region size */ int pgsz = osGetpagesize(); /* System page size */ assert( ((pgsz-1)&pgsz)==0 ); /* Page size must be a power of 2 */ if( pgsz<shmsz ) return 1; return pgsz/shmsz; } /* ** Purge the unixShmNodeList list of all entries with unixShmNode.nRef==0. ** ** This is not a VFS shared-memory method; it is a utility function called ** by VFS shared-memory methods. */ static void unixShmPurge(unixFile *pFd){ unixShmNode *p = pFd->pInode->pShmNode; assert( unixMutexHeld() ); if( p && p->nRef==0 ){ int nShmPerMap = unixShmRegionPerMap(); int i; assert( p->pInode==pFd->pInode ); sqlite3_mutex_free(p->mutex); for(i=0; i<p->nRegion; i+=nShmPerMap){ if( p->h>=0 ){ osMunmap(p->apRegion[i], p->szRegion); }else{ sqlite3_free(p->apRegion[i]); } } sqlite3_free(p->apRegion); |
︙ | ︙ | |||
4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 | int bExtend, /* True to extend file if necessary */ void volatile **pp /* OUT: Mapped memory */ ){ unixFile *pDbFd = (unixFile*)fd; unixShm *p; unixShmNode *pShmNode; int rc = SQLITE_OK; /* If the shared-memory file has not yet been opened, open it now. */ if( pDbFd->pShm==0 ){ rc = unixOpenSharedMemory(pDbFd); if( rc!=SQLITE_OK ) return rc; } p = pDbFd->pShm; pShmNode = p->pShmNode; sqlite3_mutex_enter(pShmNode->mutex); assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); assert( pShmNode->pInode==pDbFd->pInode ); assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); | > > > > > | | | 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 | int bExtend, /* True to extend file if necessary */ void volatile **pp /* OUT: Mapped memory */ ){ unixFile *pDbFd = (unixFile*)fd; unixShm *p; unixShmNode *pShmNode; int rc = SQLITE_OK; int nShmPerMap = unixShmRegionPerMap(); int nReqRegion; /* If the shared-memory file has not yet been opened, open it now. */ if( pDbFd->pShm==0 ){ rc = unixOpenSharedMemory(pDbFd); if( rc!=SQLITE_OK ) return rc; } p = pDbFd->pShm; pShmNode = p->pShmNode; sqlite3_mutex_enter(pShmNode->mutex); assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); assert( pShmNode->pInode==pDbFd->pInode ); assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); /* Minimum number of regions required to be mapped. */ nReqRegion = ((iRegion+nShmPerMap) / nShmPerMap) * nShmPerMap; if( pShmNode->nRegion<nReqRegion ){ char **apNew; /* New apRegion[] array */ int nByte = nReqRegion*szRegion; /* Minimum required file size */ struct stat sStat; /* Used by fstat() */ pShmNode->szRegion = szRegion; if( pShmNode->h>=0 ){ /* The requested region is not mapped into this processes address space. ** Check to see if it has been allocated (i.e. if the wal-index file is |
︙ | ︙ | |||
4388 4389 4390 4391 4392 4393 4394 | } } } } /* Map the requested memory region into this processes address space. */ apNew = (char **)sqlite3_realloc( | | | > > | > > | > | | 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 | } } } } /* Map the requested memory region into this processes address space. */ apNew = (char **)sqlite3_realloc( pShmNode->apRegion, nReqRegion*sizeof(char *) ); if( !apNew ){ rc = SQLITE_IOERR_NOMEM; goto shmpage_out; } pShmNode->apRegion = apNew; while( pShmNode->nRegion<nReqRegion ){ int nMap = szRegion*nShmPerMap; int i; void *pMem; if( pShmNode->h>=0 ){ pMem = osMmap(0, nMap, pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion ); if( pMem==MAP_FAILED ){ rc = unixLogError(SQLITE_IOERR_SHMMAP, "mmap", pShmNode->zFilename); goto shmpage_out; } }else{ pMem = sqlite3_malloc(szRegion); if( pMem==0 ){ rc = SQLITE_NOMEM; goto shmpage_out; } memset(pMem, 0, szRegion); } for(i=0; i<nShmPerMap; i++){ pShmNode->apRegion[pShmNode->nRegion+i] = &((char*)pMem)[szRegion*i]; } pShmNode->nRegion += nShmPerMap; } } shmpage_out: if( pShmNode->nRegion>iRegion ){ *pp = pShmNode->apRegion[iRegion]; }else{ |
︙ | ︙ | |||
4629 4630 4631 4632 4633 4634 4635 | osMunmap(pFd->pMapRegion, pFd->mmapSizeActual); pFd->pMapRegion = 0; pFd->mmapSize = 0; pFd->mmapSizeActual = 0; } } | < < < < < < < < < < < < < | 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 | osMunmap(pFd->pMapRegion, pFd->mmapSizeActual); pFd->pMapRegion = 0; pFd->mmapSize = 0; pFd->mmapSizeActual = 0; } } /* ** Attempt to set the size of the memory mapping maintained by file ** descriptor pFd to nNew bytes. Any existing mapping is discarded. ** ** If successful, this function sets the following variables: ** ** unixFile.pMapRegion |
︙ | ︙ | |||
4678 4679 4680 4681 4682 4683 4684 | assert( nNew>0 ); assert( pFd->mmapSizeActual>=pFd->mmapSize ); assert( MAP_FAILED!=0 ); if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; if( pOrig ){ | > > > | > | 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 | assert( nNew>0 ); assert( pFd->mmapSizeActual>=pFd->mmapSize ); assert( MAP_FAILED!=0 ); if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; if( pOrig ){ #if HAVE_MREMAP i64 nReuse = pFd->mmapSize; #else const int szSyspage = osGetpagesize(); i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); #endif u8 *pReq = &pOrig[nReuse]; /* Unmap any pages of the existing mapping that cannot be reused. */ if( nReuse!=nOrig ){ osMunmap(pReq, nOrig-nReuse); } |
︙ | ︙ | |||
7425 7426 7427 7428 7429 7430 7431 | UNIXVFS("unix-proxy", proxyIoFinder ), #endif }; unsigned int i; /* Loop counter */ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ | | | 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471 7472 7473 7474 7475 | UNIXVFS("unix-proxy", proxyIoFinder ), #endif }; unsigned int i; /* Loop counter */ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ assert( ArraySize(aSyscall)==25 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ sqlite3_vfs_register(&aVfs[i], i==0); } return SQLITE_OK; } |
︙ | ︙ |
Changes to src/test_syscall.c.
︙ | ︙ | |||
63 64 65 66 67 68 69 70 71 72 73 74 75 76 | ** ** test_syscall exists SYSTEM-CALL ** Return true if the named system call exists. Or false otherwise. ** ** test_syscall list ** Return a list of all system calls. The list is constructed using ** the xNextSystemCall() VFS method. */ #include "sqliteInt.h" #include "sqlite3.h" #include "tcl.h" #include <stdlib.h> #include <string.h> | > > > > > | 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | ** ** test_syscall exists SYSTEM-CALL ** Return true if the named system call exists. Or false otherwise. ** ** test_syscall list ** Return a list of all system calls. The list is constructed using ** the xNextSystemCall() VFS method. ** ** test_syscall pagesize PGSZ ** If PGSZ is a power of two greater than 256, install a wrapper around ** OS function getpagesize() that reports the system page size as PGSZ. ** Or, if PGSZ is less than zero, remove any wrapper already installed. */ #include "sqliteInt.h" #include "sqlite3.h" #include "tcl.h" #include <stdlib.h> #include <string.h> |
︙ | ︙ | |||
85 86 87 88 89 90 91 | #include <sys/types.h> #include <errno.h> static struct TestSyscallGlobal { int bPersist; /* 1 for persistent errors, 0 for transient */ int nCount; /* Fail after this many more calls */ int nFail; /* Number of failures that have occurred */ | > > | | 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | #include <sys/types.h> #include <errno.h> static struct TestSyscallGlobal { int bPersist; /* 1 for persistent errors, 0 for transient */ int nCount; /* Fail after this many more calls */ int nFail; /* Number of failures that have occurred */ int pgsz; sqlite3_syscall_ptr orig_getpagesize; } gSyscall = { 0, 0, 0, 0, 0 }; static int ts_open(const char *, int, int); static int ts_close(int fd); static int ts_access(const char *zPath, int mode); static char *ts_getcwd(char *zPath, size_t nPath); static int ts_stat(const char *zPath, struct stat *p); static int ts_fstat(int fd, struct stat *p); |
︙ | ︙ | |||
645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 | return TCL_ERROR; } pVfs = sqlite3_vfs_find(0); Tcl_SetObjResult(interp, Tcl_NewStringObj(pVfs->zName, -1)); return TCL_OK; } static int test_syscall( void * clientData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[] ){ struct SyscallCmd { const char *zName; Tcl_ObjCmdProc *xCmd; } aCmd[] = { { "fault", test_syscall_fault }, { "install", test_syscall_install }, { "uninstall", test_syscall_uninstall }, { "reset", test_syscall_reset }, { "errno", test_syscall_errno }, { "exists", test_syscall_exists }, { "list", test_syscall_list }, { "defaultvfs", test_syscall_defaultvfs }, { 0, 0 } }; int iCmd; int rc; if( objc<2 ){ Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND ..."); | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 | return TCL_ERROR; } pVfs = sqlite3_vfs_find(0); Tcl_SetObjResult(interp, Tcl_NewStringObj(pVfs->zName, -1)); return TCL_OK; } static int ts_getpagesize(void){ return gSyscall.pgsz; } static int test_syscall_pagesize( void * clientData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[] ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(0); int pgsz; if( objc!=3 ){ Tcl_WrongNumArgs(interp, 2, objv, "PGSZ"); return TCL_ERROR; } if( Tcl_GetIntFromObj(interp, objv[2], &pgsz) ){ return TCL_ERROR; } if( pgsz<0 ){ if( gSyscall.orig_getpagesize ){ pVfs->xSetSystemCall(pVfs, "getpagesize", gSyscall.orig_getpagesize); } }else{ if( pgsz<512 || (pgsz & (pgsz-1)) ){ Tcl_AppendResult(interp, "pgsz out of range", 0); return TCL_ERROR; } gSyscall.orig_getpagesize = pVfs->xGetSystemCall(pVfs, "getpagesize"); gSyscall.pgsz = pgsz; pVfs->xSetSystemCall( pVfs, "getpagesize", (sqlite3_syscall_ptr)ts_getpagesize ); } return TCL_OK; } static int test_syscall( void * clientData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[] ){ struct SyscallCmd { const char *zName; Tcl_ObjCmdProc *xCmd; } aCmd[] = { { "fault", test_syscall_fault }, { "install", test_syscall_install }, { "uninstall", test_syscall_uninstall }, { "reset", test_syscall_reset }, { "errno", test_syscall_errno }, { "exists", test_syscall_exists }, { "list", test_syscall_list }, { "defaultvfs", test_syscall_defaultvfs }, { "pagesize", test_syscall_pagesize }, { 0, 0 } }; int iCmd; int rc; if( objc<2 ){ Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND ..."); |
︙ | ︙ |
Changes to test/syscall.test.
︙ | ︙ | |||
57 58 59 60 61 62 63 64 65 66 67 68 69 70 | # Tests for the xNextSystemCall method. # foreach s { open close access getcwd stat fstat ftruncate fcntl read pread write pwrite fchmod fallocate pread64 pwrite64 unlink openDirectory mkdir rmdir statvfs fchown umask mmap munmap mremap } { if {[test_syscall exists $s]} {lappend syscall_list $s} } do_test 3.1 { lsort [test_syscall list] } [lsort $syscall_list] #------------------------------------------------------------------------- # This test verifies that if a call to open() fails and errno is set to | > | 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | # Tests for the xNextSystemCall method. # foreach s { open close access getcwd stat fstat ftruncate fcntl read pread write pwrite fchmod fallocate pread64 pwrite64 unlink openDirectory mkdir rmdir statvfs fchown umask mmap munmap mremap getpagesize } { if {[test_syscall exists $s]} {lappend syscall_list $s} } do_test 3.1 { lsort [test_syscall list] } [lsort $syscall_list] #------------------------------------------------------------------------- # This test verifies that if a call to open() fails and errno is set to |
︙ | ︙ |
Added test/wal64k.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | # 2010 April 13 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # This file implements regression tests for SQLite library. The # focus of this file is testing the operation of the library in # "PRAGMA journal_mode=WAL" mode. # set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix wal64k ifcapable !wal {finish_test ; return } db close test_syscall pagesize 65536 sqlite3 db test.db do_execsql_test 1.0 { PRAGMA journal_mode = WAL; CREATE TABLE t1(x); CREATE INDEX i1 ON t1(x); } {wal} do_test 1.1 { file size test.db-shm } {65536} do_test 1.2 { execsql BEGIN while {[file size test.db-shm]==65536} { execsql { INSERT INTO t1 VALUES( randstr(900,1100) ) } } execsql COMMIT file size test.db-shm } {131072} integrity_check 1.3 db close test_syscall pagesize -1 finish_test |