/ Check-in [538f7b25]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Experimental change to os_unix.c to delay creating a database file until it is first written.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | deferred-open
Files: files | file ages | folders
SHA1: 538f7b25e436682adba10a158f082de24782cf00
User & Date: dan 2014-02-10 19:37:03
Context
2014-02-10
21:09
Fix a bug causing the POWERSAFE_OVERWRITE device-characteristic flag to be set incorrectly if file opening is deferred. check-in: 95d0c58d user: dan tags: deferred-open
19:37
Experimental change to os_unix.c to delay creating a database file until it is first written. check-in: 538f7b25 user: dan tags: deferred-open
2014-02-08
04:24
Enable constant expression factoring even if no tables are read and no transaction is started. check-in: a45b8771 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/os_unix.c.

   199    199     sqlite3_int64 mmapSizeMax;          /* Configured FCNTL_MMAP_SIZE value */
   200    200     void *pMapRegion;                   /* Memory mapped region */
   201    201   #endif
   202    202   #ifdef __QNXNTO__
   203    203     int sectorSize;                     /* Device sector size */
   204    204     int deviceCharacteristics;          /* Precomputed device characteristics */
   205    205   #endif
   206         -#if SQLITE_ENABLE_LOCKING_STYLE
   207    206     int openFlags;                      /* The flags specified at open() */
   208         -#endif
   209    207   #if SQLITE_ENABLE_LOCKING_STYLE || defined(__APPLE__)
   210    208     unsigned fsFlags;                   /* cached details from statfs() */
   211    209   #endif
   212    210   #if OS_VXWORKS
   213    211     struct vxworksFileId *pId;          /* Unique file ID */
   214    212   #endif
   215    213   #ifdef SQLITE_DEBUG
................................................................................
   251    249   #else
   252    250   # define UNIXFILE_DIRSYNC    0x00
   253    251   #endif
   254    252   #define UNIXFILE_PSOW        0x10     /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */
   255    253   #define UNIXFILE_DELETE      0x20     /* Delete on close */
   256    254   #define UNIXFILE_URI         0x40     /* Filename might have query parameters */
   257    255   #define UNIXFILE_NOLOCK      0x80     /* Do no file locking */
   258         -#define UNIXFILE_WARNED    0x0100     /* verifyDbFile() warnings have been issued */
          256  +#define UNIXFILE_WARNED    0x0100     /* verifyDbFile() has issued warnings */
          257  +#define UNIXFILE_DEFERRED  0x0200     /* File has not yet been opened */
   259    258   
   260    259   /*
   261    260   ** Include code that is common to all os_*.c files
   262    261   */
   263    262   #include "os_common.h"
   264    263   
   265    264   /*
................................................................................
  1353   1352   ** is set to SQLITE_OK unless an I/O error occurs during lock checking.
  1354   1353   */
  1355   1354   static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){
  1356   1355     int rc = SQLITE_OK;
  1357   1356     int reserved = 0;
  1358   1357     unixFile *pFile = (unixFile*)id;
  1359   1358   
         1359  +  assert( pFile );
  1360   1360     SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; );
  1361   1361   
  1362         -  assert( pFile );
         1362  +  if( pFile->ctrlFlags==UNIXFILE_DEFERRED ){
         1363  +    *pResOut = 0;
         1364  +    return SQLITE_OK;
         1365  +  }
         1366  +
  1363   1367     unixEnterMutex(); /* Because pFile->pInode is shared across threads */
  1364   1368   
  1365   1369     /* Check if a thread in this process holds such a lock */
  1366   1370     if( pFile->pInode->eFileLock>SHARED_LOCK ){
  1367   1371       reserved = 1;
  1368   1372     }
  1369   1373   
................................................................................
  1434   1438         rc = 0;
  1435   1439       }
  1436   1440     }else{
  1437   1441       rc = osFcntl(pFile->h, F_SETLK, pLock);
  1438   1442     }
  1439   1443     return rc;
  1440   1444   }
         1445  +
         1446  +static int unixOpen(sqlite3_vfs*, const char*, sqlite3_file*, int, int *);
  1441   1447   
  1442   1448   /*
  1443   1449   ** Lock the file with the lock specified by parameter eFileLock - one
  1444   1450   ** of the following:
  1445   1451   **
  1446   1452   **     (1) SHARED_LOCK
  1447   1453   **     (2) RESERVED_LOCK
................................................................................
  1519   1525     */
  1520   1526     if( pFile->eFileLock>=eFileLock ){
  1521   1527       OSTRACE(("LOCK    %d %s ok (already held) (unix)\n", pFile->h,
  1522   1528               azFileLock(eFileLock)));
  1523   1529       return SQLITE_OK;
  1524   1530     }
  1525   1531   
         1532  +  if( pFile->ctrlFlags==UNIXFILE_DEFERRED ){
         1533  +    int eOrigLock = pFile->eFileLock;
         1534  +    if( eFileLock==SHARED_LOCK ){
         1535  +      int statrc;
         1536  +      struct stat sBuf;
         1537  +      memset(&sBuf, 0, sizeof(sBuf));
         1538  +      statrc = osStat(pFile->zPath, &sBuf);
         1539  +      if( statrc && errno==ENOENT ){
         1540  +        pFile->eFileLock = SHARED_LOCK;
         1541  +        return SQLITE_OK;
         1542  +      }
         1543  +    }
         1544  +
         1545  +    rc = unixOpen(pFile->pVfs, pFile->zPath, id, pFile->openFlags, 0);
         1546  +    if( rc==SQLITE_OK && eOrigLock ){
         1547  +      rc = unixLock(id, eOrigLock);
         1548  +    }
         1549  +    if( rc!=SQLITE_OK ) return rc;
         1550  +  }
         1551  +  assert( (pFile->ctrlFlags & UNIXFILE_DEFERRED)==0 );
         1552  +
  1526   1553     /* Make sure the locking sequence is correct.
  1527   1554     **  (1) We never move from unlocked to anything higher than shared lock.
  1528   1555     **  (2) SQLite never explicitly requests a pendig lock.
  1529   1556     **  (3) A shared lock is always held when a reserve lock is requested.
  1530   1557     */
  1531   1558     assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK );
  1532   1559     assert( eFileLock!=PENDING_LOCK );
................................................................................
  1722   1749         getpid()));
  1723   1750   
  1724   1751     assert( eFileLock<=SHARED_LOCK );
  1725   1752     if( pFile->eFileLock<=eFileLock ){
  1726   1753       return SQLITE_OK;
  1727   1754     }
  1728   1755     unixEnterMutex();
         1756  +  if( pFile->ctrlFlags==UNIXFILE_DEFERRED ) goto end_unlock;
  1729   1757     pInode = pFile->pInode;
  1730   1758     assert( pInode->nShared!=0 );
  1731   1759     if( pFile->eFileLock>SHARED_LOCK ){
  1732   1760       assert( pInode->eFileLock==pFile->eFileLock );
  1733   1761   
  1734   1762   #ifdef SQLITE_DEBUG
  1735   1763       /* When reducing a lock such that other processes can start
................................................................................
  1929   1957     int rc = SQLITE_OK;
  1930   1958     unixFile *pFile = (unixFile *)id;
  1931   1959     verifyDbFile(pFile);
  1932   1960     unixUnlock(id, NO_LOCK);
  1933   1961     unixEnterMutex();
  1934   1962   
  1935   1963     /* unixFile.pInode is always valid here. Otherwise, a different close
  1936         -  ** routine (e.g. nolockClose()) would be called instead.
  1937         -  */
  1938         -  assert( pFile->pInode->nLock>0 || pFile->pInode->bProcessLock==0 );
  1939         -  if( ALWAYS(pFile->pInode) && pFile->pInode->nLock ){
  1940         -    /* If there are outstanding locks, do not actually close the file just
  1941         -    ** yet because that would clear those locks.  Instead, add the file
  1942         -    ** descriptor to pInode->pUnused list.  It will be automatically closed 
  1943         -    ** when the last lock is cleared.
  1944         -    */
  1945         -    setPendingFd(pFile);
         1964  +  ** routine (e.g. nolockClose()) would be called instead.  */
         1965  +  assert( pFile->pInode==0 
         1966  +       || pFile->pInode->nLock>0 
         1967  +       || pFile->pInode->bProcessLock==0 
         1968  +  );
         1969  +  if( pFile->pInode ){
         1970  +    if( pFile->pInode->nLock ){
         1971  +      /* If there are outstanding locks, do not actually close the file just
         1972  +      ** yet because that would clear those locks.  Instead, add the file
         1973  +      ** descriptor to pInode->pUnused list.  It will be automatically closed 
         1974  +      ** when the last lock is cleared.
         1975  +      */
         1976  +      setPendingFd(pFile);
         1977  +    }
         1978  +    releaseInodeInfo(pFile);
  1946   1979     }
  1947         -  releaseInodeInfo(pFile);
  1948   1980     rc = closeUnixFile(id);
  1949   1981     unixLeaveMutex();
  1950   1982     return rc;
  1951   1983   }
  1952   1984   
  1953   1985   /************** End of the posix advisory lock implementation *****************
  1954   1986   ******************************************************************************/
................................................................................
  3169   3201   #if 0
  3170   3202     assert( pFile->pUnused==0
  3171   3203          || offset>=PENDING_BYTE+512
  3172   3204          || offset+amt<=PENDING_BYTE 
  3173   3205     );
  3174   3206   #endif
  3175   3207   
         3208  +  if( pFile->ctrlFlags==UNIXFILE_DEFERRED ){
         3209  +    int rc;
         3210  +    struct stat sBuf;
         3211  +    memset(&sBuf, 0, sizeof(sBuf));
         3212  +    rc = osStat(pFile->zPath, &sBuf);
         3213  +    if( rc!=0 ){
         3214  +      memset(pBuf, 0, amt);
         3215  +      rc = (errno==ENOENT ? SQLITE_IOERR_SHORT_READ : SQLITE_IOERR_FSTAT);
         3216  +    }else{
         3217  +      rc = unixOpen(pFile->pVfs, pFile->zPath, id, pFile->openFlags, 0);
         3218  +    }
         3219  +    if( rc!=SQLITE_OK ){
         3220  +      return rc;
         3221  +    }
         3222  +  }
         3223  +  assert( (pFile->ctrlFlags & UNIXFILE_DEFERRED)==0 );
         3224  +
  3176   3225   #if SQLITE_MAX_MMAP_SIZE>0
  3177   3226     /* Deal with as much of this read request as possible by transfering
  3178   3227     ** data from the memory mapping using memcpy().  */
  3179   3228     if( offset<pFile->mmapSize ){
  3180   3229       if( offset+amt <= pFile->mmapSize ){
  3181   3230         memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt);
  3182   3231         return SQLITE_OK;
................................................................................
  3621   3670     }
  3622   3671   }
  3623   3672   
  3624   3673   /*
  3625   3674   ** Determine the current size of a file in bytes
  3626   3675   */
  3627   3676   static int unixFileSize(sqlite3_file *id, i64 *pSize){
         3677  +  unixFile *pFile = (unixFile*)id;
  3628   3678     int rc;
  3629   3679     struct stat buf;
  3630   3680     assert( id );
  3631         -  rc = osFstat(((unixFile*)id)->h, &buf);
         3681  +  if( pFile->ctrlFlags==UNIXFILE_DEFERRED ){
         3682  +    rc = osStat(pFile->zPath, &buf);
         3683  +    if( rc && errno==ENOENT ){
         3684  +      rc = 0;
         3685  +      buf.st_size = 0;
         3686  +    }
         3687  +  }else{
         3688  +    rc = osFstat(pFile->h, &buf);
         3689  +  }
  3632   3690     SimulateIOError( rc=1 );
  3633   3691     if( rc!=0 ){
  3634   3692       ((unixFile*)id)->lastErrno = errno;
  3635   3693       return SQLITE_IOERR_FSTAT;
  3636   3694     }
  3637   3695     *pSize = buf.st_size;
  3638   3696   
................................................................................
  3639   3697     /* When opening a zero-size database, the findInodeInfo() procedure
  3640   3698     ** writes a single byte into that file in order to work around a bug
  3641   3699     ** in the OS-X msdos filesystem.  In order to avoid problems with upper
  3642   3700     ** layers, we need to report this file size as zero even though it is
  3643   3701     ** really 1.   Ticket #3260.
  3644   3702     */
  3645   3703     if( *pSize==1 ) *pSize = 0;
  3646         -
  3647   3704   
  3648   3705     return SQLITE_OK;
  3649   3706   }
  3650   3707   
  3651   3708   #if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__)
  3652   3709   /*
  3653   3710   ** Handler for proxy-locking file-control verbs.  Defined below in the
................................................................................
  5172   5229     /* No locking occurs in temporary files */
  5173   5230     assert( zFilename!=0 || (ctrlFlags & UNIXFILE_NOLOCK)!=0 );
  5174   5231   
  5175   5232     OSTRACE(("OPEN    %-3d %s\n", h, zFilename));
  5176   5233     pNew->h = h;
  5177   5234     pNew->pVfs = pVfs;
  5178   5235     pNew->zPath = zFilename;
  5179         -  pNew->ctrlFlags = (u8)ctrlFlags;
         5236  +  pNew->ctrlFlags = (unsigned short)ctrlFlags;
  5180   5237   #if SQLITE_MAX_MMAP_SIZE>0
  5181   5238     pNew->mmapSizeMax = sqlite3GlobalConfig.szMmap;
  5182   5239   #endif
  5183   5240     if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0),
  5184   5241                              "psow", SQLITE_POWERSAFE_OVERWRITE) ){
  5185   5242       pNew->ctrlFlags |= UNIXFILE_PSOW;
  5186   5243     }
................................................................................
  5818   5875   open_finished:
  5819   5876     if( rc!=SQLITE_OK ){
  5820   5877       sqlite3_free(p->pUnused);
  5821   5878     }
  5822   5879     return rc;
  5823   5880   }
  5824   5881   
         5882  +static int unixOpenDeferred(
         5883  +  sqlite3_vfs *pVfs,           /* The VFS for which this is the xOpen method */
         5884  +  const char *zPath,           /* Pathname of file to be opened */
         5885  +  sqlite3_file *pFile,         /* The file descriptor to be filled in */
         5886  +  int flags,                   /* Input flags to control the opening */
         5887  +  int *pOutFlags               /* Output flags returned to SQLite core */
         5888  +){
         5889  +  const int mask1 = SQLITE_OPEN_MAIN_DB | SQLITE_OPEN_READWRITE
         5890  +                  | SQLITE_OPEN_CREATE;
         5891  +  const int mask2 = SQLITE_OPEN_READONLY  | SQLITE_OPEN_DELETEONCLOSE
         5892  +                  | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_AUTOPROXY;
         5893  +  int rc = SQLITE_OK;          /* Return code */
         5894  +
         5895  +  /* If all the flags in mask1 are set, and all the flags in mask2 are
         5896  +  ** clear, then this will be a deferred open.  */
         5897  +  if( zPath && (flags & (mask1 | mask2))==mask1 ){
         5898  +    unixFile *p = (unixFile*)pFile;
         5899  +    memset(p, 0, sizeof(unixFile));
         5900  +
         5901  +    p->pMethod = (**(finder_type*)pVfs->pAppData)(0, 0);
         5902  +    p->pVfs = pVfs;
         5903  +    p->h = -1;
         5904  +    p->ctrlFlags = UNIXFILE_DEFERRED;
         5905  +    p->openFlags = flags;
         5906  +    p->zPath = zPath;
         5907  +    if( pOutFlags ) *pOutFlags = flags;
         5908  +  }else{
         5909  +    rc = unixOpen(pVfs, zPath, pFile, flags, pOutFlags);
         5910  +  }
         5911  +  return rc;
         5912  +}
  5825   5913   
  5826   5914   /*
  5827   5915   ** Delete the file at zPath. If the dirSync argument is true, fsync()
  5828   5916   ** the directory after deleting the file.
  5829   5917   */
  5830   5918   static int unixDelete(
  5831   5919     sqlite3_vfs *NotUsed,     /* VFS containing this as the xDelete method */
................................................................................
  7372   7460     #define UNIXVFS(VFSNAME, FINDER) {                        \
  7373   7461       3,                    /* iVersion */                    \
  7374   7462       sizeof(unixFile),     /* szOsFile */                    \
  7375   7463       MAX_PATHNAME,         /* mxPathname */                  \
  7376   7464       0,                    /* pNext */                       \
  7377   7465       VFSNAME,              /* zName */                       \
  7378   7466       (void*)&FINDER,       /* pAppData */                    \
  7379         -    unixOpen,             /* xOpen */                       \
         7467  +    unixOpenDeferred,     /* xOpen */                       \
  7380   7468       unixDelete,           /* xDelete */                     \
  7381   7469       unixAccess,           /* xAccess */                     \
  7382   7470       unixFullPathname,     /* xFullPathname */               \
  7383   7471       unixDlOpen,           /* xDlOpen */                     \
  7384   7472       unixDlError,          /* xDlError */                    \
  7385   7473       unixDlSym,            /* xDlSym */                      \
  7386   7474       unixDlClose,          /* xDlClose */                    \

Changes to src/test2.c.

   530    530     if( rc ){
   531    531       Tcl_AppendResult(interp, "open failed: ", sqlite3ErrName(rc), 0);
   532    532       sqlite3_free(zFile);
   533    533       return TCL_ERROR;
   534    534     }
   535    535     offset = n;
   536    536     offset *= 1024*1024;
          537  +  sqlite3OsLock(fd, SHARED_LOCK);
          538  +  sqlite3OsLock(fd, EXCLUSIVE_LOCK);
   537    539     rc = sqlite3OsWrite(fd, "Hello, World!", 14, offset);
          540  +  sqlite3OsUnlock(fd, NO_LOCK);
   538    541     sqlite3OsCloseFree(fd);
   539    542     sqlite3_free(zFile);
   540    543     if( rc ){
   541    544       Tcl_AppendResult(interp, "write failed: ", sqlite3ErrName(rc), 0);
   542    545       return TCL_ERROR;
   543    546     }
   544    547     return TCL_OK;

Changes to test/permutations.test.

   131    131   lappend ::testsuitelist xxx
   132    132   
   133    133   test_suite "veryquick" -prefix "" -description {
   134    134     "Very" quick test suite. Runs in less than 5 minutes on a workstation. 
   135    135     This test suite is the same as the "quick" tests, except that some files
   136    136     that test malloc and IO errors are omitted.
   137    137   } -files [
   138         -  test_set $allquicktests -exclude *malloc* *ioerr* *fault*
          138  +  test_set $allquicktests -exclude *malloc* *ioerr* *fault* \
          139  +    *multiplex* *quota* walbak.test
   139    140   ]
   140    141   
   141    142   test_suite "mmap" -prefix "mm-" -description {
   142    143     Similar to veryquick. Except with memory mapping disabled.
   143    144   } -presql {
   144    145     pragma mmap_size = 268435456;
   145    146   } -files [