/ Check-in [310b4b65]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Speed up xAccess() calls made on an RBU VFS when there are lots of open connections.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 310b4b65b8c8ee080760c7efb4c7e20244c6063a5dba37a4f40490105aafd29f
User & Date: dan 2018-10-01 17:33:35
References
2018-10-25
11:55
Speed up xAccess() calls made on an RBU VFS when there are lots of open connections. Cherrypick of [310b4b65]. check-in: fda8fdb0 user: dan tags: branch-3.22
Context
2018-10-25
11:55
Speed up xAccess() calls made on an RBU VFS when there are lots of open connections. Cherrypick of [310b4b65]. check-in: fda8fdb0 user: dan tags: branch-3.22
2018-10-01
21:41
Add the "PRAGMA table_xinfo" command that works like table_info but also shows hidden columns in virtual tables and adds the "hidden" boolean column. check-in: defa0515 user: drh tags: trunk
17:33
Speed up xAccess() calls made on an RBU VFS when there are lots of open connections. check-in: 310b4b65 user: dan tags: trunk
13:54
Fix a potential crash that can occur while reading an index from a corrupt database file. The corruption is a record-header-size that is larger than 0x7fffffff. Problem detected by OSSFuzz against GDAL and reported to us (with a suggested fix) by Even Rouault. The test case is in TH3. check-in: 8ac2cdda user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/rbu/sqlite3rbu.c.

   401    401   ** space used by the RBU handle.
   402    402   */
   403    403   struct rbu_vfs {
   404    404     sqlite3_vfs base;               /* rbu VFS shim methods */
   405    405     sqlite3_vfs *pRealVfs;          /* Underlying VFS */
   406    406     sqlite3_mutex *mutex;           /* Mutex to protect pMain */
   407    407     sqlite3rbu *pRbu;               /* Owner RBU object */
   408         -  rbu_file *pMain;                /* Linked list of main db files */
          408  +  rbu_file *pMain;                /* List of main db files */
          409  +  rbu_file *pMainRbu;             /* List of main db files with pRbu!=0 */
   409    410   };
   410    411   
   411    412   /*
   412    413   ** Each file opened by an rbu VFS is represented by an instance of
   413    414   ** the following structure.
   414    415   **
   415    416   ** If this is a temporary file (pRbu!=0 && flags&DELETE_ON_CLOSE), variable
................................................................................
   430    431     int nShm;                       /* Number of entries in apShm[] array */
   431    432     char **apShm;                   /* Array of mmap'd *-shm regions */
   432    433     char *zDel;                     /* Delete this when closing file */
   433    434   
   434    435     const char *zWal;               /* Wal filename for this main db file */
   435    436     rbu_file *pWalFd;               /* Wal file descriptor for this main db */
   436    437     rbu_file *pMainNext;            /* Next MAIN_DB file */
          438  +  rbu_file *pMainRbuNext;         /* Next MAIN_DB file with pRbu!=0 */
   437    439   };
   438    440   
   439    441   /*
   440    442   ** True for an RBU vacuum handle, or false otherwise.
   441    443   */
   442    444   #define rbuIsVacuum(p) ((p)->zTarget==0)
   443    445   
................................................................................
  4025   4027     i64 nDiff = nNew - pFd->sz;
  4026   4028     pRbu->szTemp += nDiff;
  4027   4029     pFd->sz = nNew;
  4028   4030     assert( pRbu->szTemp>=0 );
  4029   4031     if( pRbu->szTempLimit && pRbu->szTemp>pRbu->szTempLimit ) return SQLITE_FULL;
  4030   4032     return SQLITE_OK;
  4031   4033   }
         4034  +
         4035  +/*
         4036  +** Add an item to the main-db lists, if it is not already present.
         4037  +**
         4038  +** There are two main-db lists. One for all file descriptors, and one
         4039  +** for all file descriptors with rbu_file.pDb!=0. If the argument has
         4040  +** rbu_file.pDb!=0, then it is assumed to already be present on the
         4041  +** main list and is only added to the pDb!=0 list.
         4042  +*/
         4043  +static void rbuMainlistAdd(rbu_file *p){
         4044  +  rbu_vfs *pRbuVfs = p->pRbuVfs;
         4045  +  rbu_file *pIter;
         4046  +  assert( (p->openFlags & SQLITE_OPEN_MAIN_DB) );
         4047  +  sqlite3_mutex_enter(pRbuVfs->mutex);
         4048  +  if( p->pRbu==0 ){
         4049  +    for(pIter=pRbuVfs->pMain; pIter; pIter=pIter->pMainNext);
         4050  +    p->pMainNext = pRbuVfs->pMain;
         4051  +    pRbuVfs->pMain = p;
         4052  +  }else{
         4053  +    for(pIter=pRbuVfs->pMainRbu; pIter && pIter!=p; pIter=pIter->pMainRbuNext){}
         4054  +    if( pIter==0 ){
         4055  +      p->pMainRbuNext = pRbuVfs->pMainRbu;
         4056  +      pRbuVfs->pMainRbu = p;
         4057  +    }
         4058  +  }
         4059  +  sqlite3_mutex_leave(pRbuVfs->mutex);
         4060  +}
         4061  +
         4062  +/*
         4063  +** Remove an item from the main-db lists.
         4064  +*/
         4065  +static void rbuMainlistRemove(rbu_file *p){
         4066  +  rbu_file **pp;
         4067  +  sqlite3_mutex_enter(p->pRbuVfs->mutex);
         4068  +  for(pp=&p->pRbuVfs->pMain; *pp && *pp!=p; pp=&((*pp)->pMainNext)){}
         4069  +  if( *pp ) *pp = p->pMainNext;
         4070  +  p->pMainNext = 0;
         4071  +  for(pp=&p->pRbuVfs->pMainRbu; *pp && *pp!=p; pp=&((*pp)->pMainRbuNext)){}
         4072  +  if( *pp ) *pp = p->pMainRbuNext;
         4073  +  p->pMainRbuNext = 0;
         4074  +  sqlite3_mutex_leave(p->pRbuVfs->mutex);
         4075  +}
         4076  +
         4077  +/*
         4078  +** Given that zWal points to a buffer containing a wal file name passed to 
         4079  +** either the xOpen() or xAccess() VFS method, search the main-db list for
         4080  +** a file-handle opened by the same database connection on the corresponding
         4081  +** database file.
         4082  +**
         4083  +** If parameter bRbu is true, only search for file-descriptors with
         4084  +** rbu_file.pDb!=0.
         4085  +*/
         4086  +static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal, int bRbu){
         4087  +  rbu_file *pDb;
         4088  +  sqlite3_mutex_enter(pRbuVfs->mutex);
         4089  +  if( bRbu ){
         4090  +    for(pDb=pRbuVfs->pMainRbu; pDb && pDb->zWal!=zWal; pDb=pDb->pMainRbuNext){}
         4091  +  }else{
         4092  +    for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext){}
         4093  +  }
         4094  +  sqlite3_mutex_leave(pRbuVfs->mutex);
         4095  +  return pDb;
         4096  +}
  4032   4097   
  4033   4098   /*
  4034   4099   ** Close an rbu file.
  4035   4100   */
  4036   4101   static int rbuVfsClose(sqlite3_file *pFile){
  4037   4102     rbu_file *p = (rbu_file*)pFile;
  4038   4103     int rc;
................................................................................
  4043   4108       sqlite3_free(p->apShm[i]);
  4044   4109     }
  4045   4110     sqlite3_free(p->apShm);
  4046   4111     p->apShm = 0;
  4047   4112     sqlite3_free(p->zDel);
  4048   4113   
  4049   4114     if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
  4050         -    rbu_file **pp;
  4051         -    sqlite3_mutex_enter(p->pRbuVfs->mutex);
  4052         -    for(pp=&p->pRbuVfs->pMain; *pp!=p; pp=&((*pp)->pMainNext));
  4053         -    *pp = p->pMainNext;
  4054         -    sqlite3_mutex_leave(p->pRbuVfs->mutex);
         4115  +    rbuMainlistRemove(p);
  4055   4116       rbuUnlockShm(p);
  4056   4117       p->pReal->pMethods->xShmUnmap(p->pReal, 0);
  4057   4118     }
  4058   4119     else if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){
  4059   4120       rbuUpdateTempSize(p, 0);
  4060   4121     }
         4122  +  assert( p->pMainNext==0 && p->pRbuVfs->pMain!=p );
  4061   4123   
  4062   4124     /* Close the underlying file handle */
  4063   4125     rc = p->pReal->pMethods->xClose(p->pReal);
  4064   4126     return rc;
  4065   4127   }
  4066   4128   
  4067   4129   
................................................................................
  4312   4374         rc = xControl(p->pReal, SQLITE_FCNTL_ZIPVFS, &dummy);
  4313   4375         if( rc==SQLITE_OK ){
  4314   4376           rc = SQLITE_ERROR;
  4315   4377           pRbu->zErrmsg = sqlite3_mprintf("rbu/zipvfs setup error");
  4316   4378         }else if( rc==SQLITE_NOTFOUND ){
  4317   4379           pRbu->pTargetFd = p;
  4318   4380           p->pRbu = pRbu;
         4381  +        if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
         4382  +          rbuMainlistAdd(p);
         4383  +        }
  4319   4384           if( p->pWalFd ) p->pWalFd->pRbu = pRbu;
  4320   4385           rc = SQLITE_OK;
  4321   4386         }
  4322   4387       }
  4323   4388       return rc;
  4324   4389     }
  4325   4390     else if( op==SQLITE_FCNTL_RBUCNT ){
................................................................................
  4473   4538       /* Release the checkpointer and writer locks */
  4474   4539       rbuUnlockShm(p);
  4475   4540       rc = p->pReal->pMethods->xShmUnmap(p->pReal, delFlag);
  4476   4541     }
  4477   4542     return rc;
  4478   4543   }
  4479   4544   
  4480         -/*
  4481         -** Given that zWal points to a buffer containing a wal file name passed to 
  4482         -** either the xOpen() or xAccess() VFS method, return a pointer to the
  4483         -** file-handle opened by the same database connection on the corresponding
  4484         -** database file.
  4485         -*/
  4486         -static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal){
  4487         -  rbu_file *pDb;
  4488         -  sqlite3_mutex_enter(pRbuVfs->mutex);
  4489         -  for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext){}
  4490         -  sqlite3_mutex_leave(pRbuVfs->mutex);
  4491         -  return pDb;
  4492         -}
  4493         -
  4494   4545   /* 
  4495   4546   ** A main database named zName has just been opened. The following 
  4496   4547   ** function returns a pointer to a buffer owned by SQLite that contains
  4497   4548   ** the name of the *-wal file this db connection will use. SQLite
  4498   4549   ** happens to pass a pointer to this buffer when using xAccess()
  4499   4550   ** or xOpen() to operate on the *-wal file.  
  4500   4551   */
................................................................................
  4565   4616         ** (pFd->zWal) to point to a buffer owned by SQLite that contains
  4566   4617         ** the name of the *-wal file this db connection will use. SQLite
  4567   4618         ** happens to pass a pointer to this buffer when using xAccess()
  4568   4619         ** or xOpen() to operate on the *-wal file.  */
  4569   4620         pFd->zWal = rbuMainToWal(zName, flags);
  4570   4621       }
  4571   4622       else if( flags & SQLITE_OPEN_WAL ){
  4572         -      rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName);
         4623  +      rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName, 0);
  4573   4624         if( pDb ){
  4574   4625           if( pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){
  4575   4626             /* This call is to open a *-wal file. Intead, open the *-oal. This
  4576   4627             ** code ensures that the string passed to xOpen() is terminated by a
  4577   4628             ** pair of '\0' bytes in case the VFS attempts to extract a URI 
  4578   4629             ** parameter from it.  */
  4579   4630             const char *zBase = zName;
................................................................................
  4617   4668     }
  4618   4669     if( pFd->pReal->pMethods ){
  4619   4670       /* The xOpen() operation has succeeded. Set the sqlite3_file.pMethods
  4620   4671       ** pointer and, if the file is a main database file, link it into the
  4621   4672       ** mutex protected linked list of all such files.  */
  4622   4673       pFile->pMethods = &rbuvfs_io_methods;
  4623   4674       if( flags & SQLITE_OPEN_MAIN_DB ){
  4624         -      sqlite3_mutex_enter(pRbuVfs->mutex);
  4625         -      pFd->pMainNext = pRbuVfs->pMain;
  4626         -      pRbuVfs->pMain = pFd;
  4627         -      sqlite3_mutex_leave(pRbuVfs->mutex);
         4675  +      rbuMainlistAdd(pFd);
  4628   4676       }
  4629   4677     }else{
  4630   4678       sqlite3_free(pFd->zDel);
  4631   4679     }
  4632   4680   
  4633   4681     return rc;
  4634   4682   }
................................................................................
  4668   4716     **
  4669   4717     **   b) if the *-wal file does not exist, claim that it does anyway,
  4670   4718     **      causing SQLite to call xOpen() to open it. This call will also
  4671   4719     **      be intercepted (see the rbuVfsOpen() function) and the *-oal
  4672   4720     **      file opened instead.
  4673   4721     */
  4674   4722     if( rc==SQLITE_OK && flags==SQLITE_ACCESS_EXISTS ){
  4675         -    rbu_file *pDb = rbuFindMaindb(pRbuVfs, zPath);
         4723  +    rbu_file *pDb = rbuFindMaindb(pRbuVfs, zPath, 1);
  4676   4724       if( pDb && pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){
  4677   4725         if( *pResOut ){
  4678   4726           rc = SQLITE_CANTOPEN;
  4679   4727         }else{
  4680   4728           sqlite3_int64 sz = 0;
  4681   4729           rc = rbuVfsFileSize(&pDb->base, &sz);
  4682   4730           *pResOut = (sz>0);