SQLite

Check-in [34ab7606]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Revise the layout of filenames in the Pager object so that it is unchanged from prior versions. It turns out that some important 3rd-party software does questionable pointer manipulations on those filenames that depend on that legacy layout. Technically, this is a misuse of SQLite by the 3rd-party software, but we want to avoid unnecessary breakage.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 34ab760689fd493eda482e856047708d74e769a01cc90b69da456d79ffe39aea
User & Date: drh 2020-01-27 14:40:44
Original Comment: Revise the layout of filenames in the Pager object so that it is unchanged from prior versions. It turns out that some important 3rd-party software does questionable pointer manipulations on those filenames that depend on that legacy layout. Technical this is a misuse of SQLite by the 3rd-party software, but we want to avoid unnecessary breakage.
Context
2020-01-27
17:09
Update the version number to 3.31.1. (check-in: 6fb9a8fb user: drh tags: trunk)
14:40
Revise the layout of filenames in the Pager object so that it is unchanged from prior versions. It turns out that some important 3rd-party software does questionable pointer manipulations on those filenames that depend on that legacy layout. Technically, this is a misuse of SQLite by the 3rd-party software, but we want to avoid unnecessary breakage. (check-in: 34ab7606 user: drh tags: trunk)
2020-01-23
15:00
Fix typos in the sqlite3ext.h header file that prevent some newer APIs from being accessed from loadable extensions. (check-in: 14331989 user: drh tags: trunk)
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/main.c.

4238
4239
4240
4241
4242
4243
4244















4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258

4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273

4274
4275
4276
4277
4278
4279
4280
      break;
    }
  }
  va_end(ap);
#endif /* SQLITE_UNTESTABLE */
  return rc;
}
















/*
** This is a utility routine, useful to VFS implementations, that checks
** to see if a database file was a URI that contained a specific query 
** parameter, and if so obtains the value of the query parameter.
**
** The zFilename argument is the filename pointer passed into the xOpen()
** method of a VFS implementation.  The zParam argument is the name of the
** query parameter we seek.  This routine returns the value of the zParam
** parameter if it exists.  If the parameter does not exist, this routine
** returns a NULL pointer.
*/
const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){
  if( zFilename==0 || zParam==0 ) return 0;

  zFilename += sqlite3Strlen30(zFilename) + 1;
  while( zFilename[0] ){
    int x = strcmp(zFilename, zParam);
    zFilename += sqlite3Strlen30(zFilename) + 1;
    if( x==0 ) return zFilename;
    zFilename += sqlite3Strlen30(zFilename) + 1;
  }
  return 0;
}

/*
** Return a pointer to the name of Nth query parameter of the filename.
*/
const char *sqlite3_uri_key(const char *zFilename, int N){
  if( zFilename==0 || N<0 ) return 0;

  zFilename += sqlite3Strlen30(zFilename) + 1;
  while( zFilename[0] && (N--)>0 ){
    zFilename += sqlite3Strlen30(zFilename) + 1;
    zFilename += sqlite3Strlen30(zFilename) + 1;
  }
  return zFilename[0] ? zFilename : 0;
}







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>














>















>







4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
      break;
    }
  }
  va_end(ap);
#endif /* SQLITE_UNTESTABLE */
  return rc;
}

/*
** The Pager stores the Database filename, Journal filename, and WAL filename
** consecutively in memory, in that order.  The database filename is prefixed
** by four zero bytes.  Locate the start of the database filename by searching
** backwards for the first byte following four consecutive zero bytes.
**
** This only works if the filename passed in was obtained from the Pager.
*/
static const char *databaseName(const char *zName){
  while( zName[-1]!=0 || zName[-2]!=0 || zName[-3]!=0 || zName[-4]!=0 ){
    zName--;
  }
  return zName;
}

/*
** This is a utility routine, useful to VFS implementations, that checks
** to see if a database file was a URI that contained a specific query 
** parameter, and if so obtains the value of the query parameter.
**
** The zFilename argument is the filename pointer passed into the xOpen()
** method of a VFS implementation.  The zParam argument is the name of the
** query parameter we seek.  This routine returns the value of the zParam
** parameter if it exists.  If the parameter does not exist, this routine
** returns a NULL pointer.
*/
const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){
  if( zFilename==0 || zParam==0 ) return 0;
  zFilename = databaseName(zFilename);
  zFilename += sqlite3Strlen30(zFilename) + 1;
  while( zFilename[0] ){
    int x = strcmp(zFilename, zParam);
    zFilename += sqlite3Strlen30(zFilename) + 1;
    if( x==0 ) return zFilename;
    zFilename += sqlite3Strlen30(zFilename) + 1;
  }
  return 0;
}

/*
** Return a pointer to the name of Nth query parameter of the filename.
*/
const char *sqlite3_uri_key(const char *zFilename, int N){
  if( zFilename==0 || N<0 ) return 0;
  zFilename = databaseName(zFilename);
  zFilename += sqlite3Strlen30(zFilename) + 1;
  while( zFilename[0] && (N--)>0 ){
    zFilename += sqlite3Strlen30(zFilename) + 1;
    zFilename += sqlite3Strlen30(zFilename) + 1;
  }
  return zFilename[0] ? zFilename : 0;
}
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336

4337
4338
4339
4340





4341
4342
4343

4344





4345
4346
4347
4348
4349
4350
4351
  sqlite3_int64 v;
  if( z && sqlite3DecOrHexToI64(z, &v)==0 ){
    bDflt = v;
  }
  return bDflt;
}

/*
** The Pager stores the Journal filename, WAL filename, and Database filename
** consecutively in memory, in that order, with prefixes \000\001\000,
** \002\000, and \003\000, in that order.  Thus the three names look like query
** parameters if you start at the first prefix.
**
** This routine backs up a filename to the start of the first prefix.
**
** This only works if the filenamed passed in was obtained from the Pager.
*/
static const char *startOfNameList(const char *zName){
  while( zName[0]!='\001' || zName[1]!=0 ){
    zName -= 3;
    while( zName[0]!='\000' ){ zName--; }
    zName++;
  }
  return zName-1;
}

/*
** Translate a filename that was handed to a VFS routine into the corresponding
** database, journal, or WAL file.
**
** It is an error to pass this routine a filename string that was not
** passed into the VFS from the SQLite core.  Doing so is similar to
** passing free() a pointer that was not obtained from malloc() - it is
** an error that we cannot easily detect but that will likely cause memory
** corruption.
*/
const char *sqlite3_filename_database(const char *zFilename){

  return sqlite3_uri_parameter(zFilename - 3, "\003");
}
const char *sqlite3_filename_journal(const char *zFilename){
  const char *z = sqlite3_uri_parameter(startOfNameList(zFilename), "\001");





  return ALWAYS(z) && z[0] ? z : 0;
}
const char *sqlite3_filename_wal(const char *zFilename){

  return sqlite3_uri_parameter(startOfNameList(zFilename), "\002");





}

/*
** Return the Btree pointer identified by zDbName.  Return NULL if not found.
*/
Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){
  int iDb = zDbName ? sqlite3FindDbName(db, zDbName) : 0;







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<











>



|
>
>
>
>
>
|


>
|
>
>
>
>
>







4317
4318
4319
4320
4321
4322
4323



















4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
  sqlite3_int64 v;
  if( z && sqlite3DecOrHexToI64(z, &v)==0 ){
    bDflt = v;
  }
  return bDflt;
}




















/*
** Translate a filename that was handed to a VFS routine into the corresponding
** database, journal, or WAL file.
**
** It is an error to pass this routine a filename string that was not
** passed into the VFS from the SQLite core.  Doing so is similar to
** passing free() a pointer that was not obtained from malloc() - it is
** an error that we cannot easily detect but that will likely cause memory
** corruption.
*/
const char *sqlite3_filename_database(const char *zFilename){
  return databaseName(zFilename);
  return sqlite3_uri_parameter(zFilename - 3, "\003");
}
const char *sqlite3_filename_journal(const char *zFilename){
  zFilename = databaseName(zFilename);
  zFilename += sqlite3Strlen30(zFilename) + 1;
  while( zFilename[0] ){
    zFilename += sqlite3Strlen30(zFilename) + 1;
    zFilename += sqlite3Strlen30(zFilename) + 1;
  }
  return zFilename + 1;
}
const char *sqlite3_filename_wal(const char *zFilename){
#ifdef SQLITE_OMIT_WAL
  return 0;
#else
  zFilename = sqlite3_filename_journal(zFilename);
  zFilename += sqlite3Strlen30(zFilename) + 1;
  return zFilename;
#endif
}

/*
** Return the Btree pointer identified by zDbName.  Return NULL if not found.
*/
Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){
  int iDb = zDbName ? sqlite3FindDbName(db, zDbName) : 0;

Changes to src/pager.c.

4834
4835
4836
4837
4838
4839
4840
4841


4842
4843
4844














4845
4846
4847
4848





4849
4850
4851
4852
4853
4854
4855


4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877









4878



4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
  ** file name. The layout in memory is as follows:
  **
  **     Pager object                    (sizeof(Pager) bytes)
  **     PCache object                   (sqlite3PcacheSize() bytes)
  **     Database file handle            (pVfs->szOsFile bytes)
  **     Sub-journal file handle         (journalFileSize bytes)
  **     Main journal file handle        (journalFileSize bytes)
  **     \0\1\0 journal prefix           (3 bytes)


  **     Journal filename                (nPathname+8+1 bytes)
  **     \2\0 WAL prefix                 (2 bytes)
  **     WAL filename                    (nPathname+4+1 bytes)














  **     \3\0 database prefix            (2 bytes)
  **     Database file name              (nPathname+1 bytes)
  **     URI query parameters            (nUriByte bytes)
  **     \0\0 terminator                 (2 bytes)





  */
  pPtr = (u8 *)sqlite3MallocZero(
    ROUND8(sizeof(*pPager)) +            /* Pager structure */
    ROUND8(pcacheSize) +                 /* PCache object */
    ROUND8(pVfs->szOsFile) +             /* The main db file */
    journalFileSize * 2 +                /* The two journal files */
    3 +                                  /* Journal prefix */


    nPathname + 8 + 1 +                  /* Journal filename */
#ifndef SQLITE_OMIT_WAL
    2 +                                  /* WAL prefix */
    nPathname + 4 + 1 +                  /* WAL filename */
#endif
    2 +                                  /* Database prefix */
    nPathname + 1 +                      /* database filename */
    nUriByte +                           /* query parameters */
    2                                    /* Terminator */
  );
  assert( EIGHT_BYTE_ALIGNMENT(SQLITE_INT_TO_PTR(journalFileSize)) );
  if( !pPtr ){
    sqlite3DbFree(0, zPathname);
    return SQLITE_NOMEM_BKPT;
  }
  pPager = (Pager*)pPtr;                  pPtr += ROUND8(sizeof(*pPager));
  pPager->pPCache = (PCache*)pPtr;        pPtr += ROUND8(pcacheSize);
  pPager->fd = (sqlite3_file*)pPtr;       pPtr += ROUND8(pVfs->szOsFile);
  pPager->sjfd = (sqlite3_file*)pPtr;     pPtr += journalFileSize;
  pPager->jfd =  (sqlite3_file*)pPtr;     pPtr += journalFileSize;
  assert( EIGHT_BYTE_ALIGNMENT(pPager->jfd) );














  /* Fill in Pager.zJournal */
  pPtr[1] = '\001';                       pPtr += 3;
  if( nPathname>0 ){
    pPager->zJournal = (char*)pPtr;
    memcpy(pPtr, zPathname, nPathname);   pPtr += nPathname;
    memcpy(pPtr, "-journal",8);           pPtr += 8 + 1;
#ifdef SQLITE_ENABLE_8_3_NAMES
    sqlite3FileSuffix3(zFilename,pPager->zJournal);
    pPtr = (u8*)(pPager->zJournal + sqlite3Strlen30(pPager->zJournal)+1);
#endif
  }else{
    pPager->zJournal = 0;
    pPtr++;
  }

#ifndef SQLITE_OMIT_WAL
  /* Fill in Pager.zWal */
  pPtr[0] = '\002'; pPtr[1] = 0;          pPtr += 2;
  if( nPathname>0 ){
    pPager->zWal = (char*)pPtr;
    memcpy(pPtr, zPathname, nPathname);   pPtr += nPathname;
    memcpy(pPtr, "-wal", 4);              pPtr += 4 + 1;
#ifdef SQLITE_ENABLE_8_3_NAMES
    sqlite3FileSuffix3(zFilename, pPager->zWal);
    pPtr = (u8*)(pPager->zWal + sqlite3Strlen30(pPager->zWal)+1);
#endif
  }else{
    pPager->zWal = 0;
    pPtr++;
  }
#endif

  /* Fill in the Pager.zFilename and pPager.zQueryParam fields */
  pPtr[0] = '\003'; pPtr[1] = 0;          pPtr += 2;
  pPager->zFilename = (char*)pPtr;
  if( nPathname>0 ){
    memcpy(pPtr, zPathname, nPathname);   pPtr += nPathname + 1;
    if( zUri ){
      memcpy(pPtr, zUri, nUriByte);    /* pPtr += nUriByte; // not needed */
    }
    /* Double-zero terminator implied by the sqlite3MallocZero */
  }

  if( nPathname ) sqlite3DbFree(0, zPathname);
  pPager->pVfs = pVfs;
  pPager->vfsFlags = vfsFlags;

  /* Open the pager file.
  */
  if( zFilename && zFilename[0] ){







|
>
>

<

>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
|
|
|
>
>
>
>
>






|
>
>


<


<
<
<
|













>
>
>
>
>
>
>
>
>
|
>
>
>

<










<




<










<



<
<
<
<
<
<
<
<
<
<
<







4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844

4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879

4880
4881



4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909

4910
4911
4912
4913
4914
4915
4916
4917
4918
4919

4920
4921
4922
4923

4924
4925
4926
4927
4928
4929
4930
4931
4932
4933

4934
4935
4936











4937
4938
4939
4940
4941
4942
4943
  ** file name. The layout in memory is as follows:
  **
  **     Pager object                    (sizeof(Pager) bytes)
  **     PCache object                   (sqlite3PcacheSize() bytes)
  **     Database file handle            (pVfs->szOsFile bytes)
  **     Sub-journal file handle         (journalFileSize bytes)
  **     Main journal file handle        (journalFileSize bytes)
  **     \0\0\0\0 database prefix        (4 bytes)
  **     Database file name              (nPathname+1 bytes)
  **     URI query parameters            (nUriByte bytes)
  **     Journal filename                (nPathname+8+1 bytes)

  **     WAL filename                    (nPathname+4+1 bytes)
  **     \0\0\0 terminator               (3 bytes)
  **
  ** Some 3rd-party software, over which we have no control, depends on
  ** the specific order of the filenames and the \0 separators between them
  ** so that it can (for example) find the database filename given the WAL
  ** filename without using the sqlite3_filename_database() API.  This is a
  ** misuse of SQLite and a bug in the 3rd-party software, but the 3rd-party
  ** software is in widespread use, so we try to avoid changing the filename
  ** order and formatting if possible.  In particular, the details of the
  ** filename format expected by 3rd-party software should be as follows:
  **
  **   - Main Database Path
  **   - \0
  **   - Multiple URI components consisting of:
  **     - Key
  **     - \0
  **     - Value
  **     - \0
  **   - \0
  **   - Journal Path
  **   - \0
  **   - WAL Path (zWALName)
  **   - \0
  */
  pPtr = (u8 *)sqlite3MallocZero(
    ROUND8(sizeof(*pPager)) +            /* Pager structure */
    ROUND8(pcacheSize) +                 /* PCache object */
    ROUND8(pVfs->szOsFile) +             /* The main db file */
    journalFileSize * 2 +                /* The two journal files */
    4 +                                  /* Database prefix */
    nPathname + 1 +                      /* database filename */
    nUriByte +                           /* query parameters */
    nPathname + 8 + 1 +                  /* Journal filename */
#ifndef SQLITE_OMIT_WAL

    nPathname + 4 + 1 +                  /* WAL filename */
#endif



    3                                    /* Terminator */
  );
  assert( EIGHT_BYTE_ALIGNMENT(SQLITE_INT_TO_PTR(journalFileSize)) );
  if( !pPtr ){
    sqlite3DbFree(0, zPathname);
    return SQLITE_NOMEM_BKPT;
  }
  pPager = (Pager*)pPtr;                  pPtr += ROUND8(sizeof(*pPager));
  pPager->pPCache = (PCache*)pPtr;        pPtr += ROUND8(pcacheSize);
  pPager->fd = (sqlite3_file*)pPtr;       pPtr += ROUND8(pVfs->szOsFile);
  pPager->sjfd = (sqlite3_file*)pPtr;     pPtr += journalFileSize;
  pPager->jfd =  (sqlite3_file*)pPtr;     pPtr += journalFileSize;
  assert( EIGHT_BYTE_ALIGNMENT(pPager->jfd) );

  /* Fill in the Pager.zFilename and pPager.zQueryParam fields */
                                          pPtr += 4;  /* Skip zero prefix */
  pPager->zFilename = (char*)pPtr;
  if( nPathname>0 ){
    memcpy(pPtr, zPathname, nPathname);   pPtr += nPathname + 1;
    if( zUri ){
      memcpy(pPtr, zUri, nUriByte);       pPtr += nUriByte;
    }else{
                                          pPtr++;
    }
  }


  /* Fill in Pager.zJournal */

  if( nPathname>0 ){
    pPager->zJournal = (char*)pPtr;
    memcpy(pPtr, zPathname, nPathname);   pPtr += nPathname;
    memcpy(pPtr, "-journal",8);           pPtr += 8 + 1;
#ifdef SQLITE_ENABLE_8_3_NAMES
    sqlite3FileSuffix3(zFilename,pPager->zJournal);
    pPtr = (u8*)(pPager->zJournal + sqlite3Strlen30(pPager->zJournal)+1);
#endif
  }else{
    pPager->zJournal = 0;

  }

#ifndef SQLITE_OMIT_WAL
  /* Fill in Pager.zWal */

  if( nPathname>0 ){
    pPager->zWal = (char*)pPtr;
    memcpy(pPtr, zPathname, nPathname);   pPtr += nPathname;
    memcpy(pPtr, "-wal", 4);              pPtr += 4 + 1;
#ifdef SQLITE_ENABLE_8_3_NAMES
    sqlite3FileSuffix3(zFilename, pPager->zWal);
    pPtr = (u8*)(pPager->zWal + sqlite3Strlen30(pPager->zWal)+1);
#endif
  }else{
    pPager->zWal = 0;

  }
#endif












  if( nPathname ) sqlite3DbFree(0, zPathname);
  pPager->pVfs = pVfs;
  pPager->vfsFlags = vfsFlags;

  /* Open the pager file.
  */
  if( zFilename && zFilename[0] ){
7034
7035
7036
7037
7038
7039
7040
7041
7042
7043
7044
7045
7046
7047
7048
7049
** shared cache, it uses nullIfMemDb==0 so that in-memory databases can
** participate in shared-cache.
**
** The return value to this routine is always safe to use with
** sqlite3_uri_parameter() and sqlite3_filename_database() and friends.
*/
const char *sqlite3PagerFilename(const Pager *pPager, int nullIfMemDb){
  static const char zFake[] = { 0x00, 0x01, 0x00, 0x00, 0x00 };
  return (nullIfMemDb && pPager->memDb) ? &zFake[3] : pPager->zFilename;
}

/*
** Return the VFS structure for the pager.
*/
sqlite3_vfs *sqlite3PagerVfs(Pager *pPager){
  return pPager->pVfs;







|
|







7049
7050
7051
7052
7053
7054
7055
7056
7057
7058
7059
7060
7061
7062
7063
7064
** shared cache, it uses nullIfMemDb==0 so that in-memory databases can
** participate in shared-cache.
**
** The return value to this routine is always safe to use with
** sqlite3_uri_parameter() and sqlite3_filename_database() and friends.
*/
const char *sqlite3PagerFilename(const Pager *pPager, int nullIfMemDb){
  static const char zFake[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
  return (nullIfMemDb && pPager->memDb) ? &zFake[4] : pPager->zFilename;
}

/*
** Return the VFS structure for the pager.
*/
sqlite3_vfs *sqlite3PagerVfs(Pager *pPager){
  return pPager->pVfs;