/ Check-in [f8ca5622]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:When possible, use memcpy() to and from the mapped region instead of xWrite() and xRead().
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | experimental-mmap
Files: files | file ages | folders
SHA1: f8ca5622d99bedca957caa9ad311d798f63b3ce9
User & Date: dan 2013-03-16 20:19:21
Context
2013-03-19
19:28
Add the sqlite3_io_methods.xMremap() method to the VFS interface. Also "PRAGMA mmap_size". check-in: 6183f1bd user: dan tags: experimental-mmap
2013-03-16
20:19
When possible, use memcpy() to and from the mapped region instead of xWrite() and xRead(). check-in: f8ca5622 user: dan tags: experimental-mmap
2013-03-15
19:13
Fix a dropped error code in pager.c. check-in: 022fdc98 user: dan tags: experimental-mmap
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/pager.c.

2861
2862
2863
2864
2865
2866
2867



2868
2869
2870

2871
2872
2873
2874
2875
2876
2877
....
3830
3831
3832
3833
3834
3835
3836


3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
















3853
3854
3855
3856
3857
3858
3859
....
4221
4222
4223
4224
4225
4226
4227



4228

4229
4230
4231
4232
4233
4234
4235
....
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100

  if( pagerUseWal(pPager) ){
    /* Try to pull the page from the write-ahead log. */
    rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData);
  }
  if( rc==SQLITE_OK && !isInWal ){
    i64 iOffset = (pgno-1)*(i64)pPager->pageSize;



    rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset);
    if( rc==SQLITE_IOERR_SHORT_READ ){
      rc = SQLITE_OK;

    }
  }

  if( pgno==1 ){
    if( rc ){
      /* If the read is unsuccessful, set the dbFileVers[] to something
      ** that will never be a valid file version.  dbFileVers[] is a copy
................................................................................
static int pagerMap(Pager *pPager){
  int rc;
  i64 sz = 0;

  assert( pPager->pMap==0 && pPager->nMap==0 );

  rc = sqlite3OsFileSize(pPager->fd, &sz);


  if( rc==SQLITE_OK && sz>0 ){
    int fd;
    rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_GETFD, (void *)&fd);
    if( rc==SQLITE_OK ){
      void *pMap = mmap(0, sz, PROT_READ, MAP_SHARED, fd, 0);
      if( pMap==MAP_FAILED ){
      assert( 0 );
        return SQLITE_IOERR;
      }
      pPager->pMap = pMap;
      pPager->nMapValid = pPager->nMap = sz;
    }
  }

  return rc;
}

















static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){
  int rc;
  *ppPage = 0;

  assert( pPager->pWal==0 );

................................................................................
      assert( (pList->flags&PGHDR_NEED_SYNC)==0 );
      if( pList->pgno==1 ) pager_write_changecounter(pList);

      /* Encode the database */
      CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData);

      /* Write out the page data. */



      rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset);


      /* If page 1 was just written, update Pager.dbFileVers to match
      ** the value now stored in the database file. If writing this 
      ** page caused the database file to grow, update dbFileSize. 
      */
      if( pgno==1 ){
        memcpy(&pPager->dbFileVers, &pData[24], sizeof(pPager->dbFileVers));
................................................................................
        /* Unmap the database file. It is possible that external processes
        ** may have truncated the database file and then extended it back
        ** to its original size while this process was not holding a lock.
        ** In this case there may exist a Pager.pMap mapping that appears
        ** to be the right size but is not actually valid. Avoid this
        ** possibility by unmapping the db here. */
        pagerUnmap(pPager);
      }else if( ((i64)nPage*pPager->pageSize)!=pPager->nMap ){
        pagerUnmap(pPager);
      }
    }

    /* If there is a WAL file in the file-system, open this database in WAL
    ** mode. Otherwise, the following function call is a no-op.
    */
    rc = pagerOpenWalIfPresent(pPager);







>
>
>
|
|
|
>







 







>
>




|

<









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
|
>







 







|
|







2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
....
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848

3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
....
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
....
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125

  if( pagerUseWal(pPager) ){
    /* Try to pull the page from the write-ahead log. */
    rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData);
  }
  if( rc==SQLITE_OK && !isInWal ){
    i64 iOffset = (pgno-1)*(i64)pPager->pageSize;
    if( pPager->pMap && pPager->nMapValid>=iOffset+pPager->pageSize ){
      memcpy(pPg->pData, &((u8 *)(pPager->pMap))[iOffset], pPager->pageSize);
    }else{
      rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset);
      if( rc==SQLITE_IOERR_SHORT_READ ){
        rc = SQLITE_OK;
      }
    }
  }

  if( pgno==1 ){
    if( rc ){
      /* If the read is unsuccessful, set the dbFileVers[] to something
      ** that will never be a valid file version.  dbFileVers[] is a copy
................................................................................
static int pagerMap(Pager *pPager){
  int rc;
  i64 sz = 0;

  assert( pPager->pMap==0 && pPager->nMap==0 );

  rc = sqlite3OsFileSize(pPager->fd, &sz);
  sz = sz & ~(4096-1);

  if( rc==SQLITE_OK && sz>0 ){
    int fd;
    rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_GETFD, (void *)&fd);
    if( rc==SQLITE_OK ){
      void *pMap = mmap(0, sz, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
      if( pMap==MAP_FAILED ){

        return SQLITE_IOERR;
      }
      pPager->pMap = pMap;
      pPager->nMapValid = pPager->nMap = sz;
    }
  }

  return rc;
}

static int pagerRemap(Pager *pPager, Pgno nPage){
  i64 sz = (i64)nPage * pPager->pageSize;
  sz = sz & ~(4096-1);

  if( pPager->nMap!=sz ){
    void *pMap = mremap(pPager->pMap, pPager->nMap, sz, MREMAP_MAYMOVE);    
    if( pMap==MAP_FAILED ){
      return SQLITE_IOERR;
    }
    pPager->pMap = pMap;
    pPager->nMapValid = pPager->nMap = sz;
  }

  return SQLITE_OK;
}

static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){
  int rc;
  *ppPage = 0;

  assert( pPager->pWal==0 );

................................................................................
      assert( (pList->flags&PGHDR_NEED_SYNC)==0 );
      if( pList->pgno==1 ) pager_write_changecounter(pList);

      /* Encode the database */
      CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData);

      /* Write out the page data. */
      if( pPager->nMapValid>=(offset+pPager->pageSize) ){
        memcpy(&((u8 *)(pPager->pMap))[offset], pData, pPager->pageSize);
      }else{
        rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset);
      }

      /* If page 1 was just written, update Pager.dbFileVers to match
      ** the value now stored in the database file. If writing this 
      ** page caused the database file to grow, update dbFileSize. 
      */
      if( pgno==1 ){
        memcpy(&pPager->dbFileVers, &pData[24], sizeof(pPager->dbFileVers));
................................................................................
        /* Unmap the database file. It is possible that external processes
        ** may have truncated the database file and then extended it back
        ** to its original size while this process was not holding a lock.
        ** In this case there may exist a Pager.pMap mapping that appears
        ** to be the right size but is not actually valid. Avoid this
        ** possibility by unmapping the db here. */
        pagerUnmap(pPager);
      }else if( pPager->pMap ){
        pagerRemap(pPager, nPage);
      }
    }

    /* If there is a WAL file in the file-system, open this database in WAL
    ** mode. Otherwise, the following function call is a no-op.
    */
    rc = pagerOpenWalIfPresent(pPager);

Changes to src/test1.c.

5839
5840
5841
5842
5843
5844
5845























5846
5847
5848
5849
5850
5851
5852
....
6229
6230
6231
6232
6233
6234
6235

6236
6237
6238
6239
6240
6241
6242
      break;
    }
  }

  Tcl_ResetResult(interp);
  return TCL_OK;
}
























#if SQLITE_OS_WIN
/*
** Information passed from the main thread into the windows file locker
** background thread.
*/
struct win32FileLocker {
................................................................................
     { "sqlite3_wal_checkpoint",   test_wal_checkpoint, 0  },
     { "sqlite3_wal_checkpoint_v2",test_wal_checkpoint_v2, 0  },
     { "test_sqlite3_log",         test_sqlite3_log, 0  },
#ifndef SQLITE_OMIT_EXPLAIN
     { "print_explain_query_plan", test_print_eqp, 0  },
#endif
     { "sqlite3_test_control", test_test_control },

  };
  static int bitmask_size = sizeof(Bitmask)*8;
  int i;
  extern int sqlite3_sync_count, sqlite3_fullsync_count;
  extern int sqlite3_opentemp_count;
  extern int sqlite3_like_count;
  extern int sqlite3_xferopt_count;







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>







5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
....
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
      break;
    }
  }

  Tcl_ResetResult(interp);
  return TCL_OK;
}

#include <sys/time.h>
#include <sys/resource.h>

static int test_getrusage(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  char buf[1024];
  struct rusage r;
  memset(&r, 0, sizeof(r));
  getrusage(RUSAGE_SELF, &r);

  sprintf(buf, "ru_utime=%d.%06d ru_stime=%d.%06d ru_minflt=%d ru_majflt=%d", 
    r.ru_utime.tv_sec, r.ru_utime.tv_usec, 
    r.ru_stime.tv_sec, r.ru_stime.tv_usec, 
    r.ru_minflt, r.ru_majflt
  );
  Tcl_SetObjResult(interp, Tcl_NewStringObj(buf, -1));
  return TCL_OK;
}

#if SQLITE_OS_WIN
/*
** Information passed from the main thread into the windows file locker
** background thread.
*/
struct win32FileLocker {
................................................................................
     { "sqlite3_wal_checkpoint",   test_wal_checkpoint, 0  },
     { "sqlite3_wal_checkpoint_v2",test_wal_checkpoint_v2, 0  },
     { "test_sqlite3_log",         test_sqlite3_log, 0  },
#ifndef SQLITE_OMIT_EXPLAIN
     { "print_explain_query_plan", test_print_eqp, 0  },
#endif
     { "sqlite3_test_control", test_test_control },
     { "getrusage", test_getrusage },
  };
  static int bitmask_size = sizeof(Bitmask)*8;
  int i;
  extern int sqlite3_sync_count, sqlite3_fullsync_count;
  extern int sqlite3_opentemp_count;
  extern int sqlite3_like_count;
  extern int sqlite3_xferopt_count;