/ Check-in [7fdc78a6]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the --fsync flag to kvtest, and document the --nosync flag.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 7fdc78a672b2ea6187dcb5fdf32f809bb8e4d501e2434f2233edc3bc2e3acc7c
User & Date: drh 2017-06-02 23:32:17
Context
2017-06-03
15:17
Add the --nocheckpoint and --multitrans options to kvtest. check-in: 5828633c user: drh tags: trunk
2017-06-02
23:44
Merge all recent trunk enhancements. check-in: 1d23294d user: drh tags: lsm-vtab
23:32
Add the --fsync flag to kvtest, and document the --nosync flag. check-in: 7fdc78a6 user: drh tags: trunk
19:31
Work toward enhancing kvtest to measure write performance. check-in: fc73e7d2 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to test/kvtest.c.

86
87
88
89
90
91
92

93
94
95

96
97
98
99
100
101
102
...
537
538
539
540
541
542
543
544
545
546
547
548

549
550
551
552
553
554
555
556
557
558
559
560



561
562
563
564
565







566
567
568
569
570
571
572
...
724
725
726
727
728
729
730

731
732
733
734
735
736
737
...
811
812
813
814
815
816
817




818
819
820
821
822
823
824
...
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
"        database or a directory containing sample files.  Options:\n"
"\n"
"           --asc                  Read blobs in ascending order\n"
"           --blob-api             Use the BLOB API\n"
"           --cache-size N         Database cache size\n"
"           --count N              Read N blobs\n"
"           --desc                 Read blobs in descending order\n"

"           --integrity-check      Run 'PRAGMA integrity_check' after test\n"
"           --max-id N             Maximum blob key to use\n"
"           --mmap N               Mmap as much as N bytes of DBFILE\n"

"           --jmode MODE           Set MODE journal mode prior to starting\n"
"           --random               Read blobs in a random order\n"
"           --start N              Start reading with this blob key\n"
"           --stats                Output operating stats before exiting\n"
"           --update               To an overwrite test\n"
;

................................................................................
  return pBuf;
}

/*
** Overwrite a file with randomness.  Do not change the size of the
** file.
*/
static void updateFile(const char *zName, int *pnByte){
  FILE *out;              /* FILE from which to read content of zName */
  sqlite3_int64 sz;       /* Size of zName in bytes */
  size_t nWritten;        /* Number of bytes actually read */
  unsigned char *pBuf;    /* Content to store on disk */


  sz = fileSize(zName);
  if( sz<0 ){
    fatalError("No such file: \"%s\"", zName);
  }
  *pnByte = (int)sz;
  if( sz==0 ) return;
  pBuf = sqlite3_malloc64( sz );
  if( pBuf==0 ){
    fatalError("Cannot allocate %lld bytes\n", sz);
  }
  sqlite3_randomness((int)sz, pBuf); 



  out = fopen(zName, "wb");
  if( out==0 ){
    fatalError("Cannot open \"%s\" for writing\n", zName);
  }
  nWritten = fwrite(pBuf, 1, (size_t)sz, out);







  fclose(out);
  if( nWritten!=(size_t)sz ){
    fatalError("Wrote only %d of %d bytes to \"%s\"\n",
               (int)nWritten, (int)sz, zName);
  }
  sqlite3_free(pBuf);
}
................................................................................
  int iCache = 1000;          /* Database cache size in kibibytes */
  int bBlobApi = 0;           /* Use the incremental blob I/O API */
  int bStats = 0;             /* Print stats before exiting */
  int eOrder = ORDER_ASC;     /* Access order */
  int isUpdateTest = 0;       /* Do in-place updates rather than reads */
  int doIntegrityCk = 0;      /* Run PRAGMA integrity_check after the test */
  int noSync = 0;             /* Disable synchronous mode */

  sqlite3 *db = 0;            /* Database connection */
  sqlite3_stmt *pStmt = 0;    /* Prepared statement for SQL access */
  sqlite3_blob *pBlob = 0;    /* Handle for incremental Blob I/O */
  sqlite3_int64 tmStart;      /* Start time */
  sqlite3_int64 tmElapsed;    /* Elapsed time */
  int mmapSize = 0;           /* --mmap N argument */
  int nData = 0;              /* Bytes of data */
................................................................................
    if( strcmp(z, "-integrity-check")==0 ){
      doIntegrityCk = 1;
      continue;
    }
    if( strcmp(z, "-nosync")==0 ){
      noSync = 1;
      continue;




    }
    fatalError("unknown option: \"%s\"", argv[i]);
  }
  if( eType==PATH_DB ){
    /* Recover any prior crashes prior to starting the timer */
    sqlite3_open(zDb, &db);
    sqlite3_exec(db, "SELECT rowid FROM sqlite_master LIMIT 1", 0, 0, 0);
................................................................................
  for(i=0; i<nCount; i++){
    if( eType==PATH_DIR ){
      /* CASE 1: Reading blobs out of separate files */
      char *zKey;
      zKey = sqlite3_mprintf("%s/%06d", zDb, iKey);
      nData = 0;
      if( isUpdateTest ){
        updateFile(zKey, &nData);
      }else{
        pData = readFile(zKey, &nData);
        sqlite3_free(pData);
      }
      sqlite3_free(zKey);
    }else if( bBlobApi ){
      /* CASE 2: Reading from database using the incremental BLOB I/O API */







>
|


>







 







|




>












>
>
>
|




>
>
>
>
>
>
>







 







>







 







>
>
>
>







 







|







86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
...
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
...
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
...
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
...
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
"        database or a directory containing sample files.  Options:\n"
"\n"
"           --asc                  Read blobs in ascending order\n"
"           --blob-api             Use the BLOB API\n"
"           --cache-size N         Database cache size\n"
"           --count N              Read N blobs\n"
"           --desc                 Read blobs in descending order\n"
"           --fsync                Synchronous file writes\n"
"           --integrity-check      Run \"PRAGMA integrity_check\" after test\n"
"           --max-id N             Maximum blob key to use\n"
"           --mmap N               Mmap as much as N bytes of DBFILE\n"
"           --nosync               Set \"PRAGMA synchronous=OFF\"\n"
"           --jmode MODE           Set MODE journal mode prior to starting\n"
"           --random               Read blobs in a random order\n"
"           --start N              Start reading with this blob key\n"
"           --stats                Output operating stats before exiting\n"
"           --update               To an overwrite test\n"
;

................................................................................
  return pBuf;
}

/*
** Overwrite a file with randomness.  Do not change the size of the
** file.
*/
static void updateFile(const char *zName, int *pnByte, int doFsync){
  FILE *out;              /* FILE from which to read content of zName */
  sqlite3_int64 sz;       /* Size of zName in bytes */
  size_t nWritten;        /* Number of bytes actually read */
  unsigned char *pBuf;    /* Content to store on disk */
  const char *zMode = "wb";   /* Mode for fopen() */

  sz = fileSize(zName);
  if( sz<0 ){
    fatalError("No such file: \"%s\"", zName);
  }
  *pnByte = (int)sz;
  if( sz==0 ) return;
  pBuf = sqlite3_malloc64( sz );
  if( pBuf==0 ){
    fatalError("Cannot allocate %lld bytes\n", sz);
  }
  sqlite3_randomness((int)sz, pBuf); 
#if defined(_WIN32)
  if( doFsync ) zMode = "wbc";
#endif
  out = fopen(zName, zMode);
  if( out==0 ){
    fatalError("Cannot open \"%s\" for writing\n", zName);
  }
  nWritten = fwrite(pBuf, 1, (size_t)sz, out);
  if( doFsync ){
#if defined(_WIN32)
    fflush(out);
#else
    fsync(fileno(out));
#endif
  }
  fclose(out);
  if( nWritten!=(size_t)sz ){
    fatalError("Wrote only %d of %d bytes to \"%s\"\n",
               (int)nWritten, (int)sz, zName);
  }
  sqlite3_free(pBuf);
}
................................................................................
  int iCache = 1000;          /* Database cache size in kibibytes */
  int bBlobApi = 0;           /* Use the incremental blob I/O API */
  int bStats = 0;             /* Print stats before exiting */
  int eOrder = ORDER_ASC;     /* Access order */
  int isUpdateTest = 0;       /* Do in-place updates rather than reads */
  int doIntegrityCk = 0;      /* Run PRAGMA integrity_check after the test */
  int noSync = 0;             /* Disable synchronous mode */
  int doFsync = 0;            /* Update disk files synchronously */
  sqlite3 *db = 0;            /* Database connection */
  sqlite3_stmt *pStmt = 0;    /* Prepared statement for SQL access */
  sqlite3_blob *pBlob = 0;    /* Handle for incremental Blob I/O */
  sqlite3_int64 tmStart;      /* Start time */
  sqlite3_int64 tmElapsed;    /* Elapsed time */
  int mmapSize = 0;           /* --mmap N argument */
  int nData = 0;              /* Bytes of data */
................................................................................
    if( strcmp(z, "-integrity-check")==0 ){
      doIntegrityCk = 1;
      continue;
    }
    if( strcmp(z, "-nosync")==0 ){
      noSync = 1;
      continue;
    }
    if( strcmp(z, "-fsync")==0 ){
      doFsync = 1;
      continue;
    }
    fatalError("unknown option: \"%s\"", argv[i]);
  }
  if( eType==PATH_DB ){
    /* Recover any prior crashes prior to starting the timer */
    sqlite3_open(zDb, &db);
    sqlite3_exec(db, "SELECT rowid FROM sqlite_master LIMIT 1", 0, 0, 0);
................................................................................
  for(i=0; i<nCount; i++){
    if( eType==PATH_DIR ){
      /* CASE 1: Reading blobs out of separate files */
      char *zKey;
      zKey = sqlite3_mprintf("%s/%06d", zDb, iKey);
      nData = 0;
      if( isUpdateTest ){
        updateFile(zKey, &nData, doFsync);
      }else{
        pData = readFile(zKey, &nData);
        sqlite3_free(pData);
      }
      sqlite3_free(zKey);
    }else if( bBlobApi ){
      /* CASE 2: Reading from database using the incremental BLOB I/O API */