SQLite

Check-in [a64d96db09]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Change things so that journal2.test works with ENABLE_ATOMIC_WRITE.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | experimental
Files: files | file ages | folders
SHA1: a64d96db09ef2b7651fa4e98d3c7bf3ae5d3fe96
User & Date: dan 2010-06-21 12:34:30.000
Context
2010-06-21
12:47
Merge the experimental UNDELETABLE_WHEN_OPEN optimization into the trunk. (check-in: ee0acef1fa user: drh tags: trunk)
12:34
Change things so that journal2.test works with ENABLE_ATOMIC_WRITE. (Closed-Leaf check-in: a64d96db09 user: dan tags: experimental)
07:45
Add further pager tests. (check-in: 4104b175a8 user: dan tags: experimental)
Changes
Unified Diff Show Whitespace Changes Patch
Changes to src/pager.c.
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
  u8 noReadlock;              /* Do not bother to obtain readlocks */
  u8 noSync;                  /* Do not sync the journal if true */
  u8 fullSync;                /* Do extra syncs of the journal for robustness */
  u8 sync_flags;              /* One of SYNC_NORMAL or SYNC_FULL */
  u8 tempFile;                /* zFilename is a temporary file */
  u8 readOnly;                /* True for a read-only database */
  u8 memDb;                   /* True to inhibit all file I/O */
  u8 safeJrnlHandle;          /* True if jrnl may be held open with no lock */

  /* The following block contains those class members that are dynamically
  ** modified during normal operations. The other variables in this structure
  ** are either constant throughout the lifetime of the pager, or else
  ** used to store configuration parameters that affect the way the pager 
  ** operates.
  **







<







332
333
334
335
336
337
338

339
340
341
342
343
344
345
  u8 noReadlock;              /* Do not bother to obtain readlocks */
  u8 noSync;                  /* Do not sync the journal if true */
  u8 fullSync;                /* Do extra syncs of the journal for robustness */
  u8 sync_flags;              /* One of SYNC_NORMAL or SYNC_FULL */
  u8 tempFile;                /* zFilename is a temporary file */
  u8 readOnly;                /* True for a read-only database */
  u8 memDb;                   /* True to inhibit all file I/O */


  /* The following block contains those class members that are dynamically
  ** modified during normal operations. The other variables in this structure
  ** are either constant throughout the lifetime of the pager, or else
  ** used to store configuration parameters that affect the way the pager 
  ** operates.
  **
1216
1217
1218
1219
1220
1221
1222

1223
1224
1225
1226
1227
1228



1229
1230


1231
1232
1233
1234
1235
1236
1237
** an open journal-file, then the next time a shared-lock is obtained
** on the pager file (by this or any other process), it will be
** treated as a hot-journal and rolled back.
*/
static void pager_unlock(Pager *pPager){
  if( !pPager->exclusiveMode ){
    int rc = SQLITE_OK;          /* Return code */


    /* Always close the journal file when dropping the database lock.
    ** Otherwise, another connection with journal_mode=delete might
    ** delete the file out from under us.
    */
    if( pPager->safeJrnlHandle==0 



     || (pPager->journalMode!=PAGER_JOURNALMODE_TRUNCATE 
      && pPager->journalMode!=PAGER_JOURNALMODE_PERSIST)


    ){
      sqlite3OsClose(pPager->jfd);
    }

    sqlite3BitvecDestroy(pPager->pInJournal);
    pPager->pInJournal = 0;
    releaseAllSavepoints(pPager);







>





|
>
>
>
|
|
>
>







1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
** an open journal-file, then the next time a shared-lock is obtained
** on the pager file (by this or any other process), it will be
** treated as a hot-journal and rolled back.
*/
static void pager_unlock(Pager *pPager){
  if( !pPager->exclusiveMode ){
    int rc = SQLITE_OK;          /* Return code */
    int iDc = isOpen(pPager->fd)?sqlite3OsDeviceCharacteristics(pPager->fd):0;

    /* Always close the journal file when dropping the database lock.
    ** Otherwise, another connection with journal_mode=delete might
    ** delete the file out from under us.
    */
    assert( (PAGER_JOURNALMODE_MEMORY   & 5)!=1 );
    assert( (PAGER_JOURNALMODE_OFF      & 5)!=1 );
    assert( (PAGER_JOURNALMODE_WAL      & 5)!=1 );
    assert( (PAGER_JOURNALMODE_DELETE   & 5)!=1 );
    assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 );
    assert( (PAGER_JOURNALMODE_PERSIST  & 5)==1 );
    if( 0==(iDc & SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN)
     || 1!=(pPager->journalMode & 5)
    ){
      sqlite3OsClose(pPager->jfd);
    }

    sqlite3BitvecDestroy(pPager->pInJournal);
    pPager->pInJournal = 0;
    releaseAllSavepoints(pPager);
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
int sqlite3PagerClose(Pager *pPager){
  u8 *pTmp = (u8 *)pPager->pTmpSpace;

  disable_simulated_io_errors();
  sqlite3BeginBenignMalloc();
  pPager->errCode = 0;
  pPager->exclusiveMode = 0;
  pPager->safeJrnlHandle = 0;
#ifndef SQLITE_OMIT_WAL
  sqlite3WalClose(pPager->pWal,
    (pPager->noSync ? 0 : pPager->sync_flags), 
    pPager->pageSize, pTmp
  );
  pPager->pWal = 0;
#endif







<







3097
3098
3099
3100
3101
3102
3103

3104
3105
3106
3107
3108
3109
3110
int sqlite3PagerClose(Pager *pPager){
  u8 *pTmp = (u8 *)pPager->pTmpSpace;

  disable_simulated_io_errors();
  sqlite3BeginBenignMalloc();
  pPager->errCode = 0;
  pPager->exclusiveMode = 0;

#ifndef SQLITE_OMIT_WAL
  sqlite3WalClose(pPager->pWal,
    (pPager->noSync ? 0 : pPager->sync_flags), 
    pPager->pageSize, pTmp
  );
  pPager->pWal = 0;
#endif
3119
3120
3121
3122
3123
3124
3125

3126
3127
3128
3129
3130
3131
3132
    }
    pagerUnlockAndRollback(pPager);
  }
  sqlite3EndBenignMalloc();
  enable_simulated_io_errors();
  PAGERTRACE(("CLOSE %d\n", PAGERID(pPager)));
  IOTRACE(("CLOSE %p\n", pPager))

  sqlite3OsClose(pPager->fd);
  sqlite3PageFree(pTmp);
  sqlite3PcacheClose(pPager->pPCache);

#ifdef SQLITE_HAS_CODEC
  if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec);
#endif







>







3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
    }
    pagerUnlockAndRollback(pPager);
  }
  sqlite3EndBenignMalloc();
  enable_simulated_io_errors();
  PAGERTRACE(("CLOSE %d\n", PAGERID(pPager)));
  IOTRACE(("CLOSE %p\n", pPager))
  sqlite3OsClose(pPager->jfd);
  sqlite3OsClose(pPager->fd);
  sqlite3PageFree(pTmp);
  sqlite3PcacheClose(pPager->pPCache);

#ifdef SQLITE_HAS_CODEC
  if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec);
#endif
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
        );
#ifdef SQLITE_ENABLE_ATOMIC_WRITE
      rc = sqlite3JournalOpen(
          pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager)
      );
#else
      rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0);
      if( rc==SQLITE_OK ){
        int iDc = sqlite3OsDeviceCharacteristics(pPager->jfd);
        pPager->safeJrnlHandle = (iDc&SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN)!=0;
      }
#endif
    }
    assert( rc!=SQLITE_OK || isOpen(pPager->jfd) );
  }


  /* Write the first journal header to the journal file and open 







<
<
<
<







4508
4509
4510
4511
4512
4513
4514




4515
4516
4517
4518
4519
4520
4521
        );
#ifdef SQLITE_ENABLE_ATOMIC_WRITE
      rc = sqlite3JournalOpen(
          pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager)
      );
#else
      rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0);




#endif
    }
    assert( rc!=SQLITE_OK || isOpen(pPager->jfd) );
  }


  /* Write the first journal header to the journal file and open 
Changes to test/malloc_common.tcl.
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
  }
}
proc faultsim_save_and_close {} {
  faultsim_save
  catch { db close }
  return ""
}
proc faultsim_restore_and_reopen {} {
  catch { db close }
  foreach f [glob -nocomplain test.db*] { file delete -force $f }
  foreach f2 [glob -nocomplain sv_test.db*] {
    set f [string range $f2 3 end]
    file copy -force $f2 $f
  }
  sqlite3 db test.db
  sqlite3_extended_result_codes db 1
  sqlite3_db_config_lookaside db 0 0 0
}

proc faultsim_integrity_check {{db db}} {
  set ic [$db eval { PRAGMA integrity_check }]
  if {$ic != "ok"} { error "Integrity check: $ic" }
}

proc faultsim_delete_and_reopen {{file test.db}} {
  catch { db close }
  foreach f [glob -nocomplain test.db*] { file delete -force $f }
  sqlite3 db test.db
}


# The following procs are used as [do_one_faultsim_test] callbacks when 
# injecting OOM faults into test cases.
#
proc oom_injectstart {nRepeat iFail} {







|






|












|







132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
  }
}
proc faultsim_save_and_close {} {
  faultsim_save
  catch { db close }
  return ""
}
proc faultsim_restore_and_reopen {{dbfile test.db}} {
  catch { db close }
  foreach f [glob -nocomplain test.db*] { file delete -force $f }
  foreach f2 [glob -nocomplain sv_test.db*] {
    set f [string range $f2 3 end]
    file copy -force $f2 $f
  }
  sqlite3 db $dbfile
  sqlite3_extended_result_codes db 1
  sqlite3_db_config_lookaside db 0 0 0
}

proc faultsim_integrity_check {{db db}} {
  set ic [$db eval { PRAGMA integrity_check }]
  if {$ic != "ok"} { error "Integrity check: $ic" }
}

proc faultsim_delete_and_reopen {{file test.db}} {
  catch { db close }
  foreach f [glob -nocomplain test.db*] { file delete -force $f }
  sqlite3 db $file
}


# The following procs are used as [do_one_faultsim_test] callbacks when 
# injecting OOM faults into test cases.
#
proc oom_injectstart {nRepeat iFail} {
Changes to test/pager1.test.
224
225
226
227
228
229
230







231
232
233
234
235
236
237
#               xAccess() call to check for the presence of any master 
#               journal file is made).
#
# pager1.4.3.*: Test that the contents of a hot-journal are ignored if the
#               page-size or sector-size in the journal header appear to
#               be invalid (too large, too small or not a power of 2).
# 







do_test pager1.4.1.1 {
  faultsim_delete_and_reopen
  execsql { 
    CREATE TABLE x(y, z);
    INSERT INTO x VALUES(1, 2);
  }
  set fd [open test.db-journal w]







>
>
>
>
>
>
>







224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
#               xAccess() call to check for the presence of any master 
#               journal file is made).
#
# pager1.4.3.*: Test that the contents of a hot-journal are ignored if the
#               page-size or sector-size in the journal header appear to
#               be invalid (too large, too small or not a power of 2).
# 
# pager1.4.4.*: Test hot-journal rollback of journal file with a master
#               journal pointer generated in various "PRAGMA synchronous"
#               modes.
#
# pager1.4.5.*: Test that hot-journal rollback stops if it encounters a
#               journal-record for which the checksum fails.
# 
do_test pager1.4.1.1 {
  faultsim_delete_and_reopen
  execsql { 
    CREATE TABLE x(y, z);
    INSERT INTO x VALUES(1, 2);
  }
  set fd [open test.db-journal w]
349
350
351
352
353
354
355












































































































































356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376



377
378
379
380
381
382
383
  do_test pager1.4.3.$tn {
    faultsim_restore_and_reopen
    hexio_write test.db-journal $ofst [format %.8x $value]
    execsql { SELECT * FROM t1 }
  } $result
}
db close













































































































































#-------------------------------------------------------------------------
# The following tests deal with multi-file commits.
#
# pager1-5.1.*: The case where a multi-file cannot be committed because
#               another connection is holding a SHARED lock on one of the
#               files. After the SHARED lock is removed, the COMMIT succeeds.
#
# pager1-5.2.*: Multi-file commits with journal_mode=memory.
#
# pager1-5.3.*: Multi-file commits with journal_mode=memory.
#
# pager1-5.4.*: Check that with synchronous=normal, the master-journal file
#               name is added to a journal file immediately after the last
#               journal record. But with synchronous=full, extra unused space
#               is allocated between the last journal record and the 
#               master-journal file name so that the master-journal file
#               name does not lie on the same sector as the last journal file
#               record.
#
# pager1-5.5.*: 



#
do_test pager1-5.1.1 {
  faultsim_delete_and_reopen
  execsql {
    ATTACH 'test.db2' AS aux;
    CREATE TABLE t1(a, b);
    CREATE TABLE aux.t2(a, b);







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>




















|
>
>
>







356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
  do_test pager1.4.3.$tn {
    faultsim_restore_and_reopen
    hexio_write test.db-journal $ofst [format %.8x $value]
    execsql { SELECT * FROM t1 }
  } $result
}
db close

# Set up a VFS that snapshots the file-system just before a master journal
# file is deleted to commit a multi-file transaction. Specifically, the
# file-system is saved just before the xDelete() call to remove the 
# master journal file from the file-system.
#
testvfs tv -default 1
tv script copy_on_mj_delete
set ::mj_filename_length 0
proc copy_on_mj_delete {method filename args} {
  if {[string match *mj* [file tail $filename]]} { 
    set ::mj_filename_length [string length $filename]
    faultsim_save 
  }
  return SQLITE_OK
}

set pwd [pwd]
foreach {tn1 tcl} {
  1 { set prefix "test.db" }
  2 { 
    # This test depends on the underlying VFS being able to open paths
    # 512 bytes in length. The idea is to create a hot-journal file that
    # contains a master-journal pointer so large that it could contain
    # a valid page record (if the file page-size is 512 bytes). So as to
    # make sure SQLite doesn't get confused by this.
    #
    set nPadding [expr 511 - $::mj_filename_length]

    # We cannot just create a really long database file name to open, as
    # Linux limits a single component of a path to 255 bytes by default
    # (and presumably other systems have limits too). So create a directory
    # hierarchy to work in.
    #
    set dirname "d123456789012345678901234567890/"
    set nDir [expr $nPadding / 32]
    if { $nDir } {
      set p [string repeat $dirname $nDir]
      file mkdir $p
      cd $p
    }

    set padding [string repeat x [expr $nPadding %32]]
    set prefix "test.db${padding}"
  }
} {
  eval $tcl
  foreach {tn2 sql} {
    o { 
      PRAGMA main.synchronous=OFF;
      PRAGMA aux.synchronous=OFF;
    }
    o512 { 
      PRAGMA main.synchronous=OFF;
      PRAGMA aux.synchronous=OFF;
      PRAGMA main.page_size = 512;
      PRAGMA aux.page_size = 512;
    }
    n { 
      PRAGMA main.synchronous=NORMAL;
      PRAGMA aux.synchronous=NORMAL;
    }
    f { 
      PRAGMA main.synchronous=FULL;
      PRAGMA aux.synchronous=FULL;
    }
  } {

    set tn "${tn1}.${tn2}"
  
    # Set up a connection to have two databases, test.db (main) and 
    # test.db2 (aux). Then run a multi-file transaction on them. The
    # VFS will snapshot the file-system just before the master-journal
    # file is deleted to commit the transaction.
    #
    tv filter xDelete
    do_test pager1-4.4.$tn.1 {
      faultsim_delete_and_reopen $prefix
      execsql "
        ATTACH '${prefix}2' AS aux;
        $sql
        CREATE TABLE a(x);
        CREATE TABLE aux.b(x);
        INSERT INTO a VALUES('double-you');
        INSERT INTO a VALUES('why');
        INSERT INTO a VALUES('zed');
        INSERT INTO b VALUES('won');
        INSERT INTO b VALUES('too');
        INSERT INTO b VALUES('free');
      "
      execsql {
        BEGIN;
          INSERT INTO a SELECT * FROM b WHERE rowid<=3;
          INSERT INTO b SELECT * FROM a WHERE rowid<=3;
        COMMIT;
      }
    } {}
    tv filter {}
    
    # Check that the transaction was committed successfully.
    #
    do_execsql_test pager1-4.4.$tn.2 {
      SELECT * FROM a
    } {double-you why zed won too free}
    do_execsql_test pager1-4.4.$tn.3 {
      SELECT * FROM b
    } {won too free double-you why zed}
    
    # Restore the file-system and reopen the databases. Check that it now
    # appears that the transaction was not committed (because the file-system
    # was restored to the state where it had not been).
    #
    do_test pager1-4.4.$tn.4 {
      faultsim_restore_and_reopen $prefix
      execsql "ATTACH '${prefix}2' AS aux"
    } {}
    do_execsql_test pager1-4.4.$tn.5 {SELECT * FROM a} {double-you why zed}
    do_execsql_test pager1-4.4.$tn.6 {SELECT * FROM b} {won too free}
    
    # Restore the file-system again. This time, before reopening the databases,
    # delete the master-journal file from the file-system. It now appears that
    # the transaction was committed (no master-journal file == no rollback).
    #
    do_test pager1-4.4.$tn.7 {
      faultsim_restore_and_reopen $prefix
      foreach f [glob ${prefix}-mj*] { file delete -force $f }
      execsql "ATTACH '${prefix}2' AS aux"
    } {}
    do_execsql_test pager1-4.4.$tn.8 {
      SELECT * FROM a
    } {double-you why zed won too free}
    do_execsql_test pager1-4.4.$tn.9 {
      SELECT * FROM b
    } {won too free double-you why zed}
  }

  cd $pwd
}
db close
tv delete

#-------------------------------------------------------------------------
# The following tests deal with multi-file commits.
#
# pager1-5.1.*: The case where a multi-file cannot be committed because
#               another connection is holding a SHARED lock on one of the
#               files. After the SHARED lock is removed, the COMMIT succeeds.
#
# pager1-5.2.*: Multi-file commits with journal_mode=memory.
#
# pager1-5.3.*: Multi-file commits with journal_mode=memory.
#
# pager1-5.4.*: Check that with synchronous=normal, the master-journal file
#               name is added to a journal file immediately after the last
#               journal record. But with synchronous=full, extra unused space
#               is allocated between the last journal record and the 
#               master-journal file name so that the master-journal file
#               name does not lie on the same sector as the last journal file
#               record.
#
# pager1-5.5.*: Check that in journal_mode=PERSIST mode, a journal file is
#               truncated to zero bytes when a multi-file transaction is 
#               committed (instead of the first couple of bytes being zeroed).
#
#
do_test pager1-5.1.1 {
  faultsim_delete_and_reopen
  execsql {
    ATTACH 'test.db2' AS aux;
    CREATE TABLE t1(a, b);
    CREATE TABLE aux.t2(a, b);