SQLite

Check-in [10ea8287d0]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Do not write pages to disk to free memory after an IO error occurs. (CVS 5132)
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 10ea8287d090ae610416b4754c0838f13b51fd78
User & Date: danielk1977 2008-05-15 08:34:54.000
Context
2008-05-15
09:07
Fix a test script bug - not all database handles were being closed by ioerr5.test. (CVS 5133) (check-in: 47652e7b16 user: danielk1977 tags: trunk)
08:34
Do not write pages to disk to free memory after an IO error occurs. (CVS 5132) (check-in: 10ea8287d0 user: danielk1977 tags: trunk)
2008-05-14
16:18
Version 3.5.9 (CVS 5131) (check-in: b6129f4cc2 user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/pager.c.
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.446 2008/05/13 13:27:34 drh Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include <assert.h>
#include <string.h>

/*







|







14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.447 2008/05/15 08:34:54 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include <assert.h>
#include <string.h>

/*
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
** This routine clears the needSync field of every page current held in
** memory.
*/
static int syncJournal(Pager *pPager){
  PgHdr *pPg;
  int rc = SQLITE_OK;


  /* Sync the journal before modifying the main database
  ** (assuming there is a journal and it needs to be synced.)
  */
  if( pPager->needSync ){
    if( !pPager->tempFile ){
      int iDc = sqlite3OsDeviceCharacteristics(pPager->fd);
      assert( pPager->journalOpen );







<







2866
2867
2868
2869
2870
2871
2872

2873
2874
2875
2876
2877
2878
2879
** This routine clears the needSync field of every page current held in
** memory.
*/
static int syncJournal(Pager *pPager){
  PgHdr *pPg;
  int rc = SQLITE_OK;


  /* Sync the journal before modifying the main database
  ** (assuming there is a journal and it needs to be synced.)
  */
  if( pPager->needSync ){
    if( !pPager->tempFile ){
      int iDc = sqlite3OsDeviceCharacteristics(pPager->fd);
      assert( pPager->journalOpen );
3178
3179
3180
3181
3182
3183
3184
3185

3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203

3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
  pPg = pPager->lru.pFirstSynced;

  /* If we could not find a page that does not require an fsync()
  ** on the journal file then fsync the journal file.  This is a
  ** very slow operation, so we work hard to avoid it.  But sometimes
  ** it can't be helped.
  */
  if( pPg==0 && pPager->lru.pFirst){

    int iDc = sqlite3OsDeviceCharacteristics(pPager->fd);
    int rc = syncJournal(pPager);
    if( rc!=0 ){
      return rc;
    }
    if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){
      /* If in full-sync mode, write a new journal header into the
      ** journal file. This is done to avoid ever modifying a journal
      ** header that is involved in the rollback of pages that have
      ** already been written to the database (in case the header is
      ** trashed when the nRec field is updated).
      */
      pPager->nRec = 0;
      assert( pPager->journalOff > 0 );
      assert( pPager->doNotSync==0 );
      rc = writeJournalHdr(pPager);
      if( rc!=0 ){
        return rc;

      }
    }
    pPg = pPager->lru.pFirst;
  }

  assert( pPg->nRef==0 );

  /* Write the page to the database file if it is dirty.
  */
  if( pPg->dirty ){
    int rc;
    assert( pPg->needSync==0 );
    makeClean(pPg);
    pPg->dirty = 1;
    pPg->pDirty = 0;
    rc = pager_write_pagelist( pPg );
    pPg->dirty = 0;
    if( rc!=SQLITE_OK ){
      return rc;
    }
  }
  assert( pPg->dirty==0 );

  /* If the page we are recycling is marked as alwaysRollback, then
  ** set the global alwaysRollback flag, thus disabling the
  ** sqlite3PagerDontRollback() optimization for the rest of this transaction.
  ** It is necessary to do this because the page marked alwaysRollback
  ** might be reloaded at a later time but at that point we won't remember
  ** that is was marked alwaysRollback.  This means that all pages must







|
>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>









|











|







3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
  pPg = pPager->lru.pFirstSynced;

  /* If we could not find a page that does not require an fsync()
  ** on the journal file then fsync the journal file.  This is a
  ** very slow operation, so we work hard to avoid it.  But sometimes
  ** it can't be helped.
  */
  if( pPg==0 && pPager->lru.pFirst ){
    if( !pPager->errCode ){
      int iDc = sqlite3OsDeviceCharacteristics(pPager->fd);
      int rc = syncJournal(pPager);
      if( rc!=0 ){
        return rc;
      }
      if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){
        /* If in full-sync mode, write a new journal header into the
        ** journal file. This is done to avoid ever modifying a journal
        ** header that is involved in the rollback of pages that have
        ** already been written to the database (in case the header is
        ** trashed when the nRec field is updated).
        */
        pPager->nRec = 0;
        assert( pPager->journalOff > 0 );
        assert( pPager->doNotSync==0 );
        rc = writeJournalHdr(pPager);
        if( rc!=0 ){
          return rc;
        }
      }
    }
    pPg = pPager->lru.pFirst;
  }

  assert( pPg->nRef==0 );

  /* Write the page to the database file if it is dirty.
  */
  if( pPg->dirty && !pPager->errCode ){
    int rc;
    assert( pPg->needSync==0 );
    makeClean(pPg);
    pPg->dirty = 1;
    pPg->pDirty = 0;
    rc = pager_write_pagelist( pPg );
    pPg->dirty = 0;
    if( rc!=SQLITE_OK ){
      return rc;
    }
  }
  assert( pPg->dirty==0 || pPager->errCode );

  /* If the page we are recycling is marked as alwaysRollback, then
  ** set the global alwaysRollback flag, thus disabling the
  ** sqlite3PagerDontRollback() optimization for the rest of this transaction.
  ** It is necessary to do this because the page marked alwaysRollback
  ** might be reloaded at a later time but at that point we won't remember
  ** that is was marked alwaysRollback.  This means that all pages must
Added test/ioerr5.test.














































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
# 2008 May 12
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file tests that if sqlite3_release_memory() is called to reclaim
# memory from a pager that is in the error-state, SQLite does not 
# incorrectly write dirty pages out to the database (not safe to do
# once the pager is in error state).
#
# $Id: ioerr5.test,v 1.1 2008/05/15 08:34:54 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

ifcapable !memorymanage||!shared_cache {
  finish_test
  return
}

db close

set ::enable_shared_cache [sqlite3_enable_shared_cache 1]
set ::soft_limit [sqlite3_soft_heap_limit 1048576]

# This procedure prepares, steps and finalizes an SQL statement via the
# UTF-16 APIs. The text representation of an SQLite error code is returned
# ("SQLITE_OK", "SQLITE_IOERR" etc.). The actual results returned by the
# SQL statement, if it is a SELECT, are not available.
#
# This can be useful for testing because it forces SQLite to make an extra 
# call to sqlite3_malloc() when translating from the supplied UTF-16 to
# the UTF-8 encoding used internally.
#
proc dosql16 {zSql {db db}} {
  set sql [encoding convertto unicode $zSql]
  append sql "\00\00"
  set stmt [sqlite3_prepare16 $db $sql -1 {}]
  sqlite3_step $stmt
  set rc [sqlite3_finalize $stmt]
}

proc compilesql16 {zSql {db db}} {
  set sql [encoding convertto unicode $zSql]
  append sql "\00\00"
  set stmt [sqlite3_prepare16 $db $sql -1 {}]
  set rc [sqlite3_finalize $stmt]
}

# Open two database connections (handle db and db2) to database "test.db".
#
proc opendatabases {} {
  catch {db close}
  catch {db2 close}
  sqlite3 db test.db
  sqlite3 db2 test.db
  db2 cache size 0
  db cache size 0
  execsql {
    pragma page_size=512;
    pragma auto_vacuum=2;
    pragma cache_size=16;
  }
}

# Open two database connections and create a single table in the db.
#
do_test ioerr5-1.0 {
  opendatabases
  execsql { CREATE TABLE A(Id INTEGER, Name TEXT) }
} {}

foreach locking_mode {normal exclusive} {
  for {set iFail 1} {$iFail<200} {incr iFail} {
    sqlite3_soft_heap_limit 1048576
    opendatabases
    execsql { pragma locking_mode=exclusive }
    set nRow [db one {SELECT count(*) FROM a}]
  
    # Dirty (at least) one of the pages in the cache.
    do_test ioerr5-$locking_mode-$iFail.1 {
      execsql {
        BEGIN EXCLUSIVE;
        INSERT INTO a VALUES(1, 'ABCDEFGHIJKLMNOP');
      }
    } {}
  
    # Now try to commit the transaction. Cause an IO error to occur
    # within this operation, which moves the pager into the error state.
    #
    set ::sqlite_io_error_persist 1
    set ::sqlite_io_error_pending $iFail
    do_test ioerr5-$locking_mode-$iFail.2 {
      set rc [catchsql {COMMIT}]
      list
    } {}
    set ::sqlite_io_error_hit 0
    set ::sqlite_io_error_persist 0
    set ::sqlite_io_error_pending 0
  
    # Read the contents of the database file into a Tcl variable.
    #
    set fd [open test.db]
    fconfigure $fd -translation binary -encoding binary
    set zDatabase [read $fd]
    close $fd

    # Set a very low soft-limit and then try to compile an SQL statement 
    # from UTF-16 text. To do this, SQLite will need to reclaim memory
    # from the pager that is in error state. Including that associated
    # with the dirty page.
    #
    do_test ioerr5-$locking_mode-$iFail.3 {
      sqlite3_soft_heap_limit 1024
      compilesql16 "SELECT 10"
      set bt [btree_from_db db]
      array set stats [btree_pager_stats $bt]
      set stats(page)
    } {0}

    # Ensure that nothing was written to the database while reclaiming
    # memory from the pager in error state.
    #
    do_test ioerr5-$locking_mode-$iFail.4 {
      set fd [open test.db]
      fconfigure $fd -translation binary -encoding binary
      set zDatabase2 [read $fd]
      close $fd
      expr {$zDatabase eq $zDatabase2}
    } {1}
  
    if {$rc eq [list 0 {}]} {
      do_test ioerr5-$locking_mode-$iFail.3 {
        execsql { SELECT count(*) FROM a }
      } [expr $nRow+1]
      break
    }
  }
}

sqlite3_enable_shared_cache $::enable_shared_cache
sqlite3_soft_heap_limit $::soft_limit

finish_test