/ Check-in [839ad771]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Test auto-vacuum mode for crash-proofness. Also fix a bug related to the same. (CVS 2077)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 839ad771a6e781426c0fd624a4d1c91a5fcf8546
User & Date: danielk1977 2004-11-08 09:26:09
Context
2004-11-08
09:51
Check in the file autovacuum_crash.test that should be part of the previous check-in. (CVS 2078) check-in: 9d7cd1f7 user: danielk1977 tags: trunk
09:26
Test auto-vacuum mode for crash-proofness. Also fix a bug related to the same. (CVS 2077) check-in: 839ad771 user: danielk1977 tags: trunk
07:13
Auto-vacuum: Account for the page reserved for windows locking (PENDING_BYTE). (CVS 2076) check-in: d6335698 user: danielk1977 tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/btree.c.

5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
....
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
....
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.214 2004/11/08 07:13:14 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
**     "Sorting And Searching", pages 473-480. Addison-Wesley
**     Publishing Company, Reading, Massachusetts.
................................................................................
  if( rc!=SQLITE_OK ) goto autovacuum_out;
  put4byte(&pBt->pPage1->aData[32], 0);
  put4byte(&pBt->pPage1->aData[36], 0);
  if( rc!=SQLITE_OK ) goto autovacuum_out;
  *nTrunc = finSize;

autovacuum_out:
  /* TODO: A goto autovacuum_out; will fail to call releasePage() on 
  ** outstanding references. Fix.
  */
  assert( nRef==*sqlite3pager_stats(pPager) );
  if( rc!=SQLITE_OK ){
    sqlite3pager_rollback(pPager);
  }
  return rc;
}
#endif
................................................................................
            get4byte(&pBt->pPage1->aData[36]), "Main freelist: ");

  /* Check all the tables.
  */
  for(i=0; i<nRoot; i++){
    if( aRoot[i]==0 ) continue;
#ifndef SQLITE_OMIT_AUTOVACUUM
/* Note: This is temporary code for use during development of auto-vacuum. */
    if( pBt->autoVacuum && aRoot[i]>1 ){
      checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0, 0);
    }
#endif
    checkTreePage(&sCheck, aRoot[i], 0, "List of tree roots: ", 0,0,0,0);
  }








|







 







<
<
<







 







<







5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
....
1821
1822
1823
1824
1825
1826
1827



1828
1829
1830
1831
1832
1833
1834
....
5260
5261
5262
5263
5264
5265
5266

5267
5268
5269
5270
5271
5272
5273
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.215 2004/11/08 09:26:09 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
**     "Sorting And Searching", pages 473-480. Addison-Wesley
**     Publishing Company, Reading, Massachusetts.
................................................................................
  if( rc!=SQLITE_OK ) goto autovacuum_out;
  put4byte(&pBt->pPage1->aData[32], 0);
  put4byte(&pBt->pPage1->aData[36], 0);
  if( rc!=SQLITE_OK ) goto autovacuum_out;
  *nTrunc = finSize;

autovacuum_out:



  assert( nRef==*sqlite3pager_stats(pPager) );
  if( rc!=SQLITE_OK ){
    sqlite3pager_rollback(pPager);
  }
  return rc;
}
#endif
................................................................................
            get4byte(&pBt->pPage1->aData[36]), "Main freelist: ");

  /* Check all the tables.
  */
  for(i=0; i<nRoot; i++){
    if( aRoot[i]==0 ) continue;
#ifndef SQLITE_OMIT_AUTOVACUUM

    if( pBt->autoVacuum && aRoot[i]>1 ){
      checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0, 0);
    }
#endif
    checkTreePage(&sCheck, aRoot[i], 0, "List of tree roots: ", 0,0,0,0);
  }

Changes to src/pager.c.

14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
..
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
....
3299
3300
3301
3302
3303
3304
3305

3306
3307
3308
3309
3310
3311






3312
3313
3314
3315
3316
3317
3318
....
3337
3338
3339
3340
3341
3342
3343

















3344
3345
3346
3347
3348
3349
3350
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.175 2004/11/08 07:13:14 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>

................................................................................
** Macros for troubleshooting.  Normally turned off
*/
#if 0
#define TRACE1(X)       sqlite3DebugPrintf(X)
#define TRACE2(X,Y)     sqlite3DebugPrintf(X,Y)
#define TRACE3(X,Y,Z)   sqlite3DebugPrintf(X,Y,Z)
#define TRACE4(X,Y,Z,W) sqlite3DebugPrintf(X,Y,Z,W)
#define TRACE5(X,Y,Z,W,V) sqlite3DebugPrintf(X,Y,Z,W, V)
#else
#define TRACE1(X)
#define TRACE2(X,Y)
#define TRACE3(X,Y,Z)
#define TRACE4(X,Y,Z,W)
#define TRACE5(X,Y,Z,W,V)
#endif
................................................................................
** A transaction must be active when this routine is called, however it is 
** illegal to call this routine if a statment transaction is active.
*/
int sqlite3pager_movepage(Pager *pPager, void *pData, Pgno pgno){
  PgHdr *pPg = DATA_TO_PGHDR(pData);
  PgHdr *pPgOld; 
  int h;


  assert( !pPager->stmtInUse );
  assert( pPg->nRef>0 );

  TRACE5("MOVE %d page %d (needSync=%d) moves to %d\n", 
      PAGERID(pPager), pPg->pgno, pPg->needSync, pgno);







  /* Unlink pPg from it's hash-chain */
  unlinkHashChain(pPager, pPg);

  /* If the cache contains a page with page-number pgno, remove it
  ** from it's hash chain. Also, if the PgHdr.needSync was set for 
  ** page pgno before the 'move' operation, it needs to be retained 
................................................................................
  }
  pPg->pNextHash = pPager->aHash[h];
  pPager->aHash[h] = pPg;
  pPg->pPrevHash = 0;

  pPg->dirty = 1;
  pPager->dirtyCache = 1;


















  return SQLITE_OK;
}
#endif

#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
/*







|







 







|







 







>






>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
..
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
....
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
....
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.176 2004/11/08 09:26:10 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>

................................................................................
** Macros for troubleshooting.  Normally turned off
*/
#if 0
#define TRACE1(X)       sqlite3DebugPrintf(X)
#define TRACE2(X,Y)     sqlite3DebugPrintf(X,Y)
#define TRACE3(X,Y,Z)   sqlite3DebugPrintf(X,Y,Z)
#define TRACE4(X,Y,Z,W) sqlite3DebugPrintf(X,Y,Z,W)
#define TRACE5(X,Y,Z,W,V) sqlite3DebugPrintf(X,Y,Z,W,V)
#else
#define TRACE1(X)
#define TRACE2(X,Y)
#define TRACE3(X,Y,Z)
#define TRACE4(X,Y,Z,W)
#define TRACE5(X,Y,Z,W,V)
#endif
................................................................................
** A transaction must be active when this routine is called, however it is 
** illegal to call this routine if a statment transaction is active.
*/
int sqlite3pager_movepage(Pager *pPager, void *pData, Pgno pgno){
  PgHdr *pPg = DATA_TO_PGHDR(pData);
  PgHdr *pPgOld; 
  int h;
  Pgno needSyncPgno = 0;

  assert( !pPager->stmtInUse );
  assert( pPg->nRef>0 );

  TRACE5("MOVE %d page %d (needSync=%d) moves to %d\n", 
      PAGERID(pPager), pPg->pgno, pPg->needSync, pgno);

  if( pPg->needSync ){
    needSyncPgno = pPg->pgno;
    assert( pPg->inJournal );
    assert( pPg->dirty );
  }

  /* Unlink pPg from it's hash-chain */
  unlinkHashChain(pPager, pPg);

  /* If the cache contains a page with page-number pgno, remove it
  ** from it's hash chain. Also, if the PgHdr.needSync was set for 
  ** page pgno before the 'move' operation, it needs to be retained 
................................................................................
  }
  pPg->pNextHash = pPager->aHash[h];
  pPager->aHash[h] = pPg;
  pPg->pPrevHash = 0;

  pPg->dirty = 1;
  pPager->dirtyCache = 1;

  if( needSyncPgno ){
    /* If needSyncPgno is non-zero, then the journal file needs to be 
    ** sync()ed before any data is written to database file page needSyncPgno.
    ** Currently, no such page exists in the page-cache and the 
    ** Pager.aInJournal bit has been set. This needs to be remedied by loading
    ** the page into the pager-cache and setting the PgHdr.needSync flag.
    */
    int rc;
    void *pNeedSync;
    rc = sqlite3pager_get(pPager, needSyncPgno, &pNeedSync);
    if( rc!=SQLITE_OK ) return rc;
    DATA_TO_PGHDR(pNeedSync)->needSync = 1;
    DATA_TO_PGHDR(pNeedSync)->inJournal = 1;
    DATA_TO_PGHDR(pNeedSync)->dirty = 1;
    sqlite3pager_unref(pNeedSync);
  }

  return SQLITE_OK;
}
#endif

#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
/*

Changes to test/crash.test.

16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
..
63
64
65
66
67
68
69





70
71
72
73
74
75
76
...
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
...
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
...
320
321
322
323
324
325
326





























































# module "crashtest" compiled with the special "os_test.c" backend is used.
# The os_test.c simulates the kind of file corruption that can occur
# when writes are happening at the moment of power loss.
# 
# The special crash-test module with its os_test.c backend only works
# on Unix.
#
# $Id: crash.test,v 1.9 2004/08/21 19:20:42 drh Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# set repeats 100
set repeats 10

................................................................................
# abc changes in any way, the signature should change.  
proc signature {} {
  return [db eval {SELECT count(*), md5sum(a), md5sum(b), md5sum(c) FROM abc}]
}
proc signature2 {} {
  return [db eval {SELECT count(*), md5sum(a), md5sum(b), md5sum(c) FROM abc2}]
}






#--------------------------------------------------------------------------
# Simple crash test:
#
# crash-1.1: Create a database with a table with two rows.
# crash-1.2: Run a 'DELETE FROM abc WHERE a = 1' that crashes during
#            the first journal-sync.
................................................................................
  }
  execsql { COMMIT }
  set ::sig [signature]
  execsql { SELECT sum(a), sum(b), sum(c) from abc }
} {499500.0 999000.0 1498500.0}
do_test crash-2.2 {
  expr [file size test.db] / 1024
} {19}
do_test crash-2.3 {
  crashsql 2 test.db-journal {
    DELETE FROM abc WHERE a < 800;
  }
} {1 {child process exited abnormally}}
do_test crash-2.4 {
  signature
................................................................................
    INSERT INTO abc SELECT * FROM abc;
    INSERT INTO abc SELECT * FROM abc;
    INSERT INTO abc SELECT * FROM abc;
    INSERT INTO abc SELECT * FROM abc;
    INSERT INTO abc SELECT * FROM abc;
  }
  expr [file size test.db] / 1024
} {554}
for {set i 1} {$i < $repeats} {incr i} {
  set sig [signature]
  do_test crash-3.$i.1 {
     crashsql [expr $i%5 + 1] test.db-journal "
       BEGIN;
       SELECT random() FROM abc LIMIT $i;
       INSERT INTO abc VALUES(randstr(10,10), 0, 0);
................................................................................
  do_test crash-4.3.$i.2 {
    signature
  } $sig
  do_test crash-4.3.$i.3 {
    signature2
  } $sig2
}




































































|







 







>
>
>
>
>







 







|







 







|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
..
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
...
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
...
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
...
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
# module "crashtest" compiled with the special "os_test.c" backend is used.
# The os_test.c simulates the kind of file corruption that can occur
# when writes are happening at the moment of power loss.
# 
# The special crash-test module with its os_test.c backend only works
# on Unix.
#
# $Id: crash.test,v 1.10 2004/11/08 09:26:10 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# set repeats 100
set repeats 10

................................................................................
# abc changes in any way, the signature should change.  
proc signature {} {
  return [db eval {SELECT count(*), md5sum(a), md5sum(b), md5sum(c) FROM abc}]
}
proc signature2 {} {
  return [db eval {SELECT count(*), md5sum(a), md5sum(b), md5sum(c) FROM abc2}]
}

# This variable is set to 1 if the databases being used support auto-vacuum.
# This is because some of the tests in this file verify file-size, which is
# slightly larger for auto-vacuum databases.
set AUTOVACUUM [db eval {pragma auto_vacuum}]

#--------------------------------------------------------------------------
# Simple crash test:
#
# crash-1.1: Create a database with a table with two rows.
# crash-1.2: Run a 'DELETE FROM abc WHERE a = 1' that crashes during
#            the first journal-sync.
................................................................................
  }
  execsql { COMMIT }
  set ::sig [signature]
  execsql { SELECT sum(a), sum(b), sum(c) from abc }
} {499500.0 999000.0 1498500.0}
do_test crash-2.2 {
  expr [file size test.db] / 1024
} [expr $AUTOVACUUM ? 20 : 19]
do_test crash-2.3 {
  crashsql 2 test.db-journal {
    DELETE FROM abc WHERE a < 800;
  }
} {1 {child process exited abnormally}}
do_test crash-2.4 {
  signature
................................................................................
    INSERT INTO abc SELECT * FROM abc;
    INSERT INTO abc SELECT * FROM abc;
    INSERT INTO abc SELECT * FROM abc;
    INSERT INTO abc SELECT * FROM abc;
    INSERT INTO abc SELECT * FROM abc;
  }
  expr [file size test.db] / 1024
} [expr $AUTOVACUUM ? 557 : 554]
for {set i 1} {$i < $repeats} {incr i} {
  set sig [signature]
  do_test crash-3.$i.1 {
     crashsql [expr $i%5 + 1] test.db-journal "
       BEGIN;
       SELECT random() FROM abc LIMIT $i;
       INSERT INTO abc VALUES(randstr(10,10), 0, 0);
................................................................................
  do_test crash-4.3.$i.2 {
    signature
  } $sig
  do_test crash-4.3.$i.3 {
    signature2
  } $sig2
}

#--------------------------------------------------------------------------
# The following test cases - crash-5.* - exposes a bug that existed in the
# sqlite3pager_movepage() API used by auto-vacuum databases.
# database when a crash occurs during a multi-file transaction. See comments
# in test crash-5.3 for details.
#
db close
file delete -force test.db
sqlite3 db test.db
do_test crash-5.1 {
  execsql {
    CREATE TABLE abc(a, b, c);                          -- Root page 3
    INSERT INTO abc VALUES(randstr(1500,1500), 0, 0);   -- Overflow page 4
    INSERT INTO abc SELECT * FROM abc;
    INSERT INTO abc SELECT * FROM abc;
    INSERT INTO abc SELECT * FROM abc;
  }
} {}
do_test crash-5.2 {
  expr [file size test.db] / 1024
} [expr $AUTOVACUUM ? 11 : 10]
set sig [signature]
do_test crash-5.3 {
# The SQL below is used to expose a bug that existed in
# sqlite3pager_movepage() during development of the auto-vacuum feature. It
# functions as follows:
# 
# 1: Begin a transaction.
# 2: Put page 4 on the free-list (was the overflow page for the row deleted).
# 3: Write data to page 4 (it becomes the overflow page for the row inserted).
#    The old page 4 data has been written to the journal file, but the
#    journal file has not been sync()hronized.
# 4: Create a table, which calls sqlite3pager_movepage() to move page 4
#    to the end of the database (page 12) to make room for the new root-page.
# 5: Put pressure on the pager-cache. This results in page 4 being written
#    to the database file to make space in the cache to load a new page. The
#    bug was that page 4 was written to the database file before the journal
#    is sync()hronized.
# 6: Commit. A crash occurs during the sync of the journal file.
#
# End result: Before the bug was fixed, data has been written to page 4 of the
# database file and the journal file does not contain trustworthy rollback
# data for this page.
#
  crashsql 1 test.db-journal {
    BEGIN;                                             -- 1
    DELETE FROM abc WHERE oid = 1;                     -- 2
    INSERT INTO abc VALUES(randstr(1500,1500), 0, 0);  -- 3
    CREATE TABLE abc2(a, b, c);                        -- 4
    SELECT * FROM abc;                                 -- 5
    COMMIT;                                            -- 6
  }
} {1 {child process exited abnormally}}
integrity_check crash-5.4
do_test crash-5.5 {
  signature
} $sig



Changes to test/quick.test.

6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
..
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file runs all tests.
#
# $Id: quick.test,v 1.31 2004/11/02 12:56:41 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl
rename finish_test really_finish_test
proc finish_test {} {}
set ISQUICK 1

................................................................................
  corrupt.test
  crash.test
  malloc.test
  memleak.test
  misuse.test
  quick.test
  utf16.test

  autovacuum.test
}

if {[sqlite3 -has-codec]} {
  # lappend EXCLUDE \
  #  conflict.test
}








|







 







<
|







6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
..
28
29
30
31
32
33
34

35
36
37
38
39
40
41
42
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file runs all tests.
#
# $Id: quick.test,v 1.32 2004/11/08 09:26:10 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl
rename finish_test really_finish_test
proc finish_test {} {}
set ISQUICK 1

................................................................................
  corrupt.test
  crash.test
  malloc.test
  memleak.test
  misuse.test
  quick.test
  utf16.test

  autovacuum_crash.test
}

if {[sqlite3 -has-codec]} {
  # lappend EXCLUDE \
  #  conflict.test
}