/ Check-in [89b1b3f8]
Login
Home Timeline Branches Tags Tickets Wiki More...

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add some tests (and 2 resulting bug fixes) to incr vacuum mode. (CVS 3885)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 89b1b3f897bda1fffceb9cf72fa4d42b809ccb8e
User & Date: danielk1977 2007-04-28 15:47:44
Context
2007-04-30
16:55
Try to avoid reading pages when moving overflow chains to the free-list. (CVS 3886) check-in: 8cccec68 user: danielk1977 tags: trunk
2007-04-28
15:47
Add some tests (and 2 resulting bug fixes) to incr vacuum mode. (CVS 3885) check-in: 89b1b3f8 user: danielk1977 tags: trunk
2007-04-27
22:02
Break interior-node and leaf-node readers apart in loadSegment(). Previously, the code looped until the block was a leaf node as indicated by a leading NUL. Now the code loops until it finds a block in the range of leaf nodes for this segment, then reads it using LeavesReader. This will make it easier to traverse a range of leaves when doing a prefix search. (CVS 3884) check-in: 9466367d user: shess tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/btree.c. 

5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
....
2362
2363
2364
2365
2366
2367
2368







2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379




2380
2381
2382
2383
2384
2385
2386
....
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441

** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.360 2007/04/27 07:05:44 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
**     "Sorting And Searching", pages 473-480. Addison-Wesley
**     Publishing Company, Reading, Massachusetts.
................................................................................
      MemPage *pLastPg;

      rc = getPage(pBt, iLastPg, &pLastPg, 0);
      if( rc!=SQLITE_OK ){
        return rc;
      }








      do {
        MemPage *pFreePg;
        rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, 0, 0);
        if( rc!=SQLITE_OK ){
          releasePage(pLastPg);
          return rc;
        }
        releasePage(pFreePg);
      }while( nFin!=0 && iFreePg>nFin );
      assert( iFreePg<iLastPg );





      rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg);
      releasePage(pLastPg);
      if( rc!=SQLITE_OK ){
        return rc;
      } 
    }
  }
................................................................................
#ifndef NDEBUG
  int nRef = sqlite3PagerRefcount(pPager);
#endif

  assert(pBt->autoVacuum);
  if( !pBt->incrVacuum ){
    Pgno nFin = 0;



    if( pBt->nTrunc==0 ){
      Pgno nFree;
      Pgno nPtrmap;
      const int pgsz = pBt->pageSize;
      Pgno nOrig = sqlite3PagerPagecount(pBt->pPager);









|







 







>
>
>
>
>
>
>










|
>
>
>
>







 







<
<








5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
....
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
....
2437
2438
2439
2440
2441
2442
2443


2444
2445
2446
2447
2448
2449
2450

** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.361 2007/04/28 15:47:44 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
**     "Sorting And Searching", pages 473-480. Addison-Wesley
**     Publishing Company, Reading, Massachusetts.
................................................................................
      MemPage *pLastPg;

      rc = getPage(pBt, iLastPg, &pLastPg, 0);
      if( rc!=SQLITE_OK ){
        return rc;
      }

      /* If nFin is zero, this loop runs exactly once and page pLastPg
      ** is swapped with the first free page pulled off the free list.
      **
      ** On the other hand, if nFin is greater than zero, then keep
      ** looping until a free-page located within the first nFin pages
      ** of the file is found.
      */
      do {
        MemPage *pFreePg;
        rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, 0, 0);
        if( rc!=SQLITE_OK ){
          releasePage(pLastPg);
          return rc;
        }
        releasePage(pFreePg);
      }while( nFin!=0 && iFreePg>nFin );
      assert( iFreePg<iLastPg );
      
      rc = sqlite3PagerWrite(pLastPg->pDbPage);
      if( rc!=SQLITE_OK ){
        return rc;
      } 
      rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg);
      releasePage(pLastPg);
      if( rc!=SQLITE_OK ){
        return rc;
      } 
    }
  }
................................................................................
#ifndef NDEBUG
  int nRef = sqlite3PagerRefcount(pPager);
#endif

  assert(pBt->autoVacuum);
  if( !pBt->incrVacuum ){
    Pgno nFin = 0;



    if( pBt->nTrunc==0 ){
      Pgno nFree;
      Pgno nPtrmap;
      const int pgsz = pBt->pageSize;
      Pgno nOrig = sqlite3PagerPagecount(pBt->pPager);

Changes to src/pager.c. 

14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
....
3057
3058
3059
3060
3061
3062
3063

3064
3065
3066
3067
3068
3069
3070
....
4206
4207
4208
4209
4210
4211
4212

4213
4214
4215
4216
4217
4218
4219

** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.330 2007/04/26 12:11:28 drh Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>
................................................................................
    }else{
      rc = readDbPage(pPager, pPg, pgno);
      if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){
        pPg->pgno = 0;
        sqlite3PagerUnref(pPg);
        return rc;
      }

    }

    /* Link the page into the page hash table */
    h = pgno & (pPager->nHash-1);
    assert( pgno!=0 );
    pPg->pNextHash = pPager->aHash[h];
    pPager->aHash[h] = pPg;
................................................................................

  assert( pPg->nRef>0 );

  PAGERTRACE5("MOVE %d page %d (needSync=%d) moves to %d\n", 
      PAGERID(pPager), pPg->pgno, pPg->needSync, pgno);
  IOTRACE(("MOVE %p %d %d\n", pPager, pPg->pgno, pgno))


  if( pPg->needSync ){
    needSyncPgno = pPg->pgno;
    assert( pPg->inJournal );
    assert( pPg->dirty );
    assert( pPager->needSync );
  }









|







 







>







 







>








14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
....
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
....
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221

** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.331 2007/04/28 15:47:44 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>
................................................................................
    }else{
      rc = readDbPage(pPager, pPg, pgno);
      if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){
        pPg->pgno = 0;
        sqlite3PagerUnref(pPg);
        return rc;
      }
      pPg->needRead = 0;
    }

    /* Link the page into the page hash table */
    h = pgno & (pPager->nHash-1);
    assert( pgno!=0 );
    pPg->pNextHash = pPager->aHash[h];
    pPager->aHash[h] = pPg;
................................................................................

  assert( pPg->nRef>0 );

  PAGERTRACE5("MOVE %d page %d (needSync=%d) moves to %d\n", 
      PAGERID(pPager), pPg->pgno, pPg->needSync, pgno);
  IOTRACE(("MOVE %p %d %d\n", pPager, pPg->pgno, pgno))

  pager_get_content(pPg);
  if( pPg->needSync ){
    needSyncPgno = pPg->pgno;
    assert( pPg->inJournal );
    assert( pPg->dirty );
    assert( pPager->needSync );
  }

Changes to test/autovacuum_ioerr2.test. 

11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
...
111
112
113
114
115
116
117













118
119
120

# This file implements regression tests for SQLite library.  The
# focus of this file is testing for correct handling of I/O errors
# such as writes failing because the disk is full.
# 
# The tests in this file use special facilities that are only
# available in the SQLite test fixture.
#
# $Id: autovacuum_ioerr2.test,v 1.5 2005/01/29 09:14:05 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If this build of the library does not support auto-vacuum, omit this
# whole file.
ifcapable {!autovacuum} {
................................................................................
    DELETE FROM abc WHERE oid < 3;
    UPDATE abc SET a = randstr(100,100) WHERE oid > 2300;
    UPDATE abc SET a = randstr(1100,1100) WHERE oid = 
        (select max(oid) from abc);
    COMMIT;
  }
}














finish_test









|







 







>
>
>
>
>
>
>
>
>
>
>
>
>




11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
...
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133

# This file implements regression tests for SQLite library.  The
# focus of this file is testing for correct handling of I/O errors
# such as writes failing because the disk is full.
# 
# The tests in this file use special facilities that are only
# available in the SQLite test fixture.
#
# $Id: autovacuum_ioerr2.test,v 1.6 2007/04/28 15:47:44 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If this build of the library does not support auto-vacuum, omit this
# whole file.
ifcapable {!autovacuum} {
................................................................................
    DELETE FROM abc WHERE oid < 3;
    UPDATE abc SET a = randstr(100,100) WHERE oid > 2300;
    UPDATE abc SET a = randstr(1100,1100) WHERE oid = 
        (select max(oid) from abc);
    COMMIT;
  }
}

do_ioerr_test autovacuum-ioerr2-1 -sqlprep {
  PRAGMA auto_vacuum = 1;
  CREATE TABLE abc(a);
  INSERT INTO abc VALUES(randstr(1500,1500));
} -sqlbody {
  CREATE TABLE abc2(a);
  BEGIN;
  DELETE FROM abc;
  INSERT INTO abc VALUES(randstr(1500,1500));
  CREATE TABLE abc3(a);
  COMMIT;
}

finish_test

Changes to test/incrvacuum.test. 

7
8
9
10
11
12
13



14
15
16
17
18
19
20
21

#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the incremental vacuum feature.
#



# $Id: incrvacuum.test,v 1.3 2007/04/27 07:55:38 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If this build of the library does not support auto-vacuum, omit this
# whole file.
ifcapable {!autovacuum || !pragma} {








>
>
>
|








7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the incremental vacuum feature.
#
# Note: There are also some tests for incremental vacuum and IO 
# errors in incrvacuum_ioerr.test.
#
# $Id: incrvacuum.test,v 1.4 2007/04/28 15:47:44 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If this build of the library does not support auto-vacuum, omit this
# whole file.
ifcapable {!autovacuum || !pragma} {

Added test/incrvacuum_ioerr.test. 























































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

# 2001 October 12
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing for correct handling of I/O errors
# such as writes failing because the disk is full.
# 
# The tests in this file use special facilities that are only
# available in the SQLite test fixture.
#
# $Id: incrvacuum_ioerr.test,v 1.1 2007/04/28 15:47:45 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If this build of the library does not support auto-vacuum, omit this
# whole file.
ifcapable {!autovacuum} {
  finish_test
  return
}

do_ioerr_test incrvacuum-ioerr-1 -cksum 1 -sqlprep {
  PRAGMA auto_vacuum = 'incremental';
  CREATE TABLE abc(a);
  INSERT INTO abc VALUES(randstr(1500,1500));
} -sqlbody {
  BEGIN;
  CREATE TABLE abc2(a);
  DELETE FROM abc;
  INCREMENTAL VACUUM;
  COMMIT;
}

# do_ioerr_test incrvacuum-ioerr-3 -start 1 -cksum 1 -tclprep {
#   db eval {
#     PRAGMA auto_vacuum = 'full';
#     PRAGMA cache_size = 10;
#     BEGIN;
#     CREATE TABLE abc(a, UNIQUE(a));
#   }
#   for {set ii 0} {$ii < 25} {incr ii} {
#     db eval {INSERT INTO abc VALUES(randstr(1500,1500))}
#   }
#   db eval COMMIT
# } -sqlbody {
#   BEGIN;
#   DELETE FROM abc WHERE (oid%3)==0;
#   INSERT INTO abc SELECT a || '1234567890' FROM abc WHERE oid%2;
#   CREATE INDEX abc_i ON abc(a);
#   DELETE FROM abc WHERE (oid%2)==0;
#   DROP INDEX abc_i;
#   COMMIT;
# }


do_ioerr_test incrvacuum-ioerr-2 -start 1 -cksum 1 -tclprep {
  db eval {
    PRAGMA auto_vacuum = 'full';
    PRAGMA cache_size = 10;
    BEGIN;
    CREATE TABLE abc(a, UNIQUE(a));
  }
  for {set ii 0} {$ii < 25} {incr ii} {
    db eval {INSERT INTO abc VALUES(randstr(1500,1500))}
  }
  db eval COMMIT
} -sqlbody {
  BEGIN;
  INCREMENTAL VACUUM;
  DELETE FROM abc WHERE (oid%3)==0;
  INCREMENTAL VACUUM;
  INSERT INTO abc SELECT a || '1234567890' FROM abc WHERE oid%2;
  INCREMENTAL VACUUM;
  CREATE INDEX abc_i ON abc(a);
  DELETE FROM abc WHERE (oid%2)==0;
  INCREMENTAL VACUUM;
  DROP INDEX abc_i;
  INCREMENTAL VACUUM;
  COMMIT;
}
finish_test

Changes to test/quick.test. 

2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
..
52
53
54
55
56
57
58

59
60
61
62
63
64
65

#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file runs all tests.
#
# $Id: quick.test,v 1.52 2007/04/16 17:07:55 drh Exp $

proc lshift {lvar} {
  upvar $lvar l
  set ret [lindex $l 0]
  set l [lrange $l 1 end]
  return $ret
}
................................................................................
  memleak.test
  misc7.test
  misuse.test
  quick.test
  speed1.test
  speed2.test


  autovacuum_crash.test
  btree8.test
  utf16.test
  shared_err.test
  vtab_err.test
}









|







 







>








2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
..
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file runs all tests.
#
# $Id: quick.test,v 1.53 2007/04/28 15:47:45 danielk1977 Exp $

proc lshift {lvar} {
  upvar $lvar l
  set ret [lindex $l 0]
  set l [lrange $l 1 end]
  return $ret
}
................................................................................
  memleak.test
  misc7.test
  misuse.test
  quick.test
  speed1.test
  speed2.test

  incrvacuum_ioerr.test
  autovacuum_crash.test
  btree8.test
  utf16.test
  shared_err.test
  vtab_err.test
}

Changes to test/tester.tcl. 

7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
...
395
396
397
398
399
400
401

402
403
404
405
406
407
408

#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements some common TCL routines used for regression
# testing the SQLite library
#
# $Id: tester.tcl,v 1.79 2007/04/19 12:30:54 drh Exp $

# Make sure tclsqlite3 was compiled correctly.  Abort now with an
# error message if not.
#
if {[sqlite3 -tcl-uses-utf]} {
  if {"\u1234"=="u1234"} {
    puts stderr "***** BUILD PROBLEM *****"
................................................................................
  set ::ioerropts(-erc) 0
  set ::ioerropts(-count) 100000000
  set ::ioerropts(-persist) 1
  array set ::ioerropts $args

  set ::go 1
  for {set n $::ioerropts(-start)} {$::go} {incr n} {

    incr ::ioerropts(-count) -1
    if {$::ioerropts(-count)<0} break
 
    # Skip this IO error if it was specified with the "-exclude" option.
    if {[info exists ::ioerropts(-exclude)]} {
      if {[lsearch $::ioerropts(-exclude) $n]!=-1} continue
    }








|







 







>








7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
...
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409

#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements some common TCL routines used for regression
# testing the SQLite library
#
# $Id: tester.tcl,v 1.80 2007/04/28 15:47:45 danielk1977 Exp $

# Make sure tclsqlite3 was compiled correctly.  Abort now with an
# error message if not.
#
if {[sqlite3 -tcl-uses-utf]} {
  if {"\u1234"=="u1234"} {
    puts stderr "***** BUILD PROBLEM *****"
................................................................................
  set ::ioerropts(-erc) 0
  set ::ioerropts(-count) 100000000
  set ::ioerropts(-persist) 1
  array set ::ioerropts $args

  set ::go 1
  for {set n $::ioerropts(-start)} {$::go} {incr n} {
set ::TN $n
    incr ::ioerropts(-count) -1
    if {$::ioerropts(-count)<0} break
 
    # Skip this IO error if it was specified with the "-exclude" option.
    if {[info exists ::ioerropts(-exclude)]} {
      if {[lsearch $::ioerropts(-exclude) $n]!=-1} continue
    }

This page was generated in about 0.085s by Fossil 2.10 [89872d17d1] 2019-09-18 14:57:33