SQLite

Check-in [89b1b3f897]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add some tests (and 2 resulting bug fixes) to incr vacuum mode. (CVS 3885)
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 89b1b3f897bda1fffceb9cf72fa4d42b809ccb8e
User & Date: danielk1977 2007-04-28 15:47:44.000
Context
2007-04-30
16:55
Try to avoid reading pages when moving overflow chains to the free-list. (CVS 3886) (check-in: 8cccec68bd user: danielk1977 tags: trunk)
2007-04-28
15:47
Add some tests (and 2 resulting bug fixes) to incr vacuum mode. (CVS 3885) (check-in: 89b1b3f897 user: danielk1977 tags: trunk)
2007-04-27
22:02
Break interior-node and leaf-node readers apart in loadSegment(). Previously, the code looped until the block was a leaf node as indicated by a leading NUL. Now the code loops until it finds a block in the range of leaf nodes for this segment, then reads it using LeavesReader. This will make it easier to traverse a range of leaves when doing a prefix search. (CVS 3884) (check-in: 9466367d65 user: shess tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/btree.c.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/*
** 2004 April 6
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.360 2007/04/27 07:05:44 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
**     "Sorting And Searching", pages 473-480. Addison-Wesley
**     Publishing Company, Reading, Massachusetts.











|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/*
** 2004 April 6
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.361 2007/04/28 15:47:44 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
**     "Sorting And Searching", pages 473-480. Addison-Wesley
**     Publishing Company, Reading, Massachusetts.
2362
2363
2364
2365
2366
2367
2368







2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379




2380
2381
2382
2383
2384
2385
2386
      MemPage *pLastPg;

      rc = getPage(pBt, iLastPg, &pLastPg, 0);
      if( rc!=SQLITE_OK ){
        return rc;
      }








      do {
        MemPage *pFreePg;
        rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, 0, 0);
        if( rc!=SQLITE_OK ){
          releasePage(pLastPg);
          return rc;
        }
        releasePage(pFreePg);
      }while( nFin!=0 && iFreePg>nFin );
      assert( iFreePg<iLastPg );





      rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg);
      releasePage(pLastPg);
      if( rc!=SQLITE_OK ){
        return rc;
      } 
    }
  }







>
>
>
>
>
>
>










|
>
>
>
>







2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
      MemPage *pLastPg;

      rc = getPage(pBt, iLastPg, &pLastPg, 0);
      if( rc!=SQLITE_OK ){
        return rc;
      }

      /* If nFin is zero, this loop runs exactly once and page pLastPg
      ** is swapped with the first free page pulled off the free list.
      **
      ** On the other hand, if nFin is greater than zero, then keep
      ** looping until a free-page located within the first nFin pages
      ** of the file is found.
      */
      do {
        MemPage *pFreePg;
        rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, 0, 0);
        if( rc!=SQLITE_OK ){
          releasePage(pLastPg);
          return rc;
        }
        releasePage(pFreePg);
      }while( nFin!=0 && iFreePg>nFin );
      assert( iFreePg<iLastPg );
      
      rc = sqlite3PagerWrite(pLastPg->pDbPage);
      if( rc!=SQLITE_OK ){
        return rc;
      } 
      rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg);
      releasePage(pLastPg);
      if( rc!=SQLITE_OK ){
        return rc;
      } 
    }
  }
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
#ifndef NDEBUG
  int nRef = sqlite3PagerRefcount(pPager);
#endif

  assert(pBt->autoVacuum);
  if( !pBt->incrVacuum ){
    Pgno nFin = 0;



    if( pBt->nTrunc==0 ){
      Pgno nFree;
      Pgno nPtrmap;
      const int pgsz = pBt->pageSize;
      Pgno nOrig = sqlite3PagerPagecount(pBt->pPager);








<
<







2437
2438
2439
2440
2441
2442
2443


2444
2445
2446
2447
2448
2449
2450
#ifndef NDEBUG
  int nRef = sqlite3PagerRefcount(pPager);
#endif

  assert(pBt->autoVacuum);
  if( !pBt->incrVacuum ){
    Pgno nFin = 0;



    if( pBt->nTrunc==0 ){
      Pgno nFree;
      Pgno nPtrmap;
      const int pgsz = pBt->pageSize;
      Pgno nOrig = sqlite3PagerPagecount(pBt->pPager);

Changes to src/pager.c.
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.330 2007/04/26 12:11:28 drh Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>







|







14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.331 2007/04/28 15:47:44 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>
3057
3058
3059
3060
3061
3062
3063

3064
3065
3066
3067
3068
3069
3070
    }else{
      rc = readDbPage(pPager, pPg, pgno);
      if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){
        pPg->pgno = 0;
        sqlite3PagerUnref(pPg);
        return rc;
      }

    }

    /* Link the page into the page hash table */
    h = pgno & (pPager->nHash-1);
    assert( pgno!=0 );
    pPg->pNextHash = pPager->aHash[h];
    pPager->aHash[h] = pPg;







>







3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
    }else{
      rc = readDbPage(pPager, pPg, pgno);
      if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){
        pPg->pgno = 0;
        sqlite3PagerUnref(pPg);
        return rc;
      }
      pPg->needRead = 0;
    }

    /* Link the page into the page hash table */
    h = pgno & (pPager->nHash-1);
    assert( pgno!=0 );
    pPg->pNextHash = pPager->aHash[h];
    pPager->aHash[h] = pPg;
4206
4207
4208
4209
4210
4211
4212

4213
4214
4215
4216
4217
4218
4219

  assert( pPg->nRef>0 );

  PAGERTRACE5("MOVE %d page %d (needSync=%d) moves to %d\n", 
      PAGERID(pPager), pPg->pgno, pPg->needSync, pgno);
  IOTRACE(("MOVE %p %d %d\n", pPager, pPg->pgno, pgno))


  if( pPg->needSync ){
    needSyncPgno = pPg->pgno;
    assert( pPg->inJournal );
    assert( pPg->dirty );
    assert( pPager->needSync );
  }








>







4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221

  assert( pPg->nRef>0 );

  PAGERTRACE5("MOVE %d page %d (needSync=%d) moves to %d\n", 
      PAGERID(pPager), pPg->pgno, pPg->needSync, pgno);
  IOTRACE(("MOVE %p %d %d\n", pPager, pPg->pgno, pgno))

  pager_get_content(pPg);
  if( pPg->needSync ){
    needSyncPgno = pPg->pgno;
    assert( pPg->inJournal );
    assert( pPg->dirty );
    assert( pPager->needSync );
  }

Changes to test/autovacuum_ioerr2.test.
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# This file implements regression tests for SQLite library.  The
# focus of this file is testing for correct handling of I/O errors
# such as writes failing because the disk is full.
# 
# The tests in this file use special facilities that are only
# available in the SQLite test fixture.
#
# $Id: autovacuum_ioerr2.test,v 1.5 2005/01/29 09:14:05 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If this build of the library does not support auto-vacuum, omit this
# whole file.
ifcapable {!autovacuum} {







|







11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# This file implements regression tests for SQLite library.  The
# focus of this file is testing for correct handling of I/O errors
# such as writes failing because the disk is full.
# 
# The tests in this file use special facilities that are only
# available in the SQLite test fixture.
#
# $Id: autovacuum_ioerr2.test,v 1.6 2007/04/28 15:47:44 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If this build of the library does not support auto-vacuum, omit this
# whole file.
ifcapable {!autovacuum} {
111
112
113
114
115
116
117













118
119
120
    DELETE FROM abc WHERE oid < 3;
    UPDATE abc SET a = randstr(100,100) WHERE oid > 2300;
    UPDATE abc SET a = randstr(1100,1100) WHERE oid = 
        (select max(oid) from abc);
    COMMIT;
  }
}














finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>



111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
    DELETE FROM abc WHERE oid < 3;
    UPDATE abc SET a = randstr(100,100) WHERE oid > 2300;
    UPDATE abc SET a = randstr(1100,1100) WHERE oid = 
        (select max(oid) from abc);
    COMMIT;
  }
}

do_ioerr_test autovacuum-ioerr2-1 -sqlprep {
  PRAGMA auto_vacuum = 1;
  CREATE TABLE abc(a);
  INSERT INTO abc VALUES(randstr(1500,1500));
} -sqlbody {
  CREATE TABLE abc2(a);
  BEGIN;
  DELETE FROM abc;
  INSERT INTO abc VALUES(randstr(1500,1500));
  CREATE TABLE abc3(a);
  COMMIT;
}

finish_test

Changes to test/incrvacuum.test.
1
2
3
4
5
6
7
8
9
10
11
12
13



14
15
16
17
18
19
20
21
# 2007 April 26
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the incremental vacuum feature.
#



# $Id: incrvacuum.test,v 1.3 2007/04/27 07:55:38 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If this build of the library does not support auto-vacuum, omit this
# whole file.
ifcapable {!autovacuum || !pragma} {













>
>
>
|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# 2007 April 26
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the incremental vacuum feature.
#
# Note: There are also some tests for incremental vacuum and IO 
# errors in incrvacuum_ioerr.test.
#
# $Id: incrvacuum.test,v 1.4 2007/04/28 15:47:44 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If this build of the library does not support auto-vacuum, omit this
# whole file.
ifcapable {!autovacuum || !pragma} {
Added test/incrvacuum_ioerr.test.




















































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
# 2001 October 12
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing for correct handling of I/O errors
# such as writes failing because the disk is full.
# 
# The tests in this file use special facilities that are only
# available in the SQLite test fixture.
#
# $Id: incrvacuum_ioerr.test,v 1.1 2007/04/28 15:47:45 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If this build of the library does not support auto-vacuum, omit this
# whole file.
ifcapable {!autovacuum} {
  finish_test
  return
}

do_ioerr_test incrvacuum-ioerr-1 -cksum 1 -sqlprep {
  PRAGMA auto_vacuum = 'incremental';
  CREATE TABLE abc(a);
  INSERT INTO abc VALUES(randstr(1500,1500));
} -sqlbody {
  BEGIN;
  CREATE TABLE abc2(a);
  DELETE FROM abc;
  INCREMENTAL VACUUM;
  COMMIT;
}

# do_ioerr_test incrvacuum-ioerr-3 -start 1 -cksum 1 -tclprep {
#   db eval {
#     PRAGMA auto_vacuum = 'full';
#     PRAGMA cache_size = 10;
#     BEGIN;
#     CREATE TABLE abc(a, UNIQUE(a));
#   }
#   for {set ii 0} {$ii < 25} {incr ii} {
#     db eval {INSERT INTO abc VALUES(randstr(1500,1500))}
#   }
#   db eval COMMIT
# } -sqlbody {
#   BEGIN;
#   DELETE FROM abc WHERE (oid%3)==0;
#   INSERT INTO abc SELECT a || '1234567890' FROM abc WHERE oid%2;
#   CREATE INDEX abc_i ON abc(a);
#   DELETE FROM abc WHERE (oid%2)==0;
#   DROP INDEX abc_i;
#   COMMIT;
# }


do_ioerr_test incrvacuum-ioerr-2 -start 1 -cksum 1 -tclprep {
  db eval {
    PRAGMA auto_vacuum = 'full';
    PRAGMA cache_size = 10;
    BEGIN;
    CREATE TABLE abc(a, UNIQUE(a));
  }
  for {set ii 0} {$ii < 25} {incr ii} {
    db eval {INSERT INTO abc VALUES(randstr(1500,1500))}
  }
  db eval COMMIT
} -sqlbody {
  BEGIN;
  INCREMENTAL VACUUM;
  DELETE FROM abc WHERE (oid%3)==0;
  INCREMENTAL VACUUM;
  INSERT INTO abc SELECT a || '1234567890' FROM abc WHERE oid%2;
  INCREMENTAL VACUUM;
  CREATE INDEX abc_i ON abc(a);
  DELETE FROM abc WHERE (oid%2)==0;
  INCREMENTAL VACUUM;
  DROP INDEX abc_i;
  INCREMENTAL VACUUM;
  COMMIT;
}
finish_test

Changes to test/quick.test.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file runs all tests.
#
# $Id: quick.test,v 1.52 2007/04/16 17:07:55 drh Exp $

proc lshift {lvar} {
  upvar $lvar l
  set ret [lindex $l 0]
  set l [lrange $l 1 end]
  return $ret
}








|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file runs all tests.
#
# $Id: quick.test,v 1.53 2007/04/28 15:47:45 danielk1977 Exp $

proc lshift {lvar} {
  upvar $lvar l
  set ret [lindex $l 0]
  set l [lrange $l 1 end]
  return $ret
}
52
53
54
55
56
57
58

59
60
61
62
63
64
65
  memleak.test
  misc7.test
  misuse.test
  quick.test
  speed1.test
  speed2.test


  autovacuum_crash.test
  btree8.test
  utf16.test
  shared_err.test
  vtab_err.test
}








>







52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
  memleak.test
  misc7.test
  misuse.test
  quick.test
  speed1.test
  speed2.test

  incrvacuum_ioerr.test
  autovacuum_crash.test
  btree8.test
  utf16.test
  shared_err.test
  vtab_err.test
}

Changes to test/tester.tcl.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# 2001 September 15
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements some common TCL routines used for regression
# testing the SQLite library
#
# $Id: tester.tcl,v 1.79 2007/04/19 12:30:54 drh Exp $

# Make sure tclsqlite3 was compiled correctly.  Abort now with an
# error message if not.
#
if {[sqlite3 -tcl-uses-utf]} {
  if {"\u1234"=="u1234"} {
    puts stderr "***** BUILD PROBLEM *****"













|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# 2001 September 15
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements some common TCL routines used for regression
# testing the SQLite library
#
# $Id: tester.tcl,v 1.80 2007/04/28 15:47:45 danielk1977 Exp $

# Make sure tclsqlite3 was compiled correctly.  Abort now with an
# error message if not.
#
if {[sqlite3 -tcl-uses-utf]} {
  if {"\u1234"=="u1234"} {
    puts stderr "***** BUILD PROBLEM *****"
395
396
397
398
399
400
401

402
403
404
405
406
407
408
  set ::ioerropts(-erc) 0
  set ::ioerropts(-count) 100000000
  set ::ioerropts(-persist) 1
  array set ::ioerropts $args

  set ::go 1
  for {set n $::ioerropts(-start)} {$::go} {incr n} {

    incr ::ioerropts(-count) -1
    if {$::ioerropts(-count)<0} break
 
    # Skip this IO error if it was specified with the "-exclude" option.
    if {[info exists ::ioerropts(-exclude)]} {
      if {[lsearch $::ioerropts(-exclude) $n]!=-1} continue
    }







>







395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
  set ::ioerropts(-erc) 0
  set ::ioerropts(-count) 100000000
  set ::ioerropts(-persist) 1
  array set ::ioerropts $args

  set ::go 1
  for {set n $::ioerropts(-start)} {$::go} {incr n} {
set ::TN $n
    incr ::ioerropts(-count) -1
    if {$::ioerropts(-count)<0} break
 
    # Skip this IO error if it was specified with the "-exclude" option.
    if {[info exists ::ioerropts(-exclude)]} {
      if {[lsearch $::ioerropts(-exclude) $n]!=-1} continue
    }