/ Check-in [648dd157]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add file test/tt3_checkpoint.c that adds a multi-threaded test for blocking checkpoints to threadtest3.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | blocking-checkpoint
Files: files | file ages | folders
SHA1: 648dd157ef3b7b790764698fd4dd7107c25212c9
User & Date: dan 2010-11-19 09:58:11
Context
2010-11-19
18:51
Merge the checkpoint_fullfsync pragma and the superlock demonstration into the checkpoint-v2 experimental branch. Closed-Leaf check-in: ebf74015 user: drh tags: blocking-checkpoint
09:58
Add file test/tt3_checkpoint.c that adds a multi-threaded test for blocking checkpoints to threadtest3. check-in: 648dd157 user: dan tags: blocking-checkpoint
07:17
Add extra test cases for blocking checkpoints. check-in: ac348ae2 user: dan tags: blocking-checkpoint
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to main.mk.

526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
test:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/veryquick.test

# The next two rules are used to support the "threadtest" target. Building
# threadtest runs a few thread-safety tests that are implemented in C. This
# target is invoked by the releasetest.tcl script.
# 
threadtest3$(EXE): sqlite3.c $(TOP)/test/threadtest3.c
	$(TCCX) -O2 sqlite3.c $(TOP)/test/threadtest3.c \
		-o threadtest3$(EXE) $(THREADLIB)

threadtest: threadtest3$(EXE)
	./threadtest3$(EXE)

sqlite3_analyzer$(EXE):	$(TOP)/src/tclsqlite.c sqlite3.c $(TESTSRC) \
			$(TOP)/tool/spaceanal.tcl







|
|







526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
test:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/veryquick.test

# The next two rules are used to support the "threadtest" target. Building
# threadtest runs a few thread-safety tests that are implemented in C. This
# target is invoked by the releasetest.tcl script.
# 
threadtest3$(EXE): sqlite3.o $(TOP)/test/threadtest3.c $(TOP)/test/tt3_checkpoint.c
	$(TCCX) -O2 sqlite3.o $(TOP)/test/threadtest3.c \
		-o threadtest3$(EXE) $(THREADLIB)

threadtest: threadtest3$(EXE)
	./threadtest3$(EXE)

sqlite3_analyzer$(EXE):	$(TOP)/src/tclsqlite.c sqlite3.c $(TESTSRC) \
			$(TOP)/tool/spaceanal.tcl

Changes to test/threadtest3.c.

1390
1391
1392
1393
1394
1395
1396

1397
1398
1399
1400
1401
1402
1403
....
1404
1405
1406
1407
1408
1409
1410
1411
1412



1413
1414
1415
1416
1417
1418
1419
  launch_thread(&err, &threads, dynamic_triggers_1, 0);

  join_all_threads(&err, &threads);

  print_and_free_err(&err);
}



int main(int argc, char **argv){
  struct ThreadTest {
    void (*xTest)(int);
    const char *zTest;
    int nMs;
  } aTest[] = {
................................................................................
    { walthread1, "walthread1", 20000 },
    { walthread2, "walthread2", 20000 },
    { walthread3, "walthread3", 20000 },
    { walthread4, "walthread4", 20000 },
    { walthread5, "walthread5",  1000 },
    { walthread5, "walthread5",  1000 },
    
    { cgt_pager_1, "cgt_pager_1", 0 },
    { dynamic_triggers, "dynamic_triggers", 20000 },



  };

  int i;
  char *zTest = 0;
  int nTest = 0;
  int bTestfound = 0;
  int bPrefix = 0;







>







 







|

>
>
>







1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
....
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
  launch_thread(&err, &threads, dynamic_triggers_1, 0);

  join_all_threads(&err, &threads);

  print_and_free_err(&err);
}

#include "tt3_checkpoint.c"

int main(int argc, char **argv){
  struct ThreadTest {
    void (*xTest)(int);
    const char *zTest;
    int nMs;
  } aTest[] = {
................................................................................
    { walthread1, "walthread1", 20000 },
    { walthread2, "walthread2", 20000 },
    { walthread3, "walthread3", 20000 },
    { walthread4, "walthread4", 20000 },
    { walthread5, "walthread5",  1000 },
    { walthread5, "walthread5",  1000 },
    
    { cgt_pager_1,      "cgt_pager_1", 0 },
    { dynamic_triggers, "dynamic_triggers", 20000 },

    { checkpoint_starvation_1, "checkpoint_starvation_1", 10000 },
    { checkpoint_starvation_2, "checkpoint_starvation_2", 10000 },
  };

  int i;
  char *zTest = 0;
  int nTest = 0;
  int bTestfound = 0;
  int bPrefix = 0;

Added test/tt3_checkpoint.c.













































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
/*
** 2001 September 15
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** This file is part of the test program "threadtest3". Despite being a C
** file it is not compiled separately, but included by threadtest3.c using
** the #include directive normally used with header files.
**
** This file contains the implementation of test cases:
**
**     checkpoint_starvation_1
**     checkpoint_starvation_2
*/

/*
** Both test cases involve 1 writer/checkpointer thread and N reader threads.
** 
** Each reader thread performs a series of read transactions, one after 
** another. Each read transaction lasts for 100 ms.
**
** The writer writes transactions as fast as possible. It uses a callback
** registered with sqlite3_wal_hook() to try to keep the WAL-size limited to 
** around 50 pages.
**
** In test case checkpoint_starvation_1, the auto-checkpoint uses 
** SQLITE_CHECKPOINT_PASSIVE. In checkpoint_starvation_2, it uses RESTART.
** The expectation is that in the first case the WAL file will grow very 
** large, and in the second will be limited to the 50 pages or thereabouts.
** However, the overall transaction throughput will be lower for 
** checkpoint_starvation_2, as every checkpoint will block for up to 200 ms
** waiting for readers to clear.
*/

/* Frame limit used by the WAL hook for these tests. */
#define CHECKPOINT_STARVATION_FRAMELIMIT 50

/* Duration in ms of each read transaction */
#define CHECKPOINT_STARVATION_READMS    100

struct CheckpointStarvationCtx {
  int eMode;
  int nMaxFrame;
};
typedef struct CheckpointStarvationCtx CheckpointStarvationCtx;

static int checkpoint_starvation_walhook(
  void *pCtx, 
  sqlite3 *db, 
  const char *zDb, 
  int nFrame
){
  CheckpointStarvationCtx *p = (CheckpointStarvationCtx *)pCtx;
  if( nFrame>p->nMaxFrame ){
    p->nMaxFrame = nFrame;
  }
  if( nFrame>=CHECKPOINT_STARVATION_FRAMELIMIT ){
    sqlite3_wal_checkpoint_v2(db, zDb, p->eMode, 0, 0);
  }
  return SQLITE_OK;
}

static char *checkpoint_starvation_reader(int iTid, int iArg){
  Error err = {0};
  Sqlite db = {0};

  opendb(&err, &db, "test.db", 0);
  while( !timetostop(&err) ){
    i64 iCount1, iCount2;
    sql_script(&err, &db, "BEGIN");
    iCount1 = execsql_i64(&err, &db, "SELECT count(x) FROM t1");
    usleep(CHECKPOINT_STARVATION_READMS*1000);
    iCount2 = execsql_i64(&err, &db, "SELECT count(x) FROM t1");
    sql_script(&err, &db, "COMMIT");

    if( iCount1!=iCount2 ){
      test_error(&err, "Isolation failure - %lld %lld", iCount1, iCount2);
    }
  }
  closedb(&err, &db);

  print_and_free_err(&err);
  return 0;
}

static void checkpoint_starvation_main(int nMs, CheckpointStarvationCtx *p){
  Error err = {0};
  Sqlite db = {0};
  Threadset threads = {0};
  int nInsert = 0;
  int i;

  opendb(&err, &db, "test.db", 1);
  sql_script(&err, &db, 
      "PRAGMA page_size = 1024;"
      "PRAGMA journal_mode = WAL;"
      "CREATE TABLE t1(x);"
  );

  setstoptime(&err, nMs);

  for(i=0; i<4; i++){
    launch_thread(&err, &threads, checkpoint_starvation_reader, 0);
    usleep(CHECKPOINT_STARVATION_READMS*1000/4);
  }

  sqlite3_wal_hook(db.db, checkpoint_starvation_walhook, (void *)p);
  while( !timetostop(&err) ){
    sql_script(&err, &db, "INSERT INTO t1 VALUES(randomblob(1200))");
    nInsert++;
  }

  printf(" Checkpoint mode  : %s\n",
      p->eMode==SQLITE_CHECKPOINT_PASSIVE ? "PASSIVE" : "RESTART"
  );
  printf(" Peak WAL         : %d frames\n", p->nMaxFrame);
  printf(" Transaction count: %d transactions\n", nInsert);

  join_all_threads(&err, &threads);
  closedb(&err, &db);
  print_and_free_err(&err);
}

static void checkpoint_starvation_1(int nMs){
  Error err = {0};
  CheckpointStarvationCtx ctx = { SQLITE_CHECKPOINT_PASSIVE, 0 };
  checkpoint_starvation_main(nMs, &ctx);
  if( ctx.nMaxFrame<(CHECKPOINT_STARVATION_FRAMELIMIT*10) ){
    test_error(&err, "WAL failed to grow - %d frames", ctx.nMaxFrame);
  }
  print_and_free_err(&err);
}

static void checkpoint_starvation_2(int nMs){
  Error err = {0};
  CheckpointStarvationCtx ctx = { SQLITE_CHECKPOINT_RESTART, 0 };
  checkpoint_starvation_main(nMs, &ctx);
  if( ctx.nMaxFrame>CHECKPOINT_STARVATION_FRAMELIMIT+10 ){
    test_error(&err, "WAL grew too large - %d frames", ctx.nMaxFrame);
  }
  print_and_free_err(&err);
}


Changes to test/wal5.test.

202
203
204
205
206
207
208



209
210
211
212
213
214
215
#
#   2. Readers using part of the log file. FULL and RESTART checkpoints block
#      until readers using part (but not all) of the log file have finished.
#
#   3. Readers using any of the log file. After copying data into the
#      database file, RESTART checkpoints block until readers using any part
#      of the log file have finished.



#
foreach {tn1 checkpoint busy_on ckpt_expected expected} {
  1   PASSIVE   -   {0 5 5}   -
  2   TYPO      -   {0 5 5}   -

  3   FULL      -   {0 7 7}   2
  4   FULL      1   {1 5 5}   1







>
>
>







202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
#
#   2. Readers using part of the log file. FULL and RESTART checkpoints block
#      until readers using part (but not all) of the log file have finished.
#
#   3. Readers using any of the log file. After copying data into the
#      database file, RESTART checkpoints block until readers using any part
#      of the log file have finished.
#
# This test case involves running a checkpoint while there exist other 
# processes holding all three types of locks.
#
foreach {tn1 checkpoint busy_on ckpt_expected expected} {
  1   PASSIVE   -   {0 5 5}   -
  2   TYPO      -   {0 5 5}   -

  3   FULL      -   {0 7 7}   2
  4   FULL      1   {1 5 5}   1