SQLite

Changes On Branch threadtest3
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch threadtest3 Excluding Merge-Ins

This is equivalent to a diff from 62ef4514 to 5648af96

2014-12-16
00:20
Enhanced "stress2" testing in the threadtest3.c test program. (check-in: ae43539e user: drh tags: trunk)
00:08
Make sure the sqlite3BtreeCount() routine does not leave index cursors in an inconsistent state, as doing so might result in an assertion fault inside of sqlite3BtreeKey() called from saveAllCursors() if content is deleted out from under the statement that issued the sqlite3BtreeCount() call. (check-in: 5b1b6970 user: drh tags: trunk)
2014-12-15
20:49
Changes to threadtest3 so that "stress2" is more similar to the SDS stress test. (Closed-Leaf check-in: 5648af96 user: dan tags: threadtest3)
16:27
Add new test file e_walauto.test. (check-in: 62ef4514 user: dan tags: trunk)
08:46
Fix errors in threadtest3 tests caused by earlier tests neglecting to close database handles. (check-in: 1d44f1b1 user: dan tags: trunk)

Changes to test/threadtest3.c.

117
118
119
120
121
122
123

124


125
126
127
128
129
130
131
#  define uint32 unsigned int
#endif

struct MD5Context {
  int isInit;
  uint32 buf[4];
  uint32 bits[2];

  unsigned char in[64];


};
typedef struct MD5Context MD5Context;

/*
 * Note: this code is harmless on little-endian machines.
 */
static void byteReverse (unsigned char *buf, unsigned longs){







>
|
>
>







117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
#  define uint32 unsigned int
#endif

struct MD5Context {
  int isInit;
  uint32 buf[4];
  uint32 bits[2];
  union {
    unsigned char in[64];
    uint32 in32[16];
  } u;
};
typedef struct MD5Context MD5Context;

/*
 * Note: this code is harmless on little-endian machines.
 */
static void byteReverse (unsigned char *buf, unsigned longs){
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
  ctx->bits[1] += len >> 29;

  t = (t >> 3) & 0x3f;    /* Bytes already in shsInfo->data */

  /* Handle any leading odd-sized chunks */

  if ( t ) {
    unsigned char *p = (unsigned char *)ctx->in + t;

    t = 64-t;
    if (len < t) {
      memcpy(p, buf, len);
      return;
    }
    memcpy(p, buf, t);
    byteReverse(ctx->in, 16);
    MD5Transform(ctx->buf, (uint32 *)ctx->in);
    buf += t;
    len -= t;
  }

  /* Process data in 64-byte chunks */

  while (len >= 64) {
    memcpy(ctx->in, buf, 64);
    byteReverse(ctx->in, 16);
    MD5Transform(ctx->buf, (uint32 *)ctx->in);
    buf += 64;
    len -= 64;
  }

  /* Handle any remaining bytes of data. */

  memcpy(ctx->in, buf, len);
}

/*
 * Final wrapup - pad to 64-byte boundary with the bit pattern 
 * 1 0* (64-bit count of bits processed, MSB-first)
 */
static void MD5Final(unsigned char digest[16], MD5Context *ctx){
  unsigned count;
  unsigned char *p;

  /* Compute number of bytes mod 64 */
  count = (ctx->bits[0] >> 3) & 0x3F;

  /* Set the first char of padding to 0x80.  This is safe since there is
     always at least one byte free */
  p = ctx->in + count;
  *p++ = 0x80;

  /* Bytes of padding needed to make 64 bytes */
  count = 64 - 1 - count;

  /* Pad out to 56 mod 64 */
  if (count < 8) {
    /* Two lots of padding:  Pad the first block to 64 bytes */
    memset(p, 0, count);
    byteReverse(ctx->in, 16);
    MD5Transform(ctx->buf, (uint32 *)ctx->in);

    /* Now fill the next block with 56 bytes */
    memset(ctx->in, 0, 56);
  } else {
    /* Pad block to 56 bytes */
    memset(p, 0, count-8);
  }
  byteReverse(ctx->in, 14);

  /* Append length in bits and transform */
  ((uint32 *)ctx->in)[ 14 ] = ctx->bits[0];
  ((uint32 *)ctx->in)[ 15 ] = ctx->bits[1];

  MD5Transform(ctx->buf, (uint32 *)ctx->in);
  byteReverse((unsigned char *)ctx->buf, 4);
  memcpy(digest, ctx->buf, 16);
  memset(ctx, 0, sizeof(*ctx));    /* In case it is sensitive */
}

/*
** Convert a 128-bit MD5 digest into a 32-digit base-16 number.







|







|
|







|
|
|






|















|









|
|


|




|


|
|

|







269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
  ctx->bits[1] += len >> 29;

  t = (t >> 3) & 0x3f;    /* Bytes already in shsInfo->data */

  /* Handle any leading odd-sized chunks */

  if ( t ) {
    unsigned char *p = (unsigned char *)ctx->u.in + t;

    t = 64-t;
    if (len < t) {
      memcpy(p, buf, len);
      return;
    }
    memcpy(p, buf, t);
    byteReverse(ctx->u.in, 16);
    MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
    buf += t;
    len -= t;
  }

  /* Process data in 64-byte chunks */

  while (len >= 64) {
    memcpy(ctx->u.in, buf, 64);
    byteReverse(ctx->u.in, 16);
    MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
    buf += 64;
    len -= 64;
  }

  /* Handle any remaining bytes of data. */

  memcpy(ctx->u.in, buf, len);
}

/*
 * Final wrapup - pad to 64-byte boundary with the bit pattern 
 * 1 0* (64-bit count of bits processed, MSB-first)
 */
static void MD5Final(unsigned char digest[16], MD5Context *ctx){
  unsigned count;
  unsigned char *p;

  /* Compute number of bytes mod 64 */
  count = (ctx->bits[0] >> 3) & 0x3F;

  /* Set the first char of padding to 0x80.  This is safe since there is
     always at least one byte free */
  p = ctx->u.in + count;
  *p++ = 0x80;

  /* Bytes of padding needed to make 64 bytes */
  count = 64 - 1 - count;

  /* Pad out to 56 mod 64 */
  if (count < 8) {
    /* Two lots of padding:  Pad the first block to 64 bytes */
    memset(p, 0, count);
    byteReverse(ctx->u.in, 16);
    MD5Transform(ctx->buf, (uint32 *)ctx->u.in);

    /* Now fill the next block with 56 bytes */
    memset(ctx->u.in, 0, 56);
  } else {
    /* Pad block to 56 bytes */
    memset(p, 0, count-8);
  }
  byteReverse(ctx->u.in, 14);

  /* Append length in bits and transform */
  ctx->u.in32[14] = ctx->bits[0];
  ctx->u.in32[15] = ctx->bits[1];

  MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
  byteReverse((unsigned char *)ctx->buf, 4);
  memcpy(digest, ctx->buf, 16);
  memset(ctx, 0, sizeof(*ctx));    /* In case it is sensitive */
}

/*
** Convert a 128-bit MD5 digest into a 32-digit base-16 number.
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
    { checkpoint_starvation_1, "checkpoint_starvation_1", 10000 },
    { checkpoint_starvation_2, "checkpoint_starvation_2", 10000 },

    { create_drop_index_1, "create_drop_index_1", 10000 },
    { lookaside1,          "lookaside1", 10000 },
    { vacuum1,             "vacuum1", 10000 },
    { stress1,             "stress1", 10000 },
    { stress2,             "stress2", 10000 },
  };

  int i;
  int bTestfound = 0;

  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);







|







1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
    { checkpoint_starvation_1, "checkpoint_starvation_1", 10000 },
    { checkpoint_starvation_2, "checkpoint_starvation_2", 10000 },

    { create_drop_index_1, "create_drop_index_1", 10000 },
    { lookaside1,          "lookaside1", 10000 },
    { vacuum1,             "vacuum1", 10000 },
    { stress1,             "stress1", 10000 },
    { stress2,             "stress2", 60000 },
  };

  int i;
  int bTestfound = 0;

  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);

Changes to test/tt3_stress.c.

120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
      closedb(&err, &db);
      opendb(&err, &db, "test.db", 0);
    }
    execsql(&err, &db, "DELETE FROM t1 WHERE (rowid % 4)==:i", &i);
    i1++;
    if( err.rc ) i2++;
    clear_error(&err, SQLITE_LOCKED);
    clear_error(&err, SQLITE_ERROR);
  }
  closedb(&err, &db);
  print_and_free_err(&err);
  return sqlite3_mprintf("deleted from t1 %d/%d attempts", i2, i1);
}









<







120
121
122
123
124
125
126

127
128
129
130
131
132
133
      closedb(&err, &db);
      opendb(&err, &db, "test.db", 0);
    }
    execsql(&err, &db, "DELETE FROM t1 WHERE (rowid % 4)==:i", &i);
    i1++;
    if( err.rc ) i2++;
    clear_error(&err, SQLITE_LOCKED);

  }
  closedb(&err, &db);
  print_and_free_err(&err);
  return sqlite3_mprintf("deleted from t1 %d/%d attempts", i2, i1);
}


180
181
182
183
184
185
186
187
188



189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
** 10. Big DELETE statements.
** 11. VACUUM.
** 14. Integrity-check.
** 17. Switch the journal mode from delete to wal and back again.
** 19. Open and close database connections rapidly.
*/

#define STRESS2_TABCNT 5




static void stress2_workload1(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb, 
      "CREATE TABLE IF NOT EXISTS t%d(x PRIMARY KEY, y, z);", iTab
  );
}

static void stress2_workload2(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb, "DROP TABLE IF EXISTS t%d;", iTab);
}

static void stress2_workload3(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb, "SELECT * FROM t%d WHERE z = 'small'", iTab);
}

static void stress2_workload4(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb, "SELECT * FROM t%d WHERE z = 'big'", iTab);
}

static void stress2_workload5(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb,
      "INSERT INTO t%d VALUES(random(), hex(randomblob(57)), 'small');", iTab
  );
}

static void stress2_workload6(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb,
      "INSERT INTO t%d VALUES(random(), hex(randomblob(2000)), 'big');", iTab
  );
}

static void stress2_workload7(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb,
      "UPDATE t%d SET y = hex(randomblob(57)) "
      "WHERE (x %% 5)==(%d %% 5) AND z='small';"
      ,iTab, i
  );
}
static void stress2_workload8(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb,
      "UPDATE t%d SET y = hex(randomblob(2000)) "
      "WHERE (x %% 5)==(%d %% 5) AND z='big';"
      ,iTab, i
  );
}

static void stress2_workload9(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb,
      "DELETE FROM t%d WHERE (x %% 5)==(%d %% 5) AND z='small';"
      ,iTab, i
  );
}
static void stress2_workload10(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb,
      "DELETE FROM t%d WHERE (x %% 5)==(%d %% 5) AND z='big';"
      ,iTab, i
  );
}

static void stress2_workload11(Error *pErr, Sqlite *pDb, int i){
  sql_script(pErr, pDb, "VACUUM");
}








|

>
>
>

|






|




<
|



<
|



<
|
|




<
|
|




<

|
|
|



<

|
|
|




<

|
<



<

|
<







179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203

204
205
206
207

208
209
210
211

212
213
214
215
216
217

218
219
220
221
222
223

224
225
226
227
228
229
230

231
232
233
234
235
236
237
238

239
240

241
242
243

244
245

246
247
248
249
250
251
252
** 10. Big DELETE statements.
** 11. VACUUM.
** 14. Integrity-check.
** 17. Switch the journal mode from delete to wal and back again.
** 19. Open and close database connections rapidly.
*/

#define STRESS2_TABCNT 5          /* count1 in SDS test */

#define STRESS2_COUNT2 200        /* count2 in SDS test */
#define STRESS2_COUNT3  57        /* count2 in SDS test */

static void stress2_workload1(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % (STRESS2_TABCNT-1)) + 1;
  sql_script_printf(pErr, pDb, 
      "CREATE TABLE IF NOT EXISTS t%d(x PRIMARY KEY, y, z);", iTab
  );
}

static void stress2_workload2(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % (STRESS2_TABCNT-1)) + 1;
  sql_script_printf(pErr, pDb, "DROP TABLE IF EXISTS t%d;", iTab);
}

static void stress2_workload3(Error *pErr, Sqlite *pDb, int i){

  sql_script(pErr, pDb, "SELECT * FROM t0 WHERE z = 'small'");
}

static void stress2_workload4(Error *pErr, Sqlite *pDb, int i){

  sql_script(pErr, pDb, "SELECT * FROM t0 WHERE z = 'big'");
}

static void stress2_workload5(Error *pErr, Sqlite *pDb, int i){

  sql_script(pErr, pDb,
      "INSERT INTO t0 VALUES(hex(random()), hex(randomblob(200)), 'small');"
  );
}

static void stress2_workload6(Error *pErr, Sqlite *pDb, int i){

  sql_script(pErr, pDb,
      "INSERT INTO t0 VALUES(hex(random()), hex(randomblob(57)), 'big');"
  );
}

static void stress2_workload7(Error *pErr, Sqlite *pDb, int i){

  sql_script_printf(pErr, pDb,
      "UPDATE t0 SET y = hex(randomblob(200)) "
      "WHERE x LIKE hex((%d %% 5)) AND z='small';"
      ,i
  );
}
static void stress2_workload8(Error *pErr, Sqlite *pDb, int i){

  sql_script_printf(pErr, pDb,
      "UPDATE t0 SET y = hex(randomblob(57)) "
      "WHERE x LIKE hex(%d %% 5) AND z='big';"
      ,i
  );
}

static void stress2_workload9(Error *pErr, Sqlite *pDb, int i){

  sql_script_printf(pErr, pDb,
      "DELETE FROM t0 WHERE x LIKE hex(%d %% 5) AND z='small';", i

  );
}
static void stress2_workload10(Error *pErr, Sqlite *pDb, int i){

  sql_script_printf(pErr, pDb,
      "DELETE FROM t0 WHERE x LIKE hex(%d %% 5) AND z='big';", i

  );
}

static void stress2_workload11(Error *pErr, Sqlite *pDb, int i){
  sql_script(pErr, pDb, "VACUUM");
}

292
293
294
295
296
297
298
299
300



301
302
303
304
305


306
307
308
309
310
311
312
static char *stress2_thread_wrapper(int iTid, void *pArg){
  Stress2Ctx *pCtx = (Stress2Ctx*)pArg;
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  int i1 = 0;
  int i2 = 0;

  opendb(&err, &db, pCtx->zDb, 0);
  while( !timetostop(&err) ){



    pCtx->xProc(&err, &db, i1);
    i2 += (err.rc==SQLITE_OK);
    clear_error(&err, SQLITE_LOCKED);
    clear_error(&err, SQLITE_ERROR);
    i1++;


  }

  print_and_free_err(&err);
  return sqlite3_mprintf("ok %d/%d", i2, i1);
}

static void stress2_launch_thread_loop(







<

>
>
>
|
|
|
<
|
>
>







284
285
286
287
288
289
290

291
292
293
294
295
296
297

298
299
300
301
302
303
304
305
306
307
static char *stress2_thread_wrapper(int iTid, void *pArg){
  Stress2Ctx *pCtx = (Stress2Ctx*)pArg;
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  int i1 = 0;
  int i2 = 0;


  while( !timetostop(&err) ){
    int cnt;
    opendb(&err, &db, pCtx->zDb, 0);
    for(cnt=0; err.rc==SQLITE_OK && cnt<STRESS2_TABCNT; cnt++){
      pCtx->xProc(&err, &db, i1);
      i2 += (err.rc==SQLITE_OK);
      clear_error(&err, SQLITE_LOCKED);

      i1++;
    }
    closedb(&err, &db);
  }

  print_and_free_err(&err);
  return sqlite3_mprintf("ok %d/%d", i2, i1);
}

static void stress2_launch_thread_loop(
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356




357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
    { stress2_workload8 },
    { stress2_workload9 },
    { stress2_workload10 },
    { stress2_workload11 },
    { stress2_workload14 },
    { stress2_workload17 },
  };
  const char *azDb[] = {
    "test.db",
    "file::memory:?cache=shared"
  };
  int j;

  for(j=0; j<sizeof(azDb)/sizeof(azDb[0]); j++){
    int i;
    Error err = {0};
    Sqlite db = {0};
    Threadset threads = {0};
    const char *zDb = azDb[j];

    /* To make sure the db file is empty before commencing */
    opendb(&err, &db, zDb, 1);




    closedb(&err, &db);

    setstoptime(&err, nMs);
    sqlite3_enable_shared_cache(1);

    for(i=0; i<sizeof(aTask)/sizeof(aTask[0]); i++){
      stress2_launch_thread_loop(&err, &threads, zDb, aTask[i].x);
    }
    launch_thread(&err, &threads, stress2_workload19, (void*)zDb);
    launch_thread(&err, &threads, stress2_workload19, (void*)zDb);

    join_all_threads(&err, &threads);
    sqlite3_enable_shared_cache(0);
    print_and_free_err(&err);
    if( j==0 ){
      printf("Running stress2 (again, with in-memory db) for %d seconds...\n",
          nMs / 1000
      );
    }
  }
}











|
<
<
<
<

<
|
|
|
|
<

|
|
>
>
>
>
|

|
|

|
|
|
|
|

|
|
|
<
<
<
<
<
<





330
331
332
333
334
335
336
337




338

339
340
341
342

343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363






364
365
366
367
368
    { stress2_workload8 },
    { stress2_workload9 },
    { stress2_workload10 },
    { stress2_workload11 },
    { stress2_workload14 },
    { stress2_workload17 },
  };
  const char *zDb = "test.db";






  int i;
  Error err = {0};
  Sqlite db = {0};
  Threadset threads = {0};


  /* To make sure the db file is empty before commencing */
  opendb(&err, &db, zDb, 1);
  sql_script(&err, &db, 
      "CREATE TABLE IF NOT EXISTS t0(x PRIMARY KEY, y, z);"
      "CREATE INDEX IF NOT EXISTS i0 ON t0(y);"
  );
  closedb(&err, &db);

  setstoptime(&err, nMs);
  sqlite3_enable_shared_cache(1);

  for(i=0; i<sizeof(aTask)/sizeof(aTask[0]); i++){
    stress2_launch_thread_loop(&err, &threads, zDb, aTask[i].x);
  }
  launch_thread(&err, &threads, stress2_workload19, (void*)zDb);
  launch_thread(&err, &threads, stress2_workload19, (void*)zDb);

  join_all_threads(&err, &threads);
  sqlite3_enable_shared_cache(0);
  print_and_free_err(&err);






}