/ Check-in [fc157dd7]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a race condition to do with very large index keys in shared-cache mode.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: fc157dd7f18c94b7ae5f155e1b4a5d7714b7da8c
User & Date: dan 2014-12-11 16:38:18
Context
2014-12-12
00:52
Remove the KeyInfo cache (for now - perhaps we will add it back in later - or maybe not since it provides negligible benefit but adds a lot of complexity and thread-safety risk). Add a mutex to ATTACH to deal with a data race. check-in: 03c443ea user: drh tags: trunk
00:26
Merge the cell overflow page number cache thread race fix from trunk. check-in: cefad47e user: drh tags: threadtest4
2014-12-11
16:38
Fix a race condition to do with very large index keys in shared-cache mode. check-in: fc157dd7 user: dan tags: trunk
15:27
Fix a typo in the documentation for sqlite3_threadsafe(). check-in: 258e747b user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/btree.c.

3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
....
4203
4204
4205
4206
4207
4208
4209

4210
4211
4212
4213
4214
4215
4216
....
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
....
4269
4270
4271
4272
4273
4274
4275

4276
4277
4278
4279
4280
4281
4282
    if( pCur->pNext ){
      pCur->pNext->pPrev = pCur->pPrev;
    }
    for(i=0; i<=pCur->iPage; i++){
      releasePage(pCur->apPage[i]);
    }
    unlockBtreeIfUnused(pBt);
    sqlite3DbFree(pBtree->db, pCur->aOverflow);
    /* sqlite3_free(pCur); */
    sqlite3BtreeLeave(pBtree);
  }
  return SQLITE_OK;
}

/*
................................................................................
    rc = copyPayload(&aPayload[offset], pBuf, a, (eOp & 0x01), pPage->pDbPage);
    offset = 0;
    pBuf += a;
    amt -= a;
  }else{
    offset -= pCur->info.nLocal;
  }


  if( rc==SQLITE_OK && amt>0 ){
    const u32 ovflSize = pBt->usableSize - 4;  /* Bytes content per ovfl page */
    Pgno nextPage;

    nextPage = get4byte(&aPayload[pCur->info.nLocal]);

................................................................................
    ** in the overflow chain. The page number of the first overflow page is
    ** stored in aOverflow[0], etc. A value of 0 in the aOverflow[] array
    ** means "not yet known" (the cache is lazily populated).
    */
    if( eOp!=2 && (pCur->curFlags & BTCF_ValidOvfl)==0 ){
      int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize;
      if( nOvfl>pCur->nOvflAlloc ){
        Pgno *aNew = (Pgno*)sqlite3DbRealloc(
            pCur->pBtree->db, pCur->aOverflow, nOvfl*2*sizeof(Pgno)
        );
        if( aNew==0 ){
          rc = SQLITE_NOMEM;
        }else{
          pCur->nOvflAlloc = nOvfl*2;
          pCur->aOverflow = aNew;
        }
................................................................................
        ** function.
        **
        ** Note that the aOverflow[] array must be allocated because eOp!=2
        ** here.  If eOp==2, then offset==0 and this branch is never taken.
        */
        assert( eOp!=2 );
        assert( pCur->curFlags & BTCF_ValidOvfl );

        if( pCur->aOverflow[iIdx+1] ){
          nextPage = pCur->aOverflow[iIdx+1];
        }else{
          rc = getOverflowPage(pBt, nextPage, 0, &nextPage);
        }
        offset -= ovflSize;
      }else{







|







 







>







 







|
|







 







>







3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
....
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
....
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
....
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
    if( pCur->pNext ){
      pCur->pNext->pPrev = pCur->pPrev;
    }
    for(i=0; i<=pCur->iPage; i++){
      releasePage(pCur->apPage[i]);
    }
    unlockBtreeIfUnused(pBt);
    sqlite3_free(pCur->aOverflow);
    /* sqlite3_free(pCur); */
    sqlite3BtreeLeave(pBtree);
  }
  return SQLITE_OK;
}

/*
................................................................................
    rc = copyPayload(&aPayload[offset], pBuf, a, (eOp & 0x01), pPage->pDbPage);
    offset = 0;
    pBuf += a;
    amt -= a;
  }else{
    offset -= pCur->info.nLocal;
  }


  if( rc==SQLITE_OK && amt>0 ){
    const u32 ovflSize = pBt->usableSize - 4;  /* Bytes content per ovfl page */
    Pgno nextPage;

    nextPage = get4byte(&aPayload[pCur->info.nLocal]);

................................................................................
    ** in the overflow chain. The page number of the first overflow page is
    ** stored in aOverflow[0], etc. A value of 0 in the aOverflow[] array
    ** means "not yet known" (the cache is lazily populated).
    */
    if( eOp!=2 && (pCur->curFlags & BTCF_ValidOvfl)==0 ){
      int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize;
      if( nOvfl>pCur->nOvflAlloc ){
        Pgno *aNew = (Pgno*)sqlite3Realloc(
            pCur->aOverflow, nOvfl*2*sizeof(Pgno)
        );
        if( aNew==0 ){
          rc = SQLITE_NOMEM;
        }else{
          pCur->nOvflAlloc = nOvfl*2;
          pCur->aOverflow = aNew;
        }
................................................................................
        ** function.
        **
        ** Note that the aOverflow[] array must be allocated because eOp!=2
        ** here.  If eOp==2, then offset==0 and this branch is never taken.
        */
        assert( eOp!=2 );
        assert( pCur->curFlags & BTCF_ValidOvfl );
        assert( pCur->pBtree->db==pBt->db );
        if( pCur->aOverflow[iIdx+1] ){
          nextPage = pCur->aOverflow[iIdx+1];
        }else{
          rc = getOverflowPage(pBt, nextPage, 0, &nextPage);
        }
        offset -= ovflSize;
      }else{

Changes to src/vdbeapi.c.

576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
    ** into the database handle. This block copies the error message 
    ** from the database handle into the statement and sets the statement
    ** program counter to 0 to ensure that when the statement is 
    ** finalized or reset the parser error message is available via
    ** sqlite3_errmsg() and sqlite3_errcode().
    */
    const char *zErr = (const char *)sqlite3_value_text(db->pErr); 
    assert( zErr!=0 || db->mallocFailed );
    sqlite3DbFree(db, v->zErrMsg);
    if( !db->mallocFailed ){
      v->zErrMsg = sqlite3DbStrDup(db, zErr);
      v->rc = rc2;
    } else {
      v->zErrMsg = 0;
      v->rc = rc = SQLITE_NOMEM;







<







576
577
578
579
580
581
582

583
584
585
586
587
588
589
    ** into the database handle. This block copies the error message 
    ** from the database handle into the statement and sets the statement
    ** program counter to 0 to ensure that when the statement is 
    ** finalized or reset the parser error message is available via
    ** sqlite3_errmsg() and sqlite3_errcode().
    */
    const char *zErr = (const char *)sqlite3_value_text(db->pErr); 

    sqlite3DbFree(db, v->zErrMsg);
    if( !db->mallocFailed ){
      v->zErrMsg = sqlite3DbStrDup(db, zErr);
      v->rc = rc2;
    } else {
      v->zErrMsg = 0;
      v->rc = rc = SQLITE_NOMEM;

Changes to test/threadtest3.c.

1394
1395
1396
1397
1398
1399
1400

1401
1402
1403
1404
1405
1406
1407
....
1415
1416
1417
1418
1419
1420
1421

1422
1423
1424
1425
1426
1427
1428
  print_and_free_err(&err);
}



#include "tt3_checkpoint.c"
#include "tt3_index.c"


int main(int argc, char **argv){
  struct ThreadTest {
    void (*xTest)(int);
    const char *zTest;
    int nMs;
  } aTest[] = {
................................................................................
    { cgt_pager_1,      "cgt_pager_1", 0 },
    { dynamic_triggers, "dynamic_triggers", 20000 },

    { checkpoint_starvation_1, "checkpoint_starvation_1", 10000 },
    { checkpoint_starvation_2, "checkpoint_starvation_2", 10000 },

    { create_drop_index_1, "create_drop_index_1", 10000 },

  };

  int i;
  char *zTest = 0;
  int nTest = 0;
  int bTestfound = 0;
  int bPrefix = 0;







>







 







>







1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
....
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
  print_and_free_err(&err);
}



#include "tt3_checkpoint.c"
#include "tt3_index.c"
#include "tt3_lookaside1.c"

int main(int argc, char **argv){
  struct ThreadTest {
    void (*xTest)(int);
    const char *zTest;
    int nMs;
  } aTest[] = {
................................................................................
    { cgt_pager_1,      "cgt_pager_1", 0 },
    { dynamic_triggers, "dynamic_triggers", 20000 },

    { checkpoint_starvation_1, "checkpoint_starvation_1", 10000 },
    { checkpoint_starvation_2, "checkpoint_starvation_2", 10000 },

    { create_drop_index_1, "create_drop_index_1", 10000 },
    { lookaside1, "lookaside1", 10000 },
  };

  int i;
  char *zTest = 0;
  int nTest = 0;
  int bTestfound = 0;
  int bPrefix = 0;

Changes to test/tt3_index.c.

63
64
65
66
67
68
69
70
71
72

73
74

  sqlite3_enable_shared_cache(1);
  launch_thread(&err, &threads, create_drop_index_thread, 0);
  launch_thread(&err, &threads, create_drop_index_thread, 0);
  launch_thread(&err, &threads, create_drop_index_thread, 0);
  launch_thread(&err, &threads, create_drop_index_thread, 0);
  launch_thread(&err, &threads, create_drop_index_thread, 0);
  sqlite3_enable_shared_cache(0);

  join_all_threads(&err, &threads);

  print_and_free_err(&err);
}







<


>


63
64
65
66
67
68
69

70
71
72
73
74

  sqlite3_enable_shared_cache(1);
  launch_thread(&err, &threads, create_drop_index_thread, 0);
  launch_thread(&err, &threads, create_drop_index_thread, 0);
  launch_thread(&err, &threads, create_drop_index_thread, 0);
  launch_thread(&err, &threads, create_drop_index_thread, 0);
  launch_thread(&err, &threads, create_drop_index_thread, 0);


  join_all_threads(&err, &threads);
  sqlite3_enable_shared_cache(0);
  print_and_free_err(&err);
}

Added test/tt3_lookaside1.c.







































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
/*
** 2014 December 9
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
**
**     lookaside1
*/

/*
** The test in this file attempts to expose a specific race condition
** that is suspected to exist at time of writing.
*/

static char *lookaside1_thread_reader(int iTid, int iArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */

  opendb(&err, &db, "test.db", 0);

  while( !timetostop(&err) ){
    sqlite3_stmt *pStmt = 0;
    int rc;

    sqlite3_prepare_v2(db.db, "SELECT 1 FROM t1", -1, &pStmt, 0);
    while( sqlite3_step(pStmt)==SQLITE_ROW ){
      execsql(&err, &db, "SELECT length(x||y||z) FROM t2");
    }
    rc = sqlite3_finalize(pStmt);
    if( err.rc==SQLITE_OK && rc!=SQLITE_OK ){
      sqlite_error(&err, &db, "finalize");
    }
  }

  closedb(&err, &db);
  print_and_free_err(&err);
  return sqlite3_mprintf("ok");
}

static char *lookaside1_thread_writer(int iTid, int iArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */

  opendb(&err, &db, "test.db", 0);

  do{
    sql_script(&err, &db, 
      "BEGIN;"
        "UPDATE t3 SET i=i+1 WHERE x=1;"
      "ROLLBACK;"
    );
  }while( !timetostop(&err) );

  closedb(&err, &db);
  print_and_free_err(&err);
  return sqlite3_mprintf("ok");
}


static void lookaside1(int nMs){
  Error err = {0};
  Sqlite db = {0};
  Threadset threads = {0};

  opendb(&err, &db, "test.db", 1);
  sql_script(&err, &db, 
     "CREATE TABLE t1(x PRIMARY KEY) WITHOUT ROWID;"
     "WITH data(x,y) AS ("
     "  SELECT 1, quote(randomblob(750)) UNION ALL "
     "  SELECT x*2, y||y FROM data WHERE x<5) "
     "INSERT INTO t1 SELECT y FROM data;"

     "CREATE TABLE t3(x PRIMARY KEY,i) WITHOUT ROWID;"
     "INSERT INTO t3 VALUES(1, 1);"

     "CREATE TABLE t2(x,y,z);"
     "INSERT INTO t2 VALUES(randomblob(50), randomblob(50), randomblob(50));"
  );
  closedb(&err, &db);

  setstoptime(&err, nMs);

  sqlite3_enable_shared_cache(1);
  launch_thread(&err, &threads, lookaside1_thread_reader, 0);
  launch_thread(&err, &threads, lookaside1_thread_reader, 0);
  launch_thread(&err, &threads, lookaside1_thread_reader, 0);
  launch_thread(&err, &threads, lookaside1_thread_reader, 0);
  launch_thread(&err, &threads, lookaside1_thread_reader, 0);
  launch_thread(&err, &threads, lookaside1_thread_writer, 0);
  join_all_threads(&err, &threads);
  sqlite3_enable_shared_cache(0);
  print_and_free_err(&err);
}