/ Check-in [29887487]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Reduce the size of the large allocation (approx 8KB for every 4000 frames in the log) that occurs during checkpoint. Use the 'scratch' memory for this allocation instead of the general purpose allocation.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 29887487ed549f97c3c9b37f852bae179b6ea9a9
User & Date: dan 2010-06-25 15:16:25
Context
2010-06-25
16:34
Reduce the average (but not maximum) size of the allocations made as part of a checkpoint. check-in: 4a7fd91b user: dan tags: trunk
15:16
Reduce the size of the large allocation (approx 8KB for every 4000 frames in the log) that occurs during checkpoint. Use the 'scratch' memory for this allocation instead of the general purpose allocation. check-in: 29887487 user: dan tags: trunk
14:17
Fix up a branch in sqlite3ValueFromExpr() so that we can achieve full branch test coverage regardless of whether or not SQLITE_ENABLE_STAT2 is used. check-in: af471ed7 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/wal.c.

1357
1358
1359
1360
1361
1362
1363

1364
1365
1366
1367
1368
1369
1370
1371
1372
1373

1374
1375
1376
1377
1378
1379
1380
....
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402

1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437

1438
1439
1440
1441
1442





1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459

1460
1461
1462
1463
1464
1465
1466
1467


1468
1469
1470

1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
....
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535

  for(iList=0; iList<nList; iList++){
    nMerge = 1;
    aMerge = &aList[iList];
    for(iSub=0; iList & (1<<iSub); iSub++){
      struct Sublist *p = &aSub[iSub];
      assert( p->aList && p->nList<=(1<<iSub) );

      walMerge(aContent, p->aList, p->nList, &aMerge, &nMerge, aBuffer);
    }
    aSub[iSub].aList = aMerge;
    aSub[iSub].nList = nMerge;
  }

  for(iSub++; iSub<ArraySize(aSub); iSub++){
    if( nList & (1<<iSub) ){
      struct Sublist *p = &aSub[iSub];
      assert( p->nList<=(2<<iSub) );

      walMerge(aContent, p->aList, p->nList, &aMerge, &nMerge, aBuffer);
    }
  }
  assert( aMerge==aList );
  *pnList = nMerge;

#ifdef SQLITE_DEBUG
................................................................................
#endif
}

/* 
** Free an iterator allocated by walIteratorInit().
*/
static void walIteratorFree(WalIterator *p){
  sqlite3_free(p);
}

/*
** Map the wal-index into memory owned by this thread, if it is not
** mapped already.  Then construct a WalInterator object that can be
** used to loop over all pages in the WAL in ascending order.  
**
** On success, make *pp point to the newly allocated WalInterator object

** return SQLITE_OK.  Otherwise, leave *pp unchanged and return an error
** code.
**
** The calling routine should invoke walIteratorFree() to destroy the
** WalIterator object when it has finished with it.  The caller must
** also unmap the wal-index.  But the wal-index must not be unmapped
** prior to the WalIterator object being destroyed.
*/
static int walIteratorInit(Wal *pWal, WalIterator **pp){
  WalIterator *p;                 /* Return value */
  int nSegment;                   /* Number of segments to merge */
  u32 iLast;                      /* Last frame in log */
  int nByte;                      /* Number of bytes to allocate */
  int i;                          /* Iterator variable */
  ht_slot *aTmp;                  /* Temp space used by merge-sort */
  ht_slot *aSpace;                /* Space at the end of the allocation */

  /* This routine only runs while holding SQLITE_SHM_CHECKPOINT.  No other
  ** thread is able to write to shared memory while this routine is
  ** running (or, indeed, while the WalIterator object exists).  Hence,
  ** we can cast off the volatile qualification from shared memory
  */
  assert( pWal->ckptLock );
  iLast = pWal->hdr.mxFrame;

  /* Allocate space for the WalIterator object */
  nSegment = walFramePage(iLast) + 1;
  nByte = sizeof(WalIterator) 
        + nSegment*(sizeof(struct WalSegment))
        + (nSegment+1)*(HASHTABLE_NPAGE * sizeof(ht_slot));
  p = (WalIterator *)sqlite3_malloc(nByte);
  if( !p ){
    return SQLITE_NOMEM;
  }
  memset(p, 0, nByte);


  /* Allocate space for the WalIterator object */
  p->nSegment = nSegment;
  aSpace = (ht_slot *)&p->aSegment[nSegment];
  aTmp = &aSpace[HASHTABLE_NPAGE*nSegment];





  for(i=0; i<nSegment; i++){
    volatile ht_slot *aHash;
    int j;
    u32 iZero;
    int nEntry;
    volatile u32 *aPgno;
    int rc;

    rc = walHashGet(pWal, i, &aHash, &aPgno, &iZero);
    if( rc!=SQLITE_OK ){
      walIteratorFree(p);
      return rc;
    }
    aPgno++;
    nEntry = ((i+1)==nSegment)?iLast-iZero:(u32 *)aHash-(u32 *)aPgno;
    iZero++;


    for(j=0; j<nEntry; j++){
      aSpace[j] = j;
    }
    walMergesort((u32 *)aPgno, aTmp, aSpace, &nEntry);
    p->aSegment[i].iZero = iZero;
    p->aSegment[i].nEntry = nEntry;
    p->aSegment[i].aIndex = aSpace;
    p->aSegment[i].aPgno = (u32 *)aPgno;


    aSpace += HASHTABLE_NPAGE;
  }
  assert( aSpace==aTmp );


  /* Return the fully initialized WalIterator object */
  *pp = p;
  return SQLITE_OK ;
}

/*
** Copy as much content as we can from the WAL back into the database file
** in response to an sqlite3_wal_checkpoint() request or the equivalent.
**
** The amount of information copies from WAL to database might be limited
................................................................................
  volatile WalCkptInfo *pInfo;    /* The checkpoint status information */

  if( pWal->hdr.mxFrame==0 ) return SQLITE_OK;

  /* Allocate the iterator */
  rc = walIteratorInit(pWal, &pIter);
  if( rc!=SQLITE_OK ){
    goto walcheckpoint_out;
  }
  assert( pIter );

  /*** TODO:  Move this test out to the caller.  Make it an assert() here ***/
  if( pWal->hdr.szPage!=nBuf ){
    rc = SQLITE_CORRUPT_BKPT;
    goto walcheckpoint_out;







>









|
>







 







|



<
|
|


>
|
<


|
<
<








|

|
|
<
<

|


|


|
|
|




>

<
|
|
|
>
>
>
>
>
|





|


|
<
<
<
|
|
|
|
>
|
|
|
|
|
|
|
|
>
>
|
|
|
>
|
<

|







 







|







1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
....
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399

1400
1401
1402
1403
1404
1405

1406
1407
1408


1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420


1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436

1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454



1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474

1475
1476
1477
1478
1479
1480
1481
1482
1483
....
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537

  for(iList=0; iList<nList; iList++){
    nMerge = 1;
    aMerge = &aList[iList];
    for(iSub=0; iList & (1<<iSub); iSub++){
      struct Sublist *p = &aSub[iSub];
      assert( p->aList && p->nList<=(1<<iSub) );
      assert( p->aList==&aList[iList&~((2<<iSub)-1)] );
      walMerge(aContent, p->aList, p->nList, &aMerge, &nMerge, aBuffer);
    }
    aSub[iSub].aList = aMerge;
    aSub[iSub].nList = nMerge;
  }

  for(iSub++; iSub<ArraySize(aSub); iSub++){
    if( nList & (1<<iSub) ){
      struct Sublist *p = &aSub[iSub];
      assert( p->nList<=(1<<iSub) );
      assert( p->aList==&aList[nList&~((2<<iSub)-1)] );
      walMerge(aContent, p->aList, p->nList, &aMerge, &nMerge, aBuffer);
    }
  }
  assert( aMerge==aList );
  *pnList = nMerge;

#ifdef SQLITE_DEBUG
................................................................................
#endif
}

/* 
** Free an iterator allocated by walIteratorInit().
*/
static void walIteratorFree(WalIterator *p){
  sqlite3ScratchFree(p);
}

/*

** Construct a WalInterator object that can be used to loop over all 
** pages in the WAL in ascending order. The caller must hold the checkpoint
**
** On success, make *pp point to the newly allocated WalInterator object
** return SQLITE_OK. Otherwise, return an error code. If this routine
** returns an error, the value of *pp is undefined.

**
** The calling routine should invoke walIteratorFree() to destroy the
** WalIterator object when it has finished with it.


*/
static int walIteratorInit(Wal *pWal, WalIterator **pp){
  WalIterator *p;                 /* Return value */
  int nSegment;                   /* Number of segments to merge */
  u32 iLast;                      /* Last frame in log */
  int nByte;                      /* Number of bytes to allocate */
  int i;                          /* Iterator variable */
  ht_slot *aTmp;                  /* Temp space used by merge-sort */
  int rc = SQLITE_OK;             /* Return Code */

  /* This routine only runs while holding the checkpoint lock. And
  ** it only runs if there is actually content in the log (mxFrame>0).


  */
  assert( pWal->ckptLock && pWal->hdr.mxFrame>0 );
  iLast = pWal->hdr.mxFrame;

  /* Allocate space for the WalIterator object. */
  nSegment = walFramePage(iLast) + 1;
  nByte = sizeof(WalIterator) 
        + (nSegment-1)*(sizeof(struct WalSegment))
        + nSegment*(HASHTABLE_NPAGE * sizeof(ht_slot));
  p = (WalIterator *)sqlite3ScratchMalloc(nByte);
  if( !p ){
    return SQLITE_NOMEM;
  }
  memset(p, 0, nByte);
  p->nSegment = nSegment;


  /* Allocate temporary space used by the merge-sort routine. This block
  ** of memory will be freed before this function returns.
  */
  aTmp = (ht_slot *)sqlite3ScratchMalloc(HASHTABLE_NPAGE * sizeof(ht_slot));
  if( !aTmp ){
    rc = SQLITE_NOMEM;
  }

  for(i=0; rc==SQLITE_OK && i<nSegment; i++){
    volatile ht_slot *aHash;
    int j;
    u32 iZero;
    int nEntry;
    volatile u32 *aPgno;
    ht_slot *aIndex;

    rc = walHashGet(pWal, i, &aHash, &aPgno, &iZero);
    if( rc==SQLITE_OK ){



      aPgno++;
      nEntry = ((i+1)==nSegment)?iLast-iZero:(u32 *)aHash-(u32 *)aPgno;
      iZero++;
  
      aIndex = &((ht_slot *)&p->aSegment[p->nSegment])[i*HASHTABLE_NPAGE];
      for(j=0; j<nEntry; j++){
        aIndex[j] = j;
      }
      walMergesort((u32 *)aPgno, aTmp, aIndex, &nEntry);
      p->aSegment[i].iZero = iZero;
      p->aSegment[i].nEntry = nEntry;
      p->aSegment[i].aIndex = aIndex;
      p->aSegment[i].aPgno = (u32 *)aPgno;
    }
  }
  sqlite3ScratchFree(aTmp);

  if( rc!=SQLITE_OK ){
    walIteratorFree(p);
  }

  *pp = p;
  return rc;
}

/*
** Copy as much content as we can from the WAL back into the database file
** in response to an sqlite3_wal_checkpoint() request or the equivalent.
**
** The amount of information copies from WAL to database might be limited
................................................................................
  volatile WalCkptInfo *pInfo;    /* The checkpoint status information */

  if( pWal->hdr.mxFrame==0 ) return SQLITE_OK;

  /* Allocate the iterator */
  rc = walIteratorInit(pWal, &pIter);
  if( rc!=SQLITE_OK ){
    return rc;
  }
  assert( pIter );

  /*** TODO:  Move this test out to the caller.  Make it an assert() here ***/
  if( pWal->hdr.szPage!=nBuf ){
    rc = SQLITE_CORRUPT_BKPT;
    goto walcheckpoint_out;