Cloud Backed SQLite

Check-in [bf008eee6a]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix another problem with VACUUM and truncating the database.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: bf008eee6a6ebcd25d7172939a7dbe8478d71c016b42f0f7ea69031331734208
User & Date: dan 2022-09-29 16:35:20.693
Context
2022-09-29
18:24
Add the SQLITE_BCV_LOG_HTTPRETRY logging flag. For logging retries only. Also the equivalent 'r' flag to the command line tools and daemon. check-in: 92051dd15c user: dan tags: trunk
16:35
Fix another problem with VACUUM and truncating the database. check-in: bf008eee6a user: dan tags: trunk
2022-09-23
18:53
Improve logging of http requests and retries. check-in: 8336418e39 user: dan tags: trunk
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/blockcachevfs.c.
1812
1813
1814
1815
1816
1817
1818





























1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831

1832
1833
1834


1835

1836
1837
1838
1839
1840
1841
1842
      assert( bcvfsFileType(pFile)==BCVFS_DATABASE_FILE );
      rc = bcvfsReadWriteDatabaseWithRetry(1, pFile, (void*)pBuf, iAmt, iOfst);
      break;
    }
  }
  return rc;
}





























static int bcvfsTruncate(sqlite3_file *pFd, sqlite3_int64 size){
  int rc = SQLITE_OK;
  BcvfsFile *pFile = (BcvfsFile*)pFd;
  switch( bcvfsFileType(pFile) ){
    case BCVFS_NO_FILE:
      /* no-op */
      break;
    case BCVFS_WAL_FILE:
    case BCVFS_PASSTHRU_FILE:
      rc = pFile->pFile->pMethods->xTruncate(pFile->pFile, size);
      break;

    default: {

      int szBlk = pFile->pMan->szBlk;
      int nNew = (size + szBlk - 1) / szBlk;
      if( nNew<pFile->pManDb->nBlkLocal ){


        pFile->pManDb->nBlkLocal = nNew;

      }
      assert( pFile->pManDb && pFile->pManDb->nBlkLocalAlloc );
      assert( bcvfsFileType(pFile)==BCVFS_DATABASE_FILE );
      break;
    }
  }
  return rc;







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>













>



>
>
|
>







1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
      assert( bcvfsFileType(pFile)==BCVFS_DATABASE_FILE );
      rc = bcvfsReadWriteDatabaseWithRetry(1, pFile, (void*)pBuf, iAmt, iOfst);
      break;
    }
  }
  return rc;
}

/*
** Discard - undirty - all dirty blocks from database pDb between block iFirst
** and the end of the db, inclusive. i.e. if iFirst==0, undirty all blocks
** in the database.
*/
static void bcvfsUndirtyBlocks(
  sqlite3_bcvfs *pFs, 
  int nName, 
  ManifestDb *pDb,
  int iFirst
){
  int iBlk;
  CHECK_VFS_MUTEX;

  for(iBlk=iFirst; iBlk<pDb->nBlkLocal; iBlk++){
    int iOff = iBlk*nName;
    u8 *aBlk = &pDb->aBlkLocal[iOff];
    if( iBlk>=pDb->nBlkOrig || memcmp(aBlk, &pDb->aBlkOrig[iOff], nName) ){
      CacheEntry *pEntry = bcvfsHashFind(&pFs->c, aBlk, nName);
      assert( pEntry && pEntry->bDirty );
      pEntry->bDirty = 0;
      bcvfsLruAddIf(&pFs->c, pEntry);
      bcvfsWriteBlock(pFs, 0, 0, pEntry);
    }
  }
}


static int bcvfsTruncate(sqlite3_file *pFd, sqlite3_int64 size){
  int rc = SQLITE_OK;
  BcvfsFile *pFile = (BcvfsFile*)pFd;
  switch( bcvfsFileType(pFile) ){
    case BCVFS_NO_FILE:
      /* no-op */
      break;
    case BCVFS_WAL_FILE:
    case BCVFS_PASSTHRU_FILE:
      rc = pFile->pFile->pMethods->xTruncate(pFile->pFile, size);
      break;

    default: {
      sqlite3_bcvfs *pFs = pFile->pFs;
      int szBlk = pFile->pMan->szBlk;
      int nNew = (size + szBlk - 1) / szBlk;
      if( nNew<pFile->pManDb->nBlkLocal ){
        ENTER_VFS_MUTEX; {
          bcvfsUndirtyBlocks(pFs, NAMEBYTES(pFile->pMan), pFile->pManDb, nNew);
          pFile->pManDb->nBlkLocal = nNew;
        } LEAVE_VFS_MUTEX;
      }
      assert( pFile->pManDb && pFile->pManDb->nBlkLocalAlloc );
      assert( bcvfsFileType(pFile)==BCVFS_DATABASE_FILE );
      break;
    }
  }
  return rc;
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
    *pzErr = zErr;
  }else{
    sqlite3_free(zErr);
  }
  return rc;
}

static void bcvfsUndirtyBlocks(sqlite3_bcvfs *pFs, int nName, ManifestDb *pDb){
  int iBlk;
  CHECK_VFS_MUTEX;

  for(iBlk=0; iBlk<pDb->nBlkLocal; iBlk++){
    int iOff = iBlk*nName;
    u8 *aBlk = &pDb->aBlkLocal[iOff];
    if( iBlk>=pDb->nBlkOrig || memcmp(aBlk, &pDb->aBlkOrig[iOff], nName) ){
      CacheEntry *pEntry = bcvfsHashFind(&pFs->c, aBlk, nName);
      assert( pEntry && pEntry->bDirty );
      pEntry->bDirty = 0;
      bcvfsLruAddIf(&pFs->c, pEntry);
      bcvfsWriteBlock(pFs, 0, 0, pEntry);
    }
  }
}

/*
** This function does the bulk of the work for sqlite3_bcvfs_delete(). Before
** it is called it has already been verified that:
**
**   * pFs is not a proxy VFS, and
**   * pzErr is non-NULL.







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







5006
5007
5008
5009
5010
5011
5012
















5013
5014
5015
5016
5017
5018
5019
    *pzErr = zErr;
  }else{
    sqlite3_free(zErr);
  }
  return rc;
}


















/*
** This function does the bulk of the work for sqlite3_bcvfs_delete(). Before
** it is called it has already been verified that:
**
**   * pFs is not a proxy VFS, and
**   * pzErr is non-NULL.
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
          bcvCloseLocal(pFd);
          pFd = 0;

          /* Delete the wal file, if one is present */
          pFs->c.pVfs->xDelete(pFs->c.pVfs, zWal, 0);

          /* Undirty any dirty blocks */
          bcvfsUndirtyBlocks(pFs, nName, pDb);

          if( rc==SQLITE_OK ){
            if( pDb->nBlkLocalAlloc ){
              sqlite3_free(pDb->aBlkLocal);
              pDb->nBlkLocalAlloc = 0;
            }
            pDb->aBlkLocal = 0;







|







5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
          bcvCloseLocal(pFd);
          pFd = 0;

          /* Delete the wal file, if one is present */
          pFs->c.pVfs->xDelete(pFs->c.pVfs, zWal, 0);

          /* Undirty any dirty blocks */
          bcvfsUndirtyBlocks(pFs, nName, pDb, 0);

          if( rc==SQLITE_OK ){
            if( pDb->nBlkLocalAlloc ){
              sqlite3_free(pDb->aBlkLocal);
              pDb->nBlkLocalAlloc = 0;
            }
            pDb->aBlkLocal = 0;
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627

      /* If no error has occurred so far, discard all dirty blocks associated
      ** with the container and truncate all wal files to zero bytes in 
      ** size. */
      for(ii=0; rc==SQLITE_OK && ii<pMan->nDb; ii++){
        sqlite3_file *pFd = aWrap[ii].pWalFd;
        ManifestDb *pDb = &pMan->aDb[ii];
        bcvfsUndirtyBlocks(pFs, NAMEBYTES(pMan), pDb);
        if( pFd ){
          rc = pFd->pMethods->xTruncate(pFd, 0);
        }
      }

      if( aWrap ){
        for(ii=0; ii<pMan->nDb; ii++){







|







5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644

      /* If no error has occurred so far, discard all dirty blocks associated
      ** with the container and truncate all wal files to zero bytes in 
      ** size. */
      for(ii=0; rc==SQLITE_OK && ii<pMan->nDb; ii++){
        sqlite3_file *pFd = aWrap[ii].pWalFd;
        ManifestDb *pDb = &pMan->aDb[ii];
        bcvfsUndirtyBlocks(pFs, NAMEBYTES(pMan), pDb, 0);
        if( pFd ){
          rc = pFd->pMethods->xTruncate(pFd, 0);
        }
      }

      if( aWrap ){
        for(ii=0; ii<pMan->nDb; ii++){
Added test/bcvfs_vacuum2.test.












































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
#
# 19-July-2020
#
#-------------------------------------------------------------------------
#

source [file join [file dirname [info script]] test_common.tcl]
set testprefix bcvfs_vacuum1


#-------------------------------------------------------------------------
#
bcv_mkdir testdir
bcv_create_container cont1

bcv_upload_new_db cont1 t1.db {
  CREATE TABLE x1(x, y);
  WITH s(i) AS (
    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
  )
  INSERT INTO x1 SELECT randomblob(400), randomblob(400) FROM s;
  CREATE TABLE x2(x, y);
  INSERT INTO x2 VALUES(1, 2);
}

set cmd [bcvfs_create testdir bcvfs]
$cmd log all log_callback
proc log_callback {args} { # puts $args }

bcvfs_attach $cmd cont1
$cmd copy cont1 t1.db new.db

sqlite3 db /cont1/new.db -vfs bcvfs -uri 1
sqlite3_bcvfs_register_vtab db

do_execsql_test 1.0 { DELETE FROM x1 WHERE rowid>10 }
do_test 1.1 { 
  lindex [execsql { PRAGMA wal_checkpoint }] 0
} 0
do_execsql_test 1.2 { VACUUM }
db close

do_test 1.3 {
  $cmd upload cont1
} {}

do_test 1.4 {
  $cmd detach cont1
} {}

$cmd destroy

finish_test