SQLite

Check-in [227bb8a1]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem with UNLOCKED transactions that free pages allocated within the same transaction.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | begin-concurrent
Files: files | file ages | folders
SHA1: 227bb8a1815c4dc6084970f06b0a6bfccdff3fd2
User & Date: dan 2015-08-21 17:57:16
Context
2015-08-21
18:55
When committing an UNLOCKED transaction, try to move pages allocated at the end of the file to free slots within the file (like an incremental-vacuum operation does). (check-in: 06967916 user: dan tags: begin-concurrent)
17:57
Fix a problem with UNLOCKED transactions that free pages allocated within the same transaction. (check-in: 227bb8a1 user: dan tags: begin-concurrent)
16:22
Merge trunk changes with this branch. (check-in: deaf3b18 user: dan tags: begin-concurrent)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/btree.c.
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
  assert( sqlite3PagerIsUnlocked(pPager) );
  assert( pBt->pMap );
  rc = sqlite3PagerUpgradeSnapshot(pPager, pPage1->pDbPage);
  assert( p1==pPage1->aData );

  if( rc==SQLITE_OK ){
    Pgno nHPage = get4byte(&p1[28]);
    Pgno nFinal = nHPage;

    if( sqlite3PagerIswriteable(pPage1->pDbPage) ){
      Pgno iHTrunk = get4byte(&p1[32]);
      u32 nHFree = get4byte(&p1[36]);

      /* Attach the head database free list to the end of the current
      ** transactions free-list (if any).  */







|







3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
  assert( sqlite3PagerIsUnlocked(pPager) );
  assert( pBt->pMap );
  rc = sqlite3PagerUpgradeSnapshot(pPager, pPage1->pDbPage);
  assert( p1==pPage1->aData );

  if( rc==SQLITE_OK ){
    Pgno nHPage = get4byte(&p1[28]);
    Pgno nFinal = nHPage;         /* Size of db after transaction merge */

    if( sqlite3PagerIswriteable(pPage1->pDbPage) ){
      Pgno iHTrunk = get4byte(&p1[32]);
      u32 nHFree = get4byte(&p1[36]);

      /* Attach the head database free list to the end of the current
      ** transactions free-list (if any).  */
3888
3889
3890
3891
3892
3893
3894



3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910








3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
            put4byte((u8*)pTrunk->pData, iHTrunk);
          }
          sqlite3PagerUnref(pTrunk);
        };
      }

      if( nHPage<nOrig ){



        rc = SQLITE_CORRUPT_BKPT;
      }else{
        /* The current transaction allocated pages pMap->iFirst through
        ** nPage (inclusive) at the end of the database file. Meanwhile,
        ** other transactions have allocated (iFirst..nHPage). So move
        ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1).
        */
        Pgno iLast = MIN(nPage, nHPage);    /* Last page to move */
        Pgno iPg;

        nFinal = MAX(nPage, nHPage);   /* Final size of database */
        for(iPg=pMap->iFirst; iPg<=iLast && rc==SQLITE_OK; iPg++){
          MemPage *pPg = 0;
          Pgno iNew;              /* New page number for pPg */
          PtrmapEntry *pEntry;    /* Pointer map entry for page iPg */









          btreeGetPage(pBt, iPg, &pPg, 0);
          assert( sqlite3PagerIswriteable(pPg->pDbPage) );
          assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 );
          pEntry = &pMap->aPtr[iPg - pMap->iFirst];
          iNew = ++nFinal;

          rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent, iNew, 1);
          releasePageNotNull(pPg);
        }

        put4byte(&p1[28], nFinal);
      }
    }
    sqlite3PagerSetDbsize(pPager, nFinal);
  }

  return rc;







>
>
>










|





>
>
>
>
>
>
>
>
|
|
|
<
|
<
|
|
|
|







3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924

3925

3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
            put4byte((u8*)pTrunk->pData, iHTrunk);
          }
          sqlite3PagerUnref(pTrunk);
        };
      }

      if( nHPage<nOrig ){
        /* An unlocked transaction may not be executed on an auto-vacuum 
        ** database. Therefore the db should not have shrunk since the
        ** transaction was opened.  */
        rc = SQLITE_CORRUPT_BKPT;
      }else{
        /* The current transaction allocated pages pMap->iFirst through
        ** nPage (inclusive) at the end of the database file. Meanwhile,
        ** other transactions have allocated (iFirst..nHPage). So move
        ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1).
        */
        Pgno iLast = MIN(nPage, nHPage);    /* Last page to move */
        Pgno iPg;

        nFinal = MAX(nPage, nHPage);
        for(iPg=pMap->iFirst; iPg<=iLast && rc==SQLITE_OK; iPg++){
          MemPage *pPg = 0;
          Pgno iNew;              /* New page number for pPg */
          PtrmapEntry *pEntry;    /* Pointer map entry for page iPg */

          pEntry = &pMap->aPtr[iPg - pMap->iFirst];
          if( pEntry->eType==PTRMAP_FREEPAGE ){
            MemPage *pFree = 0;
            Pgno dummy;
            rc = allocateBtreePage(pBt, &pFree, &dummy, iPg, BTALLOC_EXACT);
            releasePage(pFree);
            assert( rc!=SQLITE_OK || dummy==iPg );
          }else{
            btreeGetPage(pBt, iPg, &pPg, 0);
            assert( sqlite3PagerIswriteable(pPg->pDbPage) );
            assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 );

            iNew = ++nFinal;

            rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent, iNew, 1);
            releasePageNotNull(pPg);
          }
        }
        put4byte(&p1[28], nFinal);
      }
    }
    sqlite3PagerSetDbsize(pPager, nFinal);
  }

  return rc;
5771
5772
5773
5774
5775
5776
5777
5778


5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809


5810
5811
5812
5813
5814
5815
5816
5817
5818
5819





5820
5821
5822
5823
5824
5825
5826
5827
  u32 n;     /* Number of pages on the freelist */
  u32 k;     /* Number of leaves on the trunk of the freelist */
  MemPage *pTrunk = 0;
  MemPage *pPrevTrunk = 0;
  Pgno mxPage;     /* Total size of the database file */

  assert( sqlite3_mutex_held(pBt->mutex) );
  assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) );


  pPage1 = pBt->pPage1;
  mxPage = btreePagecount(pBt);
  /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36
  ** stores stores the total number of pages on the freelist. */
  n = get4byte(&pPage1->aData[36]);
  testcase( n==mxPage-1 );
  if( n>=mxPage ){
    return SQLITE_CORRUPT_BKPT;
  }

  /* Ensure page 1 is writable. This function will either change the number
  ** of pages in the free-list or the size of the database file. Since both
  ** of these operations involve modifying page 1 header fields, page 1
  ** will definitely be written by this transaction. If this is an UNLOCKED
  ** transaction, ensure the BtreePtrmap structure has been allocated.  */
  assert( eMode!=BTALLOC_EXACT || sqlite3PagerIsUnlocked(pBt->pPager)==0 );
  rc = sqlite3PagerWrite(pPage1->pDbPage);
  if( rc ) return rc;

  if( n>0 ){
    /* There are pages on the freelist.  Reuse one of those pages. */
    Pgno iTrunk;
    u8 searchList = 0; /* If the free-list must be searched for 'nearby' */
    u32 nSearch = 0;   /* Count of the number of search attempts */
    
    /* If eMode==BTALLOC_EXACT and a query of the pointer-map
    ** shows that the page 'nearby' is somewhere on the free-list, then
    ** the entire-list will be searched for that page.
    */
#ifndef SQLITE_OMIT_AUTOVACUUM
    if( eMode==BTALLOC_EXACT ){


      if( nearby<=mxPage ){
        u8 eType;
        assert( nearby>0 );
        assert( pBt->autoVacuum );
        rc = ptrmapGet(pBt, nearby, &eType, 0);
        if( rc ) return rc;
        if( eType==PTRMAP_FREEPAGE ){
          searchList = 1;
        }
      }





    }else if( eMode==BTALLOC_LE ){
      searchList = 1;
    }
#endif

    /* Decrement the free-list count by 1. Set iTrunk to the index of the
    ** first free-list trunk page. iPrevTrunk is initially 1.
    */







|
>
>






|








<













<

>
>
|
|
|
|
|
|
|
|
|
|
>
>
>
>
>
|







5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804

5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817

5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
  u32 n;     /* Number of pages on the freelist */
  u32 k;     /* Number of leaves on the trunk of the freelist */
  MemPage *pTrunk = 0;
  MemPage *pPrevTrunk = 0;
  Pgno mxPage;     /* Total size of the database file */

  assert( sqlite3_mutex_held(pBt->mutex) );
  assert( eMode==BTALLOC_ANY || (nearby>0 && ISAUTOVACUUM) 
      || (eMode==BTALLOC_EXACT && sqlite3PagerIsUnlocked(pBt->pPager))
  );
  pPage1 = pBt->pPage1;
  mxPage = btreePagecount(pBt);
  /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36
  ** stores stores the total number of pages on the freelist. */
  n = get4byte(&pPage1->aData[36]);
  testcase( n==mxPage-1 );
  if( sqlite3PagerIsUnlocked(pBt->pPager)==0 && n>=mxPage ){
    return SQLITE_CORRUPT_BKPT;
  }

  /* Ensure page 1 is writable. This function will either change the number
  ** of pages in the free-list or the size of the database file. Since both
  ** of these operations involve modifying page 1 header fields, page 1
  ** will definitely be written by this transaction. If this is an UNLOCKED
  ** transaction, ensure the BtreePtrmap structure has been allocated.  */

  rc = sqlite3PagerWrite(pPage1->pDbPage);
  if( rc ) return rc;

  if( n>0 ){
    /* There are pages on the freelist.  Reuse one of those pages. */
    Pgno iTrunk;
    u8 searchList = 0; /* If the free-list must be searched for 'nearby' */
    u32 nSearch = 0;   /* Count of the number of search attempts */
    
    /* If eMode==BTALLOC_EXACT and a query of the pointer-map
    ** shows that the page 'nearby' is somewhere on the free-list, then
    ** the entire-list will be searched for that page.
    */

    if( eMode==BTALLOC_EXACT ){
      assert( ISAUTOVACUUM==!sqlite3PagerIsUnlocked(pBt->pPager) );
      if( ISAUTOVACUUM ){
        if( nearby<=mxPage ){
          u8 eType;
          assert( nearby>0 );
          assert( pBt->autoVacuum );
          rc = ptrmapGet(pBt, nearby, &eType, 0);
          if( rc ) return rc;
          if( eType==PTRMAP_FREEPAGE ){
            searchList = 1;
          }
        }
      }else{
        searchList = 1;
      }
    }
#ifndef SQLITE_OMIT_AUTOVACUUM
    else if( eMode==BTALLOC_LE ){
      searchList = 1;
    }
#endif

    /* Decrement the free-list count by 1. Set iTrunk to the index of the
    ** first free-list trunk page. iPrevTrunk is initially 1.
    */
Changes to test/unlocked2.test.
182
183
184
185
186
187
188


































189
190
191
    PRAGMA freelist_count;
} {0}
do_execsql_test 3.5.2 {
  ROLLBACK;
  PRAGMA freelist_count;
} {2}
do_execsql_test 3.5.3 { PRAGMA integrity_check } {ok}



































finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
    PRAGMA freelist_count;
} {0}
do_execsql_test 3.5.2 {
  ROLLBACK;
  PRAGMA freelist_count;
} {2}
do_execsql_test 3.5.3 { PRAGMA integrity_check } {ok}

#-------------------------------------------------------------------------
# Test that nothing goes wrong if an UNLOCKED transaction allocates a
# page at the end of the file, frees it within the same transaction, and
# then has to move the same page to avoid a conflict on COMMIT.
#
do_multiclient_test tn {
  do_test 4.$tn.1 {
    sql1 {
      PRAGMA journal_mode = wal;
      CREATE TABLE t1(x);
      CREATE TABLE t2(x);
    }
  } {wal}

  do_test 4.$tn.2 {
    sql1 {
      BEGIN UNLOCKED;
        INSERT INTO t1 VALUES(randomblob(1500));
        INSERT INTO t1 VALUES(randomblob(1500));
        DELETE FROM t1 WHERE rowid = 1;
    }

    sql2 {
      INSERT INTO t2 VALUES(randomblob(1500));
      INSERT INTO t2 VALUES(randomblob(1500));
      INSERT INTO t2 VALUES(randomblob(1500));
      INSERT INTO t2 VALUES(randomblob(1500));
      DELETE FROM t2 WHERE rowid IN (1, 2);
    }

    sql1 COMMIT
  } {}
}

finish_test