/ Check-in [2fb3fdcd]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix for ticket #9 (again). The rollback journal files should now also be byte-order independent. (CVS 705)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 2fb3fdcdf06c1206bf14da640c2f9e599455f0eb
User & Date: drh 2002-08-12 12:29:57
Context
2002-08-13
00:01
Updates prior to release 2.6.3. (CVS 706) check-in: 34c4149e user: drh tags: trunk
2002-08-12
12:29
Fix for ticket #9 (again). The rollback journal files should now also be byte-order independent. (CVS 705) check-in: 2fb3fdcd user: drh tags: trunk
2002-08-11
20:10
Fix for ticket #9: Add the ability to read little-endian database files from a big-endian machine and vice versa. (CVS 704) check-in: ce4b943b user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/pager.c.

    14     14   ** The pager is used to access a database disk file.  It implements
    15     15   ** atomic commit and rollback through the use of a journal file that
    16     16   ** is separate from the database file.  The pager also implements file
    17     17   ** locking to prevent two processes from writing the same database
    18     18   ** file simultaneously, or one process from reading the database while
    19     19   ** another is writing.
    20     20   **
    21         -** @(#) $Id: pager.c,v 1.49 2002/07/07 16:52:47 drh Exp $
           21  +** @(#) $Id: pager.c,v 1.50 2002/08/12 12:29:57 drh Exp $
    22     22   */
    23     23   #include "sqliteInt.h"
    24     24   #include "pager.h"
    25     25   #include "os.h"
    26     26   #include <assert.h>
    27     27   #include <string.h>
    28     28   
................................................................................
   120    120     u8 state;                   /* SQLITE_UNLOCK, _READLOCK or _WRITELOCK */
   121    121     u8 errMask;                 /* One of several kinds of errors */
   122    122     u8 tempFile;                /* zFilename is a temporary file */
   123    123     u8 readOnly;                /* True for a read-only database */
   124    124     u8 needSync;                /* True if an fsync() is needed on the journal */
   125    125     u8 dirtyFile;               /* True if database file has changed in any way */
   126    126     u8 alwaysRollback;          /* Disable dont_rollback() for all pages */
          127  +  u8 journalFormat;           /* Version number of the journal file */
   127    128     u8 *aInJournal;             /* One bit for each page in the database file */
   128    129     u8 *aInCkpt;                /* One bit for each page in the database */
   129    130     PgHdr *pFirst, *pLast;      /* List of free pages */
   130    131     PgHdr *pAll;                /* List of all pages */
   131    132     PgHdr *aHash[N_PG_HASH];    /* Hash table to map page number of PgHdr */
   132    133   };
   133    134   
................................................................................
   149    150     Pgno pgno;                     /* The page number */
   150    151     char aData[SQLITE_PAGE_SIZE];  /* Original data for page pgno */
   151    152   };
   152    153   
   153    154   /*
   154    155   ** Journal files begin with the following magic string.  The data
   155    156   ** was obtained from /dev/random.  It is used only as a sanity check.
          157  +**
          158  +** There are two journal formats.  The older journal format writes
          159  +** 32-bit integers in the byte-order of the host machine.  The new
          160  +** format writes integers as big-endian.  All new journals use the
          161  +** new format, but we have to be able to read an older journal in order
          162  +** to roll it back.
   156    163   */
   157         -static const unsigned char aJournalMagic[] = {
          164  +static const unsigned char aOldJournalMagic[] = {
   158    165     0xd9, 0xd5, 0x05, 0xf9, 0x20, 0xa1, 0x63, 0xd4,
   159    166   };
          167  +static const unsigned char aJournalMagic[] = {
          168  +  0xd9, 0xd5, 0x05, 0xf9, 0x20, 0xa1, 0x63, 0xd5,
          169  +};
          170  +#define SQLITE_NEW_JOURNAL_FORMAT 1
          171  +#define SQLITE_OLD_JOURNAL_FORMAT 0
          172  +
          173  +/*
          174  +** The following integer, if set, causes journals to be written in the
          175  +** old format.  This is used for testing purposes only - to make sure
          176  +** the code is able to rollback an old journal.
          177  +*/
          178  +#ifdef SQLITE_TEST
          179  +int pager_old_format = 0;
          180  +#endif
   160    181   
   161    182   /*
   162    183   ** Hash a page number
   163    184   */
   164    185   #define pager_hash(PN)  ((PN)%N_PG_HASH)
   165    186   
   166    187   /*
................................................................................
   177    198       );
   178    199       cnt++;   /* Something to set a breakpoint on */
   179    200     }
   180    201   # define REFINFO(X)  pager_refinfo(X)
   181    202   #else
   182    203   # define REFINFO(X)
   183    204   #endif
          205  +
          206  +/*
          207  +** Read a 32-bit integer from the given file descriptor
          208  +*/
          209  +static int read32bits(Pager *pPager, OsFile *fd, u32 *pRes){
          210  +  u32 res;
          211  +  int rc;
          212  +  rc = sqliteOsRead(fd, &res, sizeof(res));
          213  +  if( rc==SQLITE_OK && pPager->journalFormat==SQLITE_NEW_JOURNAL_FORMAT ){
          214  +    unsigned char ac[4];
          215  +    memcpy(ac, &res, 4);
          216  +    res = (ac[0]<<24) | (ac[1]<<16) | (ac[2]<<8) | ac[3];
          217  +  }
          218  +  *pRes = res;
          219  +  return rc;
          220  +}
          221  +
          222  +/*
          223  +** Write a 32-bit integer into the given file descriptor.  Writing
          224  +** is always done using the new journal format.
          225  +*/
          226  +static int write32bits(OsFile *fd, u32 val){
          227  +  unsigned char ac[4];
          228  +#ifdef SQLITE_TEST
          229  +  if( pager_old_format ){
          230  +    return sqliteOsWrite(fd, &val, 4);
          231  +  }
          232  +#endif
          233  +  ac[0] = (val>>24) & 0xff;
          234  +  ac[1] = (val>>16) & 0xff;
          235  +  ac[2] = (val>>8) & 0xff;
          236  +  ac[3] = val & 0xff;
          237  +  return sqliteOsWrite(fd, ac, 4);
          238  +}
          239  +
   184    240   
   185    241   /*
   186    242   ** Convert the bits in the pPager->errMask into an approprate
   187    243   ** return code.
   188    244   */
   189    245   static int pager_errcode(Pager *pPager){
   190    246     int rc = SQLITE_OK;
................................................................................
   277    333   ** jfd.  Playback this one page.
   278    334   */
   279    335   static int pager_playback_one_page(Pager *pPager, OsFile *jfd){
   280    336     int rc;
   281    337     PgHdr *pPg;              /* An existing page in the cache */
   282    338     PageRecord pgRec;
   283    339   
   284         -  rc = sqliteOsRead(jfd, &pgRec, sizeof(pgRec));
          340  +  rc = read32bits(pPager, jfd, &pgRec.pgno);
          341  +  if( rc!=SQLITE_OK ) return rc;
          342  +  rc = sqliteOsRead(jfd, &pgRec.aData, sizeof(pgRec.aData));
   285    343     if( rc!=SQLITE_OK ) return rc;
   286    344   
   287    345     /* Sanity checking on the page */
   288    346     if( pgRec.pgno>pPager->dbSize || pgRec.pgno==0 ) return SQLITE_CORRUPT;
   289    347   
   290    348     /* Playback the page.  Update the in-memory copy of the page
   291    349     ** at the same time, if there is one.
................................................................................
   343    401       goto end_playback;
   344    402     }
   345    403   
   346    404     /* Read the beginning of the journal and truncate the
   347    405     ** database file back to its original size.
   348    406     */
   349    407     rc = sqliteOsRead(&pPager->jfd, aMagic, sizeof(aMagic));
   350         -  if( rc!=SQLITE_OK || memcmp(aMagic,aJournalMagic,sizeof(aMagic))!=0 ){
          408  +  if( rc!=SQLITE_OK ){
          409  +    rc = SQLITE_PROTOCOL;
          410  +    goto end_playback;
          411  +  }
          412  +  if( memcmp(aMagic, aOldJournalMagic, sizeof(aMagic))==0 ){
          413  +    pPager->journalFormat = SQLITE_OLD_JOURNAL_FORMAT;
          414  +  }else if( memcmp(aMagic, aJournalMagic, sizeof(aMagic))==0 ){
          415  +    pPager->journalFormat = SQLITE_NEW_JOURNAL_FORMAT;
          416  +  }else{
   351    417       rc = SQLITE_PROTOCOL;
   352    418       goto end_playback;
   353    419     }
   354         -  rc = sqliteOsRead(&pPager->jfd, &mxPg, sizeof(mxPg));
          420  +  rc = read32bits(pPager, &pPager->jfd, &mxPg);
   355    421     if( rc!=SQLITE_OK ){
   356    422       goto end_playback;
   357    423     }
   358    424     rc = sqliteOsTruncate(&pPager->fd, mxPg*SQLITE_PAGE_SIZE);
   359    425     if( rc!=SQLITE_OK ){
   360    426       goto end_playback;
   361    427     }
................................................................................
   412    478       goto end_ckpt_playback;
   413    479     }
   414    480     nRec /= sizeof(PageRecord);
   415    481     
   416    482     /* Copy original pages out of the checkpoint journal and back into the
   417    483     ** database file.
   418    484     */
          485  +  pPager->journalFormat = SQLITE_NEW_JOURNAL_FORMAT;
   419    486     for(i=nRec-1; i>=0; i--){
   420    487       rc = pager_playback_one_page(pPager, &pPager->cpfd);
   421    488       if( rc!=SQLITE_OK ) goto end_ckpt_playback;
   422    489     }
   423    490   
   424    491     /* Figure out how many pages need to be copied out of the transaction
   425    492     ** journal.
................................................................................
   843    910         ** dirty free pages into the database file, thus making them
   844    911         ** clean pages and available for recycling.
   845    912         **
   846    913         ** We have to sync the journal before writing a page to the main
   847    914         ** database.  But syncing is a very slow operation.  So after a
   848    915         ** sync, it is best to write everything we can back to the main
   849    916         ** database to minimize the risk of having to sync again in the
   850         -      ** near future.  That is way we write all dirty pages after a
          917  +      ** near future.  That is why we write all dirty pages after a
   851    918         ** sync.
   852    919         */
   853    920         if( pPg==0 ){
   854    921           int rc = syncAllPages(pPager);
   855    922           if( rc!=0 ){
   856    923             sqlitepager_rollback(pPager);
   857    924             *ppPage = 0;
................................................................................
  1069   1136       pPager->journalOpen = 1;
  1070   1137       pPager->needSync = 0;
  1071   1138       pPager->dirtyFile = 0;
  1072   1139       pPager->alwaysRollback = 0;
  1073   1140       pPager->state = SQLITE_WRITELOCK;
  1074   1141       sqlitepager_pagecount(pPager);
  1075   1142       pPager->origDbSize = pPager->dbSize;
         1143  +#ifdef SQLITE_TEST
         1144  +    if( pager_old_format ){
         1145  +      rc = sqliteOsWrite(&pPager->jfd, aOldJournalMagic,
         1146  +                         sizeof(aOldJournalMagic));
         1147  +    }else{
         1148  +      rc = sqliteOsWrite(&pPager->jfd, aJournalMagic, sizeof(aJournalMagic));
         1149  +    }
         1150  +#else
  1076   1151       rc = sqliteOsWrite(&pPager->jfd, aJournalMagic, sizeof(aJournalMagic));
         1152  +#endif
  1077   1153       if( rc==SQLITE_OK ){
  1078         -      rc = sqliteOsWrite(&pPager->jfd, &pPager->dbSize, sizeof(Pgno));
         1154  +      rc = write32bits(&pPager->jfd, pPager->dbSize);
  1079   1155       }
  1080   1156       if( rc!=SQLITE_OK ){
  1081   1157         rc = pager_unwritelock(pPager);
  1082   1158         if( rc==SQLITE_OK ) rc = SQLITE_FULL;
  1083   1159       }
  1084   1160     }
  1085   1161     return rc;
................................................................................
  1140   1216     assert( pPager->journalOpen );
  1141   1217   
  1142   1218     /* The transaction journal now exists and we have a write lock on the
  1143   1219     ** main database file.  Write the current page to the transaction 
  1144   1220     ** journal if it is not there already.
  1145   1221     */
  1146   1222     if( !pPg->inJournal && (int)pPg->pgno <= pPager->origDbSize ){
  1147         -    rc = sqliteOsWrite(&pPager->jfd, &pPg->pgno, sizeof(Pgno));
         1223  +    rc = write32bits(&pPager->jfd, pPg->pgno);
  1148   1224       if( rc==SQLITE_OK ){
  1149   1225         rc = sqliteOsWrite(&pPager->jfd, pData, SQLITE_PAGE_SIZE);
  1150   1226       }
  1151   1227       if( rc!=SQLITE_OK ){
  1152   1228         sqlitepager_rollback(pPager);
  1153   1229         pPager->errMask |= PAGER_ERR_FULL;
  1154   1230         return rc;
................................................................................
  1164   1240     }
  1165   1241   
  1166   1242     /* If the checkpoint journal is open and the page is not in it,
  1167   1243     ** then write the current page to the checkpoint journal.
  1168   1244     */
  1169   1245     if( pPager->ckptInUse && !pPg->inCkpt && (int)pPg->pgno<=pPager->ckptSize ){
  1170   1246       assert( pPg->inJournal || (int)pPg->pgno>pPager->origDbSize );
  1171         -    rc = sqliteOsWrite(&pPager->cpfd, &pPg->pgno, sizeof(Pgno));
         1247  +    rc = write32bits(&pPager->cpfd, pPg->pgno);
  1172   1248       if( rc==SQLITE_OK ){
  1173   1249         rc = sqliteOsWrite(&pPager->cpfd, pData, SQLITE_PAGE_SIZE);
  1174   1250       }
  1175   1251       if( rc!=SQLITE_OK ){
  1176   1252         sqlitepager_rollback(pPager);
  1177   1253         pPager->errMask |= PAGER_ERR_FULL;
  1178   1254         return rc;

Changes to src/pager.h.

     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   *************************************************************************
    12     12   ** This header file defines the interface that the sqlite page cache
    13     13   ** subsystem.  The page cache subsystem reads and writes a file a page
    14     14   ** at a time and provides a journal for rollback.
    15     15   **
    16         -** @(#) $Id: pager.h,v 1.16 2002/03/05 12:41:20 drh Exp $
           16  +** @(#) $Id: pager.h,v 1.17 2002/08/12 12:29:57 drh Exp $
    17     17   */
    18     18   
    19     19   /*
    20     20   ** The size of one page
    21     21   **
    22     22   ** You can change this value to another (reasonable) power of two
    23     23   ** such as 512, 2048, 4096, or 8192 and things will still work.  But
................................................................................
    69     69   void sqlitepager_dont_rollback(void*);
    70     70   void sqlitepager_dont_write(Pager*, Pgno);
    71     71   int *sqlitepager_stats(Pager*);
    72     72   
    73     73   #ifdef SQLITE_TEST
    74     74   void sqlitepager_refdump(Pager*);
    75     75   int pager_refinfo_enable;
           76  +int pager_old_format;
    76     77   #endif

Changes to src/test2.c.

     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   *************************************************************************
    12     12   ** Code for testing the pager.c module in SQLite.  This code
    13     13   ** is not included in the SQLite library.  It is used for automated
    14     14   ** testing of the SQLite library.
    15     15   **
    16         -** $Id: test2.c,v 1.8 2002/05/10 05:44:56 drh Exp $
           16  +** $Id: test2.c,v 1.9 2002/08/12 12:29:57 drh Exp $
    17     17   */
    18     18   #include "sqliteInt.h"
    19     19   #include "pager.h"
    20     20   #include "tcl.h"
    21     21   #include <stdlib.h>
    22     22   #include <string.h>
    23     23   
................................................................................
   489    489     Tcl_CreateCommand(interp, "page_lookup", page_lookup, 0, 0);
   490    490     Tcl_CreateCommand(interp, "page_unref", page_unref, 0, 0);
   491    491     Tcl_CreateCommand(interp, "page_read", page_read, 0, 0);
   492    492     Tcl_CreateCommand(interp, "page_write", page_write, 0, 0);
   493    493     Tcl_CreateCommand(interp, "page_number", page_number, 0, 0);
   494    494     Tcl_LinkVar(interp, "sqlite_io_error_pending",
   495    495        (char*)&sqlite_io_error_pending, TCL_LINK_INT);
          496  +#ifdef SQLITE_TEST
          497  +  Tcl_LinkVar(interp, "pager_old_format",
          498  +     (char*)&pager_old_format, TCL_LINK_INT);
          499  +#endif
   496    500     return TCL_OK;
   497    501   }

Changes to test/trans.test.

     7      7   #    May you find forgiveness for yourself and forgive others.
     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #***********************************************************************
    11     11   # This file implements regression tests for SQLite library.  The
    12     12   # focus of this script is database locks.
    13     13   #
    14         -# $Id: trans.test,v 1.13 2002/07/07 16:52:47 drh Exp $
           14  +# $Id: trans.test,v 1.14 2002/08/12 12:29:57 drh Exp $
    15     15   
    16     16   
    17     17   set testdir [file dirname $argv0]
    18     18   source $testdir/tester.tcl
    19     19   
    20     20   
    21     21   # Create several tables to work with.
................................................................................
   853    853   }
   854    854   
   855    855   # Do rollbacks.  Make sure the signature does not change.
   856    856   #
   857    857   for {set i 2} {$i<=$limit} {incr i} {
   858    858     set ::sig [signature]
   859    859     set cnt [lindex $::sig 0]
          860  +  set ::pager_old_format [expr {($i%4)==0}]
   860    861     do_test trans-9.$i.1-$cnt {
   861    862        execsql {
   862    863          BEGIN;
   863    864          DELETE FROM t3 WHERE random()%10!=0;
   864    865          INSERT INTO t3 SELECT randstr(10,10)||x FROM t3;
   865    866          INSERT INTO t3 SELECT randstr(10,10)||x FROM t3;
   866    867          ROLLBACK;

Changes to www/index.tcl.

     1      1   #
     2      2   # Run this TCL script to generate HTML for the index.html file.
     3      3   #
     4         -set rcsid {$Id: index.tcl,v 1.61 2002/08/11 20:10:49 drh Exp $}
            4  +set rcsid {$Id: index.tcl,v 1.62 2002/08/12 12:29:58 drh Exp $}
     5      5   
     6      6   puts {<html>
     7      7   <head><title>SQLite: An SQL Database Engine In A C Library</title></head>
     8      8   <body bgcolor=white>
     9      9   <h1 align=center>SQLite: An SQL Database Engine In A C Library</h1>
    10     10   <p align=center>}
    11     11   puts "This page was last modified on [lrange $rcsid 3 4] UTC<br>"
................................................................................
   209    209   
   210    210   <li><p>A PHP module for SQLite can be found at
   211    211          <a href="http://freshmeat.net/projects/sqlite-php">
   212    212          http://freshmeat.net/projects/sqlite-php</a></li>
   213    213   </ul>}
   214    214   
   215    215   puts {
   216         -<p><hr /></p>
   217         -<p>
   218         -<a href="../index.html"><img src="/goback.jpg" border=0 />
   219         -More Open Source Software</a> from Hwaci.
   220         -</p>
   221         -
   222    216   </body></html>}