/ Check-in [5626ce0b]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Modify sqlite3_release_memory() to use a global LRU list of pages. Untested. (CVS 4301)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 5626ce0b5e249d48b56fdc4561ef663941eb23dc
User & Date: danielk1977 2007-08-27 17:27:49
Context
2007-08-27
21:10
Added the 34to35.html document describing the changes between 3.4.2 and 3.5.0. Minor interface cleanups. (CVS 4302) check-in: 0791f917 user: drh tags: trunk
17:27
Modify sqlite3_release_memory() to use a global LRU list of pages. Untested. (CVS 4301) check-in: 5626ce0b user: danielk1977 tags: trunk
2007-08-25
16:31
Create a fresh pthread_mutexattr_t every time a recursive mutex is allocated. Ticket #2588. (CVS 4300) check-in: 3d746343 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/mutex.c.

     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   *************************************************************************
    12     12   ** This file contains the C functions that implement mutexes for
    13     13   ** use by the SQLite core.
    14     14   **
    15         -** $Id: mutex.c,v 1.13 2007/08/25 16:31:30 drh Exp $
           15  +** $Id: mutex.c,v 1.14 2007/08/27 17:27:49 danielk1977 Exp $
    16     16   */
    17     17   /*
    18     18   ** If SQLITE_MUTEX_APPDEF is defined, then this whole module is
    19     19   ** omitted and equivalent functionality must be provided by the
    20     20   ** application that links against the SQLite library.
    21     21   */
    22     22   #ifndef SQLITE_MUTEX_APPDEF
................................................................................
   249    249   ** <ul>
   250    250   ** <li>  SQLITE_MUTEX_FAST
   251    251   ** <li>  SQLITE_MUTEX_RECURSIVE
   252    252   ** <li>  SQLITE_MUTEX_STATIC_MASTER
   253    253   ** <li>  SQLITE_MUTEX_STATIC_MEM
   254    254   ** <li>  SQLITE_MUTEX_STATIC_MEM2
   255    255   ** <li>  SQLITE_MUTEX_STATIC_PRNG
          256  +** <li>  SQLITE_MUTEX_STATIC_LRU
   256    257   ** </ul>
   257    258   **
   258    259   ** The first two constants cause sqlite3_mutex_alloc() to create
   259    260   ** a new mutex.  The new mutex is recursive when SQLITE_MUTEX_RECURSIVE
   260    261   ** is used but not necessarily so when SQLITE_MUTEX_FAST is used.
   261    262   ** The mutex implementation does not need to make a distinction
   262    263   ** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does
................................................................................
   277    278   ** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc()
   278    279   ** returns a different mutex on every call.  But for the static 
   279    280   ** mutex types, the same mutex is returned on every call that has
   280    281   ** the same type number.
   281    282   */
   282    283   sqlite3_mutex *sqlite3_mutex_alloc(int iType){
   283    284     static sqlite3_mutex staticMutexes[] = {
          285  +    { PTHREAD_MUTEX_INITIALIZER, },
   284    286       { PTHREAD_MUTEX_INITIALIZER, },
   285    287       { PTHREAD_MUTEX_INITIALIZER, },
   286    288       { PTHREAD_MUTEX_INITIALIZER, },
   287    289       { PTHREAD_MUTEX_INITIALIZER, },
   288    290     };
   289    291     sqlite3_mutex *p;
   290    292     switch( iType ){
................................................................................
   470    472         if( p ){
   471    473           p->id = iType;
   472    474           InitializeCriticalSection(&p->mutex);
   473    475         }
   474    476         break;
   475    477       }
   476    478       default: {
   477         -      static sqlite3_mutex staticMutexes[4];
          479  +      static sqlite3_mutex staticMutexes[5];
   478    480         static int isInit = 0;
   479    481         while( !isInit ){
   480    482           static long lock = 0;
   481    483           if( InterlockedIncrement(&lock)==1 ){
   482    484             int i;
   483    485             for(i=0; i<sizeof(staticMutexes)/sizeof(staticMutexes[0]); i++){
   484    486               InitializeCriticalSection(&staticMutexes[i].mutex);

Changes to src/pager.c.

    14     14   ** The pager is used to access a database disk file.  It implements
    15     15   ** atomic commit and rollback through the use of a journal file that
    16     16   ** is separate from the database file.  The pager also implements file
    17     17   ** locking to prevent two processes from writing the same database
    18     18   ** file simultaneously, or one process from reading the database while
    19     19   ** another is writing.
    20     20   **
    21         -** @(#) $Id: pager.c,v 1.376 2007/08/24 16:29:24 drh Exp $
           21  +** @(#) $Id: pager.c,v 1.377 2007/08/27 17:27:49 danielk1977 Exp $
    22     22   */
    23     23   #ifndef SQLITE_OMIT_DISKIO
    24     24   #include "sqliteInt.h"
    25     25   #include <assert.h>
    26     26   #include <string.h>
    27     27   
    28     28   /*
................................................................................
   127    127   
   128    128   /*
   129    129   ** This macro rounds values up so that if the value is an address it
   130    130   ** is guaranteed to be an address that is aligned to an 8-byte boundary.
   131    131   */
   132    132   #define FORCE_ALIGNMENT(X)   (((X)+7)&~7)
   133    133   
          134  +typedef struct PgHdr PgHdr;
          135  +typedef struct PagerLruLink PagerLruLink;
          136  +struct PagerLruLink {
          137  +  PgHdr *pNext;
          138  +  PgHdr *pPrev;
          139  +};
          140  +
   134    141   /*
   135    142   ** Each in-memory image of a page begins with the following header.
   136    143   ** This header is only visible to this pager module.  The client
   137    144   ** code that calls pager sees only the data that follows the header.
   138    145   **
   139    146   ** Client code should call sqlite3PagerWrite() on a page prior to making
   140    147   ** any modifications to that page.  The first time sqlite3PagerWrite()
................................................................................
   217    224   **     This flag means (when true) that the content of the page has
   218    225   **     not yet been loaded from disk.  The in-memory content is just
   219    226   **     garbage.  (Actually, we zero the content, but you should not
   220    227   **     make any assumptions about the content nevertheless.)  If the
   221    228   **     content is needed in the future, it should be read from the
   222    229   **     original database file.
   223    230   */
   224         -typedef struct PgHdr PgHdr;
   225    231   struct PgHdr {
   226    232     Pager *pPager;                 /* The pager to which this page belongs */
   227    233     Pgno pgno;                     /* The page number for this page */
   228    234     PgHdr *pNextHash, *pPrevHash;  /* Hash collision chain for PgHdr.pgno */
   229         -  PgHdr *pNextFree, *pPrevFree;  /* Freelist of pages where nRef==0 */
          235  +  PagerLruLink free;             /* Next and previous free pages */
   230    236     PgHdr *pNextAll;               /* A list of all pages */
   231    237     u8 inJournal;                  /* TRUE if has been written to journal */
   232    238     u8 dirty;                      /* TRUE if we need to write back changes */
   233    239     u8 needSync;                   /* Sync journal before writing this page */
   234    240     u8 alwaysRollback;             /* Disable DontRollback() for this page */
   235    241     u8 needRead;                   /* Read content if PagerWrite() is called */
   236    242     short int nRef;                /* Number of users of this page */
   237    243     PgHdr *pDirty, *pPrevDirty;    /* Dirty pages */
          244  +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
          245  +  PagerLruLink gfree;            /* Global list of nRef==0 pages */
          246  +#endif
   238    247     u32 notUsed;                   /* Buffer space */
   239    248   #ifdef SQLITE_CHECK_PAGES
   240    249     u32 pageHash;
   241    250   #endif
   242    251     /* pPager->pageSize bytes of page data follow this header */
   243    252     /* Pager.nExtra bytes of local data follow the page data */
   244    253   };
................................................................................
   278    287   ** and back again.
   279    288   */
   280    289   #define PGHDR_TO_DATA(P)  ((void*)(&(P)[1]))
   281    290   #define DATA_TO_PGHDR(D)  (&((PgHdr*)(D))[-1])
   282    291   #define PGHDR_TO_EXTRA(G,P) ((void*)&((char*)(&(G)[1]))[(P)->pageSize])
   283    292   #define PGHDR_TO_HIST(P,PGR)  \
   284    293               ((PgHistory*)&((char*)(&(P)[1]))[(PGR)->pageSize+(PGR)->nExtra])
          294  +
          295  +typedef struct PagerLruList PagerLruList;
          296  +struct PagerLruList {
          297  +  PgHdr *pFirst;
          298  +  PgHdr *pLast;
          299  +  PgHdr *pFirstSynced;   /* First page in list with PgHdr.needSync==0 */
          300  +};
   285    301   
   286    302   /*
   287    303   ** A open page cache is an instance of the following structure.
   288    304   **
   289    305   ** Pager.errCode may be set to SQLITE_IOERR, SQLITE_CORRUPT, or
   290    306   ** or SQLITE_FULL. Once one of the first three errors occurs, it persists
   291    307   ** and is returned as the result of every major pager API call.  The
................................................................................
   334    350     u8 *aInStmt;                /* One bit for each page in the database */
   335    351     char *zFilename;            /* Name of the database file */
   336    352     char *zJournal;             /* Name of the journal file */
   337    353     char *zDirectory;           /* Directory hold database and journal files */
   338    354     sqlite3_file *fd, *jfd;     /* File descriptors for database and journal */
   339    355     sqlite3_file *stfd;         /* File descriptor for the statement subjournal*/
   340    356     BusyHandler *pBusyHandler;  /* Pointer to sqlite.busyHandler */
   341         -  PgHdr *pFirst, *pLast;      /* List of free pages */
   342         -  PgHdr *pFirstSynced;        /* First free page with PgHdr.needSync==0 */
          357  +  PagerLruList lru;           /* LRU list of free pages */
   343    358     PgHdr *pAll;                /* List of all pages */
   344    359     PgHdr *pStmt;               /* List of pages in the statement subjournal */
   345    360     PgHdr *pDirty;              /* List of all dirty pages */
   346    361     i64 journalOff;             /* Current byte offset in the journal file */
   347    362     i64 journalHdr;             /* Byte offset to previous journal header */
   348    363     i64 stmtHdrOff;             /* First journal header written this statement */
   349    364     i64 stmtCksum;              /* cksumInit when statement was started */
................................................................................
   390    405   ** The following variable points to the head of a double-linked list
   391    406   ** of all pagers that are eligible for page stealing by the
   392    407   ** sqlite3_release_memory() interface.  Access to this list is
   393    408   ** protected by the SQLITE_MUTEX_STATIC_MEM2 mutex.
   394    409   */
   395    410   #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
   396    411   static Pager *sqlite3PagerList = 0;
          412  +static PagerLruList sqlite3LruPageList = {0, 0, 0};
   397    413   #endif
   398    414   
   399    415   
   400    416   /*
   401    417   ** Journal files begin with the following magic string.  The data
   402    418   ** was obtained from /dev/random.  It is used only as a sanity check.
   403    419   **
................................................................................
   510    526       );
   511    527       cnt++;   /* Something to set a breakpoint on */
   512    528     }
   513    529   # define REFINFO(X)  pager_refinfo(X)
   514    530   #else
   515    531   # define REFINFO(X)
   516    532   #endif
          533  +
          534  +static void listAdd(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){
          535  +  pLink->pNext = 0;
          536  +  pLink->pPrev = pList->pLast;
          537  +
          538  +  if( pList->pLast ){
          539  +    int iOff = (char *)pLink - (char *)pPg;
          540  +    PagerLruLink *pLastLink = (PagerLruLink *)(&((u8 *)pList->pLast)[iOff]);
          541  +    pLastLink->pNext = pPg;
          542  +  }else{
          543  +    assert(!pList->pFirst);
          544  +    pList->pFirst = pPg;
          545  +  }
          546  +
          547  +  pList->pLast = pPg;
          548  +  if( !pList->pFirstSynced && pPg->needSync==0 ){
          549  +    pList->pFirstSynced = pPg;
          550  +  }
          551  +}
          552  +
          553  +static void listRemove(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){
          554  +  int iOff = (char *)pLink - (char *)pPg;
          555  +
          556  +  if( pPg==pList->pFirst ){
          557  +    pList->pFirst = pLink->pNext;
          558  +  }
          559  +  if( pPg==pList->pLast ){
          560  +    pList->pLast = pLink->pPrev;
          561  +  }
          562  +  if( pLink->pPrev ){
          563  +    PagerLruLink *pPrevLink = (PagerLruLink *)(&((u8 *)pLink->pPrev)[iOff]);
          564  +    pPrevLink->pNext = pLink->pNext;
          565  +  }
          566  +  if( pLink->pNext ){
          567  +    PagerLruLink *pNextLink = (PagerLruLink *)(&((u8 *)pLink->pNext)[iOff]);
          568  +    pNextLink->pPrev = pLink->pPrev;
          569  +  }
          570  +  if( pPg==pList->pFirstSynced ){
          571  +    PgHdr *p = pLink->pNext;
          572  +    while( p && p->needSync ){
          573  +      PagerLruLink *pL = (PagerLruLink *)(&((u8 *)p)[iOff]);
          574  +      p = pL->pNext;
          575  +    }
          576  +    pList->pFirstSynced = p;
          577  +  }
          578  +
          579  +  pLink->pNext = pLink->pPrev = 0;
          580  +}
          581  +
          582  +/* 
          583  +** Add page to the free-list 
          584  +*/
          585  +static void lruListAdd(PgHdr *pPg){
          586  +  listAdd(&pPg->pPager->lru, &pPg->free, pPg);
          587  +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
          588  +  if( !pPg->pPager->memDb ){
          589  +    sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
          590  +    listAdd(&sqlite3LruPageList, &pPg->gfree, pPg);
          591  +    sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
          592  +  }
          593  +#endif
          594  +}
          595  +
          596  +/* 
          597  +** Remove page from free-list 
          598  +*/
          599  +static void lruListRemove(PgHdr *pPg){
          600  +  listRemove(&pPg->pPager->lru, &pPg->free, pPg);
          601  +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
          602  +  if( !pPg->pPager->memDb ){
          603  +    sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
          604  +    listRemove(&sqlite3LruPageList, &pPg->gfree, pPg);
          605  +    sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
          606  +  }
          607  +#endif
          608  +}
          609  +
          610  +/* 
          611  +** Set the Pager.pFirstSynced variable 
          612  +*/
          613  +static void lruListSetFirstSynced(Pager *pPager){
          614  +  pPager->lru.pFirstSynced = pPager->lru.pFirst;
          615  +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
          616  +  if( !pPager->memDb ){
          617  +    PgHdr *p;
          618  +    sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
          619  +    for(p=sqlite3LruPageList.pFirst; p && p->needSync; p=p->gfree.pNext);
          620  +    assert(p==pPager->lru.pFirstSynced || p==sqlite3LruPageList.pFirstSynced);
          621  +    sqlite3LruPageList.pFirstSynced = p;
          622  +    sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
          623  +  }
          624  +#endif
          625  +}
   517    626   
   518    627   /*
   519    628   ** Return true if page *pPg has already been written to the statement
   520    629   ** journal (or statement snapshot has been created, if *pPg is part
   521    630   ** of an in-memory database).
   522    631   */
   523    632   static int pageInStatement(PgHdr *pPg){
................................................................................
  1077   1186   static void pager_reset(Pager *pPager){
  1078   1187     PgHdr *pPg, *pNext;
  1079   1188     if( pPager->errCode ) return;
  1080   1189     for(pPg=pPager->pAll; pPg; pPg=pNext){
  1081   1190       IOTRACE(("PGFREE %p %d\n", pPager, pPg->pgno));
  1082   1191       PAGER_INCR(sqlite3_pager_pgfree_count);
  1083   1192       pNext = pPg->pNextAll;
         1193  +    lruListRemove(pPg);
  1084   1194       sqlite3_free(pPg);
  1085   1195     }
         1196  +  assert(pPager->lru.pFirst==0);
         1197  +  assert(pPager->lru.pFirstSynced==0);
         1198  +  assert(pPager->lru.pLast==0);
  1086   1199     pPager->pStmt = 0;
  1087         -  pPager->pFirst = 0;
  1088         -  pPager->pFirstSynced = 0;
  1089         -  pPager->pLast = 0;
  1090   1200     pPager->pAll = 0;
  1091   1201     pPager->nHash = 0;
  1092   1202     sqlite3_free(pPager->aHash);
  1093   1203     pPager->nPage = 0;
  1094   1204     pPager->aHash = 0;
  1095   1205     pPager->nRef = 0;
  1096   1206   }
................................................................................
  1161   1271       pPager->state = PAGER_SHARED;
  1162   1272     }else if( pPager->state==PAGER_SYNCED ){
  1163   1273       pPager->state = PAGER_EXCLUSIVE;
  1164   1274     }
  1165   1275     pPager->origDbSize = 0;
  1166   1276     pPager->setMaster = 0;
  1167   1277     pPager->needSync = 0;
  1168         -  pPager->pFirstSynced = pPager->pFirst;
         1278  +  lruListSetFirstSynced(pPager);
  1169   1279     pPager->dbSize = -1;
  1170   1280   
  1171   1281     return (rc==SQLITE_OK?rc2:rc);
  1172   1282   }
  1173   1283   
  1174   1284   /*
  1175   1285   ** Compute and return a checksum for the page of data.
................................................................................
  2280   2390   /*
  2281   2391   ** Unlink a page from the free list (the list of all pages where nRef==0)
  2282   2392   ** and from its hash collision chain.
  2283   2393   */
  2284   2394   static void unlinkPage(PgHdr *pPg){
  2285   2395     Pager *pPager = pPg->pPager;
  2286   2396   
  2287         -  /* Keep the pFirstSynced pointer pointing at the first synchronized page */
  2288         -  if( pPg==pPager->pFirstSynced ){
  2289         -    PgHdr *p = pPg->pNextFree;
  2290         -    while( p && p->needSync ){ p = p->pNextFree; }
  2291         -    pPager->pFirstSynced = p;
  2292         -  }
  2293         -
  2294         -  /* Unlink from the freelist */
  2295         -  if( pPg->pPrevFree ){
  2296         -    pPg->pPrevFree->pNextFree = pPg->pNextFree;
  2297         -  }else{
  2298         -    assert( pPager->pFirst==pPg );
  2299         -    pPager->pFirst = pPg->pNextFree;
  2300         -  }
  2301         -  if( pPg->pNextFree ){
  2302         -    pPg->pNextFree->pPrevFree = pPg->pPrevFree;
  2303         -  }else{
  2304         -    assert( pPager->pLast==pPg );
  2305         -    pPager->pLast = pPg->pPrevFree;
  2306         -  }
  2307         -  pPg->pNextFree = pPg->pPrevFree = 0;
         2397  +  /* Unlink from free page list */
         2398  +  lruListRemove(pPg);
  2308   2399   
  2309   2400     /* Unlink from the pgno hash table */
  2310   2401     unlinkHashChain(pPager, pPg);
  2311   2402   }
  2312   2403   
  2313   2404   /*
  2314   2405   ** This routine is used to truncate the cache when a database
................................................................................
  2494   2585   ** For non-test systems, page_ref() is a macro that calls _page_ref()
  2495   2586   ** online of the reference count is zero.  For test systems, page_ref()
  2496   2587   ** is a real function so that we can set breakpoints and trace it.
  2497   2588   */
  2498   2589   static void _page_ref(PgHdr *pPg){
  2499   2590     if( pPg->nRef==0 ){
  2500   2591       /* The page is currently on the freelist.  Remove it. */
  2501         -    if( pPg==pPg->pPager->pFirstSynced ){
  2502         -      PgHdr *p = pPg->pNextFree;
  2503         -      while( p && p->needSync ){ p = p->pNextFree; }
  2504         -      pPg->pPager->pFirstSynced = p;
  2505         -    }
  2506         -    if( pPg->pPrevFree ){
  2507         -      pPg->pPrevFree->pNextFree = pPg->pNextFree;
  2508         -    }else{
  2509         -      pPg->pPager->pFirst = pPg->pNextFree;
  2510         -    }
  2511         -    if( pPg->pNextFree ){
  2512         -      pPg->pNextFree->pPrevFree = pPg->pPrevFree;
  2513         -    }else{
  2514         -      pPg->pPager->pLast = pPg->pPrevFree;
  2515         -    }
         2592  +    lruListRemove(pPg);
  2516   2593       pPg->pPager->nRef++;
  2517   2594     }
  2518   2595     pPg->nRef++;
  2519   2596     REFINFO(pPg);
  2520   2597   }
  2521   2598   #ifdef SQLITE_DEBUG
  2522   2599     static void page_ref(PgHdr *pPg){
................................................................................
  2631   2708       pPager->needSync = 0;
  2632   2709   
  2633   2710       /* Erase the needSync flag from every page.
  2634   2711       */
  2635   2712       for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
  2636   2713         pPg->needSync = 0;
  2637   2714       }
  2638         -    pPager->pFirstSynced = pPager->pFirst;
         2715  +    lruListSetFirstSynced(pPager);
  2639   2716     }
  2640   2717   
  2641   2718   #ifndef NDEBUG
  2642   2719     /* If the Pager.needSync flag is clear then the PgHdr.needSync
  2643   2720     ** flag must also be clear for all pages.  Verify that this
  2644   2721     ** invariant is true.
  2645   2722     */
  2646   2723     else{
  2647   2724       for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
  2648   2725         assert( pPg->needSync==0 );
  2649   2726       }
  2650         -    assert( pPager->pFirstSynced==pPager->pFirst );
         2727  +    assert( pPager->lru.pFirstSynced==pPager->lru.pFirst );
  2651   2728     }
  2652   2729   #endif
  2653   2730   
  2654   2731     return rc;
  2655   2732   }
  2656   2733   
  2657   2734   /*
................................................................................
  2853   2930     *ppPg = 0;
  2854   2931   
  2855   2932     assert(!MEMDB);
  2856   2933   
  2857   2934     /* Find a page to recycle.  Try to locate a page that does not
  2858   2935     ** require us to do an fsync() on the journal.
  2859   2936     */
  2860         -  pPg = pPager->pFirstSynced;
         2937  +  pPg = pPager->lru.pFirstSynced;
  2861   2938   
  2862   2939     /* If we could not find a page that does not require an fsync()
  2863   2940     ** on the journal file then fsync the journal file.  This is a
  2864   2941     ** very slow operation, so we work hard to avoid it.  But sometimes
  2865   2942     ** it can't be helped.
  2866   2943     */
  2867         -  if( pPg==0 && pPager->pFirst && syncOk && !MEMDB){
         2944  +  if( pPg==0 && pPager->lru.pFirst && syncOk && !MEMDB){
  2868   2945       int iDc = sqlite3OsDeviceCharacteristics(pPager->fd);
  2869   2946       int rc = syncJournal(pPager);
  2870   2947       if( rc!=0 ){
  2871   2948         return rc;
  2872   2949       }
  2873   2950       if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){
  2874   2951         /* If in full-sync mode, write a new journal header into the
................................................................................
  2881   2958         assert( pPager->journalOff > 0 );
  2882   2959         assert( pPager->doNotSync==0 );
  2883   2960         rc = writeJournalHdr(pPager);
  2884   2961         if( rc!=0 ){
  2885   2962           return rc;
  2886   2963         }
  2887   2964       }
  2888         -    pPg = pPager->pFirst;
         2965  +    pPg = pPager->lru.pFirst;
  2889   2966     }
  2890   2967     if( pPg==0 ){
  2891   2968       return SQLITE_OK;
  2892   2969     }
  2893   2970   
  2894   2971     assert( pPg->nRef==0 );
  2895   2972   
................................................................................
  2941   3018   ** been released, the function returns. The return value is the total number 
  2942   3019   ** of bytes of memory released.
  2943   3020   */
  2944   3021   int sqlite3PagerReleaseMemory(int nReq){
  2945   3022     int nReleased = 0;          /* Bytes of memory released so far */
  2946   3023     sqlite3_mutex *mutex;       /* The MEM2 mutex */
  2947   3024     Pager *pPager;              /* For looping over pagers */
  2948         -  int i;                      /* Passes over pagers */
         3025  +  int rc = SQLITE_OK;
  2949   3026   
  2950   3027     /* Acquire the memory-management mutex
  2951   3028     */
  2952   3029     mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MEM2);
  2953   3030     sqlite3_mutex_enter(mutex);
  2954   3031   
  2955   3032     /* Signal all database connections that memory management wants
  2956   3033     ** to have access to the pagers.
  2957   3034     */
  2958   3035     for(pPager=sqlite3PagerList; pPager; pPager=pPager->pNext){
  2959   3036        pPager->iInUseMM = 1;
  2960   3037     }
  2961   3038   
  2962         -  /* Outermost loop runs for at most two iterations. First iteration we
  2963         -  ** try to find memory that can be released without calling fsync(). Second
  2964         -  ** iteration (which only runs if the first failed to free nReq bytes of
  2965         -  ** memory) is permitted to call fsync(). This is of course much more 
  2966         -  ** expensive.
  2967         -  */
  2968         -  for(i=0; i<=1; i++){
  2969         -
  2970         -    /* Loop through all the SQLite pagers opened by the current thread. */
  2971         -    Pager *pPager = sqlite3PagerList;
  2972         -    for( ; pPager && (nReq<0 || nReleased<nReq); pPager=pPager->pNext){
  2973         -      PgHdr *pPg;
  2974         -      int rc = SQLITE_OK;
  2975         -
  2976         -      /* In-memory databases should not appear on the pager list */
  2977         -      assert( !MEMDB );
  2978         -
  2979         -      /* Skip pagers that are currently in use by the b-tree layer */
  2980         -      if( pPager->iInUseDB ) continue;
  2981         -
  2982         -      /* For each pager, try to free as many pages as possible (without 
  2983         -      ** calling fsync() if this is the first iteration of the outermost 
  2984         -      ** loop).
         3039  +  while( rc==SQLITE_OK && (nReq<0 || nReleased<nReq) ){
         3040  +    PgHdr *pPg;
         3041  +    PgHdr *pRecycled;
         3042  + 
         3043  +    /* Try to find a page to recycle that does not require a sync(). If
         3044  +    ** this is not possible, find one that does require a sync().
         3045  +    */
         3046  +    sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
         3047  +    pPg = sqlite3LruPageList.pFirstSynced;
         3048  +    while( pPg && (pPg->needSync || pPg->pPager->iInUseDB) ){
         3049  +      pPg = pPg->gfree.pNext;
         3050  +    }
         3051  +    if( !pPg ){
         3052  +      pPg = sqlite3LruPageList.pFirst;
         3053  +      while( pPg && pPg->pPager->iInUseDB ){
         3054  +        pPg = pPg->gfree.pNext;
         3055  +      }
         3056  +    }
         3057  +    sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
         3058  +
         3059  +    if( !pPg ) break;
         3060  +
         3061  +    pPager = pPg->pPager;
         3062  +    assert(!pPg->needSync || pPg==pPager->lru.pFirst);
         3063  +    assert(pPg->needSync || pPg==pPager->lru.pFirstSynced);
         3064  +  
         3065  +    rc = pager_recycle(pPager, 1, &pRecycled);
         3066  +    assert(pRecycled==pPg || rc!=SQLITE_OK);
         3067  +    if( rc==SQLITE_OK ){
         3068  +      /* We've found a page to free. At this point the page has been 
         3069  +      ** removed from the page hash-table, free-list and synced-list 
         3070  +      ** (pFirstSynced). It is still in the all pages (pAll) list. 
         3071  +      ** Remove it from this list before freeing.
         3072  +      **
         3073  +      ** Todo: Check the Pager.pStmt list to make sure this is Ok. It 
         3074  +      ** probably is though.
         3075  +      */
         3076  +      PgHdr *pTmp;
         3077  +      assert( pPg );
         3078  +      if( pPg==pPager->pAll ){
         3079  +         pPager->pAll = pPg->pNextAll;
         3080  +      }else{
         3081  +        for( pTmp=pPager->pAll; pTmp->pNextAll!=pPg; pTmp=pTmp->pNextAll ){}
         3082  +        pTmp->pNextAll = pPg->pNextAll;
         3083  +      }
         3084  +      nReleased += (
         3085  +          sizeof(*pPg) + pPager->pageSize
         3086  +          + sizeof(u32) + pPager->nExtra
         3087  +          + MEMDB*sizeof(PgHistory) 
         3088  +      );
         3089  +      IOTRACE(("PGFREE %p %d *\n", pPager, pPg->pgno));
         3090  +      PAGER_INCR(sqlite3_pager_pgfree_count);
         3091  +      sqlite3_free(pPg);
         3092  +    }else{
         3093  +      /* An error occured whilst writing to the database file or 
         3094  +      ** journal in pager_recycle(). The error is not returned to the 
         3095  +      ** caller of this function. Instead, set the Pager.errCode variable.
         3096  +      ** The error will be returned to the user (or users, in the case 
         3097  +      ** of a shared pager cache) of the pager for which the error occured.
  2985   3098         */
  2986         -      while( (nReq<0 || nReleased<nReq) &&
  2987         -             SQLITE_OK==(rc = pager_recycle(pPager, i, &pPg)) &&
  2988         -             pPg
  2989         -      ) {
  2990         -        /* We've found a page to free. At this point the page has been 
  2991         -        ** removed from the page hash-table, free-list and synced-list 
  2992         -        ** (pFirstSynced). It is still in the all pages (pAll) list. 
  2993         -        ** Remove it from this list before freeing.
  2994         -        **
  2995         -        ** Todo: Check the Pager.pStmt list to make sure this is Ok. It 
  2996         -        ** probably is though.
  2997         -        */
  2998         -        PgHdr *pTmp;
  2999         -        assert( pPg );
  3000         -        if( pPg==pPager->pAll ){
  3001         -           pPager->pAll = pPg->pNextAll;
  3002         -        }else{
  3003         -          for( pTmp=pPager->pAll; pTmp->pNextAll!=pPg; pTmp=pTmp->pNextAll ){}
  3004         -          pTmp->pNextAll = pPg->pNextAll;
  3005         -        }
  3006         -        nReleased += (
  3007         -            sizeof(*pPg) + pPager->pageSize
  3008         -            + sizeof(u32) + pPager->nExtra
  3009         -            + MEMDB*sizeof(PgHistory) 
  3010         -        );
  3011         -        IOTRACE(("PGFREE %p %d *\n", pPager, pPg->pgno));
  3012         -        PAGER_INCR(sqlite3_pager_pgfree_count);
  3013         -        sqlite3_free(pPg);
  3014         -      }
  3015         -
  3016         -      if( rc!=SQLITE_OK ){
  3017         -        /* An error occured whilst writing to the database file or 
  3018         -        ** journal in pager_recycle(). The error is not returned to the 
  3019         -        ** caller of this function. Instead, set the Pager.errCode variable.
  3020         -        ** The error will be returned to the user (or users, in the case 
  3021         -        ** of a shared pager cache) of the pager for which the error occured.
  3022         -        */
  3023         -        assert(
  3024         -            (rc&0xff)==SQLITE_IOERR ||
  3025         -            rc==SQLITE_FULL ||
  3026         -            rc==SQLITE_BUSY
  3027         -        );
  3028         -        assert( pPager->state>=PAGER_RESERVED );
  3029         -        pager_error(pPager, rc);
  3030         -      }
         3099  +      assert(
         3100  +          (rc&0xff)==SQLITE_IOERR ||
         3101  +          rc==SQLITE_FULL ||
         3102  +          rc==SQLITE_BUSY
         3103  +      );
         3104  +      assert( pPager->state>=PAGER_RESERVED );
         3105  +      pager_error(pPager, rc);
  3031   3106       }
  3032   3107     }
  3033   3108   
  3034   3109     /* Clear the memory management flags and release the mutex
  3035   3110     */
  3036   3111     for(pPager=sqlite3PagerList; pPager; pPager=pPager->pNext){
  3037   3112        pPager->iInUseMM = 0;
................................................................................
  3248   3323   static int pagerAllocatePage(Pager *pPager, PgHdr **ppPg){
  3249   3324     int rc = SQLITE_OK;
  3250   3325     PgHdr *pPg;
  3251   3326   
  3252   3327     /* Create a new PgHdr if any of the four conditions defined 
  3253   3328     ** above are met: */
  3254   3329     if( pPager->nPage<pPager->mxPage
  3255         -   || pPager->pFirst==0 
         3330  +   || pPager->lru.pFirst==0 
  3256   3331      || MEMDB
  3257         -   || (pPager->pFirstSynced==0 && pPager->doNotSync)
         3332  +   || (pPager->lru.pFirstSynced==0 && pPager->doNotSync)
  3258   3333     ){
  3259   3334       if( pPager->nPage>=pPager->nHash ){
  3260   3335         pager_resize_hash_table(pPager,
  3261   3336            pPager->nHash<256 ? 256 : pPager->nHash*2);
  3262   3337         if( pPager->nHash==0 ){
  3263   3338           rc = SQLITE_NOMEM;
  3264   3339           goto pager_allocate_out;
................................................................................
  3537   3612   
  3538   3613     CHECK_PAGE(pPg);
  3539   3614   
  3540   3615     /* When the number of references to a page reach 0, call the
  3541   3616     ** destructor and add the page to the freelist.
  3542   3617     */
  3543   3618     if( pPg->nRef==0 ){
  3544         -    Pager *pPager;
  3545         -    pPager = pPg->pPager;
  3546         -    pPg->pNextFree = 0;
  3547         -    pPg->pPrevFree = pPager->pLast;
  3548         -    pPager->pLast = pPg;
  3549         -    if( pPg->pPrevFree ){
  3550         -      pPg->pPrevFree->pNextFree = pPg;
  3551         -    }else{
  3552         -      pPager->pFirst = pPg;
  3553         -    }
  3554         -    if( pPg->needSync==0 && pPager->pFirstSynced==0 ){
  3555         -      pPager->pFirstSynced = pPg;
  3556         -    }
         3619  +    Pager *pPager = pPg->pPager;
         3620  +
         3621  +    lruListAdd(pPg);
  3557   3622       if( pPager->xDestructor ){
  3558   3623         pPager->xDestructor(pPg, pPager->pageSize);
  3559   3624       }
  3560   3625     
  3561   3626       /* When all pages reach the freelist, drop the read lock from
  3562   3627       ** the database file.
  3563   3628       */

Changes to src/sqlite.h.in.

    26     26   ** on how SQLite interfaces are suppose to operate.
    27     27   **
    28     28   ** The name of this file under configuration management is "sqlite.h.in".
    29     29   ** The makefile makes some minor changes to this file (such as inserting
    30     30   ** the version number) and changes its name to "sqlite3.h" as
    31     31   ** part of the build process.
    32     32   **
    33         -** @(#) $Id: sqlite.h.in,v 1.242 2007/08/25 16:21:30 drh Exp $
           33  +** @(#) $Id: sqlite.h.in,v 1.243 2007/08/27 17:27:49 danielk1977 Exp $
    34     34   */
    35     35   #ifndef _SQLITE3_H_
    36     36   #define _SQLITE3_H_
    37     37   #include <stdarg.h>     /* Needed for the definition of va_list */
    38     38   
    39     39   /*
    40     40   ** Make sure we can call this stuff from C++.
................................................................................
  3279   3279   ** <ul>
  3280   3280   ** <li>  SQLITE_MUTEX_FAST
  3281   3281   ** <li>  SQLITE_MUTEX_RECURSIVE
  3282   3282   ** <li>  SQLITE_MUTEX_STATIC_MASTER
  3283   3283   ** <li>  SQLITE_MUTEX_STATIC_MEM
  3284   3284   ** <li>  SQLITE_MUTEX_STATIC_MEM2
  3285   3285   ** <li>  SQLITE_MUTEX_STATIC_PRNG
         3286  +** <li>  SQLITE_MUTEX_STATIC_LRU
  3286   3287   ** </ul>
  3287   3288   **
  3288   3289   ** The first two constants cause sqlite3_mutex_alloc() to create
  3289   3290   ** a new mutex.  The new mutex is recursive when SQLITE_MUTEX_RECURSIVE
  3290   3291   ** is used but not necessarily so when SQLITE_MUTEX_FAST is used.
  3291   3292   ** The mutex implementation does not need to make a distinction
  3292   3293   ** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does
................................................................................
  3383   3384   */
  3384   3385   #define SQLITE_MUTEX_FAST             0
  3385   3386   #define SQLITE_MUTEX_RECURSIVE        1
  3386   3387   #define SQLITE_MUTEX_STATIC_MASTER    2
  3387   3388   #define SQLITE_MUTEX_STATIC_MEM       3  /* sqlite3_malloc() */
  3388   3389   #define SQLITE_MUTEX_STATIC_MEM2      4  /* sqlite3_release_memory() */
  3389   3390   #define SQLITE_MUTEX_STATIC_PRNG      5  /* sqlite3_random() */
         3391  +#define SQLITE_MUTEX_STATIC_LRU       6  /* lru page list */
  3390   3392   
  3391   3393   
  3392   3394   /*
  3393   3395   ** Undo the hack that converts floating point types to integer for
  3394   3396   ** builds on processors without floating point support.
  3395   3397   */
  3396   3398   #ifdef SQLITE_OMIT_FLOATING_POINT