** a database write does not have to wait for (sometimes slow) disk I/O
** to occur.  The write seems to happen very quickly, though in reality
** it is happening at its usual slow pace in the background.
**
** Asynchronous I/O appears to give better responsiveness, but at a price.
** You lose the Durable property.  With the default I/O backend of SQLite,
** once a write completes, you know that the information you wrote is
** safely on disk.  With the asynchronous I/O, this is no the case.  If
** your program crashes or if you take a power lose after the database
** write but before the asynchronous write thread has completed, then the
** database change might never make it to disk and the next user of the
** database might not see your change.
**
** You lose Durability with asynchronous I/O, but you still retain the
** other parts of ACID:  Atomic,  Consistent, and Isolated.  Many
** appliations get along fine without the Durablity.
**
** HOW IT WORKS
**
** Asynchronous I/O works by overloading the OS-layer disk I/O routines
** with modified versions that store the data to be written in queue of
** pending write operations.  Look at the asyncEnable() subroutine to see
** how overloading works.  Six os-layer routines are overloaded:

**
**     sqlite3OsOpenReadWrite;
**     sqlite3OsOpenReadOnly;
**     sqlite3OsOpenExclusive;
**     sqlite3OsDelete;
**     sqlite3OsFileExists;
**     sqlite3OsSyncDirectory;
**
** The original implementations of these routines are saved and are
** used by the writer thread to do the real I/O.  The substitute
** implementations typically put the I/O operation on a queue
** to be handled later by the writer thread, though read operations
** must be handled right away, obviously.
**
** Asynchronous I/O is disabled by setting the os-layer interface routines
** back to their original values.
**
** LIMITATIONS
**
** This demonstration code is deliberately kept simple in order to keep
** the main ideas clear and easy to understand.  Real applications that
** want to do asynchronous I/O might want to add additional capabilities.
** For example, in this demonstration if writes are happening at a steady
................................................................................
** the quantity of pending writes and stop accepting new write requests
** when the buffer gets to be too big.
*/

/* 
** If this symbol is defined, then file-system locks are obtained as
** required. This slows things down, but allows multiple processes
** to access the database concurrently.



*/
#define ENABLE_FILE_LOCKING

#include "sqliteInt.h"
#include <tcl.h>

/*
................................................................................

/*
** THREAD SAFETY NOTES
**
** Basic rules:
**
**     * Both read and write access to the global write-op queue must be 
**       protected by the async.queueMutex.




**
**     * The file handles from the underlying system are assumed not to 
**       be thread safe.
**
**     * See the last two paragraphs under "The Writer Thread" for
**       an assumption to do with file-handle synchronization by the Os.
**
................................................................................
  int op;                      /* One of ASYNC_xxx etc. */
  i64 iOffset;        /* See above */
  int nByte;          /* See above */
  char *zBuf;         /* Data to write to file (or NULL if op!=ASYNC_WRITE) */
  AsyncWrite *pNext;  /* Next write operation (to any file) */
};

/*

























** An instance of the following structure is allocated along with each
** AsyncFileData structure (see AsyncFileData.lock), but is only used if the
** file was opened with the SQLITE_OPEN_MAIN_DB.
**
** The global async.aLock[] hash table maps from database file-name to a
** linked-list of AsyncFileLock structures corresponding to handles opened on
** the file. The AsyncFileLock structures are linked into the list when the
** file is opened and removed when it is closed. Mutex async.lockMutex must be
** held before accessing any AsyncFileLock structure or the async.aLock[]
** table.
*/
struct AsyncFileLock {
  int eLock;                /* Internally visible lock state (sqlite pov) */
  int eAsyncLock;           /* Lock-state with write-queue unlock */
  AsyncFileLock *pNext;
};

struct AsyncLock {
  sqlite3_file *pFile;
  int eLock;
  AsyncFileLock *pList;
};

/* 
** The AsyncFile structure is a subclass of sqlite3_file used for 
** asynchronous IO. 
**
** All of the actual data for the structure is stored in the structure
** pointed to by AsyncFile.pData, which is allocated as part of the
** sqlite3OsOpen() using sqlite3_malloc(). The reason for this is that the
................................................................................
  int eRequired = 0;

  if( pLock->pFile ){
    for(pIter=pLock->pList; pIter; pIter=pIter->pNext){
      assert(pIter->eAsyncLock>=pIter->eLock);
      if( pIter->eAsyncLock>eRequired ){
        eRequired = pIter->eAsyncLock;

      }
    }
    if( eRequired>pLock->eLock ){
      rc = sqlite3OsLock(pLock->pFile, eRequired);
    }else if(eRequired<pLock->eLock){
      rc = sqlite3OsUnlock(pLock->pFile, eRequired);
    }
................................................................................
    }
  }

  return rc;
}

/*
** No disk locking is performed.  We keep track of locks locally in
** the async.aLock hash table.  Locking should appear to work the same
** as with standard (unmodified) SQLite as long as all connections 
** come from this one process.  Connections from external processes
** cannot see our internal hash table (obviously) and will thus not
** honor our locks.
*/
static int asyncLock(sqlite3_file *pFile, int eLock){
  int rc = SQLITE_OK;
  AsyncFileData *p = ((AsyncFile *)pFile)->pData;

  pthread_mutex_lock(&async.lockMutex);
  if( p->lock.eLock<eLock ){
................................................................................
        (eLock==SQLITE_LOCK_SHARED && pIter->eLock>=SQLITE_LOCK_PENDING)
      )){
        rc = SQLITE_BUSY;
      }
    }
    if( rc==SQLITE_OK ){
      p->lock.eLock = eLock;
      if( eLock>p->lock.eAsyncLock ){
        p->lock.eAsyncLock = eLock;
      }
    }
    assert(p->lock.eAsyncLock>=p->lock.eLock);
    if( rc==SQLITE_OK ){
      rc = getFileLock(pLock);
    }
  }
  pthread_mutex_unlock(&async.lockMutex);
................................................................................
  ASYNC_TRACE(("LOCK %d (%s) rc=%d\n", eLock, p->zName, rc));
  return rc;
}
static int asyncUnlock(sqlite3_file *pFile, int eLock){
  AsyncFileData *p = ((AsyncFile *)pFile)->pData;
  AsyncFileLock *pLock = &p->lock;
  pthread_mutex_lock(&async.lockMutex);
  if( pLock->eLock>eLock ){
    pLock->eLock = eLock;
  }
  pthread_mutex_unlock(&async.lockMutex);
  return addNewAsyncWrite(p, ASYNC_UNLOCK, 0, eLock, 0);
}

/*
** This function is called when the pager layer first opens a database file
** and is checking for a hot-journal.
................................................................................
**
** This routine is not even remotely threadsafe.  Do not call
** this routine while any SQLite database connections are open.
*/
static void asyncEnable(int enable){
  if( enable ){
    if( !async_vfs.pAppData ){

      async_vfs.pAppData = (void *)sqlite3_vfs_find(0);
      async_vfs.mxPathname = ((sqlite3_vfs *)async_vfs.pAppData)->mxPathname;
      sqlite3_vfs_register(&async_vfs, 1);

      sqlite3HashInit(&async.aLock, SQLITE_HASH_BINARY, 1);


    }
  }else{
    if( async_vfs.pAppData ){
      sqlite3_vfs_unregister(&async_vfs);
      async_vfs.pAppData = 0;
      sqlite3HashClear(&async.aLock);
    }
  }
}

/* 
** This procedure runs in a separate thread, reading messages off of the
** write queue and processing them one by one.  
................................................................................
        ** structures for this file. Obtain the async.lockMutex mutex 
        ** before doing so.
        */
        pthread_mutex_lock(&async.lockMutex);
        pLock = sqlite3HashFind(&async.aLock, pData->zName, pData->nName);
        for(ppIter=&pLock->pList; *ppIter; ppIter=&((*ppIter)->pNext)){
          if( (*ppIter)==&pData->lock ){
            *ppIter = (*ppIter)->pNext;
            break;
          }
        }
        if( !pLock->pList ){

          if( pLock->pFile ) sqlite3OsClose(pLock->pFile);

          sqlite3_free(pLock);
          sqlite3HashInsert(&async.aLock, pData->zName, pData->nName, 0);



        }else{
          rc = getFileLock(pLock);
        }
        pthread_mutex_unlock(&async.lockMutex);

        sqlite3_free(pData);
        break;
................................................................................
      }

      case ASYNC_UNLOCK: {
        AsyncLock *pLock;
        AsyncFileData *pData = p->pFileData;
        int eLock = p->nByte;
        pthread_mutex_lock(&async.lockMutex);
        if( pData->lock.eAsyncLock>eLock ){
          if( pData->lock.eLock>eLock ){
            pData->lock.eAsyncLock = pData->lock.eLock;
          }else{
            pData->lock.eAsyncLock = eLock;
          }
        }

        assert(pData->lock.eAsyncLock>=pData->lock.eLock);
        pLock = sqlite3HashFind(&async.aLock, pData->zName, pData->nName);
        rc = getFileLock(pLock);
        pthread_mutex_unlock(&async.lockMutex);
        break;
      }

** a database write does not have to wait for (sometimes slow) disk I/O
** to occur.  The write seems to happen very quickly, though in reality
** it is happening at its usual slow pace in the background.
**
** Asynchronous I/O appears to give better responsiveness, but at a price.
** You lose the Durable property.  With the default I/O backend of SQLite,
** once a write completes, you know that the information you wrote is
** safely on disk.  With the asynchronous I/O, this is not the case.  If
** your program crashes or if a power lose occurs after the database
** write but before the asynchronous write thread has completed, then the
** database change might never make it to disk and the next user of the
** database might not see your change.
**
** You lose Durability with asynchronous I/O, but you still retain the
** other parts of ACID:  Atomic,  Consistent, and Isolated.  Many
** appliations get along fine without the Durablity.
**
** HOW IT WORKS
**
** Asynchronous I/O works by creating a special SQLite "vfs" structure
** and registering it with sqlite3_vfs_register(). When files opened via 
** this vfs are written to (using sqlite3OsWrite()), the data is not 
** written directly to disk, but is placed in the "write-queue" to be
** handled by the background thread.
**

** The special vfs is registered (and unregistered) by calls to 
** function asyncEnable() (see below).












**
** LIMITATIONS
**
** This demonstration code is deliberately kept simple in order to keep
** the main ideas clear and easy to understand.  Real applications that
** want to do asynchronous I/O might want to add additional capabilities.
** For example, in this demonstration if writes are happening at a steady
................................................................................
** the quantity of pending writes and stop accepting new write requests
** when the buffer gets to be too big.
*/

/* 
** If this symbol is defined, then file-system locks are obtained as
** required. This slows things down, but allows multiple processes
** to access the database concurrently. If this symbol is not defined,
** then connections from within a single process will respect each
** others database locks, but external connections will not - leading
** to database corruption.
*/
#define ENABLE_FILE_LOCKING

#include "sqliteInt.h"
#include <tcl.h>

/*
................................................................................

/*
** THREAD SAFETY NOTES
**
** Basic rules:
**
**     * Both read and write access to the global write-op queue must be 
**       protected by the async.queueMutex. As are the async.ioError and
**       async.nFile variables.
**
**     * The async.aLock hash-table and all AsyncLock and AsyncFileLock
**       structures must be protected by teh async.lockMutex mutex.
**
**     * The file handles from the underlying system are assumed not to 
**       be thread safe.
**
**     * See the last two paragraphs under "The Writer Thread" for
**       an assumption to do with file-handle synchronization by the Os.
**
................................................................................
  int op;                      /* One of ASYNC_xxx etc. */
  i64 iOffset;        /* See above */
  int nByte;          /* See above */
  char *zBuf;         /* Data to write to file (or NULL if op!=ASYNC_WRITE) */
  AsyncWrite *pNext;  /* Next write operation (to any file) */
};

/*
** An instance of this structure is created for each distinct open file 
** (i.e. if two handles are opened on the one file, only one of these
** structures is allocated) and stored in the async.aLock hash table. The
** keys for async.aLock are the full pathnames of the opened files.
**
** AsyncLock.pList points to the head of a linked list of AsyncFileLock
** structures, one for each handle currently open on the file.
**
** If the opened file is not a main-database (the SQLITE_OPEN_MAIN_DB is
** not passed to the sqlite3OsOpen() call), or if ENABLE_FILE_LOCKING is 
** not defined at compile time, variables AsyncLock.pFile and 
** AsyncLock.eLock are never used. Otherwise, pFile is a file handle
** opened on the file in question and used to obtain the file-system 
** locks required by database connections within this process.
**
** See comments above the asyncLock() function for more details on 
** the implementation of database locking used by this backend.
*/
struct AsyncLock {
  sqlite3_file *pFile;
  int eLock;
  AsyncFileLock *pList;
};

/*
** An instance of the following structure is allocated along with each
** AsyncFileData structure (see AsyncFileData.lock), but is only used if the
** file was opened with the SQLITE_OPEN_MAIN_DB.







*/
struct AsyncFileLock {
  int eLock;                /* Internally visible lock state (sqlite pov) */
  int eAsyncLock;           /* Lock-state with write-queue unlock */
  AsyncFileLock *pNext;
};







/* 
** The AsyncFile structure is a subclass of sqlite3_file used for 
** asynchronous IO. 
**
** All of the actual data for the structure is stored in the structure
** pointed to by AsyncFile.pData, which is allocated as part of the
** sqlite3OsOpen() using sqlite3_malloc(). The reason for this is that the
................................................................................
  int eRequired = 0;

  if( pLock->pFile ){
    for(pIter=pLock->pList; pIter; pIter=pIter->pNext){
      assert(pIter->eAsyncLock>=pIter->eLock);
      if( pIter->eAsyncLock>eRequired ){
        eRequired = pIter->eAsyncLock;
        assert(eRequired>=0 && eRequired<=SQLITE_LOCK_EXCLUSIVE);
      }
    }
    if( eRequired>pLock->eLock ){
      rc = sqlite3OsLock(pLock->pFile, eRequired);
    }else if(eRequired<pLock->eLock){
      rc = sqlite3OsUnlock(pLock->pFile, eRequired);
    }
................................................................................
    }
  }

  return rc;
}

/*
** The following two methods - asyncLock() and asyncUnlock() - are used
** to obtain and release locks on database files opened with the
** asynchronous backend.



*/
static int asyncLock(sqlite3_file *pFile, int eLock){
  int rc = SQLITE_OK;
  AsyncFileData *p = ((AsyncFile *)pFile)->pData;

  pthread_mutex_lock(&async.lockMutex);
  if( p->lock.eLock<eLock ){
................................................................................
        (eLock==SQLITE_LOCK_SHARED && pIter->eLock>=SQLITE_LOCK_PENDING)
      )){
        rc = SQLITE_BUSY;
      }
    }
    if( rc==SQLITE_OK ){
      p->lock.eLock = eLock;

      p->lock.eAsyncLock = MAX(p->lock.eAsyncLock, eLock);

    }
    assert(p->lock.eAsyncLock>=p->lock.eLock);
    if( rc==SQLITE_OK ){
      rc = getFileLock(pLock);
    }
  }
  pthread_mutex_unlock(&async.lockMutex);
................................................................................
  ASYNC_TRACE(("LOCK %d (%s) rc=%d\n", eLock, p->zName, rc));
  return rc;
}
static int asyncUnlock(sqlite3_file *pFile, int eLock){
  AsyncFileData *p = ((AsyncFile *)pFile)->pData;
  AsyncFileLock *pLock = &p->lock;
  pthread_mutex_lock(&async.lockMutex);

  pLock->eLock = MIN(pLock->eLock, eLock);

  pthread_mutex_unlock(&async.lockMutex);
  return addNewAsyncWrite(p, ASYNC_UNLOCK, 0, eLock, 0);
}

/*
** This function is called when the pager layer first opens a database file
** and is checking for a hot-journal.
................................................................................
**
** This routine is not even remotely threadsafe.  Do not call
** this routine while any SQLite database connections are open.
*/
static void asyncEnable(int enable){
  if( enable ){
    if( !async_vfs.pAppData ){
      static int hashTableInit = 0;
      async_vfs.pAppData = (void *)sqlite3_vfs_find(0);
      async_vfs.mxPathname = ((sqlite3_vfs *)async_vfs.pAppData)->mxPathname;
      sqlite3_vfs_register(&async_vfs, 1);
      if( !hashTableInit ){
        sqlite3HashInit(&async.aLock, SQLITE_HASH_BINARY, 1);
        hashTableInit = 1;
      }
    }
  }else{
    if( async_vfs.pAppData ){
      sqlite3_vfs_unregister(&async_vfs);
      async_vfs.pAppData = 0;

    }
  }
}

/* 
** This procedure runs in a separate thread, reading messages off of the
** write queue and processing them one by one.  
................................................................................
        ** structures for this file. Obtain the async.lockMutex mutex 
        ** before doing so.
        */
        pthread_mutex_lock(&async.lockMutex);
        pLock = sqlite3HashFind(&async.aLock, pData->zName, pData->nName);
        for(ppIter=&pLock->pList; *ppIter; ppIter=&((*ppIter)->pNext)){
          if( (*ppIter)==&pData->lock ){
            *ppIter = pData->lock.pNext;
            break;
          }
        }
        if( !pLock->pList ){
          if( pLock->pFile ){
            sqlite3OsClose(pLock->pFile);
          }
          sqlite3_free(pLock);
          sqlite3HashInsert(&async.aLock, pData->zName, pData->nName, 0);
          if( !sqliteHashFirst(&async.aLock) ){
            sqlite3HashClear(&async.aLock);
          }
        }else{
          rc = getFileLock(pLock);
        }
        pthread_mutex_unlock(&async.lockMutex);

        sqlite3_free(pData);
        break;
................................................................................
      }

      case ASYNC_UNLOCK: {
        AsyncLock *pLock;
        AsyncFileData *pData = p->pFileData;
        int eLock = p->nByte;
        pthread_mutex_lock(&async.lockMutex);


        pData->lock.eAsyncLock = MIN(

            pData->lock.eAsyncLock, MAX(pData->lock.eLock, eLock)


        );
        assert(pData->lock.eAsyncLock>=pData->lock.eLock);
        pLock = sqlite3HashFind(&async.aLock, pData->zName, pData->nName);
        rc = getFileLock(pLock);
        pthread_mutex_unlock(&async.lockMutex);
        break;
      }

#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file runs all tests.
#
# $Id: async.test,v 1.9 2007/09/04 18:28:44 danielk1977 Exp $


if {[catch {sqlite3async_enable}]} {
  # The async logic is not built into this system
  return
}


set testdir [file dirname $argv0]
source $testdir/tester.tcl
rename finish_test really_finish_test
proc finish_test {} {}




set ISQUICK 1

set INCLUDE {
  select1.test
  select2.test
  select3.test
  select4.test
................................................................................
  insert2.test
  insert3.test
  trans.test
  lock.test
  lock3.test
  lock2.test
}
# set INCLUDE lock4.test

# Enable asynchronous IO.
sqlite3async_enable 1

rename do_test really_do_test
proc do_test {name args} {
  uplevel really_do_test async_io-$name $args
................................................................................
  sqlite3async_wait
}

foreach testfile [lsort -dictionary [glob $testdir/*.test]] {
  set tail [file tail $testfile]
  if {[lsearch -exact $INCLUDE $tail]<0} continue
  source $testfile
  catch {db close}









}

# Flush the write-queue and disable asynchronous IO. This should ensure
# all allocated memory is cleaned up.
set sqlite3async_trace 1
sqlite3async_halt idle
sqlite3async_start

#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file runs all tests.
#
# $Id: async.test,v 1.10 2007/09/05 11:34:54 danielk1977 Exp $


if {[catch {sqlite3async_enable}]} {
  # The async logic is not built into this system
  return
}


set testdir [file dirname $argv0]
source $testdir/tester.tcl
rename finish_test really_finish_test
proc finish_test {} {
  catch {db close}
  catch {db2 close}
  catch {db3 close}
}
set ISQUICK 1

set INCLUDE {
  select1.test
  select2.test
  select3.test
  select4.test
................................................................................
  insert2.test
  insert3.test
  trans.test
  lock.test
  lock3.test
  lock2.test
}


# Enable asynchronous IO.
sqlite3async_enable 1

rename do_test really_do_test
proc do_test {name args} {
  uplevel really_do_test async_io-$name $args
................................................................................
  sqlite3async_wait
}

foreach testfile [lsort -dictionary [glob $testdir/*.test]] {
  set tail [file tail $testfile]
  if {[lsearch -exact $INCLUDE $tail]<0} continue
  source $testfile


  # Make sure everything is flushed through. This is because [source]ing 
  # the next test file will delete the database file on disk (using
  # [file delete]). If the asynchronous backend still has the file
  # open, it will become confused.
  #
  sqlite3async_halt idle
  sqlite3async_start
  sqlite3async_wait
}

# Flush the write-queue and disable asynchronous IO. This should ensure
# all allocated memory is cleaned up.
set sqlite3async_trace 1
sqlite3async_halt idle
sqlite3async_start

#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# $Id: async2.test,v 1.6 2007/08/30 10:49:55 danielk1977 Exp $


set testdir [file dirname $argv0]
source $testdir/tester.tcl

if {
  [info commands sqlite3async_enable]=="" ||
................................................................................
      ioerr             { set ::sqlite_io_error_pending $n }
      malloc-persistent { sqlite3_memdebug_fail $n -repeat 1 }
      malloc-transient  { sqlite3_memdebug_fail $n -repeat 0 }
    }
    sqlite3async_halt idle
    sqlite3async_start
    sqlite3async_wait

  
    set ::sqlite_io_error_pending 0
    sqlite3_memdebug_fail -1

    sqlite3 db test.db
    set c [db eval {SELECT c FROM counter LIMIT 1}]
    switch -- $c {
      1 {
................................................................................
        } {klmnopqrst and seven}
      }
      FIN {
        set ::go 0
      }
    }
  
    sqlite3async_enable 0
  }
}

catch {db close}
sqlite3async_halt idle
sqlite3async_start
sqlite3async_wait

finish_test

#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# $Id: async2.test,v 1.7 2007/09/05 11:34:54 danielk1977 Exp $


set testdir [file dirname $argv0]
source $testdir/tester.tcl

if {
  [info commands sqlite3async_enable]=="" ||
................................................................................
      ioerr             { set ::sqlite_io_error_pending $n }
      malloc-persistent { sqlite3_memdebug_fail $n -repeat 1 }
      malloc-transient  { sqlite3_memdebug_fail $n -repeat 0 }
    }
    sqlite3async_halt idle
    sqlite3async_start
    sqlite3async_wait
    sqlite3async_enable 0

    set ::sqlite_io_error_pending 0
    sqlite3_memdebug_fail -1

    sqlite3 db test.db
    set c [db eval {SELECT c FROM counter LIMIT 1}]
    switch -- $c {
      1 {
................................................................................
        } {klmnopqrst and seven}
      }
      FIN {
        set ::go 0
      }
    }
  
    db close
  }
}

catch {db close}
sqlite3async_halt idle
sqlite3async_start
sqlite3async_wait

finish_test