/ Check-in [f5b3ce94]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the socketvfs test extension.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | socketvfs
Files: files | file ages | folders
SHA3-256: f5b3ce9404ffc8afdab6d8e57f806afba0b3ee75520d5e03335335d0f93d4a4b
User & Date: dan 2019-04-12 20:33:17
Context
2019-04-12
20:33
Add the socketvfs test extension. Leaf check-in: f5b3ce94 user: dan tags: socketvfs
16:25
Test case changes so that they work with both Tcl8.6 and Tcl8.7. check-in: 7b771405 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Added ext/misc/socketvfs.c.

            1  +/*
            2  +** 2019 April 12
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +**
           13  +** A read-only VFS that reads data from a server instead of a file
           14  +** using a custom protocol over a tcp/ip socket. The VFS is named 
           15  +** "socket". The filename passed to sqlite3_open() is of the
           16  +** form "host:portnumber". For example, to connect to the server
           17  +** on port 23456 on the localhost:
           18  +**
           19  +**   sqlite3_open_v2("localhost:23456", &db, SQLITE_OPEN_READONLY, "socket");
           20  +**
           21  +** Or, if using URIs:
           22  +**
           23  +**   sqlite3_open("file:localhost:23456?vfs=socket", &db);
           24  +** 
           25  +** The protocol is:
           26  +**
           27  +**   * Client connects to tcp/ip server. Server immediately sends the
           28  +**     database file-size in bytes as a 64-bit big-endian integer.
           29  +**
           30  +**   * To read from the file, client sends the byte offset and amount
           31  +**     of data required in bytes, both as 64-bit big-endian integers
           32  +**     (i.e. a 16-byte message). Server sends back the requested data.
           33  +**
           34  +** As well as the usual SQLite loadable extension entry point, this file
           35  +** exports one more function:
           36  +**
           37  +**     sqlite3_vfs *sqlite3_socketvfs(void);
           38  +**
           39  +** To install the "socket" VFS without loading the extension, link this file
           40  +** into the application and invoke:
           41  +**
           42  +**     int bDefault = 0;          // Do not make "socket" the default VFS
           43  +**     sqlite3_vfs_register(sqlite3_socketvfs(), bDefault);
           44  +**
           45  +*/
           46  +
           47  +#include "sqlite3ext.h"
           48  +SQLITE_EXTENSION_INIT1
           49  +
           50  +#include <assert.h>
           51  +#include <string.h>
           52  +
           53  +#if defined(_WIN32)
           54  +# if defined(_WIN32_WINNT)
           55  +#  undef _WIN32_WINNT
           56  +# endif
           57  +# define _WIN32_WINNT 0x501
           58  +#endif
           59  +#ifndef __EXTENSIONS__
           60  +# define __EXTENSIONS__ 1  /* IPv6 won't compile on Solaris without this */
           61  +#endif
           62  +#if defined(_WIN32)
           63  +#  include <winsock2.h>
           64  +#  include <ws2tcpip.h>
           65  +#  include <Windows.h>
           66  +#  include <time.h>
           67  +#else
           68  +#  include <netinet/in.h>
           69  +#  include <arpa/inet.h>
           70  +#  include <sys/socket.h>
           71  +#  include <netdb.h>
           72  +#  include <time.h>
           73  +#endif
           74  +#include <assert.h>
           75  +#include <sys/types.h>
           76  +#include <signal.h>
           77  +
           78  +#if !defined(_WIN32)
           79  +# include <unistd.h>
           80  +#endif
           81  +
           82  +/*
           83  +** When using this VFS, the sqlite3_file* handles that SQLite uses are
           84  +** actually pointers to instances of type SocketFile.
           85  +*/
           86  +typedef struct SocketFile SocketFile;
           87  +struct SocketFile {
           88  +  sqlite3_file base;              /* Base class. Must be first. */
           89  +  int iSocket;                    /* Socket used to talk to server. */
           90  +  sqlite3_int64 szFile;           /* Size of file in bytes */
           91  +};
           92  +
           93  +static sqlite3_uint64 socketGetU64(const unsigned char *a){
           94  +  return (((sqlite3_uint64)(a[0])) << 56)
           95  +       + (((sqlite3_uint64)(a[1])) << 48)
           96  +       + (((sqlite3_uint64)(a[2])) << 40)
           97  +       + (((sqlite3_uint64)(a[3])) << 32)
           98  +       + (((sqlite3_uint64)(a[4])) << 24)
           99  +       + (((sqlite3_uint64)(a[5])) << 16)
          100  +       + (((sqlite3_uint64)(a[6])) <<  8)
          101  +       + (((sqlite3_uint64)(a[7])) <<  0);
          102  +}
          103  +
          104  +static void socketPutU64(unsigned char *a, sqlite3_int64 i){
          105  +  a[0] = ((i >> 56) & 0xFF);
          106  +  a[1] = ((i >> 48) & 0xFF);
          107  +  a[2] = ((i >> 40) & 0xFF);
          108  +  a[3] = ((i >> 32) & 0xFF);
          109  +  a[4] = ((i >> 24) & 0xFF);
          110  +  a[5] = ((i >> 16) & 0xFF);
          111  +  a[6] = ((i >>  8) & 0xFF);
          112  +  a[7] = ((i >>  0) & 0xFF);
          113  +}
          114  +
          115  +static void socket_close(int iSocket){
          116  +  if( iSocket>=0 ){
          117  +#if defined(_WIN32)
          118  +    if( shutdown(iSocket,1)==0 ) shutdown(iSocket,0);
          119  +    closesocket(iSocket);
          120  +#else
          121  +    close(iSocket);
          122  +#endif
          123  +  }
          124  +}
          125  +
          126  +/*
          127  +** Write nData bytes of data from buffer aData to socket iSocket. If
          128  +** successful, return SQLITE_OK. Otherwise, SQLITE_IOERR_WRITE.
          129  +*/
          130  +static int socket_send(int iSocket, const unsigned char *aData, int nData){
          131  +  int nWrite = 0;
          132  +  do{
          133  +    int res = send(iSocket, (const char*)&aData[nWrite], nData-nWrite, 0);
          134  +    if( res<=0 ) return SQLITE_IOERR_WRITE;
          135  +    nWrite += res;
          136  +  }while( nWrite<nData );
          137  +  return SQLITE_OK;
          138  +}
          139  +
          140  +/*
          141  +** Read nData bytes of data from socket iSocket into buffer aData. If
          142  +** successful, return SQLITE_OK. Otherwise, SQLITE_IOERR_READ.
          143  +*/
          144  +static int socket_recv(int iSocket, unsigned char *aData, int nData){
          145  +  int nRead = 0;
          146  +  do{
          147  +    int res = recv(iSocket, (char*)&aData[nRead], nData-nRead, 0);
          148  +    if( res<=0 ) return SQLITE_IOERR_READ;
          149  +    nRead += res;
          150  +  }while( nRead<nData );
          151  +  return SQLITE_OK;
          152  +}
          153  +
          154  +/*
          155  +** Close a SocketFile file.
          156  +*/
          157  +static int socketClose(sqlite3_file *pFile){
          158  +  SocketFile *pSock = (SocketFile*)pFile;
          159  +  socket_close(pSock->iSocket);
          160  +  pSock->iSocket = -1;
          161  +  return SQLITE_OK;
          162  +}
          163  +
          164  +/*
          165  +** Read data from a SocketFile file.
          166  +*/
          167  +static int socketRead(
          168  +  sqlite3_file *pFile, 
          169  +  void *zBuf, 
          170  +  int iAmt, 
          171  +  sqlite3_int64 iOfst
          172  +){
          173  +  SocketFile *pSock = (SocketFile*)pFile;
          174  +  unsigned char aRequest[16];
          175  +  int rc = SQLITE_OK;
          176  +  int nRead = iAmt;
          177  +
          178  +  if( iOfst+nRead>pSock->szFile ){
          179  +    nRead = (int)(pSock->szFile - iOfst);
          180  +    memset(zBuf, 0, iAmt);
          181  +    rc = SQLITE_IOERR_SHORT_READ;
          182  +  }
          183  +
          184  +  if( nRead>0 ){
          185  +    socketPutU64(&aRequest[0], (sqlite3_uint64)iOfst);
          186  +    socketPutU64(&aRequest[8], (sqlite3_uint64)nRead);
          187  +    rc = socket_send(pSock->iSocket, aRequest, sizeof(aRequest));
          188  +    if( rc==SQLITE_OK ){
          189  +      rc = socket_recv(pSock->iSocket, zBuf, nRead);
          190  +    }
          191  +  }
          192  +
          193  +  return rc;
          194  +}
          195  +
          196  +/*
          197  +** Write to a file. This is a no-op, as this VFS is always opens files
          198  +** read-only.
          199  +*/
          200  +static int socketWrite(
          201  +  sqlite3_file *pFile, 
          202  +  const void *zBuf, 
          203  +  int iAmt, 
          204  +  sqlite3_int64 iOfst
          205  +){
          206  +  return SQLITE_IOERR_WRITE;
          207  +}
          208  +
          209  +/*
          210  +** Truncate a file. This is a no-op, as this VFS is always opens files
          211  +** read-only.
          212  +*/
          213  +static int socketTruncate(sqlite3_file *pFile, sqlite3_int64 size){
          214  +  return SQLITE_IOERR_TRUNCATE;
          215  +}
          216  +
          217  +/*
          218  +** Synk a file. This is a no-op, as this VFS is always opens files
          219  +** read-only.
          220  +*/
          221  +static int socketSync(sqlite3_file *pFile, int flags){
          222  +  return SQLITE_IOERR_FSYNC;
          223  +}
          224  +
          225  +/*
          226  +** Write the size of the file in bytes to *pSize.
          227  +*/
          228  +static int socketFileSize(sqlite3_file *pFile, sqlite3_int64 *pSize){
          229  +  SocketFile *pSock = (SocketFile*)pFile;
          230  +  *pSize = pSock->szFile;
          231  +  return SQLITE_OK;
          232  +}
          233  +
          234  +/*
          235  +** Locking functions. All no-ops.
          236  +*/
          237  +static int socketLock(sqlite3_file *pFile, int eLock){
          238  +  return SQLITE_OK;
          239  +}
          240  +static int socketUnlock(sqlite3_file *pFile, int eLock){
          241  +  return SQLITE_OK;
          242  +}
          243  +static int socketCheckReservedLock(sqlite3_file *pFile, int *pResOut){
          244  +  *pResOut = 0;
          245  +  return SQLITE_OK;
          246  +}
          247  +
          248  +/*
          249  +** No xFileControl() verbs are implemented by this VFS.
          250  +*/
          251  +static int socketFileControl(sqlite3_file *pFile, int op, void *pArg){
          252  +  return SQLITE_OK;
          253  +}
          254  +
          255  +/*
          256  +** The xSectorSize() and xDeviceCharacteristics() methods. These two
          257  +** may return special values allowing SQLite to optimize file-system 
          258  +** access to some extent. But it is also safe to simply return 0.
          259  +*/
          260  +static int socketSectorSize(sqlite3_file *pFile){
          261  +  return 0;
          262  +}
          263  +static int socketDeviceCharacteristics(sqlite3_file *pFile){
          264  +  return 0;
          265  +}
          266  +
          267  +/*
          268  +** Open a SocketFile file.
          269  +*/
          270  +static int socketOpen(
          271  +  sqlite3_vfs *pVfs,              /* VFS */
          272  +  const char *zName,              /* File to open, or 0 for a temp file */
          273  +  sqlite3_file *pFile,            /* Pointer to SocketFile struct to populate */
          274  +  int flags,                      /* Input SQLITE_OPEN_XXX flags */
          275  +  int *pOutFlags                  /* Output SQLITE_OPEN_XXX flags (or NULL) */
          276  +){
          277  +  static const sqlite3_io_methods socketio = {
          278  +    1,                            /* iVersion */
          279  +    socketClose,                  /* xClose */
          280  +    socketRead,                   /* xRead */
          281  +    socketWrite,                  /* xWrite */
          282  +    socketTruncate,               /* xTruncate */
          283  +    socketSync,                   /* xSync */
          284  +    socketFileSize,               /* xFileSize */
          285  +    socketLock,                   /* xLock */
          286  +    socketUnlock,                 /* xUnlock */
          287  +    socketCheckReservedLock,      /* xCheckReservedLock */
          288  +    socketFileControl,            /* xFileControl */
          289  +    socketSectorSize,             /* xSectorSize */
          290  +    socketDeviceCharacteristics   /* xDeviceCharacteristics */
          291  +  };
          292  +
          293  +  SocketFile *pSock = (SocketFile*)pFile;
          294  +
          295  +  char zHost[1024];
          296  +  const char *zPort;
          297  +  int i;
          298  +
          299  +  struct addrinfo hints;
          300  +  struct addrinfo *ai = 0;
          301  +  struct addrinfo *pInfo;
          302  +  unsigned char aFileSize[8];
          303  +
          304  +  pSock->iSocket = -1;
          305  +  if( (flags & SQLITE_OPEN_MAIN_DB)==0 ) return SQLITE_CANTOPEN;
          306  +
          307  +  /* Parse the argument and copy the results to zHost and zPort. It should be
          308  +  ** "hostname:port". Anything else is an error.  */
          309  +  assert( sizeof(zHost)>=pVfs->mxPathname );
          310  +  if( zName==0 ) return SQLITE_CANTOPEN;
          311  +  for(i=0; zName[i] && zName[i]!=':'; i++);
          312  +  if( zName[i]==0 ) return SQLITE_CANTOPEN;
          313  +  memcpy(zHost, zName, i);
          314  +  zHost[i] = '\0';
          315  +  zPort = &zName[i+1];
          316  +
          317  +  /* Resolve the address */
          318  +  memset(&hints, 0, sizeof(hints));
          319  +  hints.ai_family = AF_UNSPEC;
          320  +  hints.ai_socktype = SOCK_STREAM;
          321  +  hints.ai_protocol = IPPROTO_TCP;
          322  +  if( getaddrinfo(zHost, zPort, &hints, &ai) ){
          323  +    return SQLITE_CANTOPEN;
          324  +  }
          325  +
          326  +  /* Connect to the resolved address. Set SocketFile.iSocket to the tcp/ip
          327  +  ** socket and return SQLITE_OK.  */
          328  +  for(pInfo=ai; pInfo; pInfo=pInfo->ai_next){
          329  +    int sd = socket(pInfo->ai_family, pInfo->ai_socktype, pInfo->ai_protocol);
          330  +    if( sd<0 ) continue;
          331  +    if( connect(sd, pInfo->ai_addr, pInfo->ai_addrlen)<0 ){
          332  +      socket_close(sd);
          333  +      continue;
          334  +    }
          335  +    pSock->iSocket = sd;
          336  +    break;
          337  +  }
          338  +
          339  +  if( ai ) freeaddrinfo(ai);
          340  +  if( pSock->iSocket<0 ) return SQLITE_CANTOPEN;
          341  +
          342  +  /* The server sends back the file size as a 64-bit big-endian */
          343  +  if( socket_recv(pSock->iSocket, aFileSize, 8) ){
          344  +    socket_close(pSock->iSocket);
          345  +    return SQLITE_CANTOPEN;
          346  +  }
          347  +  pSock->szFile = (sqlite3_int64)socketGetU64(aFileSize);
          348  +
          349  +  *pOutFlags = flags & ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE);
          350  +  *pOutFlags |= SQLITE_OPEN_READONLY;
          351  +  pSock->base.pMethods = &socketio;
          352  +  return SQLITE_OK;
          353  +}
          354  +
          355  +/*
          356  +** Another no-op. This is a read-only VFS.
          357  +*/
          358  +static int socketDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
          359  +  return SQLITE_IOERR_DELETE;
          360  +}
          361  +
          362  +/*
          363  +** This is used by SQLite to detect journal and wal files. Which cannot
          364  +** exist for this VFS. So always set the output to false and return 
          365  +** SQLITE_OK.
          366  +*/
          367  +static int socketAccess(
          368  +  sqlite3_vfs *pVfs, 
          369  +  const char *zPath, 
          370  +  int flags, 
          371  +  int *pResOut
          372  +){
          373  +  *pResOut = 0;
          374  +  return SQLITE_OK;
          375  +}
          376  +
          377  +/*
          378  +** A no-op. Copy the input to the output.
          379  +*/
          380  +static int socketFullPathname(
          381  +  sqlite3_vfs *pVfs,              /* VFS */
          382  +  const char *zPath,              /* Input path (possibly a relative path) */
          383  +  int nPathOut,                   /* Size of output buffer in bytes */
          384  +  char *zPathOut                  /* Pointer to output buffer */
          385  +){
          386  +  int nByte = strlen(zPath);
          387  +  if( nByte>=pVfs->mxPathname ) return SQLITE_IOERR;
          388  +  memcpy(zPathOut, zPath, nByte+1);
          389  +  return SQLITE_OK;
          390  +}
          391  +
          392  +/*
          393  +** The following four VFS methods:
          394  +**
          395  +**   xDlOpen
          396  +**   xDlError
          397  +**   xDlSym
          398  +**   xDlClose
          399  +**
          400  +** are supposed to implement the functionality needed by SQLite to load
          401  +** extensions compiled as shared objects. This simple VFS does not support
          402  +** this functionality, so the following functions are no-ops.
          403  +*/
          404  +static void *socketDlOpen(sqlite3_vfs *pVfs, const char *zPath){
          405  +  return 0;
          406  +}
          407  +static void socketDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
          408  +  sqlite3_snprintf(nByte, zErrMsg, "Loadable extensions are not supported");
          409  +  zErrMsg[nByte-1] = '\0';
          410  +}
          411  +static void (*socketDlSym(sqlite3_vfs *pVfs, void *pH, const char *z))(void){
          412  +  return 0;
          413  +}
          414  +static void socketDlClose(sqlite3_vfs *pVfs, void *pHandle){
          415  +  return;
          416  +}
          417  +
          418  +/*
          419  +** Parameter zByte points to a buffer nByte bytes in size. Populate this
          420  +** buffer with pseudo-random data.
          421  +*/
          422  +static int socketRandomness(sqlite3_vfs *pVfs, int nByte, char *zByte){
          423  +  memset(zByte, 0, nByte);
          424  +  return SQLITE_OK;
          425  +}
          426  +
          427  +/*
          428  +** Sleep for at least nMicro microseconds. Return the (approximate) number 
          429  +** of microseconds slept for.
          430  +*/
          431  +static int socketSleep(sqlite3_vfs *pVfs, int nMicro){
          432  +#ifdef _WIN32
          433  +  Sleep(nMicro/1000);
          434  +#else
          435  +  sleep(nMicro / 1000000);
          436  +  usleep(nMicro % 1000000);
          437  +#endif
          438  +  return nMicro;
          439  +}
          440  +
          441  +/*
          442  +** Set *pTime to the current UTC time expressed as a Julian day. Return
          443  +** SQLITE_OK if successful, or an error code otherwise.
          444  +**
          445  +**   http://en.wikipedia.org/wiki/Julian_day
          446  +**
          447  +** This implementation is not very good. The current time is rounded to
          448  +** an integer number of seconds. Also, assuming time_t is a signed 32-bit 
          449  +** value, it will stop working some time in the year 2038 AD (the so-called
          450  +** "year 2038" problem that afflicts systems that store time this way). 
          451  +*/
          452  +static int socketCurrentTime(sqlite3_vfs *pVfs, double *pTime){
          453  +  time_t t = time(0);
          454  +  *pTime = t/86400.0 + 2440587.5; 
          455  +  return SQLITE_OK;
          456  +}
          457  +
          458  +/*
          459  +** This function returns a pointer to the VFS implemented in this file.
          460  +** To make the VFS available to SQLite:
          461  +**
          462  +**   sqlite3_vfs_register(sqlite3_socketvfs(), 0);
          463  +*/
          464  +sqlite3_vfs *sqlite3_socketvfs(void){
          465  +  static sqlite3_vfs socketvfs = {
          466  +    1,                            /* iVersion */
          467  +    sizeof(SocketFile),           /* szOsFile */
          468  +    512,                          /* mxPathname */
          469  +    0,                            /* pNext */
          470  +    "socket",                     /* zName */
          471  +    0,                            /* pAppData */
          472  +    socketOpen,                   /* xOpen */
          473  +    socketDelete,                 /* xDelete */
          474  +    socketAccess,                 /* xAccess */
          475  +    socketFullPathname,           /* xFullPathname */
          476  +    socketDlOpen,                 /* xDlOpen */
          477  +    socketDlError,                /* xDlError */
          478  +    socketDlSym,                  /* xDlSym */
          479  +    socketDlClose,                /* xDlClose */
          480  +    socketRandomness,             /* xRandomness */
          481  +    socketSleep,                  /* xSleep */
          482  +    socketCurrentTime,            /* xCurrentTime */
          483  +  };
          484  +  return &socketvfs;
          485  +}
          486  +
          487  +/*
          488  +** Register the amatch virtual table
          489  +*/
          490  +#ifdef _WIN32
          491  +__declspec(dllexport)
          492  +#endif
          493  +int sqlite3_socketvfs_init(
          494  +  sqlite3 *db, 
          495  +  char **pzErrMsg, 
          496  +  const sqlite3_api_routines *pApi
          497  +){
          498  +  int rc = SQLITE_OK;
          499  +  SQLITE_EXTENSION_INIT2(pApi);
          500  +  (void)pzErrMsg;  /* Not used */
          501  +  sqlite3_vfs_register(sqlite3_socketvfs(), 0);
          502  +  return SQLITE_OK_LOAD_PERMANENTLY;
          503  +}
          504  +

Added ext/misc/socketvfs_server.tcl.

            1  +#!/bin/sh
            2  +# \
            3  +exec tclsh "$0" ${1+"$@"}
            4  +
            5  +if {[llength $argv]!=2} {
            6  +  puts stderr "Usage: $argv0 <filename> <port>"
            7  +  exit -1
            8  +}
            9  +set G(filename) [lindex $argv 0]
           10  +set G(port) [lindex $argv 1]
           11  +
           12  +proc new_message {chan} {
           13  +  global G
           14  +  if {[eof $chan]} {
           15  +    close $chan
           16  +    puts "Close channel $chan"
           17  +  } else {
           18  +    set msg [read $chan 16]
           19  +    if {[string length $msg]>0} {
           20  +      binary scan $msg WW offset amt
           21  +      # puts "Request from $chan for $amt bytes at offset $offset"
           22  +      seek $G(fd) $offset
           23  +      set data [read $G(fd) $amt]
           24  +      puts -nonewline $chan $data
           25  +      flush $chan
           26  +    }
           27  +  }
           28  +}
           29  +
           30  +proc new_connection {chan addr port} {
           31  +  global G
           32  +
           33  +  set sz [file size $G(filename)]
           34  +
           35  +  puts -nonewline "$addr:$port connects! "
           36  +  puts "Sending file size ($sz) as a 64-bit big-endian integer."
           37  +  set bin [binary format W $sz]
           38  +  puts -nonewline $chan $bin
           39  +  flush $chan
           40  +
           41  +  fconfigure $chan -encoding binary
           42  +  fconfigure $chan -translation binary
           43  +  fileevent $chan readable [list new_message $chan]
           44  +}  
           45  +
           46  +set G(fd) [open $G(filename) r]
           47  +fconfigure $G(fd) -encoding binary
           48  +fconfigure $G(fd) -translation binary
           49  +
           50  +socket -server new_connection $G(port)
           51  +vwait forever
           52  +