/ Check-in [f5b3ce94]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the socketvfs test extension.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | socketvfs
Files: files | file ages | folders
SHA3-256: f5b3ce9404ffc8afdab6d8e57f806afba0b3ee75520d5e03335335d0f93d4a4b
User & Date: dan 2019-04-12 20:33:17
Context
2019-04-12
20:33
Add the socketvfs test extension. Leaf check-in: f5b3ce94 user: dan tags: socketvfs
16:25
Test case changes so that they work with both Tcl8.6 and Tcl8.7. check-in: 7b771405 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Added ext/misc/socketvfs.c.

















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
/*
** 2019 April 12
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
**
** A read-only VFS that reads data from a server instead of a file
** using a custom protocol over a tcp/ip socket. The VFS is named 
** "socket". The filename passed to sqlite3_open() is of the
** form "host:portnumber". For example, to connect to the server
** on port 23456 on the localhost:
**
**   sqlite3_open_v2("localhost:23456", &db, SQLITE_OPEN_READONLY, "socket");
**
** Or, if using URIs:
**
**   sqlite3_open("file:localhost:23456?vfs=socket", &db);
** 
** The protocol is:
**
**   * Client connects to tcp/ip server. Server immediately sends the
**     database file-size in bytes as a 64-bit big-endian integer.
**
**   * To read from the file, client sends the byte offset and amount
**     of data required in bytes, both as 64-bit big-endian integers
**     (i.e. a 16-byte message). Server sends back the requested data.
**
** As well as the usual SQLite loadable extension entry point, this file
** exports one more function:
**
**     sqlite3_vfs *sqlite3_socketvfs(void);
**
** To install the "socket" VFS without loading the extension, link this file
** into the application and invoke:
**
**     int bDefault = 0;          // Do not make "socket" the default VFS
**     sqlite3_vfs_register(sqlite3_socketvfs(), bDefault);
**
*/

#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1

#include <assert.h>
#include <string.h>

#if defined(_WIN32)
# if defined(_WIN32_WINNT)
#  undef _WIN32_WINNT
# endif
# define _WIN32_WINNT 0x501
#endif
#ifndef __EXTENSIONS__
# define __EXTENSIONS__ 1  /* IPv6 won't compile on Solaris without this */
#endif
#if defined(_WIN32)
#  include <winsock2.h>
#  include <ws2tcpip.h>
#  include <Windows.h>
#  include <time.h>
#else
#  include <netinet/in.h>
#  include <arpa/inet.h>
#  include <sys/socket.h>
#  include <netdb.h>
#  include <time.h>
#endif
#include <assert.h>
#include <sys/types.h>
#include <signal.h>

#if !defined(_WIN32)
# include <unistd.h>
#endif

/*
** When using this VFS, the sqlite3_file* handles that SQLite uses are
** actually pointers to instances of type SocketFile.
*/
typedef struct SocketFile SocketFile;
struct SocketFile {
  sqlite3_file base;              /* Base class. Must be first. */
  int iSocket;                    /* Socket used to talk to server. */
  sqlite3_int64 szFile;           /* Size of file in bytes */
};

static sqlite3_uint64 socketGetU64(const unsigned char *a){
  return (((sqlite3_uint64)(a[0])) << 56)
       + (((sqlite3_uint64)(a[1])) << 48)
       + (((sqlite3_uint64)(a[2])) << 40)
       + (((sqlite3_uint64)(a[3])) << 32)
       + (((sqlite3_uint64)(a[4])) << 24)
       + (((sqlite3_uint64)(a[5])) << 16)
       + (((sqlite3_uint64)(a[6])) <<  8)
       + (((sqlite3_uint64)(a[7])) <<  0);
}

static void socketPutU64(unsigned char *a, sqlite3_int64 i){
  a[0] = ((i >> 56) & 0xFF);
  a[1] = ((i >> 48) & 0xFF);
  a[2] = ((i >> 40) & 0xFF);
  a[3] = ((i >> 32) & 0xFF);
  a[4] = ((i >> 24) & 0xFF);
  a[5] = ((i >> 16) & 0xFF);
  a[6] = ((i >>  8) & 0xFF);
  a[7] = ((i >>  0) & 0xFF);
}

static void socket_close(int iSocket){
  if( iSocket>=0 ){
#if defined(_WIN32)
    if( shutdown(iSocket,1)==0 ) shutdown(iSocket,0);
    closesocket(iSocket);
#else
    close(iSocket);
#endif
  }
}

/*
** Write nData bytes of data from buffer aData to socket iSocket. If
** successful, return SQLITE_OK. Otherwise, SQLITE_IOERR_WRITE.
*/
static int socket_send(int iSocket, const unsigned char *aData, int nData){
  int nWrite = 0;
  do{
    int res = send(iSocket, (const char*)&aData[nWrite], nData-nWrite, 0);
    if( res<=0 ) return SQLITE_IOERR_WRITE;
    nWrite += res;
  }while( nWrite<nData );
  return SQLITE_OK;
}

/*
** Read nData bytes of data from socket iSocket into buffer aData. If
** successful, return SQLITE_OK. Otherwise, SQLITE_IOERR_READ.
*/
static int socket_recv(int iSocket, unsigned char *aData, int nData){
  int nRead = 0;
  do{
    int res = recv(iSocket, (char*)&aData[nRead], nData-nRead, 0);
    if( res<=0 ) return SQLITE_IOERR_READ;
    nRead += res;
  }while( nRead<nData );
  return SQLITE_OK;
}

/*
** Close a SocketFile file.
*/
static int socketClose(sqlite3_file *pFile){
  SocketFile *pSock = (SocketFile*)pFile;
  socket_close(pSock->iSocket);
  pSock->iSocket = -1;
  return SQLITE_OK;
}

/*
** Read data from a SocketFile file.
*/
static int socketRead(
  sqlite3_file *pFile, 
  void *zBuf, 
  int iAmt, 
  sqlite3_int64 iOfst
){
  SocketFile *pSock = (SocketFile*)pFile;
  unsigned char aRequest[16];
  int rc = SQLITE_OK;
  int nRead = iAmt;

  if( iOfst+nRead>pSock->szFile ){
    nRead = (int)(pSock->szFile - iOfst);
    memset(zBuf, 0, iAmt);
    rc = SQLITE_IOERR_SHORT_READ;
  }

  if( nRead>0 ){
    socketPutU64(&aRequest[0], (sqlite3_uint64)iOfst);
    socketPutU64(&aRequest[8], (sqlite3_uint64)nRead);
    rc = socket_send(pSock->iSocket, aRequest, sizeof(aRequest));
    if( rc==SQLITE_OK ){
      rc = socket_recv(pSock->iSocket, zBuf, nRead);
    }
  }

  return rc;
}

/*
** Write to a file. This is a no-op, as this VFS is always opens files
** read-only.
*/
static int socketWrite(
  sqlite3_file *pFile, 
  const void *zBuf, 
  int iAmt, 
  sqlite3_int64 iOfst
){
  return SQLITE_IOERR_WRITE;
}

/*
** Truncate a file. This is a no-op, as this VFS is always opens files
** read-only.
*/
static int socketTruncate(sqlite3_file *pFile, sqlite3_int64 size){
  return SQLITE_IOERR_TRUNCATE;
}

/*
** Synk a file. This is a no-op, as this VFS is always opens files
** read-only.
*/
static int socketSync(sqlite3_file *pFile, int flags){
  return SQLITE_IOERR_FSYNC;
}

/*
** Write the size of the file in bytes to *pSize.
*/
static int socketFileSize(sqlite3_file *pFile, sqlite3_int64 *pSize){
  SocketFile *pSock = (SocketFile*)pFile;
  *pSize = pSock->szFile;
  return SQLITE_OK;
}

/*
** Locking functions. All no-ops.
*/
static int socketLock(sqlite3_file *pFile, int eLock){
  return SQLITE_OK;
}
static int socketUnlock(sqlite3_file *pFile, int eLock){
  return SQLITE_OK;
}
static int socketCheckReservedLock(sqlite3_file *pFile, int *pResOut){
  *pResOut = 0;
  return SQLITE_OK;
}

/*
** No xFileControl() verbs are implemented by this VFS.
*/
static int socketFileControl(sqlite3_file *pFile, int op, void *pArg){
  return SQLITE_OK;
}

/*
** The xSectorSize() and xDeviceCharacteristics() methods. These two
** may return special values allowing SQLite to optimize file-system 
** access to some extent. But it is also safe to simply return 0.
*/
static int socketSectorSize(sqlite3_file *pFile){
  return 0;
}
static int socketDeviceCharacteristics(sqlite3_file *pFile){
  return 0;
}

/*
** Open a SocketFile file.
*/
static int socketOpen(
  sqlite3_vfs *pVfs,              /* VFS */
  const char *zName,              /* File to open, or 0 for a temp file */
  sqlite3_file *pFile,            /* Pointer to SocketFile struct to populate */
  int flags,                      /* Input SQLITE_OPEN_XXX flags */
  int *pOutFlags                  /* Output SQLITE_OPEN_XXX flags (or NULL) */
){
  static const sqlite3_io_methods socketio = {
    1,                            /* iVersion */
    socketClose,                  /* xClose */
    socketRead,                   /* xRead */
    socketWrite,                  /* xWrite */
    socketTruncate,               /* xTruncate */
    socketSync,                   /* xSync */
    socketFileSize,               /* xFileSize */
    socketLock,                   /* xLock */
    socketUnlock,                 /* xUnlock */
    socketCheckReservedLock,      /* xCheckReservedLock */
    socketFileControl,            /* xFileControl */
    socketSectorSize,             /* xSectorSize */
    socketDeviceCharacteristics   /* xDeviceCharacteristics */
  };

  SocketFile *pSock = (SocketFile*)pFile;

  char zHost[1024];
  const char *zPort;
  int i;

  struct addrinfo hints;
  struct addrinfo *ai = 0;
  struct addrinfo *pInfo;
  unsigned char aFileSize[8];

  pSock->iSocket = -1;
  if( (flags & SQLITE_OPEN_MAIN_DB)==0 ) return SQLITE_CANTOPEN;

  /* Parse the argument and copy the results to zHost and zPort. It should be
  ** "hostname:port". Anything else is an error.  */
  assert( sizeof(zHost)>=pVfs->mxPathname );
  if( zName==0 ) return SQLITE_CANTOPEN;
  for(i=0; zName[i] && zName[i]!=':'; i++);
  if( zName[i]==0 ) return SQLITE_CANTOPEN;
  memcpy(zHost, zName, i);
  zHost[i] = '\0';
  zPort = &zName[i+1];

  /* Resolve the address */
  memset(&hints, 0, sizeof(hints));
  hints.ai_family = AF_UNSPEC;
  hints.ai_socktype = SOCK_STREAM;
  hints.ai_protocol = IPPROTO_TCP;
  if( getaddrinfo(zHost, zPort, &hints, &ai) ){
    return SQLITE_CANTOPEN;
  }

  /* Connect to the resolved address. Set SocketFile.iSocket to the tcp/ip
  ** socket and return SQLITE_OK.  */
  for(pInfo=ai; pInfo; pInfo=pInfo->ai_next){
    int sd = socket(pInfo->ai_family, pInfo->ai_socktype, pInfo->ai_protocol);
    if( sd<0 ) continue;
    if( connect(sd, pInfo->ai_addr, pInfo->ai_addrlen)<0 ){
      socket_close(sd);
      continue;
    }
    pSock->iSocket = sd;
    break;
  }

  if( ai ) freeaddrinfo(ai);
  if( pSock->iSocket<0 ) return SQLITE_CANTOPEN;

  /* The server sends back the file size as a 64-bit big-endian */
  if( socket_recv(pSock->iSocket, aFileSize, 8) ){
    socket_close(pSock->iSocket);
    return SQLITE_CANTOPEN;
  }
  pSock->szFile = (sqlite3_int64)socketGetU64(aFileSize);

  *pOutFlags = flags & ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE);
  *pOutFlags |= SQLITE_OPEN_READONLY;
  pSock->base.pMethods = &socketio;
  return SQLITE_OK;
}

/*
** Another no-op. This is a read-only VFS.
*/
static int socketDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
  return SQLITE_IOERR_DELETE;
}

/*
** This is used by SQLite to detect journal and wal files. Which cannot
** exist for this VFS. So always set the output to false and return 
** SQLITE_OK.
*/
static int socketAccess(
  sqlite3_vfs *pVfs, 
  const char *zPath, 
  int flags, 
  int *pResOut
){
  *pResOut = 0;
  return SQLITE_OK;
}

/*
** A no-op. Copy the input to the output.
*/
static int socketFullPathname(
  sqlite3_vfs *pVfs,              /* VFS */
  const char *zPath,              /* Input path (possibly a relative path) */
  int nPathOut,                   /* Size of output buffer in bytes */
  char *zPathOut                  /* Pointer to output buffer */
){
  int nByte = strlen(zPath);
  if( nByte>=pVfs->mxPathname ) return SQLITE_IOERR;
  memcpy(zPathOut, zPath, nByte+1);
  return SQLITE_OK;
}

/*
** The following four VFS methods:
**
**   xDlOpen
**   xDlError
**   xDlSym
**   xDlClose
**
** are supposed to implement the functionality needed by SQLite to load
** extensions compiled as shared objects. This simple VFS does not support
** this functionality, so the following functions are no-ops.
*/
static void *socketDlOpen(sqlite3_vfs *pVfs, const char *zPath){
  return 0;
}
static void socketDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
  sqlite3_snprintf(nByte, zErrMsg, "Loadable extensions are not supported");
  zErrMsg[nByte-1] = '\0';
}
static void (*socketDlSym(sqlite3_vfs *pVfs, void *pH, const char *z))(void){
  return 0;
}
static void socketDlClose(sqlite3_vfs *pVfs, void *pHandle){
  return;
}

/*
** Parameter zByte points to a buffer nByte bytes in size. Populate this
** buffer with pseudo-random data.
*/
static int socketRandomness(sqlite3_vfs *pVfs, int nByte, char *zByte){
  memset(zByte, 0, nByte);
  return SQLITE_OK;
}

/*
** Sleep for at least nMicro microseconds. Return the (approximate) number 
** of microseconds slept for.
*/
static int socketSleep(sqlite3_vfs *pVfs, int nMicro){
#ifdef _WIN32
  Sleep(nMicro/1000);
#else
  sleep(nMicro / 1000000);
  usleep(nMicro % 1000000);
#endif
  return nMicro;
}

/*
** Set *pTime to the current UTC time expressed as a Julian day. Return
** SQLITE_OK if successful, or an error code otherwise.
**
**   http://en.wikipedia.org/wiki/Julian_day
**
** This implementation is not very good. The current time is rounded to
** an integer number of seconds. Also, assuming time_t is a signed 32-bit 
** value, it will stop working some time in the year 2038 AD (the so-called
** "year 2038" problem that afflicts systems that store time this way). 
*/
static int socketCurrentTime(sqlite3_vfs *pVfs, double *pTime){
  time_t t = time(0);
  *pTime = t/86400.0 + 2440587.5; 
  return SQLITE_OK;
}

/*
** This function returns a pointer to the VFS implemented in this file.
** To make the VFS available to SQLite:
**
**   sqlite3_vfs_register(sqlite3_socketvfs(), 0);
*/
sqlite3_vfs *sqlite3_socketvfs(void){
  static sqlite3_vfs socketvfs = {
    1,                            /* iVersion */
    sizeof(SocketFile),           /* szOsFile */
    512,                          /* mxPathname */
    0,                            /* pNext */
    "socket",                     /* zName */
    0,                            /* pAppData */
    socketOpen,                   /* xOpen */
    socketDelete,                 /* xDelete */
    socketAccess,                 /* xAccess */
    socketFullPathname,           /* xFullPathname */
    socketDlOpen,                 /* xDlOpen */
    socketDlError,                /* xDlError */
    socketDlSym,                  /* xDlSym */
    socketDlClose,                /* xDlClose */
    socketRandomness,             /* xRandomness */
    socketSleep,                  /* xSleep */
    socketCurrentTime,            /* xCurrentTime */
  };
  return &socketvfs;
}

/*
** Register the amatch virtual table
*/
#ifdef _WIN32
__declspec(dllexport)
#endif
int sqlite3_socketvfs_init(
  sqlite3 *db, 
  char **pzErrMsg, 
  const sqlite3_api_routines *pApi
){
  int rc = SQLITE_OK;
  SQLITE_EXTENSION_INIT2(pApi);
  (void)pzErrMsg;  /* Not used */
  sqlite3_vfs_register(sqlite3_socketvfs(), 0);
  return SQLITE_OK_LOAD_PERMANENTLY;
}

Added ext/misc/socketvfs_server.tcl.









































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
#!/bin/sh
# \
exec tclsh "$0" ${1+"$@"}

if {[llength $argv]!=2} {
  puts stderr "Usage: $argv0 <filename> <port>"
  exit -1
}
set G(filename) [lindex $argv 0]
set G(port) [lindex $argv 1]

proc new_message {chan} {
  global G
  if {[eof $chan]} {
    close $chan
    puts "Close channel $chan"
  } else {
    set msg [read $chan 16]
    if {[string length $msg]>0} {
      binary scan $msg WW offset amt
      # puts "Request from $chan for $amt bytes at offset $offset"
      seek $G(fd) $offset
      set data [read $G(fd) $amt]
      puts -nonewline $chan $data
      flush $chan
    }
  }
}

proc new_connection {chan addr port} {
  global G

  set sz [file size $G(filename)]

  puts -nonewline "$addr:$port connects! "
  puts "Sending file size ($sz) as a 64-bit big-endian integer."
  set bin [binary format W $sz]
  puts -nonewline $chan $bin
  flush $chan

  fconfigure $chan -encoding binary
  fconfigure $chan -translation binary
  fileevent $chan readable [list new_message $chan]
}  

set G(fd) [open $G(filename) r]
fconfigure $G(fd) -encoding binary
fconfigure $G(fd) -translation binary

socket -server new_connection $G(port)
vwait forever