/ Check-in [d9d5cc62]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Modifications to the zonefile module to make it easier to add a cache of uncompressed frame content.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | zonefile
Files: files | file ages | folders
SHA3-256:d9d5cc62f11058f9ba560381367ff4765dbbde08184e55abdb50ae1b6bf4a016
User & Date: dan 2018-02-21 21:15:45
Context
2018-02-22
16:46
Add an LRU cache of uncompressed frame content to the zonefile virtual table implementation. check-in: 883e7e75 user: dan tags: zonefile
2018-02-21
21:15
Modifications to the zonefile module to make it easier to add a cache of uncompressed frame content. check-in: d9d5cc62 user: dan tags: zonefile
16:36
Have zonefile store encryption keys in a hash-table instead of a linked list. Add extra tests for key management. check-in: 3a63ea65 user: dan tags: zonefile
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/zonefile/zonefile.c.

12
13
14
15
16
17
18

19
20






















































21
22
23
24
25
26
27
..
35
36
37
38
39
40
41

















42
43
44
45
46
47
48
...
153
154
155
156
157
158
159



160
161
162
163
164
165
166
...
298
299
300
301
302
303
304
305

306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325

326
327
328
329
330
331




332
333
334
335
336
337
338




339
340
341
342
343
344
345
...
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675



676
677
678
679
680
681
682
...
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
...
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
....
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142

1143

1144
1145
1146
1147
1148
1149
1150
....
1889
1890
1891
1892
1893
1894
1895

1896
1897
1898
1899
1900
1901
1902











































































1903
1904
1905
1906
1907
1908
1909
....
1924
1925
1926
1927
1928
1929
1930

1931
1932
1933
1934
1935
1936
1937
....
1961
1962
1963
1964
1965
1966
1967




1968
1969
1970
1971
1972
1973
1974
....
2272
2273
2274
2275
2276
2277
2278
2279























































2280










2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345

2346
2347
2348





2349




2350
2351
2352
2353



















2354


2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
....
2403
2404
2405
2406
2407
2408
2409
2410




2411
2412
2413
2414
2415
2416
2417
*/

#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#ifndef SQLITE_OMIT_VIRTUALTABLE

#include <stdio.h>

#include <string.h>
#include <assert.h>























































#ifndef SQLITE_AMALGAMATION
typedef sqlite3_int64 i64;
typedef sqlite3_uint64 u64;
typedef unsigned char u8;
typedef unsigned short u16;
typedef unsigned long u32;
................................................................................
# define NEVER(X)       ((X)?(assert(0),1):0)
#else
# define ALWAYS(X)      (X)
# define NEVER(X)       (X)
#endif
#endif   /* SQLITE_AMALGAMATION */


















#ifndef SQLITE_HAVE_ZONEFILE_CODEC
typedef struct ZonefileCodec ZonefileCodec;

struct ZonefileCodec {
  u8 aKey[16];
};

................................................................................
  i64 iFileid;                    /* File id */
  const char *zKey;               /* Key buffer */
  int nKey;                       /* Size of zKey in bytes */
  u32 iHash;                      /* zonefileKeyHash() value */
  ZonefileKey *pHashNext;         /* Next colliding key in hash table */
};




static u32 zonefileKeyHash(
  const char *zDb,
  const char *zTab,
  i64 iFileid
){
  u32 iHash = 0;
  int i;
................................................................................
      sqlite3_free(pKey);
    }
  }
  sqlite3_free(pGlobal->aHash);
  sqlite3_free(pGlobal);
}



#define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024)
#define ZONEFILE_DEFAULT_ENCRYPTION       1
#define ZONEFILE_DEFAULT_COMPRESSION      0
#define ZONEFILE_DEFAULT_DICTSIZE         (64*1024)

#define ZONEFILE_MAGIC_NUMBER 0x464B3138

#define ZONEFILE_SZ_HEADER           32
#define ZONEFILE_SZ_KEYOFFSETS_ENTRY 20


#define ZONEFILE_COMPRESSION_NONE             0
#define ZONEFILE_COMPRESSION_ZSTD             1
#define ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT 2
#define ZONEFILE_COMPRESSION_ZLIB             3
#define ZONEFILE_COMPRESSION_BROTLI           4
#define ZONEFILE_COMPRESSION_LZ4              5
#define ZONEFILE_COMPRESSION_LZ4HC            6



static void zonefilePut32(u8 *aBuf, u32 v){
  aBuf[0] = (v >> 24) & 0xFF;
  aBuf[1] = (v >> 16) & 0xFF;
  aBuf[2] = (v >>  8) & 0xFF;
  aBuf[3] = v & 0xFF;
}




static u32 zonefileGet32(const u8 *aBuf){
  return (((u32)aBuf[0]) << 24)
       + (((u32)aBuf[1]) << 16)
       + (((u32)aBuf[2]) <<  8)
       + (((u32)aBuf[3]) <<  0);
}





static int zfGenericOpen(void **pp, u8 *aDict, int nDict){
  *pp = 0;
  return SQLITE_OK;
}
static void zfGenericClose(void *p){
}
static int zfGenericUncompressSize(
................................................................................
  return 0;
}

/* End of code for compression routines
**************************************************************************/



#define ZONEFILE_SCHEMA          \
  "CREATE TABLE z1("             \
  "  k INTEGER PRIMARY KEY,"     \
  "  v BLOB,"                    \
  "  fileid INTEGER,"            \
  "  sz INTEGER"                 \
  ")"

#define ZONEFILE_FILES_SCHEMA    \
  "CREATE TABLE z2("             \
  "  filename TEXT,"             \
  "  ekey BLOB,"                 \
  "  header JSON HIDDEN"         \
  ")"



/*
** A structure to store the parameters for a single zonefile_write()
** invocation. 



*/
typedef struct ZonefileParam ZonefileParam;
struct ZonefileParam {
  ZonefileCompress *pCmpIdx;      /* For compressing the index */
  ZonefileCompress *pCmpData;     /* For compressing each frame */
  int encryptionType;
  int maxAutoFrameSize;
................................................................................
  u8 encryptionType;
  u8 encryptionKeyIdx;
  u8 extendedHeaderVersion;
  u8 extendedHeaderSize;
};

/*
** Buffer structure used by the zonefile_write() implementation.
*/
typedef struct ZonefileBuffer ZonefileBuffer;
struct ZonefileBuffer {
  u8 *a;
  int n;
  int nAlloc;
};
................................................................................
    sqlite3_free(zSql);
  }else{
    rc = SQLITE_NOMEM;
  }
  return rc;
}


static sqlite3_stmt *zonefileCtxPrepare(
  sqlite3_context *pCtx,
  const char *zFmt,
  ...
){
  sqlite3_stmt *pRet = 0;
  va_list ap;
  char *zSql;
  va_start(ap, zFmt);
  zSql = sqlite3_vmprintf(zFmt, ap);
  if( zSql ){
    sqlite3 *db = sqlite3_context_db_handle(pCtx);
    int rc = sqlite3_prepare(db, zSql, -1, &pRet, 0);
    if( rc!=SQLITE_OK ){
      zonefileTransferError(pCtx);
    }
    sqlite3_free(zSql);
  }else{
    sqlite3_result_error_nomem(pCtx);
  }
  return pRet;
}

/*
** Return zero if the two SQL values passed as arguments are equal, or
** non-zero otherwise. Values with different types are considered unequal,
** even if they both contain the same numeric value (e.g. 2 and 2.0).
*/
static int zonefileCompareValue(sqlite3_value *p1, sqlite3_value *p2){
  int eType;
................................................................................
      zonefileCtxError(pCtx, "error in compressor construction");
      goto zone_write_out;
    }
  }

  /* Prepare the SQL statement used to read data from the source table. This
  ** also serves to verify the suitability of the source table schema. */
  pStmt = zonefileCtxPrepare(pCtx, 
      "SELECT k, frame, v FROM %Q ORDER BY frame, idx, k", zTbl
  );
  if( pStmt==0 ) goto zone_write_out;

  /* Open the file-handle used to write out the zonefile */ 

  pFd = zonefileFileOpen(zFile, 1, &zErr);

  if( pFd==0 ){
    sqlite3_result_error(pCtx, zErr, -1);
    sqlite3_free(zErr);
    goto zone_write_out;
  }

  /* If the data compressor uses a global dictionary, create the dictionary
................................................................................
struct ZonefileTab {
  sqlite3_vtab base;         /* Base class - must be first */
  sqlite3 *db;
  sqlite3_stmt *pIdToName;   /* Translate fileid to filename */
  ZonefileGlobal *pGlobal;
  char *zName;               /* Name of this table */
  char *zDb;                 /* Name of db containing this table */

};

typedef struct ZonefileCsr ZonefileCsr;
struct ZonefileCsr {
  sqlite3_vtab_cursor base;  /* Base class - must be first */
  sqlite3_stmt *pSelect;     /* SELECT on %_shadow_idx table */
};












































































/*
** This function does the work of xCreate (if bCreate!=0) or xConnect
** (if bCreate==0) for the zonefile module.
**
**   argv[0]   -> module name  ("zonefile")
**   argv[1]   -> database name
................................................................................
  int nDb = strlen(zDb);
  int rc = SQLITE_OK;

  p = (ZonefileTab*)sqlite3_malloc(sizeof(ZonefileTab) + nName+1 + nDb+1);
  if( !p ){
    rc = SQLITE_NOMEM;
  }else{

    memset(p, 0, sizeof(ZonefileTab));
    p->zName = (char*)&p[1];
    memcpy(p->zName, zName, nName+1);
    p->zDb = &p->zName[nName+1];
    memcpy(p->zDb, zDb, nDb+1);
    p->db = db;
    p->pGlobal = (ZonefileGlobal*)pAux;
................................................................................
        sqlite3_free(zSql);
      }
    }
    
    if( rc==SQLITE_OK ){
      rc = sqlite3_declare_vtab(db, ZONEFILE_SCHEMA);
    }




  }

  if( rc!=SQLITE_OK ){
    sqlite3_free(p);
    p = 0;
  }
  *ppVtab = (sqlite3_vtab*)p;
................................................................................
}

static void zonefileReleaseFile(ZonefileCsr *pCsr){
  ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab;
  sqlite3_reset(pTab->pIdToName);
}

static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){























































  ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab;










  const char *zFile = 0;
  char *zErr = 0;
  FILE *pFd = 0;
  int rc = SQLITE_OK;
  ZonefileHeader hdr;
  ZonefileCompress *pCmpMethod = 0;
  ZonefileCodec *pCodec = 0;
  void *pCmp = 0;

  /* Open the file to read the blob from */
  rc = zonefileGetFile(pCtx, pCsr, &zFile);
  if( rc==SQLITE_OK ){
    pFd = zonefileFileOpen(zFile, 0, &zErr);
    if( pFd==0 ) rc = SQLITE_ERROR;
  }

  /* Read the zonefile header */
  if( rc==SQLITE_OK ){
    rc = zonefileReadHeader(pFd, zFile, &hdr, &zErr);
  }

  /* Find the compression method and open the compressor handle. */
  if( rc==SQLITE_OK ){
    rc = zfFindCompress(hdr.compressionTypeContent, &pCmpMethod, &zErr);
  }
  if( pCmpMethod ){
    int nDict = 0;
    u8 *aDict = 0;
    assert( rc==SQLITE_OK );
    if( hdr.byteOffsetDictionary ){
      nDict = hdr.byteOffsetFrames - hdr.byteOffsetDictionary;
      aDict = sqlite3_malloc(nDict);
      if( aDict==0 ){
        rc = SQLITE_NOMEM;
      }else{
        rc = zonefileFileRead(pFd, aDict, nDict, hdr.byteOffsetDictionary);
      }
    }
    if( rc==SQLITE_OK ){
      rc = pCmpMethod->xOpen(&pCmp, aDict, nDict);
    }
    sqlite3_free(aDict);
  }

  /* Find the encryption method and key. */
  if( hdr.encryptionType ){
    i64 iFileid = sqlite3_column_int64(pCsr->pSelect, 1);
    const char *z = 0;
    int n = zonefileKeyFind(pTab->pGlobal, pTab->zDb, pTab->zName, iFileid, &z);
    if( n==0 ){
      zErr = sqlite3_mprintf("missing encryption key for file \"%s\"", zFile);
      rc = SQLITE_ERROR;
    }else{
      rc = zonefileCodecCreate(hdr.encryptionType, (u8*)z, n, &pCodec, &zErr);
    }
  }

  /* Read some data into memory. If the data is uncompressed, then just
  ** the required record is read. Otherwise, the entire frame is read
  ** into memory.  */
  if( rc==SQLITE_OK ){
    i64 iFrameOff = sqlite3_column_int64(pCsr->pSelect, 2);
    int szFrame = sqlite3_column_int(pCsr->pSelect, 3);
    u8 *aBuf = sqlite3_malloc(szFrame);


    if( aBuf==0 ){
      rc = SQLITE_NOMEM;
    }else{





      rc = zonefileFileRead(pFd, aBuf, szFrame, iFrameOff);




      if( rc!=SQLITE_OK ){
        zErr = sqlite3_mprintf(
            "failed to read %d bytes at offset %d from file \"%s\"", 
            szFrame, iFrameOff, zFile



















        );


      }
    }

    if( rc==SQLITE_OK ){
      if( pCodec==0 && pCmpMethod==0 ){
        sqlite3_result_blob(pCtx, aBuf, szFrame, zonefileFree);
        aBuf = 0;
      }else{
        i64 iOff = sqlite3_column_int64(pCsr->pSelect, 4);
        int sz = sqlite3_column_int(pCsr->pSelect, 5);

        /* Decrypt the data if necessary */
        if( pCodec ){
          zonefileCodecDecode(pCodec, aBuf, szFrame);
          szFrame -= zonefileCodecNonceSize(pCodec);
        }
        if( pCmpMethod ){
          rc = zonefileCtxUncompress(
              pCtx, pCmpMethod, pCmp, aBuf, szFrame, iOff, sz
          );
        }else{
          sqlite3_result_blob(pCtx, &aBuf[iOff], sz, SQLITE_TRANSIENT);
        }
      }
    }
    sqlite3_free(aBuf);
  }

  zonefileReleaseFile(pCsr);
  if( zErr ){
    assert( rc!=SQLITE_OK );
    sqlite3_result_error(pCtx, zErr, -1);
    sqlite3_free(zErr);
  }
  zonefileFileClose(pFd);
  zonefileCodecDestroy(pCodec);
  if( pCmpMethod ) pCmpMethod->xClose(pCmp);
  return rc;
}

/* 
** zonefile virtual table module xColumn method.
*/
static int zonefileColumn(
................................................................................
  ZonefileCsr *pCsr = (ZonefileCsr*)cur;
  int rc = SQLITE_OK;
  switch( i ){
    case 0: /* k */
      sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pSelect, 0));
      break;
    case 1: /* v */
      rc = zonefileGetValue(pCtx, pCsr);




      break;
    case 2: /* fileid */
      sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pSelect, 1));
      break;
    default: { /* sz */
      int iCol;
      if( sqlite3_column_type(pCsr->pSelect, 5)==SQLITE_NULL ){







>


>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>







 







<
>
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
>






>
>
>
>







>
>
>
>







 







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<


<
>
>
>







 







|







 







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







|


<


>
|
>







 







>







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>







 







>
>
>
>







 







|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

>
>
>
>
>
>
>
>
>
>
|
|
|
<
|
|
|
|

|
|
|
|
|
|

|
|
|
|

|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|

|
|
<
|
|
|
|
|
|
|
|
|

|
<
<
|
<
|
<

>
|
|
|
>
>
>
>
>
|
>
>
>
>
|
<
<
<
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

>
>



|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<
<
<
<
<
<
<
<
<
<
<
<







 







|
>
>
>
>







12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
..
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
...
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
...
373
374
375
376
377
378
379

380
381



















382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
...
713
714
715
716
717
718
719


















720
721

722
723
724
725
726
727
728
729
730
731
...
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
...
802
803
804
805
806
807
808
























809
810
811
812
813
814
815
....
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164

1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
....
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
....
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
....
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
....
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455

2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497

2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508


2509

2510

2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526



2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573












2574
2575
2576
2577
2578
2579
2580
....
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
*/

#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#ifndef SQLITE_OMIT_VIRTUALTABLE

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>

/*
** Default values for various zonefile_write() parameters.
*/
#define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024)
#define ZONEFILE_DEFAULT_ENCRYPTION       1
#define ZONEFILE_DEFAULT_COMPRESSION      0
#define ZONEFILE_DEFAULT_DICTSIZE         (64*1024)

/* 
** Value to use for the first 4 bytes of a zonefile file header.
*/
#define ZONEFILE_MAGIC_NUMBER 0x464B3138

/*
** Size of a zonefile header. And of each entry in the 
** ZonefileIndex.keyOffsets array.
*/
#define ZONEFILE_SZ_HEADER           32
#define ZONEFILE_SZ_KEYOFFSETS_ENTRY 20

/*
** Constants for supported compression types. These are copied from the
** published file format spec.
*/
#define ZONEFILE_COMPRESSION_NONE             0
#define ZONEFILE_COMPRESSION_ZSTD             1
#define ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT 2
#define ZONEFILE_COMPRESSION_ZLIB             3
#define ZONEFILE_COMPRESSION_BROTLI           4
#define ZONEFILE_COMPRESSION_LZ4              5
#define ZONEFILE_COMPRESSION_LZ4HC            6

/*
** Schema for "zonefile" virtual table.
*/
#define ZONEFILE_SCHEMA          \
  "CREATE TABLE z1("             \
  "  k INTEGER PRIMARY KEY,"     \
  "  v BLOB,"                    \
  "  fileid INTEGER,"            \
  "  sz INTEGER"                 \
  ")"

/*
** Schema for "zonefile_files" virtual table.
*/
#define ZONEFILE_FILES_SCHEMA    \
  "CREATE TABLE z2("             \
  "  filename TEXT,"             \
  "  ekey BLOB,"                 \
  "  header JSON HIDDEN"         \
  ")"


#ifndef SQLITE_AMALGAMATION
typedef sqlite3_int64 i64;
typedef sqlite3_uint64 u64;
typedef unsigned char u8;
typedef unsigned short u16;
typedef unsigned long u32;
................................................................................
# define NEVER(X)       ((X)?(assert(0),1):0)
#else
# define ALWAYS(X)      (X)
# define NEVER(X)       (X)
#endif
#endif   /* SQLITE_AMALGAMATION */

/*
** Forward declarations for encryption/decryption functions.
**
** If this module is not compiled with SQLITE_HAVE_ZONEFILE_CODEC, then
** implementations of the following type and functions that support the
** mock encryption method "xor" only are provided. Alternatively, the
** application may append a more functional implementation of the following 
** type and functions to this file before compiling it with
** SQLITE_HAVE_ZONEFILE_CODEC defined.
*/
typedef struct ZonefileCodec ZonefileCodec;
static int zonefileCodecCreate(int,unsigned char*,int,ZonefileCodec**,char**);
static int zonefileCodecNonceSize(ZonefileCodec*);
static void zonefileCodecEncode(ZonefileCodec*, unsigned char*, int);
static void zonefileCodecDecode(ZonefileCodec*, unsigned char*, int);
static void zonefileCodecDestroy(ZonefileCodec*);

#ifndef SQLITE_HAVE_ZONEFILE_CODEC
typedef struct ZonefileCodec ZonefileCodec;

struct ZonefileCodec {
  u8 aKey[16];
};

................................................................................
  i64 iFileid;                    /* File id */
  const char *zKey;               /* Key buffer */
  int nKey;                       /* Size of zKey in bytes */
  u32 iHash;                      /* zonefileKeyHash() value */
  ZonefileKey *pHashNext;         /* Next colliding key in hash table */
};

/*
** Return a 32-bit hash value for the three arguments.
*/
static u32 zonefileKeyHash(
  const char *zDb,
  const char *zTab,
  i64 iFileid
){
  u32 iHash = 0;
  int i;
................................................................................
      sqlite3_free(pKey);
    }
  }
  sqlite3_free(pGlobal->aHash);
  sqlite3_free(pGlobal);
}


/*
** Write value v to buffer aBuf as an unsigned 32-bit big-endian integer.



















*/
static void zonefilePut32(u8 *aBuf, u32 v){
  aBuf[0] = (v >> 24) & 0xFF;
  aBuf[1] = (v >> 16) & 0xFF;
  aBuf[2] = (v >>  8) & 0xFF;
  aBuf[3] = v & 0xFF;
}

/*
** Read and return an unsigned 32-bit big-endian integer from buffer aBuf.
*/
static u32 zonefileGet32(const u8 *aBuf){
  return (((u32)aBuf[0]) << 24)
       + (((u32)aBuf[1]) << 16)
       + (((u32)aBuf[2]) <<  8)
       + (((u32)aBuf[3]) <<  0);
}

/*
** Generic xOpen, xClose and xUncompressSize methods used by a few
** different compression method bindings.
*/
static int zfGenericOpen(void **pp, u8 *aDict, int nDict){
  *pp = 0;
  return SQLITE_OK;
}
static void zfGenericClose(void *p){
}
static int zfGenericUncompressSize(
................................................................................
  return 0;
}

/* End of code for compression routines
**************************************************************************/




















/*
** A structure to store the parameters for a single zonefile_write()

** invocation. An instance of this structure is populated based on
** the json parameters passed to zonefile_write() by function
** zonefileGetParams();
*/
typedef struct ZonefileParam ZonefileParam;
struct ZonefileParam {
  ZonefileCompress *pCmpIdx;      /* For compressing the index */
  ZonefileCompress *pCmpData;     /* For compressing each frame */
  int encryptionType;
  int maxAutoFrameSize;
................................................................................
  u8 encryptionType;
  u8 encryptionKeyIdx;
  u8 extendedHeaderVersion;
  u8 extendedHeaderSize;
};

/*
** Resizable buffer structure used by the zonefile_write() implementation.
*/
typedef struct ZonefileBuffer ZonefileBuffer;
struct ZonefileBuffer {
  u8 *a;
  int n;
  int nAlloc;
};
................................................................................
    sqlite3_free(zSql);
  }else{
    rc = SQLITE_NOMEM;
  }
  return rc;
}

























/*
** Return zero if the two SQL values passed as arguments are equal, or
** non-zero otherwise. Values with different types are considered unequal,
** even if they both contain the same numeric value (e.g. 2 and 2.0).
*/
static int zonefileCompareValue(sqlite3_value *p1, sqlite3_value *p2){
  int eType;
................................................................................
      zonefileCtxError(pCtx, "error in compressor construction");
      goto zone_write_out;
    }
  }

  /* Prepare the SQL statement used to read data from the source table. This
  ** also serves to verify the suitability of the source table schema. */
  rc = zonefilePrepare(sqlite3_context_db_handle(pCtx), &pStmt, &zErr,
      "SELECT k, frame, v FROM %Q ORDER BY frame, idx, k", zTbl
  );


  /* Open the file-handle used to write out the zonefile */ 
  if( rc==SQLITE_OK ){
    pFd = zonefileFileOpen(zFile, 1, &zErr);
  }
  if( pFd==0 ){
    sqlite3_result_error(pCtx, zErr, -1);
    sqlite3_free(zErr);
    goto zone_write_out;
  }

  /* If the data compressor uses a global dictionary, create the dictionary
................................................................................
struct ZonefileTab {
  sqlite3_vtab base;         /* Base class - must be first */
  sqlite3 *db;
  sqlite3_stmt *pIdToName;   /* Translate fileid to filename */
  ZonefileGlobal *pGlobal;
  char *zName;               /* Name of this table */
  char *zDb;                 /* Name of db containing this table */
  int nCacheSize;
};

typedef struct ZonefileCsr ZonefileCsr;
struct ZonefileCsr {
  sqlite3_vtab_cursor base;  /* Base class - must be first */
  sqlite3_stmt *pSelect;     /* SELECT on %_shadow_idx table */
};

typedef struct ZonefileFrame ZonefileFrame;
struct ZonefileFrame {
  i64 iFileid;               /* Fileid for this frame */
  i64 iFrameOff;             /* Offset of frame in file */
  int nBuf;                  /* Size of aBuf[] in bytes */
  u8 *aBuf;                  /* Buffer containing uncompressed frame data */
};

/*
** Attempt to interpret the contents of string z as an integer. If
** successful, set (*piVal) to the integer value and return SQLITE_OK.
** Otherwise, return SQLITE_ERROR.
*/
static int zonefileParseInteger(const char *z, int *piVal){
  *piVal = atoi(z);
  return SQLITE_OK;
}

/*
** Return true character i is considered to be whitespace.
*/
static int zonefile_isspace(char i){
  return (i==' ');
}

/*
** This function is called as part of constructing zonefile virtual table
** pTab. Argument zOption is the full text of a parameter (column name)
** specified as part of the CREATE VIRTUAL TABLE statement. This function
** attempts to interpret the parameter and update structure pTab 
** accordingly. If successful, SQLITE_OK is returned. Otherwise, an
** SQLite error code is returned and (*pzErr) is left pointing to
** a buffer containing an English language error message. It is the
** responsibility of the caller to eventually free this buffer using
** sqlite3_free().
*/
static int zonefileParseOption(
  ZonefileTab *pTab,              /* Zonefile vtab under construction */
  const char *zOption,            /* Text of option (column name) */
  char **pzErr                    /* OUT: Error message */
){
  const char *z = zOption;
  const char *zOpt;
  int nOpt;
  const char *zVal;
  int rc = SQLITE_OK;

  /* Skip leading whitespace */
  while( zonefile_isspace(*z) ) z++;
  zOpt = z;

  /* Skip until EOF, whitespace or "=" */
  while( *z && !zonefile_isspace(*z) && *z!='=' ) z++;
  nOpt = z-zOpt;

  /* Skip whitespace. Then check there is an "=". */
  while( zonefile_isspace(*z) ) z++;
  if( *z!='=' ) goto parse_error;
  z++;
  while( zonefile_isspace(*z) ) z++;
  zVal = z;

  if( nOpt==9 && sqlite3_strnicmp(zOpt, "cachesize", 9)==0 ){
    rc = zonefileParseInteger(zVal, &pTab->nCacheSize);
  }else{
    goto parse_error;
  }

  return rc;

 parse_error:
  *pzErr = sqlite3_mprintf("parse error in option: %s", zOption);
  return SQLITE_ERROR;
}

/*
** This function does the work of xCreate (if bCreate!=0) or xConnect
** (if bCreate==0) for the zonefile module.
**
**   argv[0]   -> module name  ("zonefile")
**   argv[1]   -> database name
................................................................................
  int nDb = strlen(zDb);
  int rc = SQLITE_OK;

  p = (ZonefileTab*)sqlite3_malloc(sizeof(ZonefileTab) + nName+1 + nDb+1);
  if( !p ){
    rc = SQLITE_NOMEM;
  }else{
    int i;
    memset(p, 0, sizeof(ZonefileTab));
    p->zName = (char*)&p[1];
    memcpy(p->zName, zName, nName+1);
    p->zDb = &p->zName[nName+1];
    memcpy(p->zDb, zDb, nDb+1);
    p->db = db;
    p->pGlobal = (ZonefileGlobal*)pAux;
................................................................................
        sqlite3_free(zSql);
      }
    }
    
    if( rc==SQLITE_OK ){
      rc = sqlite3_declare_vtab(db, ZONEFILE_SCHEMA);
    }

    for(i=3; i<argc && rc==SQLITE_OK; i++){
      zonefileParseOption(p, argv[i], pzErr);
    }
  }

  if( rc!=SQLITE_OK ){
    sqlite3_free(p);
    p = 0;
  }
  *ppVtab = (sqlite3_vtab*)p;
................................................................................
}

static void zonefileReleaseFile(ZonefileCsr *pCsr){
  ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab;
  sqlite3_reset(pTab->pIdToName);
}

static int zonefileValueReadDirect(sqlite3_context *pCtx, ZonefileCsr *pCsr){
  i64 iOff = sqlite3_column_int64(pCsr->pSelect, 2);
  int sz = sqlite3_column_int(pCsr->pSelect, 3);

  FILE *pFd = 0;                  /* File handle open on zonefile */
  u8 *aBuf;                       /* Buffer to read blob into */
  int rc;                         /* Return code */

  aBuf = sqlite3_malloc(sz);
  if( aBuf==0 ){
    rc = SQLITE_NOMEM;
  }else{
    const char *zFile = 0;
    /* Open the file to read the blob from */
    rc = zonefileGetFile(pCtx, pCsr, &zFile);
    if( rc==SQLITE_OK ){
      char *zErr = 0;
      pFd = zonefileFileOpen(zFile, 0, &zErr);
      if( pFd==0 ){ 
        sqlite3_result_error(pCtx, zErr, -1);
        sqlite3_free(zErr);
        rc = SQLITE_ERROR;
      }
      zonefileReleaseFile(pCsr);
    }
  }

  if( rc==SQLITE_OK ){
    rc = zonefileFileRead(pFd, aBuf, sz, iOff);
    if( rc==SQLITE_OK ){
      sqlite3_result_blob(pCtx, aBuf, sz, zonefileFree);
      aBuf = 0;
    }
  }

  zonefileFileClose(pFd);
  sqlite3_free(aBuf);
  return rc;
}

static ZonefileFrame *zonefileCacheFind(
  ZonefileTab *pTab, 
  i64 iFile, 
  i64 iFrameOff
){
  return 0;
}

static void zonefileCacheStore(
  ZonefileTab *pTab,
  ZonefileFrame *pFrame
){
}

static int zonefileValueReadCache(sqlite3_context *pCtx, ZonefileCsr *pCsr){
  int rc = SQLITE_OK;
  ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab;
  ZonefileFrame *pFrame = 0;
  i64 iFile = sqlite3_column_int64(pCsr->pSelect, 1);
  i64 iFrameOff = sqlite3_column_int64(pCsr->pSelect, 2);
  i64 iKeyOff = sqlite3_column_int64(pCsr->pSelect, 4);
  int nKeySz = sqlite3_column_int64(pCsr->pSelect, 5);

  /* Check if this frame is already in the cache. If not, read it from 
  ** the file.  */
  pFrame = zonefileCacheFind(pTab, iFile, iFrameOff);
  if( pFrame==0 ){
    const char *zFile = 0;
    char *zErr = 0;
    FILE *pFd = 0;

    ZonefileHeader hdr;
    ZonefileCompress *pCmpMethod = 0;
    ZonefileCodec *pCodec = 0;
    void *pCmp = 0;

    /* Open the file to read the blob from */
    rc = zonefileGetFile(pCtx, pCsr, &zFile);
    if( rc==SQLITE_OK ){
      pFd = zonefileFileOpen(zFile, 0, &zErr);
      if( pFd==0 ) rc = SQLITE_ERROR;
    }

    /* Read the zonefile header */
    if( rc==SQLITE_OK ){
      rc = zonefileReadHeader(pFd, zFile, &hdr, &zErr);
    }

    /* Find the compression method and open the compressor handle. */
    if( rc==SQLITE_OK ){
      rc = zfFindCompress(hdr.compressionTypeContent, &pCmpMethod, &zErr);
    }
    if( pCmpMethod ){
      int nDict = 0;
      u8 *aDict = 0;
      assert( rc==SQLITE_OK );
      if( hdr.byteOffsetDictionary ){
        nDict = hdr.byteOffsetFrames - hdr.byteOffsetDictionary;
        aDict = sqlite3_malloc(nDict);
        if( aDict==0 ){
          rc = SQLITE_NOMEM;
        }else{
          rc = zonefileFileRead(pFd, aDict, nDict, hdr.byteOffsetDictionary);
        }
      }
      if( rc==SQLITE_OK ){
        rc = pCmpMethod->xOpen(&pCmp, aDict, nDict);
      }
      sqlite3_free(aDict);
    }

    /* Find the encryption method and key. */
    if( hdr.encryptionType ){

      const char *z = 0;
      int n = zonefileKeyFind(pTab->pGlobal, pTab->zDb, pTab->zName, iFile, &z);
      if( n==0 ){
        zErr = sqlite3_mprintf("missing encryption key for file \"%s\"", zFile);
        rc = SQLITE_ERROR;
      }else{
        rc = zonefileCodecCreate(hdr.encryptionType, (u8*)z, n, &pCodec, &zErr);
      }
    }

    /* Read some data into memory. */


    if( rc==SQLITE_OK ){

      int szFrame = sqlite3_column_int(pCsr->pSelect, 3);


      pFrame = (ZonefileFrame*)sqlite3_malloc(szFrame + sizeof(ZonefileFrame));
      if( pFrame==0 ){
        rc = SQLITE_NOMEM;
      }else{
        memset(pFrame, 0, sizeof(ZonefileFrame));
        pFrame->aBuf = (u8*)&pFrame[1];
        pFrame->nBuf = szFrame;
        pFrame->iFrameOff = iFrameOff;
        pFrame->iFileid = iFile;
        rc = zonefileFileRead(pFd, pFrame->aBuf, szFrame, iFrameOff);
      }
    }

    /* Decrypt data if necessary */
    if( rc==SQLITE_OK && pCodec ){



      zonefileCodecDecode(pCodec, pFrame->aBuf, pFrame->nBuf);
      pFrame->nBuf -= zonefileCodecNonceSize(pCodec);
    }

    /* Uncompress data if required */
    if( rc==SQLITE_OK && pCmpMethod ){
      ZonefileFrame *p = 0;
      int nOut = pCmpMethod->xUncompressSize(pCmp, pFrame->aBuf, pFrame->nBuf);

      p = (ZonefileFrame*)sqlite3_malloc(nOut + sizeof(ZonefileFrame));
      if( p==0 ){
        rc = SQLITE_NOMEM;
      }else{
        p->aBuf = (u8*)&p[1];
        p->nBuf = nOut;
        p->iFrameOff = iFrameOff;
        p->iFileid = iFile;
        rc = pCmpMethod->xUncompress(
            pCmp, p->aBuf, p->nBuf, pFrame->aBuf, pFrame->nBuf
        );
        sqlite3_free(pFrame);
        pFrame = p;
      }
    }

    if( rc!=SQLITE_OK ){
      sqlite3_free(pFrame);
      pFrame = 0;
    }
    zonefileReleaseFile(pCsr);
    zonefileFileClose(pFd);
    zonefileCodecDestroy(pCodec);
    if( pCmpMethod ) pCmpMethod->xClose(pCmp);

    if( zErr ){
      assert( rc!=SQLITE_OK );
      sqlite3_result_error(pCtx, zErr, -1);
      sqlite3_free(zErr);
    }
  }

  if( pFrame ){
    assert( rc==SQLITE_OK );
    sqlite3_result_blob(pCtx, &pFrame->aBuf[iKeyOff], nKeySz, SQLITE_TRANSIENT);
    sqlite3_free(pFrame);
  }













  return rc;
}

/* 
** zonefile virtual table module xColumn method.
*/
static int zonefileColumn(
................................................................................
  ZonefileCsr *pCsr = (ZonefileCsr*)cur;
  int rc = SQLITE_OK;
  switch( i ){
    case 0: /* k */
      sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pSelect, 0));
      break;
    case 1: /* v */
      if( sqlite3_column_type(pCsr->pSelect, 5)==SQLITE_NULL ){
        rc = zonefileValueReadDirect(pCtx, pCsr);
      }else{
        rc = zonefileValueReadCache(pCtx, pCsr);
      }
      break;
    case 2: /* fileid */
      sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pSelect, 1));
      break;
    default: { /* sz */
      int iCol;
      if( sqlite3_column_type(pCsr->pSelect, 5)==SQLITE_NULL ){