#define ZONEFILE_SZ_HEADER           32
#define ZONEFILE_SZ_KEYOFFSETS_ENTRY 20

#define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024)
#define ZONEFILE_DEFAULT_ENCRYPTION       0
#define ZONEFILE_DEFAULT_COMPRESSION      0






































































































































#define ZONEFILE_SCHEMA          \
  "CREATE TABLE z1("             \
  "  k INTEGER PRIMARY KEY,"     \
  "  v BLOB,"                    \
  "  fileid INTEGER,"            \
  "  frame INTEGER,"             \
................................................................................

/*
** A structure to store the parameters for a single zonefile_write()
** invocation. 
*/
typedef struct ZonefileWrite ZonefileWrite;
struct ZonefileWrite {
  int compressionTypeIndexData;
  int compressionTypeContent;
  int encryptionType;
  int maxAutoFrameSize;
};

/*
** A structure to store a deserialized zonefile header in.
*/
................................................................................
  sqlite3_stmt *pStmt = 0;
  char *zErr = 0;
  int rc = SQLITE_OK;
  int rc2;

  memset(p, 0, sizeof(ZonefileWrite));
  p->maxAutoFrameSize = ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE;


  rc = zonefilePrepare(db, &pStmt, &zErr,"SELECT key, value FROM json_each(?)");
  if( rc==SQLITE_OK ){
    sqlite3_bind_text(pStmt, 1, zJson, -1, SQLITE_STATIC);
  }
  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
    const char *zKey = (const char*)sqlite3_column_text(pStmt, 0);
    int iVal = sqlite3_column_int(pStmt, 1);
    if( sqlite3_stricmp("maxAutoFrameSize", zKey)==0 ){
      p->maxAutoFrameSize = iVal;
    }else
    if( sqlite3_stricmp("compressionTypeIndexData", zKey)==0 ){
      p->compressionTypeIndexData = iVal;





    }else
    if( sqlite3_stricmp("compressionTypeContent", zKey)==0 ){
      p->compressionTypeContent = iVal;






    }else
    if( sqlite3_stricmp("encryptionType", zKey)==0 ){
      p->encryptionType = iVal;
    }else{
      rc = SQLITE_ERROR;
      zErr = sqlite3_mprintf("unknown parameter name: \"%s\"", zKey);
    }
................................................................................
}

static void zonefileBufferFree(ZonefileBuffer *pBuf){
  sqlite3_free(pBuf->a);
  memset(pBuf, 0, sizeof(ZonefileBuffer));
}

static void zonefilePut32(u8 *aBuf, u32 v){
  aBuf[0] = (v >> 24) & 0xFF;
  aBuf[1] = (v >> 16) & 0xFF;
  aBuf[2] = (v >>  8) & 0xFF;
  aBuf[3] = v & 0xFF;
}

static u32 zonefileGet32(u8 *aBuf){
  return (((u32)aBuf[0]) << 24)
       + (((u32)aBuf[1]) << 16)
       + (((u32)aBuf[2]) <<  8)
       + (((u32)aBuf[3]) <<  0);
}

static u64 zonefileGet64(u8 *aBuf){
  return (((u64)aBuf[0]) << 56)
       + (((u64)aBuf[1]) << 48)
       + (((u64)aBuf[2]) << 40)
       + (((u64)aBuf[3]) << 32) 
       + (((u64)aBuf[4]) << 24)
       + (((u64)aBuf[5]) << 16)
................................................................................
  }
  return pFd;
}

static void zonefileFileClose(FILE *pFd){
  if( pFd ) fclose(pFd);
}


























/*
** Function:     zonefile_write(F,T[,J])
*/
static void zonefileWriteFunc(
  sqlite3_context *pCtx,       /* Context object */
  int objc,                       /* Number of SQL arguments */
................................................................................
){
  const char *zFile = 0;          /* File to write to */
  const char *zTbl = 0;           /* Database object to read from */
  const char *zJson = 0;          /* JSON configuration parameters */
  ZonefileWrite sWrite;           /* Decoded JSON parameters */
  int nKey = 0;                   /* Number of keys in new zonefile */
  int nFrame = 0;                 /* Number of frames in new zonefile */
  int szFrame = 0;                /* Size of current frame */
  sqlite3_stmt *pStmt = 0;        /* SQL used to read data from source table */
  FILE *pFd = 0;
  int rc;
  sqlite3_value *pPrev = 0;
  char *zErr = 0;

  ZonefileBuffer sFrameIdx = {0, 0, 0};
  ZonefileBuffer sKeyIdx = {0, 0, 0};

  ZonefileBuffer sFrames = {0, 0, 0};

  u8 aHdr[ZONEFILE_SZ_HEADER];    /* Space to assemble zonefile header */

  assert( objc==2 || objc==3 );
  zFile = (const char*)sqlite3_value_text(objv[0]);
  zTbl = (const char*)sqlite3_value_text(objv[1]);
  if( objc==3 ){
    zJson = (const char*)sqlite3_value_text(objv[2]);
................................................................................
    sqlite3_int64 k = sqlite3_column_int64(pStmt, 0);
    sqlite3_value *pFrame = sqlite3_column_value(pStmt, 1);
    int nBlob = sqlite3_column_bytes(pStmt, 2);
    const u8 *pBlob = (const u8*)sqlite3_column_blob(pStmt, 2);

    int bAuto = zonefileIsAutoFrame(pFrame);
    if( zonefileCompareValue(pFrame, pPrev) 
     || (bAuto && szFrame && (szFrame+nBlob)>sWrite.maxAutoFrameSize)
    ){
      /* Add new entry to sFrameIdx */
      szFrame = 0;
      if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) ) goto zone_write_out;





      zonefileAppend32(&sFrameIdx, sFrames.n);
      sqlite3_value_free(pPrev);
      pPrev = sqlite3_value_dup(pFrame);
      if( pPrev==0 ){
        sqlite3_result_error_nomem(pCtx);
        goto zone_write_out;
      }
      nFrame++;
................................................................................

    /* Add new entry to sKeyIdx */
    if( zonefileBufferGrow(pCtx, &sKeyIdx, ZONEFILE_SZ_KEYOFFSETS_ENTRY) ){
      goto zone_write_out;
    }
    zonefileAppend64(&sKeyIdx, k);
    zonefileAppend32(&sKeyIdx, nFrame-1);
    zonefileAppend32(&sKeyIdx, szFrame);
    zonefileAppend32(&sKeyIdx, nBlob);

    /* Add data for new entry to sFrames */
    if( zonefileBufferGrow(pCtx, &sFrames, nBlob) ) goto zone_write_out;
    zonefileAppendBlob(&sFrames, pBlob, nBlob);
    szFrame += nBlob;
    nKey++;





  }
  sqlite3_value_free(pPrev);
  pPrev = 0;











  /* Create the zonefile header in the in-memory buffer */
  memset(aHdr, 0, ZONEFILE_SZ_HEADER);
  zonefilePut32(&aHdr[0], ZONEFILE_MAGIC_NUMBER);
  aHdr[4] = sWrite.compressionTypeIndexData;
  aHdr[5] = sWrite.compressionTypeContent;
  zonefilePut32(&aHdr[6], 0);     /* Compression dictionary byte offset */
  zonefilePut32(&aHdr[10], ZONEFILE_SZ_HEADER + sFrameIdx.n + sKeyIdx.n); 
  zonefilePut32(&aHdr[14], nFrame);
  zonefilePut32(&aHdr[18], nKey);
  aHdr[22] = sWrite.encryptionType;
  aHdr[23] = 0;                   /* Encryption key index */
  aHdr[24] = 0;                   /* extended header version */
  aHdr[25] = 0;                   /* extended header size */
  assert( ZONEFILE_SZ_HEADER>=26 );

  rc = zonefileFileWrite(pFd, aHdr, ZONEFILE_SZ_HEADER);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sFrameIdx.a, sFrameIdx.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sKeyIdx.a, sKeyIdx.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sFrames.a, sFrames.n);
  if( rc ){
    zonefileCtxError(pCtx, "error writing file \"%s\" (fwrite())", zFile);
    goto zone_write_out;
  }

  if( fclose(pFd) ){
    zonefileCtxError(pCtx, "error writing file \"%s\" (fclose())", zFile);
................................................................................
  pFd = 0;

 zone_write_out:
  if( pFd ) fclose(pFd);
  sqlite3_finalize(pStmt);
  zonefileBufferFree(&sFrameIdx);
  zonefileBufferFree(&sKeyIdx);
  zonefileBufferFree(&sFrames);

}

typedef struct ZonefileFilesTab ZonefileFilesTab;
struct ZonefileFilesTab {
  sqlite3_vtab base;              /* Base class - must be first */
  sqlite3 *db;
  char *zBase;                    /* Name of this table */
................................................................................
    *pzErr = sqlite3_mprintf(
        "failed to read zonefile header from file \"%s\"", zFile
    );
  }

  return rc;
}



























































































static int zonefilePopulateIndex(
  ZonefileFilesTab *pTab,
  const char *zFile,
  i64 iFileid
){
  ZonefileHeader hdr;
................................................................................
  if( pFd==0 ){
    rc = SQLITE_ERROR;
  }else{
    rc = zonefileReadHeader(pFd, zFile, &hdr, &pTab->base.zErrMsg);
  }

  if( rc==SQLITE_OK && hdr.numKeys>0 ){
    /* TODO: Deal with encrypted and compressed ZonefileIndex objects */
    i64 iOff;                     /* Offset of keyOffsets array */
    u8 *aKey;                     /* Entire KeyOffsets array */
    int nKey;                     /* Size of buffer aKey[] in bytes */
    int i;
    assert( hdr.encryptionType==0 && hdr.compressionTypeIndexData==0 );


    iOff = ZONEFILE_SZ_HEADER + (hdr.numFrames * 4);
    nKey = ZONEFILE_SZ_KEYOFFSETS_ENTRY * hdr.numKeys;
    aKey = (u8*)sqlite3_malloc(nKey);
    if( aKey==0 ){
      rc = SQLITE_NOMEM;
    }else{
      rc = zonefileFileRead(pFd, aKey, nKey, iOff);
    }

    if( rc==SQLITE_OK && pTab->pInsertIdx==0 ){
      rc = zonefilePrepare(pTab->db, &pTab->pInsertIdx, &pTab->base.zErrMsg,
          "INSERT INTO %Q.'%q_shadow_idx'(k, fileid, frame, ofst, sz)"
          "VALUES(?,?,?,?,?)",
          pTab->zDb, pTab->zBase
      );
    }

    for(i=0; i<hdr.numKeys && rc==SQLITE_OK; i++){
      u8 *aEntry = &aKey[ZONEFILE_SZ_KEYOFFSETS_ENTRY * i];

      sqlite3_bind_int64(pTab->pInsertIdx, 1, (i64)zonefileGet64(&aEntry[0]));
      sqlite3_bind_int64(pTab->pInsertIdx, 2, iFileid);
      sqlite3_bind_int64(pTab->pInsertIdx, 3, (i64)zonefileGet32(&aEntry[8]));
      sqlite3_bind_int64(pTab->pInsertIdx, 4, (i64)zonefileGet32(&aEntry[12]));
      sqlite3_bind_int64(pTab->pInsertIdx, 5, (i64)zonefileGet32(&aEntry[16]));

................................................................................
  ZonefileCsr *pCsr = (ZonefileCsr*)cur;
  return (pCsr->pSelect==0);
}

static void zonefileFree(void *p){
  sqlite3_free(p);
}






















static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){
  ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab;
  const char *zFile = 0;
  char *zErr = 0;
  FILE *pFd = 0;
  int rc = SQLITE_OK;
  u32 iOff = 0;                   /* Offset of frame in file */



  ZonefileHeader hdr;


  if( pTab->pIdToName==0 ){
    rc = zonefilePrepare(pTab->db, &pTab->pIdToName, &pTab->base.zErrMsg, 
        "SELECT filename FROM %Q.'%q_shadow_file' WHERE fileid=?",
        pTab->zDb, pTab->zName
    );
    if( rc!=SQLITE_OK ){
      zonefileTransferError(pCtx);
      return SQLITE_ERROR;
    }
  }




  /* Open the file to read the blob from */
  sqlite3_bind_int64(pTab->pIdToName, 1, sqlite3_column_int64(pCsr->pSelect,1));
  if( SQLITE_ROW==sqlite3_step(pTab->pIdToName) ){
    zFile = (const char*)sqlite3_column_text(pTab->pIdToName, 0);
    pFd = zonefileFileOpen(zFile, 0, &zErr);
  }
................................................................................
  }

  /* Read the zonefile header */
  if( rc==SQLITE_OK ){
    rc = zonefileReadHeader(pFd, zFile, &hdr, &zErr);
  }

  /* Calculate the offset of the frame to read the blob from */
  if( rc==SQLITE_OK ){
    u8 aOff[4] = {0,0,0,0};





    int iFrame = sqlite3_column_int(pCsr->pSelect, 2);








    rc = zonefileFileRead(pFd, aOff, 4, ZONEFILE_SZ_HEADER + 4 * iFrame);

    iOff = zonefileGet32(aOff);





  }


  /* Read the blob */




  if( rc==SQLITE_OK ){
    int sz = sqlite3_column_int(pCsr->pSelect, 4);
    int ofst = sqlite3_column_int(pCsr->pSelect, 3);


    u8 *aBuf = sqlite3_malloc(sz);
    if( aBuf==0 ){
      rc = SQLITE_NOMEM;
    }else{
      rc = zonefileFileRead(pFd, aBuf, sz, hdr.byteOffsetFrames + iOff + ofst);
      if( rc==SQLITE_OK ){

        sqlite3_result_blob(pCtx, aBuf, sz, zonefileFree);

      }else{



        zErr = sqlite3_mprintf(
            "failed to read %d bytes from file \"%s\"", sz, zFile

        );
      }

    }
  }

  sqlite3_reset(pTab->pIdToName);
  if( zErr ){
    assert( rc!=SQLITE_OK );
    sqlite3_result_error(pCtx, zErr, -1);

#define ZONEFILE_SZ_HEADER           32
#define ZONEFILE_SZ_KEYOFFSETS_ENTRY 20

#define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024)
#define ZONEFILE_DEFAULT_ENCRYPTION       0
#define ZONEFILE_DEFAULT_COMPRESSION      0

#define ZONEFILE_COMPRESSION_NONE             0
#define ZONEFILE_COMPRESSION_ZSTD             1
#define ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT 2
#define ZONEFILE_COMPRESSION_ZLIB             3
#define ZONEFILE_COMPRESSION_BROTLI           4
#define ZONEFILE_COMPRESSION_LZ4              5
#define ZONEFILE_COMPRESSION_LZ4HC            6


static void zonefilePut32(u8 *aBuf, u32 v){
  aBuf[0] = (v >> 24) & 0xFF;
  aBuf[1] = (v >> 16) & 0xFF;
  aBuf[2] = (v >>  8) & 0xFF;
  aBuf[3] = v & 0xFF;
}
static u32 zonefileGet32(const u8 *aBuf){
  return (((u32)aBuf[0]) << 24)
       + (((u32)aBuf[1]) << 16)
       + (((u32)aBuf[2]) <<  8)
       + (((u32)aBuf[3]) <<  0);
}

#ifdef SQLITE_HAVE_ZLIB 

#include <zlib.h>

static int zfZlibOpen(void **pp){
  *pp = 0;
  return SQLITE_OK;
}
static void zfZlibClose(void *p){
}
static int zfZlibCompressBound(void *p, int nSrc){
  return (int)compressBound((uLong)nSrc) + 4;
}
static int zfZlibCompress(
  void *p, 
  u8 *aDest, int *pnDest, 
  const u8 *aSrc, int nSrc
){
  uLongf destLen = (uLongf)(*pnDest)-4;
  int rc = compress(&aDest[4], &destLen, aSrc, (uLong)nSrc);
  *pnDest = (int)(destLen+4);
  zonefilePut32(aDest, nSrc);
  return rc==Z_OK ? SQLITE_OK : SQLITE_ERROR;
}
static int zfZlibUncompressSize(
  void *p, 
  const u8 *aSrc, int nSrc
){
  return (int)zonefileGet32(aSrc);
}
static int zfZlibUncompress(
  void *p, 
  u8 *aDest, int *pnDest, 
  const u8 *aSrc, int nSrc
){
  uLongf destLen = (uLongf)(*pnDest);
  int rc = uncompress(aDest, &destLen, &aSrc[4], (uLong)nSrc-4);
  *pnDest = (int)destLen;
  return rc==Z_OK ? SQLITE_OK : SQLITE_ERROR;
}
#endif

typedef struct ZonefileCompress ZonefileCompress;
static struct ZonefileCompress {
  int eType;
  const char *zName;
  int (*xOpen)(void**);
  void (*xClose)(void*);
  int (*xCompressBound)(void*, int nSrc);
  int (*xCompress)(void*, u8 *aDest, int *pnDest, const u8 *aSrc, int nSrc);
  int (*xUncompressSize)(void*, const u8 *aSrc, int nSrc);
  int (*xUncompress)(void*, u8 *aDest, int *pnDest,const u8 *aSrc,int nSrc);
} aZonefileCompress[] = {
  { ZONEFILE_COMPRESSION_NONE,             "none",
    0, 0, 0, 0, 0, 0
  },
#ifdef SQLITE_HAVE_ZSTD
  { ZONEFILE_COMPRESSION_ZSTD,             "zstd",
    0, 0, 0, 0, 0, 0
  },
  { ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT, "zstd_global_dict",
    0, 0, 0, 0, 0, 0
  },
#endif /* SQLITE_HAVE_ZSTD */
#ifdef SQLITE_HAVE_ZLIB
  { ZONEFILE_COMPRESSION_ZLIB,             "zlib",
    zfZlibOpen, zfZlibClose, 
    zfZlibCompressBound, zfZlibCompress, 
    zfZlibUncompressSize, zfZlibUncompress
  },
#endif /* SQLITE_HAVE_ZLIB */
#ifdef SQLITE_HAVE_BROTLI
  { ZONEFILE_COMPRESSION_BROTLI,           "brotli",
    0, 0, 0, 0, 0, 0
  },
#endif /* SQLITE_HAVE_BROTLI */
#ifdef SQLITE_HAVE_LZ4
  { ZONEFILE_COMPRESSION_LZ4,              "lz4",
    0, 0, 0, 0, 0, 0
  },
  { ZONEFILE_COMPRESSION_LZ4HC,            "lz4hc",
    0, 0, 0, 0, 0, 0
  },
#endif /* SQLITE_HAVE_LZ4 */
};

static ZonefileCompress *zonefileCompress(const char *zName){
  int i;
  for(i=0; i<sizeof(aZonefileCompress)/sizeof(aZonefileCompress[0]); i++){
    if( sqlite3_stricmp(aZonefileCompress[i].zName, zName)==0 ){
      return &aZonefileCompress[i];
    }
  }
  return 0;
}

static ZonefileCompress *zonefileCompressByValue(int eType){
  int i;
  for(i=0; i<sizeof(aZonefileCompress)/sizeof(aZonefileCompress[0]); i++){
    if( aZonefileCompress[i].eType==eType ){
      return &aZonefileCompress[i];
    }
  }
  return 0;
}

/* End of code for compression routines
**************************************************************************/



#define ZONEFILE_SCHEMA          \
  "CREATE TABLE z1("             \
  "  k INTEGER PRIMARY KEY,"     \
  "  v BLOB,"                    \
  "  fileid INTEGER,"            \
  "  frame INTEGER,"             \
................................................................................

/*
** A structure to store the parameters for a single zonefile_write()
** invocation. 
*/
typedef struct ZonefileWrite ZonefileWrite;
struct ZonefileWrite {
  ZonefileCompress *pCmpIdx;      /* For compressing the index */
  ZonefileCompress *pCmpData;     /* For compressing each frame */
  int encryptionType;
  int maxAutoFrameSize;
};

/*
** A structure to store a deserialized zonefile header in.
*/
................................................................................
  sqlite3_stmt *pStmt = 0;
  char *zErr = 0;
  int rc = SQLITE_OK;
  int rc2;

  memset(p, 0, sizeof(ZonefileWrite));
  p->maxAutoFrameSize = ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE;
  p->pCmpData = p->pCmpIdx = zonefileCompressByValue(0);

  rc = zonefilePrepare(db, &pStmt, &zErr,"SELECT key, value FROM json_each(?)");
  if( rc==SQLITE_OK ){
    sqlite3_bind_text(pStmt, 1, zJson, -1, SQLITE_STATIC);
  }
  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
    const char *zKey = (const char*)sqlite3_column_text(pStmt, 0);
    int iVal = sqlite3_column_int(pStmt, 1);
    if( sqlite3_stricmp("maxAutoFrameSize", zKey)==0 ){
      p->maxAutoFrameSize = iVal;
    }else
    if( sqlite3_stricmp("compressionTypeIndexData", zKey)==0 ){
      const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
      p->pCmpIdx = zonefileCompress(zName);
      if( p->pCmpIdx==0 ){
        rc = SQLITE_ERROR;
        zErr = sqlite3_mprintf("unknown compression scheme: \"%s\"", zName);
      }
    }else
    if( sqlite3_stricmp("compressionTypeContent", zKey)==0 ){

      const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
      p->pCmpData = zonefileCompress(zName);
      if( p->pCmpData==0 ){
        rc = SQLITE_ERROR;
        zErr = sqlite3_mprintf("unknown compression scheme: \"%s\"", zName);
      }
    }else
    if( sqlite3_stricmp("encryptionType", zKey)==0 ){
      p->encryptionType = iVal;
    }else{
      rc = SQLITE_ERROR;
      zErr = sqlite3_mprintf("unknown parameter name: \"%s\"", zKey);
    }
................................................................................
}

static void zonefileBufferFree(ZonefileBuffer *pBuf){
  sqlite3_free(pBuf->a);
  memset(pBuf, 0, sizeof(ZonefileBuffer));
}















static u64 zonefileGet64(u8 *aBuf){
  return (((u64)aBuf[0]) << 56)
       + (((u64)aBuf[1]) << 48)
       + (((u64)aBuf[2]) << 40)
       + (((u64)aBuf[3]) << 32) 
       + (((u64)aBuf[4]) << 24)
       + (((u64)aBuf[5]) << 16)
................................................................................
  }
  return pFd;
}

static void zonefileFileClose(FILE *pFd){
  if( pFd ) fclose(pFd);
}

static int zonefileAppendCompressed(
  sqlite3_context *pCtx,          /* Leave any error message here */
  ZonefileCompress *pCmp,
  ZonefileBuffer *pTo,            /* Append new data here */
  ZonefileBuffer *pFrom           /* Input buffer */
){
  int rc = SQLITE_OK;
  if( pCmp->eType==ZONEFILE_COMPRESSION_NONE ){
    if( zonefileBufferGrow(pCtx, pTo, pFrom->n) ){
      rc = SQLITE_ERROR;
    }else{
      zonefileAppendBlob(pTo, pFrom->a, pFrom->n);
    }
  }else{
    int nReq = pCmp->xCompressBound(0, pFrom->n);
    if( zonefileBufferGrow(pCtx, pTo, nReq) ) return SQLITE_ERROR;
    rc = pCmp->xCompress(0, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n);
    pTo->n += nReq;
    if( rc!=SQLITE_OK ){
      return rc;
    }
  }
  return SQLITE_OK;
}

/*
** Function:     zonefile_write(F,T[,J])
*/
static void zonefileWriteFunc(
  sqlite3_context *pCtx,       /* Context object */
  int objc,                       /* Number of SQL arguments */
................................................................................
){
  const char *zFile = 0;          /* File to write to */
  const char *zTbl = 0;           /* Database object to read from */
  const char *zJson = 0;          /* JSON configuration parameters */
  ZonefileWrite sWrite;           /* Decoded JSON parameters */
  int nKey = 0;                   /* Number of keys in new zonefile */
  int nFrame = 0;                 /* Number of frames in new zonefile */

  sqlite3_stmt *pStmt = 0;        /* SQL used to read data from source table */
  FILE *pFd = 0;
  int rc;
  sqlite3_value *pPrev = 0;
  char *zErr = 0;

  ZonefileBuffer sFrameIdx = {0, 0, 0};
  ZonefileBuffer sKeyIdx = {0, 0, 0};
  ZonefileBuffer sData = {0, 0, 0};         /* All completed frames so far */
  ZonefileBuffer sFrame = {0, 0, 0};        /* Current frame (uncompressed) */

  u8 aHdr[ZONEFILE_SZ_HEADER];    /* Space to assemble zonefile header */

  assert( objc==2 || objc==3 );
  zFile = (const char*)sqlite3_value_text(objv[0]);
  zTbl = (const char*)sqlite3_value_text(objv[1]);
  if( objc==3 ){
    zJson = (const char*)sqlite3_value_text(objv[2]);
................................................................................
    sqlite3_int64 k = sqlite3_column_int64(pStmt, 0);
    sqlite3_value *pFrame = sqlite3_column_value(pStmt, 1);
    int nBlob = sqlite3_column_bytes(pStmt, 2);
    const u8 *pBlob = (const u8*)sqlite3_column_blob(pStmt, 2);

    int bAuto = zonefileIsAutoFrame(pFrame);
    if( zonefileCompareValue(pFrame, pPrev) 
     || (bAuto && sFrame.n && (sFrame.n+nBlob)>sWrite.maxAutoFrameSize)
    ){
      /* Add new entry to sFrame */

      if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) 
       || zonefileAppendCompressed(pCtx, sWrite.pCmpData, &sData, &sFrame) 
      ){
        goto zone_write_out;
      }
      sFrame.n = 0;
      zonefileAppend32(&sFrameIdx, sData.n);
      sqlite3_value_free(pPrev);
      pPrev = sqlite3_value_dup(pFrame);
      if( pPrev==0 ){
        sqlite3_result_error_nomem(pCtx);
        goto zone_write_out;
      }
      nFrame++;
................................................................................

    /* Add new entry to sKeyIdx */
    if( zonefileBufferGrow(pCtx, &sKeyIdx, ZONEFILE_SZ_KEYOFFSETS_ENTRY) ){
      goto zone_write_out;
    }
    zonefileAppend64(&sKeyIdx, k);
    zonefileAppend32(&sKeyIdx, nFrame-1);
    zonefileAppend32(&sKeyIdx, sFrame.n);
    zonefileAppend32(&sKeyIdx, nBlob);

    /* Add uncompressed data for new entry to sFrame */
    if( zonefileBufferGrow(pCtx, &sFrame, nBlob) ) goto zone_write_out;
    zonefileAppendBlob(&sFrame, pBlob, nBlob);

    nKey++;
  }
  if( sFrame.n>0 ){
    if( zonefileAppendCompressed(pCtx, sWrite.pCmpData, &sData, &sFrame) ){
      goto zone_write_out;
    }
  }
  sqlite3_value_free(pPrev);
  pPrev = 0;

  /* If a compression method was specified, compress the key-index here */
  if( sWrite.pCmpIdx->eType!=ZONEFILE_COMPRESSION_NONE ){
    if( zonefileBufferGrow(pCtx, &sFrameIdx, sKeyIdx.n) ) goto zone_write_out;
    zonefileAppendBlob(&sFrameIdx, sKeyIdx.a, sKeyIdx.n);
    zonefileBufferFree(&sKeyIdx);
    rc = zonefileAppendCompressed(pCtx, sWrite.pCmpIdx, &sKeyIdx, &sFrameIdx);
    sFrameIdx.n = 0;
    if( rc ) goto zone_write_out;
  }

  /* Create the zonefile header in the in-memory buffer */
  memset(aHdr, 0, ZONEFILE_SZ_HEADER);
  zonefilePut32(&aHdr[0], ZONEFILE_MAGIC_NUMBER);
  aHdr[4] = sWrite.pCmpIdx->eType;
  aHdr[5] = sWrite.pCmpData->eType;
  zonefilePut32(&aHdr[6], 0);     /* Compression dictionary byte offset */
  zonefilePut32(&aHdr[10], ZONEFILE_SZ_HEADER + sFrameIdx.n + sKeyIdx.n); 
  zonefilePut32(&aHdr[14], nFrame);
  zonefilePut32(&aHdr[18], nKey);
  aHdr[22] = sWrite.encryptionType;
  aHdr[23] = 0;                   /* Encryption key index */
  aHdr[24] = 0;                   /* extended header version */
  aHdr[25] = 0;                   /* extended header size */
  assert( ZONEFILE_SZ_HEADER>=26 );

  rc = zonefileFileWrite(pFd, aHdr, ZONEFILE_SZ_HEADER);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sFrameIdx.a, sFrameIdx.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sKeyIdx.a, sKeyIdx.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sData.a, sData.n);
  if( rc ){
    zonefileCtxError(pCtx, "error writing file \"%s\" (fwrite())", zFile);
    goto zone_write_out;
  }

  if( fclose(pFd) ){
    zonefileCtxError(pCtx, "error writing file \"%s\" (fclose())", zFile);
................................................................................
  pFd = 0;

 zone_write_out:
  if( pFd ) fclose(pFd);
  sqlite3_finalize(pStmt);
  zonefileBufferFree(&sFrameIdx);
  zonefileBufferFree(&sKeyIdx);
  zonefileBufferFree(&sFrame);
  zonefileBufferFree(&sData);
}

typedef struct ZonefileFilesTab ZonefileFilesTab;
struct ZonefileFilesTab {
  sqlite3_vtab base;              /* Base class - must be first */
  sqlite3 *db;
  char *zBase;                    /* Name of this table */
................................................................................
    *pzErr = sqlite3_mprintf(
        "failed to read zonefile header from file \"%s\"", zFile
    );
  }

  return rc;
}

static int zonefileUncompress(
  ZonefileCompress *pCmp,
  u8 *aIn, int nIn,
  u8 **paOut, int *pnOut
){
  int rc;
  int nOut = pCmp->xUncompressSize(0, aIn, nIn);
  u8 *aOut = sqlite3_malloc(nOut);

  assert( pCmp->eType!=ZONEFILE_COMPRESSION_NONE );
  if( aOut==0 ){
    rc = SQLITE_NOMEM;
  }else{
    rc = pCmp->xUncompress(0, aOut, &nOut, aIn, nIn);
    if( rc ){
      sqlite3_free(aOut);
      aOut = 0;
    }
  }

  *paOut = aOut;
  *pnOut = nOut;
  return rc;
}

static int zfFindCompress(int eType, ZonefileCompress **pp, char **pzErr){
  int rc = SQLITE_OK;
  ZonefileCompress *pCmp;
  pCmp = zonefileCompressByValue(eType);
  if( pCmp==0 ){
    *pzErr = sqlite3_mprintf("unsupported compression method: %d", eType);
    rc = SQLITE_ERROR;
  }else if( pCmp->eType==ZONEFILE_COMPRESSION_NONE ){
    pCmp = 0;
  }
  *pp = pCmp;
  return rc;
}

static int zonefileLoadIndex(
  ZonefileHeader *pHdr,
  FILE *pFd,
  u8 **paIdx, int *pnIdx,
  char **pzErr
){
  ZonefileCompress *pCmp = 0;
  int rc;
  u8 *aIdx = 0;
  int nIdx = 0;
    
  rc = zfFindCompress(pHdr->compressionTypeIndexData, &pCmp, pzErr);
  if( rc==SQLITE_OK ){
    if( pHdr->byteOffsetDictionary ){
      nIdx = pHdr->byteOffsetDictionary - ZONEFILE_SZ_HEADER;
    }else{
      nIdx = pHdr->byteOffsetFrames - ZONEFILE_SZ_HEADER;
    }
    aIdx = (u8*)sqlite3_malloc(nIdx);

    if( aIdx==0 ){
      rc = SQLITE_NOMEM;
    }else{
      rc = zonefileFileRead(pFd, aIdx, nIdx, ZONEFILE_SZ_HEADER);
    }
  }

  if( rc==SQLITE_OK && pCmp ){
    u8 *aUn = 0;
    int nUn = 0;
    rc = zonefileUncompress(pCmp, aIdx, nIdx, &aUn, &nUn);
    if( rc==SQLITE_ERROR ){
      *pzErr = sqlite3_mprintf("failed to uncompress index");
    }
    sqlite3_free(aIdx);
    aIdx = aUn;
    nIdx = nUn;
  }

  if( rc!=SQLITE_OK ){
    sqlite3_free(aIdx);
    aIdx = 0;
    nIdx = 0;
  }

  *paIdx = aIdx;
  *pnIdx = nIdx;
  return SQLITE_OK;
}


static int zonefilePopulateIndex(
  ZonefileFilesTab *pTab,
  const char *zFile,
  i64 iFileid
){
  ZonefileHeader hdr;
................................................................................
  if( pFd==0 ){
    rc = SQLITE_ERROR;
  }else{
    rc = zonefileReadHeader(pFd, zFile, &hdr, &pTab->base.zErrMsg);
  }

  if( rc==SQLITE_OK && hdr.numKeys>0 ){


    u8 *aKey;                     /* Entire KeyOffsets array */
    int nKey;                     /* Size of buffer aKey[] in bytes */
    int i;


    assert( hdr.encryptionType==0 );
    rc = zonefileLoadIndex(&hdr, pFd, &aKey, &nKey, &pTab->base.zErrMsg);








    if( rc==SQLITE_OK && pTab->pInsertIdx==0 ){
      rc = zonefilePrepare(pTab->db, &pTab->pInsertIdx, &pTab->base.zErrMsg,
          "INSERT INTO %Q.'%q_shadow_idx'(k, fileid, frame, ofst, sz)"
          "VALUES(?,?,?,?,?)",
          pTab->zDb, pTab->zBase
      );
    }

    for(i=0; i<hdr.numKeys && rc==SQLITE_OK; i++){
      u8 *aEntry = &aKey[4*hdr.numFrames + ZONEFILE_SZ_KEYOFFSETS_ENTRY * i];

      sqlite3_bind_int64(pTab->pInsertIdx, 1, (i64)zonefileGet64(&aEntry[0]));
      sqlite3_bind_int64(pTab->pInsertIdx, 2, iFileid);
      sqlite3_bind_int64(pTab->pInsertIdx, 3, (i64)zonefileGet32(&aEntry[8]));
      sqlite3_bind_int64(pTab->pInsertIdx, 4, (i64)zonefileGet32(&aEntry[12]));
      sqlite3_bind_int64(pTab->pInsertIdx, 5, (i64)zonefileGet32(&aEntry[16]));

................................................................................
  ZonefileCsr *pCsr = (ZonefileCsr*)cur;
  return (pCsr->pSelect==0);
}

static void zonefileFree(void *p){
  sqlite3_free(p);
}

static int zonefileCtxUncompress(
  sqlite3_context *pCtx, 
  ZonefileCompress *pCmp, 
  u8 *aBuf, int nBuf,
  int iKeyOff, int nKey
){
  int rc;
  u8 *aUn = 0;
  int nUn = 0;
    
  rc = zonefileUncompress(pCmp, aBuf, nBuf, &aUn, &nUn);
  if( rc==SQLITE_OK ){
    sqlite3_result_blob(pCtx, &aUn[iKeyOff], nKey, SQLITE_TRANSIENT);
  }else if( rc==SQLITE_ERROR ){
    zonefileCtxError(pCtx, "failed to uncompress frame");
  }

  sqlite3_free(aUn);
  return rc;
}

static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){
  ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab;
  const char *zFile = 0;
  char *zErr = 0;
  FILE *pFd = 0;
  int rc = SQLITE_OK;
  u32 iOff = 0;                   /* Offset of frame in file */
  u32 szFrame = 0;                /* Size of frame in bytes */
  int iKeyOff = 0;                /* Offset of record within frame */
  int szKey = 0;                  /* Uncompressed size of record in bytes */
  ZonefileHeader hdr;
  ZonefileCompress *pCmp = 0;

  if( pTab->pIdToName==0 ){
    rc = zonefilePrepare(pTab->db, &pTab->pIdToName, &pTab->base.zErrMsg, 
        "SELECT filename FROM %Q.'%q_shadow_file' WHERE fileid=?",
        pTab->zDb, pTab->zName
    );
    if( rc!=SQLITE_OK ){
      zonefileTransferError(pCtx);
      return SQLITE_ERROR;
    }
  }

  iKeyOff = sqlite3_column_int(pCsr->pSelect, 3);
  szKey = sqlite3_column_int(pCsr->pSelect, 4);

  /* Open the file to read the blob from */
  sqlite3_bind_int64(pTab->pIdToName, 1, sqlite3_column_int64(pCsr->pSelect,1));
  if( SQLITE_ROW==sqlite3_step(pTab->pIdToName) ){
    zFile = (const char*)sqlite3_column_text(pTab->pIdToName, 0);
    pFd = zonefileFileOpen(zFile, 0, &zErr);
  }
................................................................................
  }

  /* Read the zonefile header */
  if( rc==SQLITE_OK ){
    rc = zonefileReadHeader(pFd, zFile, &hdr, &zErr);
  }

  /* Find the compression method */
  if( rc==SQLITE_OK ){
    rc = zfFindCompress(hdr.compressionTypeContent, &pCmp, &zErr);
  }

  /* Find the offset (iOff) and size (szFrame) of the frame that the
  ** record is stored in.  */
  if( rc==SQLITE_OK ){
    int iFrame = sqlite3_column_int(pCsr->pSelect, 2);
    u8 aSpace[8] = {0,0,0,0,0,0,0,0};
    u8 *aOff = aSpace;
    u8 *aFree = 0;
    if( hdr.compressionTypeIndexData ){
      int nFree = 0;
      rc = zonefileLoadIndex(&hdr, pFd, &aFree, &nFree, &zErr);
      if( rc==SQLITE_OK ) aOff = &aFree[4*iFrame];
    }else{
      rc = zonefileFileRead(pFd, aOff, 8, ZONEFILE_SZ_HEADER + 4 * iFrame);
    }
    iOff = zonefileGet32(aOff);
    if( iFrame+1<hdr.numFrames ){
      szFrame = zonefileGet32(&aOff[4]) - iOff;
    }else{
      fseek(pFd, 0, SEEK_END);
      szFrame = (u32)ftell(pFd) - iOff - hdr.byteOffsetFrames;
    }
    sqlite3_free(aFree);
  }


  /* Read some data into memory. If the data is uncompressed, then just
  ** the required record is read. Otherwise, the entire frame is read
  ** into memory.  */
  if( rc==SQLITE_OK ){


    int sz = (pCmp ? (int)szFrame : szKey);
    i64 ofst = iOff + (pCmp ? 0 : iKeyOff);
    u8 *aBuf = sqlite3_malloc(sz);
    if( aBuf==0 ){
      rc = SQLITE_NOMEM;
    }else{
      rc = zonefileFileRead(pFd, aBuf, sz, hdr.byteOffsetFrames + ofst);
      if( rc==SQLITE_OK ){
        if( pCmp==0 ){
          sqlite3_result_blob(pCtx, aBuf, szKey, zonefileFree);
          aBuf = 0;
        }else{
          rc = zonefileCtxUncompress(pCtx, pCmp, aBuf, szFrame, iKeyOff, szKey);
        }
      }else{
        zErr = sqlite3_mprintf(
            "failed to read %d bytes at offset %d from file \"%s\"", 
            sz, (int)(hdr.byteOffsetFrames+ofst), zFile
        );
      }
      sqlite3_free(aBuf);
    }
  }

  sqlite3_reset(pTab->pIdToName);
  if( zErr ){
    assert( rc!=SQLITE_OK );
    sqlite3_result_error(pCtx, zErr, -1);

do_execsql_test 1.6 { SELECT count(*) FROM z1_shadow_idx } 0

do_execsql_test 1.7 { DROP TABLE z1 }

#-------------------------------------------------------------------------







reset_db
load_static_extension db zonefile

do_execsql_test 2.0 {
  CREATE TABLE zz(
    k INTEGER PRIMARY KEY, 
    frame INTEGER DEFAULT -1, 
    idx INTEGER DEFAULT -1, 
    v BLOB
  );
  CREATE TABLE rt(k INTEGER PRIMARY KEY, v BLOB);
  CREATE VIRTUAL TABLE zone USING zonefile;
}

set nMinByte   0
set nMaxByte 444
foreach {zonefile lKey} {
  test1.zonefile {195 1238 298 405 297}
  test2.zonefile {124 1624 82 1929}
  test3.zonefile {932 683 1751 410 41}
  test4.zonefile {427 1491}
  test5.zonefile {1004 473 801 394 1672 816 1577}
  test6.zonefile {1374 1454 1005}
  test7.zonefile {450 241 319 133}
  test8.zonefile {1414 900 1406 1917 127 673}
  test9.zonefile {1192 226 988 1292 718 1345 1675}
  test10.zonefile {314}
  test11.zonefile {1177 1597 60 532 291 1164 812}
  test12.zonefile {1168 1290 1585 939 1916}
  test13.zonefile {644 1784 1476 1283 433 506}
  test14.zonefile {1141 1547 1506 364}
  test15.zonefile {1756 1885 844 1880 1896 354}
  test16.zonefile {1383 1928 1371}
  test17.zonefile {93}
  test18.zonefile {1067}
  test19.zonefile {642}
  test20.zonefile {1380 1857}
  test21.zonefile {288 293 1968 1207 1739 231 300}
  test22.zonefile {651 1007 607 830 299 1431}
  test23.zonefile {81 1651 543 1949 256 119 1088}
  test24.zonefile {1278 2024 682 1115 194 636 1804}
  test25.zonefile {514 1155 171 2015 791}
  test26.zonefile {1615 1228 147 1464}
  test27.zonefile {55 1130 781 678 78}
  test28.zonefile {1981 1401 1178}
  test29.zonefile {1754 864 183 1953 1901}
  test30.zonefile {1461 817}
  test31.zonefile {1720 1722 686 1833}
} {
  forcedelete $zonefile
  execsql { DELETE FROM zz; }
  foreach k $lKey {
    execsql { INSERT INTO zz(k, v) VALUES($k, randomblob($k)) }
  }
  execsql { INSERT INTO rt SELECT k, v FROM zz }
  breakpoint
  execsql { 





    SELECT zonefile_write($zonefile, 'zz', '{"maxAutoFrameSize":2000}');
    INSERT INTO zone_files(filename) VALUES($zonefile);
  }
}

do_execsql_test 2.1 {
  SELECT k FROM zone JOIN rt USING (k) WHERE zone.v!=rt.v
}
do_execsql_test 2.2 {
  SELECT count(*) FROM zone JOIN rt USING (k);
} {135}
do_execsql_test 2.3 {
  SELECT filename, 
         json_extract(header, '$.numKeys'),
         json_extract(header, '$.numFrames')
  FROM zone_files 
  WHERE filename IN ('test19.zonefile', 'test20.zonefile', 'test21.zonefile')
  ORDER BY 1
} {
  test19.zonefile 1 1
  test20.zonefile 2 2
  test21.zonefile 7 4
}



finish_test

do_execsql_test 1.6 { SELECT count(*) FROM z1_shadow_idx } 0

do_execsql_test 1.7 { DROP TABLE z1 }

#-------------------------------------------------------------------------

foreach {tn cmp cmpidx} {
  1 none none
  2 zlib none
  3 none zlib
  4 zlib zlib
} {
  reset_db
  load_static_extension db zonefile
  
  do_execsql_test 2.$tn.0 {
    CREATE TABLE zz(
      k INTEGER PRIMARY KEY, 
      frame INTEGER DEFAULT -1, 
      idx INTEGER DEFAULT -1, 
      v BLOB
    );
    CREATE TABLE rt(k INTEGER PRIMARY KEY, v BLOB);
    CREATE VIRTUAL TABLE zone USING zonefile;
  }
  
  set nMinByte   0
  set nMaxByte 444
  foreach {zonefile lKey} {
    test1.zonefile {195 1238 298 405 297}
    test2.zonefile {124 1624 82 1929}
    test3.zonefile {932 683 1751 410 41}
    test4.zonefile {427 1491}
    test5.zonefile {1004 473 801 394 1672 816 1577}
    test6.zonefile {1374 1454 1005}
    test7.zonefile {450 241 319 133}
    test8.zonefile {1414 900 1406 1917 127 673}
    test9.zonefile {1192 226 988 1292 718 1345 1675}
    test10.zonefile {314}
    test11.zonefile {1177 1597 60 532 291 1164 812}
    test12.zonefile {1168 1290 1585 939 1916}
    test13.zonefile {644 1784 1476 1283 433 506}
    test14.zonefile {1141 1547 1506 364}
    test15.zonefile {1756 1885 844 1880 1896 354}
    test16.zonefile {1383 1928 1371}
    test17.zonefile {93}
    test18.zonefile {1067}
    test19.zonefile {642}
    test20.zonefile {1380 1857}
    test21.zonefile {288 293 1968 1207 1739 231 300}
    test22.zonefile {651 1007 607 830 299 1431}
    test23.zonefile {81 1651 543 1949 256 119 1088}
    test24.zonefile {1278 2024 682 1115 194 636 1804}
    test25.zonefile {514 1155 171 2015 791}
    test26.zonefile {1615 1228 147 1464}
    test27.zonefile {55 1130 781 678 78}
    test28.zonefile {1981 1401 1178}
    test29.zonefile {1754 864 183 1953 1901}
    test30.zonefile {1461 817}
    test31.zonefile {1720 1722 686 1833}
  } {
    forcedelete $zonefile
    execsql { DELETE FROM zz; }
    foreach k $lKey {
      execsql { INSERT INTO zz(k, v) VALUES($k, randomblob($k)) }
    }
    execsql { INSERT INTO rt SELECT k, v FROM zz }

    execsql { 
      WITH p(n,v) AS (
        VALUES('maxAutoFrameSize', 2000) UNION ALL
        VALUES('compressionTypeIndexData', $cmpidx) UNION ALL
        VALUES('compressionTypeContent', $cmp)
      )
      SELECT zonefile_write($zonefile, 'zz', json_group_object(n, v)) FROM p;
      INSERT INTO zone_files(filename) VALUES($zonefile);
    }
  }
  
  do_execsql_test 2.$tn.1 {
    SELECT k FROM zone JOIN rt USING (k) WHERE zone.v!=rt.v
  }
  do_execsql_test 2.$tn.2 {
    SELECT count(*) FROM zone JOIN rt USING (k);
  } {135}
  do_execsql_test 2.$tn.3 {
    SELECT filename, 
           json_extract(header, '$.numKeys'),
           json_extract(header, '$.numFrames')
    FROM zone_files 
    WHERE filename IN ('test19.zonefile', 'test20.zonefile', 'test21.zonefile')
    ORDER BY 1
  } {
    test19.zonefile 1 1
    test20.zonefile 2 2
    test21.zonefile 7 4
  }
}


finish_test