/ Check-in [bbe5b21f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add support for compression types "lz4" and "lz4hc" to the zonefile module.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | zonefile
Files: files | file ages | folders
SHA3-256: bbe5b21ffab3cd312680ca9f179c5847790c17fb91d4174985153c6c398d48e3
User & Date: dan 2018-02-17 19:38:02
Context
2018-02-17
20:22
Add support for "brotli" compression to the zonefile module. check-in: 3eb25b3f user: dan tags: zonefile
19:38
Add support for compression types "lz4" and "lz4hc" to the zonefile module. check-in: bbe5b21f user: dan tags: zonefile
18:33
Add support for compression methods "zstd" and "zstd_global_dict". check-in: a993a50b user: dan tags: zonefile
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/zonefile/zonefile.c.

   128    128   ){
   129    129     size_t szDest = (size_t)(*pnDest);
   130    130     size_t rc = ZSTD_compress(aDest, szDest, aSrc, (size_t)nSrc, 1);
   131    131     if( ZSTD_isError(rc) ) return SQLITE_ERROR;
   132    132     *pnDest = (int)rc;
   133    133     return SQLITE_OK;
   134    134   }
   135         -static int zfZstdUncompressSize(
   136         -  void *p, 
   137         -  const u8 *aSrc, int nSrc
   138         -){
          135  +static int zfZstdUncompressSize(void *p, const u8 *aSrc, int nSrc){
   139    136     return (int)ZSTD_getFrameContentSize(aSrc, (size_t)nSrc);
   140    137   }
   141    138   static int zfZstdUncompress(
   142    139     void *p, 
   143    140     u8 *aDest, int nDest, 
   144    141     const u8 *aSrc, int nSrc
   145    142   ){
................................................................................
   215    212     rc = ZSTD_compress_usingCDict(
   216    213         pCmp->pCCtx, aDest, szDest, aSrc, (size_t)nSrc, pCmp->pCDict
   217    214     );
   218    215     if( ZSTD_isError(rc) ) return SQLITE_ERROR;
   219    216     *pnDest = (int)rc;
   220    217     return SQLITE_OK;
   221    218   }
   222         -static int zfZstddictUncompressSize(
   223         -  void *p, 
   224         -  const u8 *aSrc, int nSrc
   225         -){
          219  +static int zfZstddictUncompressSize(void *p, const u8 *aSrc, int nSrc){
   226    220     return (int)ZSTD_getFrameContentSize(aSrc, (size_t)nSrc);
   227    221   }
   228    222   static int zfZstddictUncompress(
   229    223     void *p, 
   230    224     u8 *aDest, int nDest, 
   231    225     const u8 *aSrc, int nSrc
   232    226   ){
................................................................................
   234    228     size_t rc = ZSTD_decompress_usingDDict(
   235    229         pCmp->pDCtx, aDest, (size_t)nDest, aSrc, (size_t)nSrc, pCmp->pDDict
   236    230     );
   237    231     if( rc!=(size_t)nDest ) return SQLITE_ERROR;
   238    232     return SQLITE_OK;
   239    233   }
   240    234   #endif
          235  +
          236  +#ifdef SQLITE_HAVE_LZ4 
          237  +#include <lz4.h>
          238  +#include <lz4hc.h>
          239  +static int zfLz4Open(void **pp, u8 *aDict, int nDict){
          240  +  *pp = 0;
          241  +  return SQLITE_OK;
          242  +}
          243  +static void zfLz4Close(void *p){
          244  +}
          245  +static int zfLz4CompressBound(void *p, int nSrc){
          246  +  return (int)LZ4_compressBound((uLong)nSrc) + 4;
          247  +}
          248  +static int zfLz4Uncompress(
          249  +  void *p, 
          250  +  u8 *aDest, int nDest, 
          251  +  const u8 *aSrc, int nSrc
          252  +){
          253  +  int rc = LZ4_decompress_safe(
          254  +      (const char*)&aSrc[4], (char*)aDest, nSrc-4, nDest
          255  +  );
          256  +  return rc==nDest ? SQLITE_OK : SQLITE_ERROR;
          257  +}
          258  +static int zfLz4Compress(
          259  +  void *p, 
          260  +  u8 *aDest, int *pnDest, 
          261  +  const u8 *aSrc, int nSrc
          262  +){
          263  +  int rc = LZ4_compress_default(
          264  +      (const char*)aSrc, (char*)&aDest[4], nSrc, (*pnDest - 4)
          265  +  );
          266  +  *pnDest = rc+4;
          267  +  zonefilePut32(aDest, nSrc);
          268  +  return rc==0 ? SQLITE_ERROR : SQLITE_OK;
          269  +}
          270  +static int zfLz4UncompressSize(void *p, const u8 *aSrc, int nSrc){
          271  +  return (int)zonefileGet32(aSrc);
          272  +}
          273  +static int zfLz4hcCompress(
          274  +  void *p, 
          275  +  u8 *aDest, int *pnDest, 
          276  +  const u8 *aSrc, int nSrc
          277  +){
          278  +  int rc = LZ4_compress_HC(
          279  +      (const char*)aSrc, (char*)&aDest[4], nSrc, *pnDest, 0
          280  +  );
          281  +  *pnDest = rc+4;
          282  +  zonefilePut32(aDest, nSrc);
          283  +  return rc==0 ? SQLITE_ERROR : SQLITE_OK;
          284  +}
          285  +#endif
   241    286   
   242    287   typedef struct ZonefileCompress ZonefileCompress;
   243    288   static struct ZonefileCompress {
   244    289     int eType;
   245    290     const char *zName;
   246    291     int (*xOpen)(void**, u8 *aDict, int nDict);
   247    292     void (*xClose)(void*);
................................................................................
   279    324   #ifdef SQLITE_HAVE_BROTLI
   280    325     { ZONEFILE_COMPRESSION_BROTLI,           "brotli",
   281    326       0, 0, 0, 0, 0, 0, 0
   282    327     },
   283    328   #endif /* SQLITE_HAVE_BROTLI */
   284    329   #ifdef SQLITE_HAVE_LZ4
   285    330     { ZONEFILE_COMPRESSION_LZ4,              "lz4",
   286         -    0, 0, 0, 0, 0, 0, 0
          331  +    zfLz4Open, zfLz4Close, 
          332  +    0,
          333  +    zfLz4CompressBound, zfLz4Compress, 
          334  +    zfLz4UncompressSize, zfLz4Uncompress
   287    335     },
   288    336     { ZONEFILE_COMPRESSION_LZ4HC,            "lz4hc",
   289         -    0, 0, 0, 0, 0, 0, 0
          337  +    zfLz4Open, zfLz4Close, 
          338  +    0,
          339  +    zfLz4CompressBound, zfLz4hcCompress, 
          340  +    zfLz4UncompressSize, zfLz4Uncompress
   290    341     },
   291    342   #endif /* SQLITE_HAVE_LZ4 */
   292    343   };
   293    344   
   294    345   static ZonefileCompress *zonefileCompress(const char *zName){
   295    346     int i;
   296    347     for(i=0; i<sizeof(aZonefileCompress)/sizeof(aZonefileCompress[0]); i++){
................................................................................
   630    681     sqlite3_context *pCtx,          /* Leave any error message here */
   631    682     ZonefileCompress *pMethod,      /* Compression method object */
   632    683     void *pCmp,                     /* Compression handle */
   633    684     ZonefileBuffer *pTo,            /* Append new data here */
   634    685     ZonefileBuffer *pFrom           /* Input buffer */
   635    686   ){
   636    687     int rc = SQLITE_OK;
   637         -  if( pMethod->eType==ZONEFILE_COMPRESSION_NONE ){
   638         -    if( zonefileBufferGrow(pCtx, pTo, pFrom->n) ){
   639         -      rc = SQLITE_ERROR;
          688  +  if( pFrom->n ){
          689  +    if( pMethod->eType==ZONEFILE_COMPRESSION_NONE ){
          690  +      if( zonefileBufferGrow(pCtx, pTo, pFrom->n) ){
          691  +        rc = SQLITE_ERROR;
          692  +      }else{
          693  +        zonefileAppendBlob(pTo, pFrom->a, pFrom->n);
          694  +      }
   640    695       }else{
   641         -      zonefileAppendBlob(pTo, pFrom->a, pFrom->n);
   642         -    }
   643         -  }else{
   644         -    int nReq = pMethod->xCompressBound(pCmp, pFrom->n);
   645         -    if( zonefileBufferGrow(pCtx, pTo, nReq) ) return SQLITE_ERROR;
   646         -    rc = pMethod->xCompress(pCmp, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n);
   647         -    pTo->n += nReq;
   648         -    if( rc!=SQLITE_OK ){
   649         -      return rc;
          696  +      int nReq = pMethod->xCompressBound(pCmp, pFrom->n);
          697  +      if( zonefileBufferGrow(pCtx, pTo, nReq) ) return SQLITE_ERROR;
          698  +      rc = pMethod->xCompress(pCmp, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n);
          699  +      pTo->n += nReq;
          700  +      if( rc!=SQLITE_OK ){
          701  +        return rc;
          702  +      }
   650    703       }
   651    704     }
   652    705     return SQLITE_OK;
   653    706   }
   654    707   
   655    708   /*
   656    709   ** Function:     zonefile_write(F,T[,J])
................................................................................
  1278   1331       sqlite3_free(aIdx);
  1279   1332       aIdx = 0;
  1280   1333       nIdx = 0;
  1281   1334     }
  1282   1335   
  1283   1336     *paIdx = aIdx;
  1284   1337     *pnIdx = nIdx;
  1285         -  return SQLITE_OK;
         1338  +  return rc;
  1286   1339   }
  1287   1340   
  1288   1341   
  1289   1342   static int zonefilePopulateIndex(
  1290   1343     ZonefileFilesTab *pTab,
  1291   1344     const char *zFile,
  1292   1345     i64 iFileid