/ Check-in [4dbe0cba]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Modify the zonefile format in order to avoid depending on the filesize to determine the extent of the final frame. See README.md for details.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | zonefile
Files: files | file ages | folders
SHA3-256: 4dbe0cba3fad9a752834d795127cf35eed21fab63b18a48f75d5c1e96ca77447
User & Date: dan 2018-02-19 14:27:24
Context
2018-02-19
16:28
Add support for the ExtendedHeaderSize header field to zonefile. check-in: 78267a09 user: dan tags: zonefile
14:27
Modify the zonefile format in order to avoid depending on the filesize to determine the extent of the final frame. See README.md for details. check-in: 4dbe0cba user: dan tags: zonefile
2018-02-17
20:22
Add support for "brotli" compression to the zonefile module. check-in: 3eb25b3f user: dan tags: zonefile
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/zonefile/README.md.

   107    107     *  Multi-byte integer values are big-endian.
   108    108   
   109    109     *  The offsets in the ZoneFileIndex.byteOffsetZoneFrame[] array are
   110    110        relative to the offset in ZoneFileHeader.byteOffsetFrames. This is
   111    111        necessary as we may not know the offset of the start of the frame data
   112    112        until after the ZoneFileIndex structure is compressed.
   113    113   
   114         -  *  Currently there is no support at all for encryption or compression.
          114  +  *  The offsets in the ZoneFileIndex.byteOffsetZoneFrame[] array are the
          115  +     offsets for the first byte past the end of the corresponding frame.
          116  +     For example, byteOffsetZoneFrame[] identifies the first byte of the
          117  +     second frame, and byteOffsetZoneFrame[numFrames-1] is one byte past
          118  +     the end of the last frame in the file.
          119  +
          120  +     This is better as if we store the starting offset of each frame, there
          121  +     is no way to determine the size of the last frame in the file without
          122  +     trusting the filesize itself.
          123  +
          124  +  *  Currently there is no support at all for encryption.
   115    125   
   116    126     *  Zonefile currently uses json1 to parse the json argument to
   117    127        zonefile\_write(). And so must be used with an SQLITE\_ENABLE\_JSON1
   118    128        or otherwise json1-enabled SQLite.
   119    129   
   120    130   

Changes to ext/zonefile/zonefile.c.

   784    784     /* Prepare the SQL statement used to read data from the source table. This
   785    785     ** also serves to verify the suitability of the source table schema. */
   786    786     pStmt = zonefileCtxPrepare(pCtx, 
   787    787         "SELECT k, frame, v FROM %Q ORDER BY frame, idx, k", zTbl
   788    788     );
   789    789     if( pStmt==0 ) goto zone_write_out;
   790    790   
   791         -  /* Open a file-handle used to write out the zonefile */ 
          791  +  /* Open the file-handle used to write out the zonefile */ 
   792    792     pFd = zonefileFileOpen(zFile, 1, &zErr);
   793    793     if( pFd==0 ){
   794    794       sqlite3_result_error(pCtx, zErr, -1);
   795    795       sqlite3_free(zErr);
   796    796       goto zone_write_out;
   797    797     }
   798    798   
   799    799     /* If the data compressor uses a global dictionary, create the dictionary
   800         -  ** now.  */
          800  +  ** and store it in buffer sDict.  */
   801    801     if( sWrite.pCmpData->xTrain ){
   802    802       int nSample = 0;
   803    803   
   804    804       while( SQLITE_ROW==sqlite3_step(pStmt) ){
   805    805         int nByte = sqlite3_column_bytes(pStmt, 2);
   806    806         const u8 *aByte = (const u8*)sqlite3_column_blob(pStmt, 2);
   807    807         if( zonefileBufferGrow(pCtx, &sSample, nByte) ){
................................................................................
   843    843     while( SQLITE_ROW==sqlite3_step(pStmt) ){
   844    844       sqlite3_int64 k = sqlite3_column_int64(pStmt, 0);
   845    845       sqlite3_value *pFrame = sqlite3_column_value(pStmt, 1);
   846    846       int nBlob = sqlite3_column_bytes(pStmt, 2);
   847    847       const u8 *pBlob = (const u8*)sqlite3_column_blob(pStmt, 2);
   848    848   
   849    849       int bAuto = zonefileIsAutoFrame(pFrame);
   850         -    if( zonefileCompareValue(pFrame, pPrev) 
   851         -     || (bAuto && sFrame.n && (sFrame.n+nBlob)>sWrite.maxAutoFrameSize)
   852         -    ){
   853         -      /* Add new entry to sFrame */
   854         -      if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) 
   855         -       || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame)
          850  +    if( sFrame.n>0 ){
          851  +      if( zonefileCompareValue(pFrame, pPrev) 
          852  +       || (bAuto && (sFrame.n+nBlob)>sWrite.maxAutoFrameSize)
   856    853         ){
   857         -        goto zone_write_out;
          854  +        /* Add new entry to sFrame */
          855  +        if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) 
          856  +         || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp,&sData,&sFrame)
          857  +        ){
          858  +          goto zone_write_out;
          859  +        }
          860  +        sFrame.n = 0;
          861  +        zonefileAppend32(&sFrameIdx, sData.n);
          862  +        sqlite3_value_free(pPrev);
          863  +        pPrev = 0;
          864  +        nFrame++;
   858    865         }
   859         -      sFrame.n = 0;
   860         -      zonefileAppend32(&sFrameIdx, sData.n);
   861         -      sqlite3_value_free(pPrev);
          866  +    }
          867  +
          868  +    if( pPrev==0 ){
   862    869         pPrev = sqlite3_value_dup(pFrame);
   863    870         if( pPrev==0 ){
   864    871           sqlite3_result_error_nomem(pCtx);
   865    872           goto zone_write_out;
   866    873         }
   867         -      nFrame++;
   868    874       }
   869    875   
   870    876       /* Add new entry to sKeyIdx */
   871    877       if( zonefileBufferGrow(pCtx, &sKeyIdx, ZONEFILE_SZ_KEYOFFSETS_ENTRY) ){
   872    878         goto zone_write_out;
   873    879       }
   874    880       zonefileAppend64(&sKeyIdx, k);
   875         -    zonefileAppend32(&sKeyIdx, nFrame-1);
          881  +    zonefileAppend32(&sKeyIdx, nFrame);
   876    882       zonefileAppend32(&sKeyIdx, sFrame.n);
   877    883       zonefileAppend32(&sKeyIdx, nBlob);
   878    884   
   879    885       /* Add uncompressed data for new entry to sFrame */
   880    886       if( zonefileBufferGrow(pCtx, &sFrame, nBlob) ) goto zone_write_out;
   881    887       zonefileAppendBlob(&sFrame, pBlob, nBlob);
   882    888       nKey++;
   883    889     }
   884         -  if( sFrame.n>0
   885         -   && zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame) 
   886         -  ){
   887         -    goto zone_write_out;
          890  +
          891  +  if( sFrame.n>0 ){
          892  +    if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) 
          893  +     || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame)
          894  +    ){
          895  +      goto zone_write_out;
          896  +    }
          897  +    zonefileAppend32(&sFrameIdx, sData.n);
          898  +    nFrame++;
   888    899     }
   889         -  sqlite3_value_free(pPrev);
   890         -  pPrev = 0;
   891    900   
   892    901     /* If a compression method was specified, compress the key-index here */
   893    902     if( sWrite.pCmpIdx->eType!=ZONEFILE_COMPRESSION_NONE ){
   894    903       if( zonefileBufferGrow(pCtx, &sFrameIdx, sKeyIdx.n) ) goto zone_write_out;
   895    904       zonefileAppendBlob(&sFrameIdx, sKeyIdx.a, sKeyIdx.n);
   896    905       zonefileBufferFree(&sKeyIdx);
   897    906       rc = zonefileAppendCompressed(pCtx, sWrite.pCmpIdx, 0, &sKeyIdx,&sFrameIdx);
................................................................................
   929    938       zonefileCtxError(pCtx, "error writing file \"%s\" (fclose())", zFile);
   930    939     }
   931    940     pFd = 0;
   932    941   
   933    942    zone_write_out:
   934    943     if( pCmp ) sWrite.pCmpData->xClose(pCmp);
   935    944     if( pFd ) fclose(pFd);
          945  +  sqlite3_value_free(pPrev);
   936    946     sqlite3_finalize(pStmt);
   937    947     zonefileBufferFree(&sFrameIdx);
   938    948     zonefileBufferFree(&sKeyIdx);
   939    949     zonefileBufferFree(&sFrame);
   940    950     zonefileBufferFree(&sDict);
   941    951     zonefileBufferFree(&sData);
   942    952     zonefileBufferFree(&sSample);
................................................................................
  1904   1914       int iFrame = sqlite3_column_int(pCsr->pSelect, 2);
  1905   1915       u8 aSpace[8] = {0,0,0,0,0,0,0,0};
  1906   1916       u8 *aOff = aSpace;
  1907   1917       u8 *aFree = 0;
  1908   1918       if( hdr.compressionTypeIndexData ){
  1909   1919         int nFree = 0;
  1910   1920         rc = zonefileLoadIndex(&hdr, pFd, &aFree, &nFree, &zErr);
  1911         -      if( rc==SQLITE_OK ) aOff = &aFree[4*iFrame];
         1921  +      if( rc==SQLITE_OK ) aOff = &aFree[4*(iFrame-1)];
  1912   1922       }else{
  1913         -      rc = zonefileFileRead(pFd, aOff, 8, ZONEFILE_SZ_HEADER + 4 * iFrame);
         1923  +      rc = zonefileFileRead(pFd, aOff, 8, ZONEFILE_SZ_HEADER + 4 * (iFrame-1));
  1914   1924       }
  1915         -    iOff = zonefileGet32(aOff);
  1916         -    if( iFrame+1<hdr.numFrames ){
  1917         -      szFrame = zonefileGet32(&aOff[4]) - iOff;
  1918         -    }else{
  1919         -      fseek(pFd, 0, SEEK_END);
  1920         -      szFrame = (u32)ftell(pFd) - iOff - hdr.byteOffsetFrames;
         1925  +    szFrame = zonefileGet32(&aOff[4]);
         1926  +    if( iFrame>0 ){
         1927  +      iOff = zonefileGet32(aOff);
         1928  +      szFrame = szFrame - iOff;
  1921   1929       }
  1922   1930       sqlite3_free(aFree);
  1923   1931     }
  1924   1932   
  1925   1933     /* Read some data into memory. If the data is uncompressed, then just
  1926   1934     ** the required record is read. Otherwise, the entire frame is read
  1927   1935     ** into memory.  */