Index: ext/zonefile/README.md ================================================================== --- ext/zonefile/README.md +++ ext/zonefile/README.md @@ -109,11 +109,21 @@ * The offsets in the ZoneFileIndex.byteOffsetZoneFrame[] array are relative to the offset in ZoneFileHeader.byteOffsetFrames. This is necessary as we may not know the offset of the start of the frame data until after the ZoneFileIndex structure is compressed. - * Currently there is no support at all for encryption or compression. + * The offsets in the ZoneFileIndex.byteOffsetZoneFrame[] array are the + offsets for the first byte past the end of the corresponding frame. + For example, byteOffsetZoneFrame[] identifies the first byte of the + second frame, and byteOffsetZoneFrame[numFrames-1] is one byte past + the end of the last frame in the file. + + This is better as if we store the starting offset of each frame, there + is no way to determine the size of the last frame in the file without + trusting the filesize itself. + + * Currently there is no support at all for encryption. * Zonefile currently uses json1 to parse the json argument to zonefile\_write(). And so must be used with an SQLITE\_ENABLE\_JSON1 or otherwise json1-enabled SQLite. Index: ext/zonefile/zonefile.c ================================================================== --- ext/zonefile/zonefile.c +++ ext/zonefile/zonefile.c @@ -786,20 +786,20 @@ pStmt = zonefileCtxPrepare(pCtx, "SELECT k, frame, v FROM %Q ORDER BY frame, idx, k", zTbl ); if( pStmt==0 ) goto zone_write_out; - /* Open a file-handle used to write out the zonefile */ + /* Open the file-handle used to write out the zonefile */ pFd = zonefileFileOpen(zFile, 1, &zErr); if( pFd==0 ){ sqlite3_result_error(pCtx, zErr, -1); sqlite3_free(zErr); goto zone_write_out; } /* If the data compressor uses a global dictionary, create the dictionary - ** now. */ + ** and store it in buffer sDict. */ if( sWrite.pCmpData->xTrain ){ int nSample = 0; while( SQLITE_ROW==sqlite3_step(pStmt) ){ int nByte = sqlite3_column_bytes(pStmt, 2); @@ -845,51 +845,60 @@ sqlite3_value *pFrame = sqlite3_column_value(pStmt, 1); int nBlob = sqlite3_column_bytes(pStmt, 2); const u8 *pBlob = (const u8*)sqlite3_column_blob(pStmt, 2); int bAuto = zonefileIsAutoFrame(pFrame); - if( zonefileCompareValue(pFrame, pPrev) - || (bAuto && sFrame.n && (sFrame.n+nBlob)>sWrite.maxAutoFrameSize) - ){ - /* Add new entry to sFrame */ - if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) - || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame) - ){ - goto zone_write_out; - } - sFrame.n = 0; - zonefileAppend32(&sFrameIdx, sData.n); - sqlite3_value_free(pPrev); + if( sFrame.n>0 ){ + if( zonefileCompareValue(pFrame, pPrev) + || (bAuto && (sFrame.n+nBlob)>sWrite.maxAutoFrameSize) + ){ + /* Add new entry to sFrame */ + if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) + || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp,&sData,&sFrame) + ){ + goto zone_write_out; + } + sFrame.n = 0; + zonefileAppend32(&sFrameIdx, sData.n); + sqlite3_value_free(pPrev); + pPrev = 0; + nFrame++; + } + } + + if( pPrev==0 ){ pPrev = sqlite3_value_dup(pFrame); if( pPrev==0 ){ sqlite3_result_error_nomem(pCtx); goto zone_write_out; } - nFrame++; } /* Add new entry to sKeyIdx */ if( zonefileBufferGrow(pCtx, &sKeyIdx, ZONEFILE_SZ_KEYOFFSETS_ENTRY) ){ goto zone_write_out; } zonefileAppend64(&sKeyIdx, k); - zonefileAppend32(&sKeyIdx, nFrame-1); + zonefileAppend32(&sKeyIdx, nFrame); zonefileAppend32(&sKeyIdx, sFrame.n); zonefileAppend32(&sKeyIdx, nBlob); /* Add uncompressed data for new entry to sFrame */ if( zonefileBufferGrow(pCtx, &sFrame, nBlob) ) goto zone_write_out; zonefileAppendBlob(&sFrame, pBlob, nBlob); nKey++; } - if( sFrame.n>0 - && zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame) - ){ - goto zone_write_out; + + if( sFrame.n>0 ){ + if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) + || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame) + ){ + goto zone_write_out; + } + zonefileAppend32(&sFrameIdx, sData.n); + nFrame++; } - sqlite3_value_free(pPrev); - pPrev = 0; /* If a compression method was specified, compress the key-index here */ if( sWrite.pCmpIdx->eType!=ZONEFILE_COMPRESSION_NONE ){ if( zonefileBufferGrow(pCtx, &sFrameIdx, sKeyIdx.n) ) goto zone_write_out; zonefileAppendBlob(&sFrameIdx, sKeyIdx.a, sKeyIdx.n); @@ -931,10 +940,11 @@ pFd = 0; zone_write_out: if( pCmp ) sWrite.pCmpData->xClose(pCmp); if( pFd ) fclose(pFd); + sqlite3_value_free(pPrev); sqlite3_finalize(pStmt); zonefileBufferFree(&sFrameIdx); zonefileBufferFree(&sKeyIdx); zonefileBufferFree(&sFrame); zonefileBufferFree(&sDict); @@ -1906,20 +1916,18 @@ u8 *aOff = aSpace; u8 *aFree = 0; if( hdr.compressionTypeIndexData ){ int nFree = 0; rc = zonefileLoadIndex(&hdr, pFd, &aFree, &nFree, &zErr); - if( rc==SQLITE_OK ) aOff = &aFree[4*iFrame]; - }else{ - rc = zonefileFileRead(pFd, aOff, 8, ZONEFILE_SZ_HEADER + 4 * iFrame); - } - iOff = zonefileGet32(aOff); - if( iFrame+10 ){ + iOff = zonefileGet32(aOff); + szFrame = szFrame - iOff; } sqlite3_free(aFree); } /* Read some data into memory. If the data is uncompressed, then just