Index: ext/misc/zipfile.c ================================================================== --- ext/misc/zipfile.c +++ ext/misc/zipfile.c @@ -101,10 +101,12 @@ #define ZIPFILE_SIGNATURE_CDS 0x02014b50 #define ZIPFILE_SIGNATURE_LFH 0x04034b50 #define ZIPFILE_SIGNATURE_EOCD 0x06054b50 #define ZIPFILE_LFH_FIXED_SZ 30 +#define ZIPFILE_EOCD_FIXED_SZ 22 + /* ** Set the error message contained in context ctx to the results of ** vprintf(zFmt, ...). */ static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ @@ -643,11 +645,11 @@ u8 *aRead; char **pzErr = &pTab->base.zErrMsg; int rc = SQLITE_OK; if( aBlob==0 ){ - aRead = (u8*)pTab->aBuffer; + aRead = pTab->aBuffer; rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr); }else{ aRead = (u8*)&aBlob[iOff]; } @@ -1092,14 +1094,11 @@ zipfileResetCursor(pCsr); if( pTab->zFile ){ zFile = pTab->zFile; }else if( idxNum==0 ){ - /* Error. This is an eponymous virtual table and the user has not - ** supplied a file name. */ - zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument"); - return SQLITE_ERROR; + bInMemory = 1; }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]); int nBlob = sqlite3_value_bytes(argv[0]); assert( pTab->pFirstEntry==0 ); rc = zipfileLoadDirectory(pTab, aBlob, nBlob); @@ -1164,64 +1163,72 @@ } return SQLITE_OK; } -static ZipfileEntry *zipfileNewEntry(const char *zPath){ +static ZipfileEntry *zipfileNewEntry(const char *zPath, int nData){ ZipfileEntry *pNew; - pNew = sqlite3_malloc(sizeof(ZipfileEntry)); + pNew = sqlite3_malloc(sizeof(ZipfileEntry) + nData); if( pNew ){ memset(pNew, 0, sizeof(ZipfileEntry)); + if( nData ){ + pNew->aData = (u8*)&pNew[1]; + } pNew->cds.zFile = sqlite3_mprintf("%s", zPath); if( pNew->cds.zFile==0 ){ sqlite3_free(pNew); pNew = 0; } } return pNew; } -static int zipfileAppendEntry( - ZipfileTab *pTab, - ZipfileCDS *pCds, - const char *zPath, /* Path for new entry */ - int nPath, /* strlen(zPath) */ - const u8 *pData, - int nData, - u32 mTime -){ - u8 *aBuf = pTab->aBuffer; - int rc; +static int zipfileSerializeLFH(ZipfileEntry *pEntry, u8 *aBuf){ + ZipfileCDS *pCds = &pEntry->cds; + u8 *a = aBuf; pCds->nExtra = 9; - zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_LFH); - zipfileWrite16(aBuf, pCds->iVersionExtract); - zipfileWrite16(aBuf, pCds->flags); - zipfileWrite16(aBuf, pCds->iCompression); - zipfileWrite16(aBuf, pCds->mTime); - zipfileWrite16(aBuf, pCds->mDate); - zipfileWrite32(aBuf, pCds->crc32); - zipfileWrite32(aBuf, pCds->szCompressed); - zipfileWrite32(aBuf, pCds->szUncompressed); - zipfileWrite16(aBuf, (u16)nPath); - zipfileWrite16(aBuf, pCds->nExtra); - assert( aBuf==&pTab->aBuffer[ZIPFILE_LFH_FIXED_SZ] ); - rc = zipfileAppendData(pTab, pTab->aBuffer, (int)(aBuf - pTab->aBuffer)); - if( rc==SQLITE_OK ){ - rc = zipfileAppendData(pTab, (const u8*)zPath, nPath); - } - - if( rc==SQLITE_OK && pCds->nExtra ){ - aBuf = pTab->aBuffer; - zipfileWrite16(aBuf, ZIPFILE_EXTRA_TIMESTAMP); - zipfileWrite16(aBuf, 5); - *aBuf++ = 0x01; - zipfileWrite32(aBuf, mTime); - rc = zipfileAppendData(pTab, pTab->aBuffer, 9); - } - + /* Write the LFH itself */ + zipfileWrite32(a, ZIPFILE_SIGNATURE_LFH); + zipfileWrite16(a, pCds->iVersionExtract); + zipfileWrite16(a, pCds->flags); + zipfileWrite16(a, pCds->iCompression); + zipfileWrite16(a, pCds->mTime); + zipfileWrite16(a, pCds->mDate); + zipfileWrite32(a, pCds->crc32); + zipfileWrite32(a, pCds->szCompressed); + zipfileWrite32(a, pCds->szUncompressed); + zipfileWrite16(a, (u16)pCds->nFile); + zipfileWrite16(a, pCds->nExtra); + assert( a==&aBuf[ZIPFILE_LFH_FIXED_SZ] ); + + /* Add the file name */ + memcpy(a, pCds->zFile, (int)pCds->nFile); + a += (int)pCds->nFile; + + /* The "extra" data */ + zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP); + zipfileWrite16(a, 5); + *a++ = 0x01; + zipfileWrite32(a, pEntry->mUnixTime); + + return a-aBuf; +} + +static int zipfileAppendEntry( + ZipfileTab *pTab, + ZipfileEntry *pEntry, + const u8 *pData, + int nData +){ + u8 *aBuf = pTab->aBuffer; + int nBuf; + int rc; + + nBuf = zipfileSerializeLFH(pEntry, aBuf); + rc = zipfileAppendData(pTab, aBuf, nBuf); if( rc==SQLITE_OK ){ rc = zipfileAppendData(pTab, pData, nData); } return rc; @@ -1301,12 +1308,11 @@ char *zFree = 0; /* Also free this */ ZipfileEntry *pOld = 0; int bIsDir = 0; u32 iCrc32 = 0; - assert( pTab->zFile ); - assert( pTab->pWriteFd ); + assert( (pTab->zFile==0)==(pTab->pWriteFd==0) ); /* If this is a DELETE or UPDATE, find the archive entry to delete. */ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ const char *zDelete = (const char*)sqlite3_value_text(apVal[0]); int nDelete = (int)strlen(zDelete); @@ -1405,11 +1411,11 @@ } } if( rc==SQLITE_OK ){ /* Create the new CDS record. */ - pNew = zipfileNewEntry(zPath); + pNew = zipfileNewEntry(zPath, pTab->zFile ? 0 : (nData+1)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ pNew->cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; pNew->cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; @@ -1421,13 +1427,15 @@ pNew->cds.szUncompressed = (u32)sz; pNew->cds.iExternalAttr = (mode<<16); pNew->cds.iOffset = (u32)pTab->szCurrent; pNew->cds.nFile = nPath; pNew->mUnixTime = (u32)mTime; - rc = zipfileAppendEntry( - pTab, &pNew->cds, zPath, nPath, pData, nData, pNew->mUnixTime - ); + if( pTab->zFile ){ + rc = zipfileAppendEntry(pTab, pNew, pData, nData); + }else{ + memcpy(pNew->aData, pData, nData); + } zipfileAddEntry(pTab, pOld, pNew); } } } @@ -1447,60 +1455,56 @@ sqlite3_free(pFree); sqlite3_free(zFree); return rc; } + +static int zipfileSerializeEOCD(ZipfileEOCD *p, u8 *aBuf){ + u8 *a = aBuf; + zipfileWrite32(a, ZIPFILE_SIGNATURE_EOCD); + zipfileWrite16(a, p->iDisk); + zipfileWrite16(a, p->iFirstDisk); + zipfileWrite16(a, p->nEntry); + zipfileWrite16(a, p->nEntryTotal); + zipfileWrite32(a, p->nSize); + zipfileWrite32(a, p->iOffset); + zipfileWrite16(a, 0); /* Size of trailing comment in bytes*/ + + return a-aBuf; +} static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){ - u8 *aBuf = pTab->aBuffer; - - zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_EOCD); - zipfileWrite16(aBuf, p->iDisk); - zipfileWrite16(aBuf, p->iFirstDisk); - zipfileWrite16(aBuf, p->nEntry); - zipfileWrite16(aBuf, p->nEntryTotal); - zipfileWrite32(aBuf, p->nSize); - zipfileWrite32(aBuf, p->iOffset); - zipfileWrite16(aBuf, 0); /* Size of trailing comment in bytes*/ - - assert( (aBuf-pTab->aBuffer)==22 ); - return zipfileAppendData(pTab, pTab->aBuffer, (int)(aBuf - pTab->aBuffer)); + int nBuf = zipfileSerializeEOCD(p, pTab->aBuffer); + assert( nBuf==ZIPFILE_EOCD_FIXED_SZ ); + return zipfileAppendData(pTab, pTab->aBuffer, nBuf); } static int zipfileBegin(sqlite3_vtab *pVtab){ ZipfileTab *pTab = (ZipfileTab*)pVtab; int rc = SQLITE_OK; assert( pTab->pWriteFd==0 ); - - /* This table is only writable if a default archive path was specified - ** as part of the CREATE VIRTUAL TABLE statement. */ - if( pTab->zFile==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "zipfile: writing requires a default archive" - ); - return SQLITE_ERROR; - } - - /* Open a write fd on the file. Also load the entire central directory - ** structure into memory. During the transaction any new file data is - ** appended to the archive file, but the central directory is accumulated - ** in main-memory until the transaction is committed. */ - pTab->pWriteFd = fopen(pTab->zFile, "ab+"); - if( pTab->pWriteFd==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "zipfile: failed to open file %s for writing", pTab->zFile - ); - rc = SQLITE_ERROR; - }else{ - fseek(pTab->pWriteFd, 0, SEEK_END); - pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); - rc = zipfileLoadDirectory(pTab, 0, 0); - } - - if( rc!=SQLITE_OK ){ - zipfileCleanupTransaction(pTab); + if( pTab->zFile ){ + /* Open a write fd on the file. Also load the entire central directory + ** structure into memory. During the transaction any new file data is + ** appended to the archive file, but the central directory is accumulated + ** in main-memory until the transaction is committed. */ + pTab->pWriteFd = fopen(pTab->zFile, "ab+"); + if( pTab->pWriteFd==0 ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "zipfile: failed to open file %s for writing", pTab->zFile + ); + rc = SQLITE_ERROR; + }else{ + fseek(pTab->pWriteFd, 0, SEEK_END); + pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); + rc = zipfileLoadDirectory(pTab, 0, 0); + } + + if( rc!=SQLITE_OK ){ + zipfileCleanupTransaction(pTab); + } } return rc; } @@ -1641,10 +1645,77 @@ sqlite3_free(zRes); } } } +static void zipfileFree(void *p) { sqlite3_free(p); } + +static void zipfileFunctionBlob( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + ZipfileCsr *pCsr; + ZipfileTab *pTab = (ZipfileTab*)sqlite3_user_data(context); + ZipfileEntry *p; + int nBody = 0; + int nCds = 0; + int nEocd = ZIPFILE_EOCD_FIXED_SZ; + ZipfileEOCD eocd; + + u8 *aZip; + int nZip; + + u8 *aBody; + u8 *aCds; + + pCsr = zipfileFindCursor(pTab, sqlite3_value_int64(argv[0])); + if( pCsr->pFile || pTab->zFile ){ + sqlite3_result_error(context, "illegal use of zipfile_blob()", -1); + return; + } + + /* Figure out how large the final file will be */ + for(p=pTab->pFirstEntry; p; p=p->pNext){ + nBody += ZIPFILE_LFH_FIXED_SZ + p->cds.nFile + 9 + p->cds.szCompressed; + nCds += ZIPFILE_CDS_FIXED_SZ + p->cds.nFile + 9; + } + + /* Allocate space to create the serialized file */ + nZip = nBody + nCds + nEocd; + aZip = (u8*)sqlite3_malloc(nZip); + if( aZip==0 ){ + sqlite3_result_error_nomem(context); + return; + } + aBody = aZip; + aCds = &aZip[nBody]; + + /* Populate the body and CDS */ + memset(&eocd, 0, sizeof(eocd)); + for(p=pTab->pFirstEntry; p; p=p->pNext){ + p->cds.iOffset = (aBody - aZip); + aBody += zipfileSerializeLFH(p, aBody); + if( p->cds.szCompressed ){ + memcpy(aBody, p->aData, p->cds.szCompressed); + aBody += p->cds.szCompressed; + } + aCds += zipfileSerializeCDS(p, aCds); + eocd.nEntry++; + } + + /* Append the EOCD record */ + assert( aBody==&aZip[nBody] ); + assert( aCds==&aZip[nBody+nCds] ); + eocd.nEntryTotal = eocd.nEntry; + eocd.nSize = nCds; + eocd.iOffset = nBody; + zipfileSerializeEOCD(&eocd, aCds); + + sqlite3_result_blob(context, aZip, nZip, zipfileFree); +} + /* ** xFindFunction method. */ static int zipfileFindFunction( @@ -1657,10 +1728,15 @@ if( nArg>0 ){ if( sqlite3_stricmp("zipfile_cds", zName)==0 ){ *pxFunc = zipfileFunctionCds; *ppArg = (void*)pVtab; return 1; + } + if( sqlite3_stricmp("zipfile_blob", zName)==0 ){ + *pxFunc = zipfileFunctionBlob; + *ppArg = (void*)pVtab; + return 1; } } return 0; } @@ -1691,13 +1767,12 @@ zipfileFindFunction, /* xFindMethod */ 0, /* xRename */ }; int rc = sqlite3_create_module(db, "zipfile" , &zipfileModule, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3_overload_function(db, "zipfile_cds", -1); - } + if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_cds", -1); + if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_blob", -1); return rc; } #else /* SQLITE_OMIT_VIRTUALTABLE */ # define zipfileRegister(x) SQLITE_OK #endif Index: test/zipfile.test ================================================================== --- test/zipfile.test +++ test/zipfile.test @@ -30,12 +30,27 @@ set fd [open $file] fconfigure $fd -translation binary -encoding binary set data [read $fd] close $fd - set res2 [db eval { SELECT name,mode,mtime,method,quote(data) FROM zipfile($data) }] - uplevel [list do_test $tn [list set {} $res2] $res1] + set res2 [db eval { + SELECT name,mode,mtime,method,quote(data) FROM zipfile($data) + }] + + uplevel [list do_test $tn.1 [list set {} $res2] $res1] + + set T "$file.test_zip" + set fd [open $T w] + fconfigure $fd -translation binary -encoding binary + puts -nonewline $fd $data + close $fd + + set res3 [ + db eval { SELECT name,mode,mtime,method,quote(data) FROM zipfile($T) } + ] + + uplevel [list do_test $tn.2 [list set {} $res3] $res1] } forcedelete test.zip do_execsql_test 1.0 { CREATE VIRTUAL TABLE temp.zz USING zipfile('test.zip'); @@ -268,9 +283,30 @@ } do_catchsql_test 3.2 { SELECT rowid FROM x1 } {1 {no such column: rowid}} + +#------------------------------------------------------------------------- +reset_db +forcedelete test.zip +load_static_extension db zipfile + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x2 USING zipfile(); + INSERT INTO x2(name, data) VALUES('dir1/', NULL); + INSERT INTO x2(name, data) VALUES('file1', '1234'); + INSERT INTO x2(name, data) VALUES('dir1/file2', '5678'); + SELECT name, data FROM x2 +} { + dir1/ {} file1 1234 dir1/file2 5678 +} + +do_test 4.1 { + set data [db one {SELECT zipfile_blob(z) FROM x2 LIMIT 1}] + db eval { SELECT name, data FROM zipfile($data) } +} {dir1/ {} file1 1234 dir1/file2 5678} + finish_test