Index: ext/misc/zipfile.c ================================================================== --- ext/misc/zipfile.c +++ ext/misc/zipfile.c @@ -462,14 +462,51 @@ } /* ** Magic numbers used to read CDS records. */ -#define ZIPFILE_CDS_FIXED_SZ 46 -#define ZIPFILE_CDS_NFILE_OFF 28 +#define ZIPFILE_CDS_FIXED_SZ 46 +#define ZIPFILE_CDS_NFILE_OFF 28 -static int zipfileReadCDS(ZipfileCsr *pCsr){ +/* +** Decode the CDS record in buffer aBuf into (*pCDS). Return SQLITE_ERROR +** if the record is not well-formed, or SQLITE_OK otherwise. +*/ +static int zipfileReadCDS(u8 *aBuf, ZipfileCDS *pCDS){ + u8 *aRead = aBuf; + u32 sig = zipfileRead32(aRead); + int rc = SQLITE_OK; + if( sig!=ZIPFILE_SIGNATURE_CDS ){ + rc = SQLITE_ERROR; + }else{ + pCDS->iVersionMadeBy = zipfileRead16(aRead); + pCDS->iVersionExtract = zipfileRead16(aRead); + pCDS->flags = zipfileRead16(aRead); + pCDS->iCompression = zipfileRead16(aRead); + pCDS->mTime = zipfileRead16(aRead); + pCDS->mDate = zipfileRead16(aRead); + pCDS->crc32 = zipfileRead32(aRead); + pCDS->szCompressed = zipfileRead32(aRead); + pCDS->szUncompressed = zipfileRead32(aRead); + assert( aRead==&aBuf[ZIPFILE_CDS_NFILE_OFF] ); + pCDS->nFile = zipfileRead16(aRead); + pCDS->nExtra = zipfileRead16(aRead); + pCDS->nComment = zipfileRead16(aRead); + pCDS->iDiskStart = zipfileRead16(aRead); + pCDS->iInternalAttr = zipfileRead16(aRead); + pCDS->iExternalAttr = zipfileRead32(aRead); + pCDS->iOffset = zipfileRead32(aRead); + assert( aRead==&aBuf[ZIPFILE_CDS_FIXED_SZ] ); + } + + return rc; +} + +/* +** Read the CDS record for the current entry from disk into pCsr->cds. +*/ +static int zipfileCsrReadCDS(ZipfileCsr *pCsr){ char **pzErr = &pCsr->base.pVtab->zErrMsg; u8 *aRead; int rc = SQLITE_OK; sqlite3_free(pCsr->cds.zFile); @@ -483,45 +520,23 @@ }else{ aRead = pCsr->pCurrent->aCdsEntry; } if( rc==SQLITE_OK ){ - u32 sig = zipfileRead32(aRead); - if( sig!=ZIPFILE_SIGNATURE_CDS ){ + rc = zipfileReadCDS(aRead, &pCsr->cds); + if( rc!=SQLITE_OK ){ assert( pCsr->pCurrent==0 ); zipfileSetErrmsg(pCsr,"failed to read CDS at offset %lld",pCsr->iNextOff); - rc = SQLITE_ERROR; }else{ int nRead; - pCsr->cds.iVersionMadeBy = zipfileRead16(aRead); - pCsr->cds.iVersionExtract = zipfileRead16(aRead); - pCsr->cds.flags = zipfileRead16(aRead); - pCsr->cds.iCompression = zipfileRead16(aRead); - pCsr->cds.mTime = zipfileRead16(aRead); - pCsr->cds.mDate = zipfileRead16(aRead); - pCsr->cds.crc32 = zipfileRead32(aRead); - pCsr->cds.szCompressed = zipfileRead32(aRead); - pCsr->cds.szUncompressed = zipfileRead32(aRead); - assert( pCsr->pCurrent - || aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_NFILE_OFF - ); - pCsr->cds.nFile = zipfileRead16(aRead); - pCsr->cds.nExtra = zipfileRead16(aRead); - pCsr->cds.nComment = zipfileRead16(aRead); - pCsr->cds.iDiskStart = zipfileRead16(aRead); - pCsr->cds.iInternalAttr = zipfileRead16(aRead); - pCsr->cds.iExternalAttr = zipfileRead32(aRead); - pCsr->cds.iOffset = zipfileRead32(aRead); - assert( pCsr->pCurrent - || aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_FIXED_SZ - ); - if( pCsr->pCurrent==0 ){ nRead = pCsr->cds.nFile + pCsr->cds.nExtra; aRead = zipfileCsrBuffer(pCsr); pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ; rc = zipfileReadData(pCsr->pFile, aRead, nRead, pCsr->iNextOff, pzErr); + }else{ + aRead = &aRead[ZIPFILE_CDS_FIXED_SZ]; } if( rc==SQLITE_OK ){ pCsr->cds.zFile = sqlite3_mprintf("%.*s", (int)pCsr->cds.nFile, aRead); pCsr->iNextOff += pCsr->cds.nFile; @@ -568,41 +583,51 @@ static FILE *zipfileGetFd(ZipfileCsr *pCsr){ if( pCsr->pFile ) return pCsr->pFile; return ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd; } -static int zipfileReadLFH(ZipfileCsr *pCsr){ - FILE *pFile = zipfileGetFd(pCsr); - char **pzErr = &pCsr->base.pVtab->zErrMsg; +static int zipfileReadLFH( + FILE *pFd, + i64 iOffset, + u8 *aTmp, + ZipfileLFH *pLFH, + char **pzErr +){ + u8 *aRead = aTmp; static const int szFix = ZIPFILE_LFH_FIXED_SZ; - u8 *aRead = zipfileCsrBuffer(pCsr); int rc; - rc = zipfileReadData(pFile, aRead, szFix, pCsr->cds.iOffset, pzErr); + rc = zipfileReadData(pFd, aRead, szFix, iOffset, pzErr); if( rc==SQLITE_OK ){ u32 sig = zipfileRead32(aRead); if( sig!=ZIPFILE_SIGNATURE_LFH ){ - zipfileSetErrmsg(pCsr, "failed to read LFH at offset %d", - (int)pCsr->cds.iOffset - ); + *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", (int)iOffset); rc = SQLITE_ERROR; }else{ - pCsr->lfh.iVersionExtract = zipfileRead16(aRead); - pCsr->lfh.flags = zipfileRead16(aRead); - pCsr->lfh.iCompression = zipfileRead16(aRead); - pCsr->lfh.mTime = zipfileRead16(aRead); - pCsr->lfh.mDate = zipfileRead16(aRead); - pCsr->lfh.crc32 = zipfileRead32(aRead); - pCsr->lfh.szCompressed = zipfileRead32(aRead); - pCsr->lfh.szUncompressed = zipfileRead32(aRead); - pCsr->lfh.nFile = zipfileRead16(aRead); - pCsr->lfh.nExtra = zipfileRead16(aRead); - assert( aRead==zipfileCsrBuffer(pCsr)+szFix ); - pCsr->iDataOff = pCsr->cds.iOffset+szFix+pCsr->lfh.nFile+pCsr->lfh.nExtra; + pLFH->iVersionExtract = zipfileRead16(aRead); + pLFH->flags = zipfileRead16(aRead); + pLFH->iCompression = zipfileRead16(aRead); + pLFH->mTime = zipfileRead16(aRead); + pLFH->mDate = zipfileRead16(aRead); + pLFH->crc32 = zipfileRead32(aRead); + pLFH->szCompressed = zipfileRead32(aRead); + pLFH->szUncompressed = zipfileRead32(aRead); + pLFH->nFile = zipfileRead16(aRead); + pLFH->nExtra = zipfileRead16(aRead); + assert( aRead==&aTmp[szFix] ); } } + return rc; +} +static int zipfileCsrReadLFH(ZipfileCsr *pCsr){ + FILE *pFile = zipfileGetFd(pCsr); + char **pzErr = &pCsr->base.pVtab->zErrMsg; + u8 *aRead = zipfileCsrBuffer(pCsr); + int rc = zipfileReadLFH(pFile, pCsr->cds.iOffset, aRead, &pCsr->lfh, pzErr); + pCsr->iDataOff = pCsr->cds.iOffset + ZIPFILE_LFH_FIXED_SZ; + pCsr->iDataOff += pCsr->lfh.nFile+pCsr->lfh.nExtra; return rc; } /* @@ -627,13 +652,13 @@ pCsr->bEof = 1; } } if( pCsr->bEof==0 ){ - rc = zipfileReadCDS(pCsr); + rc = zipfileCsrReadCDS(pCsr); if( rc==SQLITE_OK ){ - rc = zipfileReadLFH(pCsr); + rc = zipfileCsrReadLFH(pCsr); } } return rc; } @@ -768,10 +793,13 @@ sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ int i /* Which column to return */ ){ ZipfileCsr *pCsr = (ZipfileCsr*)cur; int rc = SQLITE_OK; + if( i>=3 && sqlite3_vtab_nochange(ctx) ){ + return SQLITE_OK; + } switch( i ){ case 0: /* name */ sqlite3_result_text(ctx, pCsr->cds.zFile, -1, SQLITE_TRANSIENT); break; case 1: /* mode */ @@ -987,19 +1015,30 @@ /* ** Add object pNew to the end of the linked list that begins at ** ZipfileTab.pFirstEntry and ends with pLastEntry. */ -static void zipfileAddEntry(ZipfileTab *pTab, ZipfileEntry *pNew){ +static void zipfileAddEntry( + ZipfileTab *pTab, + ZipfileEntry *pBefore, + ZipfileEntry *pNew +){ assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) ); assert( pNew->pNext==0 ); - if( pTab->pFirstEntry==0 ){ - pTab->pFirstEntry = pTab->pLastEntry = pNew; + if( pBefore==0 ){ + if( pTab->pFirstEntry==0 ){ + pTab->pFirstEntry = pTab->pLastEntry = pNew; + }else{ + assert( pTab->pLastEntry->pNext==0 ); + pTab->pLastEntry->pNext = pNew; + pTab->pLastEntry = pNew; + } }else{ - assert( pTab->pLastEntry->pNext==0 ); - pTab->pLastEntry->pNext = pNew; - pTab->pLastEntry = pNew; + ZipfileEntry **pp; + for(pp=&pTab->pFirstEntry; *pp!=pBefore; pp=&((*pp)->pNext)); + pNew->pNext = pBefore; + *pp = pNew; } } static int zipfileLoadDirectory(ZipfileTab *pTab){ ZipfileEOCD eocd; @@ -1042,11 +1081,11 @@ memcpy(pNew->zPath, &aRec[ZIPFILE_CDS_FIXED_SZ], nFile); pNew->zPath[nFile] = '\0'; pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1]; pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment; memcpy(pNew->aCdsEntry, aRec, pNew->nCdsEntry); - zipfileAddEntry(pTab, pNew); + zipfileAddEntry(pTab, 0, pNew); } iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment; } @@ -1166,11 +1205,11 @@ ){ const char *z = (const char*)sqlite3_value_text(pVal); u32 mode = 0; if( z==0 ){ mode = defaultMode; - }else if( z[0]>=0 && z[0]<=9 ){ + }else if( z[0]>='0' && z[0]<='9' ){ mode = (unsigned int)sqlite3_value_int(pVal); }else{ const char zTemplate[11] = "-rwxrwxrwx"; int i; if( strlen(z)!=10 ) goto parse_error; @@ -1229,83 +1268,108 @@ int nData = 0; /* Size of pData buffer in bytes */ int iMethod = 0; /* Compression method for new entry */ u8 *pFree = 0; /* Free this */ char *zFree = 0; /* Also free this */ ZipfileCDS cds; /* New Central Directory Structure entry */ - + ZipfileEntry *pOld = 0; int bIsDir = 0; - int mNull; - assert( pTab->zFile ); assert( pTab->pWriteFd ); if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ - if( nVal>1 ){ - return SQLITE_CONSTRAINT; - }else{ - const char *zDelete = (const char*)sqlite3_value_text(apVal[0]); - int nDelete = strlen(zDelete); - ZipfileEntry *p; - for(p=pTab->pFirstEntry; p; p=p->pNext){ - if( zipfileComparePath(p->zPath, zDelete, nDelete)==0 ){ - p->bDeleted = 1; - break; + const char *zDelete = (const char*)sqlite3_value_text(apVal[0]); + int nDelete = strlen(zDelete); + for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){ + if( pOld->bDeleted ) continue; + if( zipfileComparePath(pOld->zPath, zDelete, nDelete)==0 ){ + pOld->bDeleted = 1; + break; + } + assert( pOld->pNext ); + } + if( nVal==1 ) return SQLITE_OK; + } + + if( sqlite3_value_nochange(apVal[5]) && sqlite3_value_nochange(apVal[6]) + && sqlite3_value_nochange(apVal[7]) && sqlite3_value_nochange(apVal[8]) + ){ + /* Reuse the data from the existing entry. */ + FILE *pFile = pTab->pWriteFd; + ZipfileCDS cds; + zipfileReadCDS(pOld->aCdsEntry, &cds); + + bIsDir = ((cds.iExternalAttr>>16) & S_IFDIR) ? 1 : 0; + sz = cds.szUncompressed; + iMethod = cds.iCompression; + if( sz>0 ){ + char **pzErr = &pTab->base.zErrMsg; + ZipfileLFH lfh; + rc = zipfileReadLFH(pFile, cds.iOffset, pTab->aBuffer, &lfh, pzErr); + if( rc==SQLITE_OK ){ + nData = lfh.szCompressed; + pData = pFree = sqlite3_malloc(nData); + if( pFree==NULL ){ + rc = SQLITE_NOMEM; + }else{ + i64 iRead = cds.iOffset + ZIPFILE_LFH_FIXED_SZ + lfh.nFile+lfh.nExtra; + rc = zipfileReadData(pFile, pFree, nData, iRead, pzErr); } } - return SQLITE_OK; + } + }else{ + int mNull; + mNull = (sqlite3_value_type(apVal[5])==SQLITE_NULL ? 0x0 : 0x8) /* sz */ + + (sqlite3_value_type(apVal[6])==SQLITE_NULL ? 0x0 : 0x4) /* rawdata */ + + (sqlite3_value_type(apVal[7])==SQLITE_NULL ? 0x0 : 0x2) /* data */ + + (sqlite3_value_type(apVal[8])==SQLITE_NULL ? 0x0 : 0x1); /* method */ + if( mNull==0x00 ){ + /* All four are NULL - this must be a directory */ + bIsDir = 1; } - } - - mNull = (sqlite3_value_type(apVal[5])==SQLITE_NULL ? 0x0 : 0x8) /* sz */ - + (sqlite3_value_type(apVal[6])==SQLITE_NULL ? 0x0 : 0x4) /* rawdata */ - + (sqlite3_value_type(apVal[7])==SQLITE_NULL ? 0x0 : 0x2) /* data */ - + (sqlite3_value_type(apVal[8])==SQLITE_NULL ? 0x0 : 0x1); /* method */ - if( mNull==0x00 ){ - /* All four are NULL - this must be a directory */ - bIsDir = 1; - } - else if( mNull==0x2 || mNull==0x3 ){ - /* Value specified for "data", and possibly "method". This must be - ** a regular file or a symlink. */ - const u8 *aIn = sqlite3_value_blob(apVal[7]); - int nIn = sqlite3_value_bytes(apVal[7]); - int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL; - - iMethod = sqlite3_value_int(apVal[8]); - sz = nIn; - if( iMethod!=0 && iMethod!=8 ){ - rc = SQLITE_CONSTRAINT; - }else if( bAuto || iMethod ){ - rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nData); - if( rc==SQLITE_OK ){ - if( iMethod || nData65535 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "zipfile: invalid compression method: %d", iMethod - ); - rc = SQLITE_ERROR; + else if( mNull==0x0D ){ + /* Values specified for "sz", "rawdata" and "method". In other words, + ** pre-compressed data is being inserted. */ + pData = sqlite3_value_blob(apVal[6]); + nData = sqlite3_value_bytes(apVal[6]); + sz = sqlite3_value_int(apVal[5]); + iMethod = sqlite3_value_int(apVal[8]); + if( iMethod<0 || iMethod>65535 ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "zipfile: invalid compression method: %d", iMethod + ); + rc = SQLITE_ERROR; + } } - } - else{ - rc = SQLITE_CONSTRAINT; + else{ + rc = SQLITE_CONSTRAINT; + } } if( rc==SQLITE_OK ){ rc = zipfileGetMode(pTab, apVal[3], (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)), &mode @@ -1342,10 +1406,11 @@ /* Check that we're not inserting a duplicate entry */ if( rc==SQLITE_OK ){ ZipfileEntry *p; for(p=pTab->pFirstEntry; p; p=p->pNext){ + if( p->bDeleted ) continue; if( zipfileComparePath(p->zPath, zPath, nPath)==0 ){ rc = SQLITE_CONSTRAINT; break; } } @@ -1366,19 +1431,22 @@ cds.iOffset = (u32)pTab->szCurrent; pNew = zipfileNewEntry(&cds, zPath, nPath, (u32)mTime); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ - zipfileAddEntry(pTab, pNew); + zipfileAddEntry(pTab, pOld, pNew); } } /* Append the new header+file to the archive */ if( rc==SQLITE_OK ){ rc = zipfileAppendEntry(pTab, &cds, zPath, nPath, pData, nData, (u32)mTime); } + if( rc!=SQLITE_OK && pOld ){ + pOld->bDeleted = 0; + } sqlite3_free(pFree); sqlite3_free(zFree); return rc; } Index: test/zipfile.test ================================================================== --- test/zipfile.test +++ test/zipfile.test @@ -74,15 +74,82 @@ COMMIT; } {f.txt g.txt h.txt i.txt} do_execsql_test 1.5.2 { SELECT name FROM zz; } {f.txt g.txt h.txt i.txt} +do_execsql_test 1.5.3 { + SELECT data FROM zz WHERE name='i.txt'; +} {zxcvb} do_execsql_test 1.6.0 { DELETE FROM zz WHERE name='g.txt'; SELECT name FROM zz; } {f.txt h.txt i.txt} + +do_execsql_test 1.6.1 { + SELECT name, mode, mtime, data, method FROM zipfile('test.zip'); +} { + f.txt 33188 1000000000 abcde 0 + h.txt 33188 1000000004 aaaaaaaaaabbbbbbbbbb 8 + i.txt 33188 1000000006 zxcvb 0 +} + +do_execsql_test 1.6.2 { + UPDATE zz SET mtime=4 WHERE name='i.txt'; + SELECT name, mode, mtime, data, method FROM zipfile('test.zip'); +} { + f.txt 33188 1000000000 abcde 0 + h.txt 33188 1000000004 aaaaaaaaaabbbbbbbbbb 8 + i.txt 33188 4 zxcvb 0 +} + +do_execsql_test 1.6.3 { + UPDATE zz SET mode='-rw-r--r-x' WHERE name='h.txt'; + SELECT name, mode, mtime, data, method FROM zipfile('test.zip'); +} { + f.txt 33188 1000000000 abcde 0 + h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8 + i.txt 33188 4 zxcvb 0 +} + +do_execsql_test 1.6.4 { + UPDATE zz SET name = 'blue.txt' WHERE name='f.txt'; + SELECT name, mode, mtime, data, method FROM zipfile('test.zip'); +} { + blue.txt 33188 1000000000 abcde 0 + h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8 + i.txt 33188 4 zxcvb 0 +} + +do_execsql_test 1.6.5 { + UPDATE zz SET data = 'edcba' WHERE name='blue.txt'; + SELECT name, mode, mtime, data, method FROM zipfile('test.zip'); +} { + blue.txt 33188 1000000000 edcba 0 + h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8 + i.txt 33188 4 zxcvb 0 +} + +do_execsql_test 1.6.6 { + UPDATE zz SET mode=NULL, data = NULL WHERE name='blue.txt'; + SELECT name, mode, mtime, data, method FROM zipfile('test.zip'); +} { + blue.txt/ 16877 1000000000 {} 0 + h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8 + i.txt 33188 4 zxcvb 0 +} + +do_catchsql_test 1.6.7 { + UPDATE zz SET data=NULL WHERE name='i.txt' +} {1 {constraint failed}} +do_execsql_test 1.6.8 { + SELECT name, mode, mtime, data, method FROM zipfile('test.zip'); +} { + blue.txt/ 16877 1000000000 {} 0 + h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8 + i.txt 33188 4 zxcvb 0 +} #------------------------------------------------------------------------- db close forcedelete test.zip reset_db @@ -102,29 +169,29 @@ dirname2/file1.txt 33188 abcdefghijklmnop } do_catchsql_test 2.3 { UPDATE zzz SET name = 'dirname3' WHERE name = 'dirname/'; -} {1 {constraint failed}} +} {0 {}} do_execsql_test 2.4 { SELECT name, mode, data FROM zzz; } { - dirname/ 16877 {} + dirname3/ 16877 {} dirname2/ 16877 {} dirname2/file1.txt 33188 abcdefghijklmnop } # If on unix, check that the [unzip] utility can unpack our archive. # if {$::tcl_platform(platform)=="unix"} { do_test 2.5.1 { forcedelete dirname - forcedelete dirname2 - set rc [catch { exec unzip test.zip > /dev/null } msg] - list $rc $msg + forcedelete dirname2 + set rc [catch { exec unzip test.zip > /dev/null } msg] + list $rc $msg } {0 {}} - do_test 2.5.2 { file isdir dirname } 1 + do_test 2.5.2 { file isdir dirname3 } 1 do_test 2.5.3 { file isdir dirname2 } 1 do_test 2.5.4 { file isdir dirname2/file1.txt } 0 do_test 2.5.5 { set fd [open dirname2/file1.txt] set data [read $fd]