Index: ext/misc/zipfile.c ================================================================== --- ext/misc/zipfile.c +++ ext/misc/zipfile.c @@ -198,52 +198,58 @@ u32 szCompressed; u32 szUncompressed; u16 nFile; u16 nExtra; }; + +typedef struct ZipfileEntry ZipfileEntry; +struct ZipfileEntry { + char *zPath; /* Path of zipfile entry */ + i64 iRowid; /* Rowid for this value if queried */ + u8 *aCdsEntry; /* Buffer containing entire CDS entry */ + int nCdsEntry; /* Size of buffer aCdsEntry[] in bytes */ + int bDeleted; /* True if entry has been deleted */ + ZipfileEntry *pNext; /* Next element in in-memory CDS */ +}; /* ** Cursor type for recursively iterating through a directory structure. */ typedef struct ZipfileCsr ZipfileCsr; - struct ZipfileCsr { sqlite3_vtab_cursor base; /* Base class - must be first */ - i64 iRowid; /* Rowid for current row */ - FILE *pFile; /* Zip file */ - i64 nByte; /* Size of zip file on disk */ int bEof; /* True when at EOF */ + + /* Used outside of write transactions */ + FILE *pFile; /* Zip file */ i64 iNextOff; /* Offset of next record in central directory */ ZipfileEOCD eocd; /* Parse of central directory record */ + + /* Used inside write transactions */ + ZipfileEntry *pCurrent; + ZipfileCDS cds; /* Central Directory Structure */ ZipfileLFH lfh; /* Local File Header for current entry */ i64 iDataOff; /* Offset in zipfile to data */ u32 mTime; /* Extended mtime value */ int flags; /* Flags byte (see below for bits) */ + ZipfileCsr *pCsrNext; /* Next cursor on same virtual table */ }; /* ** Values for ZipfileCsr.flags. */ #define ZIPFILE_MTIME_VALID 0x0001 -typedef struct ZipfileEntry ZipfileEntry; -struct ZipfileEntry { - char *zPath; /* Path of zipfile entry */ - i64 iRowid; /* Rowid for this value if queried */ - u8 *aCdsEntry; /* Buffer containing entire CDS entry */ - int nCdsEntry; /* Size of buffer aCdsEntry[] in bytes */ - ZipfileEntry *pNext; /* Next element in in-memory CDS */ -}; - typedef struct ZipfileTab ZipfileTab; struct ZipfileTab { sqlite3_vtab base; /* Base class - must be first */ char *zFile; /* Zip file this table accesses (may be NULL) */ u8 *aBuffer; /* Temporary buffer used for various tasks */ /* The following are used by write transactions only */ + ZipfileCsr *pCsrList; /* List of cursors */ ZipfileEntry *pFirstEntry; /* Linked list of all files (if pWriteFd!=0) */ ZipfileEntry *pLastEntry; /* Last element in pFirstEntry list */ FILE *pWriteFd; /* File handle open on zip archive */ i64 szCurrent; /* Current size of zip archive */ i64 szOrig; /* Size of archive at start of transaction */ @@ -333,13 +339,20 @@ /* ** Reset a cursor back to the state it was in when first returned ** by zipfileOpen(). */ static void zipfileResetCursor(ZipfileCsr *pCsr){ + ZipfileTab *pTab = (ZipfileTab*)(pCsr->base.pVtab); + ZipfileCsr **pp; + + /* Remove this cursor from the ZipfileTab.pCsrList list. */ + for(pp=&pTab->pCsrList; *pp; pp=&((*pp)->pCsrNext)){ + if( *pp==pCsr ) *pp = pCsr->pCsrNext; + } + sqlite3_free(pCsr->cds.zFile); pCsr->cds.zFile = 0; - pCsr->iRowid = 0; pCsr->bEof = 0; if( pCsr->pFile ){ fclose(pCsr->pFile); pCsr->pFile = 0; } @@ -436,22 +449,29 @@ #define ZIPFILE_CDS_FIXED_SZ 46 #define ZIPFILE_CDS_NFILE_OFF 28 static int zipfileReadCDS(ZipfileCsr *pCsr){ char **pzErr = &pCsr->base.pVtab->zErrMsg; - u8 *aRead = zipfileCsrBuffer(pCsr); - int rc; + u8 *aRead; + int rc = SQLITE_OK; sqlite3_free(pCsr->cds.zFile); pCsr->cds.zFile = 0; - rc = zipfileReadData( - pCsr->pFile, aRead, ZIPFILE_CDS_FIXED_SZ, pCsr->iNextOff, pzErr - ); + if( pCsr->pCurrent==0 ){ + aRead = zipfileCsrBuffer(pCsr); + rc = zipfileReadData( + pCsr->pFile, aRead, ZIPFILE_CDS_FIXED_SZ, pCsr->iNextOff, pzErr + ); + }else{ + aRead = pCsr->pCurrent->aCdsEntry; + } + if( rc==SQLITE_OK ){ u32 sig = zipfileRead32(aRead); if( sig!=ZIPFILE_SIGNATURE_CDS ){ + assert( pCsr->pCurrent==0 ); zipfileSetErrmsg(pCsr,"failed to read CDS at offset %lld",pCsr->iNextOff); rc = SQLITE_ERROR; }else{ int nRead; pCsr->cds.iVersionMadeBy = zipfileRead16(aRead); @@ -461,25 +481,30 @@ pCsr->cds.mTime = zipfileRead16(aRead); pCsr->cds.mDate = zipfileRead16(aRead); pCsr->cds.crc32 = zipfileRead32(aRead); pCsr->cds.szCompressed = zipfileRead32(aRead); pCsr->cds.szUncompressed = zipfileRead32(aRead); - assert( aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_NFILE_OFF ); + assert( pCsr->pCurrent + || aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_NFILE_OFF + ); pCsr->cds.nFile = zipfileRead16(aRead); pCsr->cds.nExtra = zipfileRead16(aRead); pCsr->cds.nComment = zipfileRead16(aRead); pCsr->cds.iDiskStart = zipfileRead16(aRead); pCsr->cds.iInternalAttr = zipfileRead16(aRead); pCsr->cds.iExternalAttr = zipfileRead32(aRead); pCsr->cds.iOffset = zipfileRead32(aRead); + assert( pCsr->pCurrent + || aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_FIXED_SZ + ); - assert( aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_FIXED_SZ ); - - nRead = pCsr->cds.nFile + pCsr->cds.nExtra; - aRead = zipfileCsrBuffer(pCsr); - pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ; - rc = zipfileReadData(pCsr->pFile, aRead, nRead, pCsr->iNextOff, pzErr); + if( pCsr->pCurrent==0 ){ + nRead = pCsr->cds.nFile + pCsr->cds.nExtra; + aRead = zipfileCsrBuffer(pCsr); + pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ; + rc = zipfileReadData(pCsr->pFile, aRead, nRead, pCsr->iNextOff, pzErr); + } if( rc==SQLITE_OK ){ pCsr->cds.zFile = sqlite3_mprintf("%.*s", (int)pCsr->cds.nFile, aRead); pCsr->iNextOff += pCsr->cds.nFile; pCsr->iNextOff += pCsr->cds.nExtra; @@ -519,18 +544,24 @@ } } return rc; } + +static FILE *zipfileGetFd(ZipfileCsr *pCsr){ + if( pCsr->pFile ) return pCsr->pFile; + return ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd; +} static int zipfileReadLFH(ZipfileCsr *pCsr){ + FILE *pFile = zipfileGetFd(pCsr); char **pzErr = &pCsr->base.pVtab->zErrMsg; static const int szFix = ZIPFILE_LFH_FIXED_SZ; u8 *aRead = zipfileCsrBuffer(pCsr); int rc; - rc = zipfileReadData(pCsr->pFile, aRead, szFix, pCsr->cds.iOffset, pzErr); + rc = zipfileReadData(pFile, aRead, szFix, pCsr->cds.iOffset, pzErr); if( rc==SQLITE_OK ){ u32 sig = zipfileRead32(aRead); if( sig!=ZIPFILE_SIGNATURE_LFH ){ zipfileSetErrmsg(pCsr, "failed to read LFH at offset %d", (int)pCsr->cds.iOffset @@ -559,23 +590,35 @@ /* ** Advance an ZipfileCsr to its next row of output. */ static int zipfileNext(sqlite3_vtab_cursor *cur){ ZipfileCsr *pCsr = (ZipfileCsr*)cur; - i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize; int rc = SQLITE_OK; + pCsr->flags = 0; - if( pCsr->iNextOff>=iEof ){ - pCsr->bEof = 1; + if( pCsr->pCurrent==0 ){ + i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize; + if( pCsr->iNextOff>=iEof ){ + pCsr->bEof = 1; + } }else{ - pCsr->iRowid++; - pCsr->flags = 0; + assert( pCsr->pFile==0 ); + do { + pCsr->pCurrent = pCsr->pCurrent->pNext; + }while( pCsr->pCurrent && pCsr->pCurrent->bDeleted ); + if( pCsr->pCurrent==0 ){ + pCsr->bEof = 1; + } + } + + if( pCsr->bEof==0 ){ rc = zipfileReadCDS(pCsr); if( rc==SQLITE_OK ){ rc = zipfileReadLFH(pCsr); } } + return rc; } /* ** "Standard" MS-DOS time format: @@ -656,11 +699,12 @@ if( sz>0 ){ u8 *aBuf = sqlite3_malloc(sz); if( aBuf==0 ){ rc = SQLITE_NOMEM; }else{ - rc = zipfileReadData(pCsr->pFile, aBuf, sz, pCsr->iDataOff, + FILE *pFile = zipfileGetFd(pCsr); + rc = zipfileReadData(pFile, aBuf, sz, pCsr->iDataOff, &pCsr->base.pVtab->zErrMsg ); } if( rc==SQLITE_OK ){ sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT); @@ -676,17 +720,19 @@ return SQLITE_OK; } /* -** Return the rowid for the current row. In this implementation, the -** first row returned is assigned rowid value 1, and each subsequent -** row a value 1 more than that of the previous. +** Return the rowid for the current row. */ static int zipfileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ ZipfileCsr *pCsr = (ZipfileCsr*)cur; - *pRowid = pCsr->iRowid; + if( pCsr->pCurrent ){ + *pRowid = pCsr->pCurrent->iRowid; + }else{ + *pRowid = pCsr->cds.iOffset; + } return SQLITE_OK; } /* ** Return TRUE if the cursor has been moved off of the last @@ -769,36 +815,43 @@ const char *zFile; /* Zip file to scan */ int rc = SQLITE_OK; /* Return Code */ zipfileResetCursor(pCsr); - assert( idxNum==argc && (idxNum==0 || idxNum==1) ); - if( idxNum==0 ){ - ZipfileTab *pTab = (ZipfileTab*)cur->pVtab; + if( pTab->zFile ){ zFile = pTab->zFile; - if( zFile==0 ){ - /* Error. This is an eponymous virtual table and the user has not - ** supplied a file name. */ - zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument"); - return SQLITE_ERROR; - } + }else if( idxNum==0 ){ + /* Error. This is an eponymous virtual table and the user has not + ** supplied a file name. */ + zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument"); + return SQLITE_ERROR; }else{ zFile = (const char*)sqlite3_value_text(argv[0]); } - pCsr->pFile = fopen(zFile, "rb"); - if( pCsr->pFile==0 ){ - zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile); - rc = SQLITE_ERROR; - }else{ - rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd); - if( rc==SQLITE_OK ){ - pCsr->iNextOff = pCsr->eocd.iOffset; - rc = zipfileNext(cur); - }else if( rc==SQLITE_EMPTY ){ - rc = SQLITE_OK; - pCsr->bEof = 1; + + if( pTab->pWriteFd==0 ){ + pCsr->pFile = fopen(zFile, "rb"); + if( pCsr->pFile==0 ){ + zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile); + rc = SQLITE_ERROR; + }else{ + rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd); + if( rc==SQLITE_OK ){ + pCsr->iNextOff = pCsr->eocd.iOffset; + rc = zipfileNext(cur); + }else if( rc==SQLITE_EMPTY ){ + rc = SQLITE_OK; + pCsr->bEof = 1; + } } + }else{ + ZipfileEntry e; + memset(&e, 0, sizeof(e)); + e.pNext = pTab->pFirstEntry; + pCsr->pCurrent = &e; + rc = zipfileNext(cur); + assert( pCsr->pCurrent!=&e ); } return rc; } @@ -838,13 +891,15 @@ */ static void zipfileAddEntry(ZipfileTab *pTab, ZipfileEntry *pNew){ assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) ); assert( pNew->pNext==0 ); if( pTab->pFirstEntry==0 ){ + pNew->iRowid = 1; pTab->pFirstEntry = pTab->pLastEntry = pNew; }else{ assert( pTab->pLastEntry->pNext==0 ); + pNew->iRowid = pTab->pLastEntry->iRowid+1; pTab->pLastEntry->pNext = pNew; pTab->pLastEntry = pNew; } } @@ -882,16 +937,16 @@ + ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment ); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ + memset(pNew, 0, sizeof(ZipfileEntry)); pNew->zPath = (char*)&pNew[1]; memcpy(pNew->zPath, &aBuf[ZIPFILE_CDS_FIXED_SZ], nFile); pNew->zPath[nFile] = '\0'; pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1]; pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment; - pNew->pNext = 0; memcpy(pNew->aCdsEntry, aRec, pNew->nCdsEntry); zipfileAddEntry(pTab, pNew); } iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment; @@ -920,14 +975,14 @@ nPath+1 + ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra ); if( pNew ){ + memset(pNew, 0, sizeof(ZipfileEntry)); pNew->zPath = (char*)&pNew[1]; pNew->aCdsEntry = (u8*)&pNew->zPath[nPath+1]; pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra; - pNew->pNext = 0; memcpy(pNew->zPath, zPath, nPath+1); aWrite = pNew->aCdsEntry; zipfileWrite32(aWrite, ZIPFILE_SIGNATURE_CDS); zipfileWrite16(aWrite, pCds->iVersionMadeBy); @@ -1055,42 +1110,23 @@ int nData; /* Size of pData buffer in bytes */ int iMethod = 0; /* Compression method for new entry */ u8 *pFree = 0; /* Free this */ ZipfileCDS cds; /* New Central Directory Structure entry */ + assert( pTab->zFile ); + assert( pTab->pWriteFd ); + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "zipfile: UPDATE/DELETE not yet supported" - ); - return SQLITE_ERROR; - } - - /* This table is only writable if a default archive path was specified - ** as part of the CREATE VIRTUAL TABLE statement. */ - if( pTab->zFile==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "zipfile: writing requires a default archive" - ); - return SQLITE_ERROR; - } - - /* Open a write fd on the file. Also load the entire central directory - ** structure into memory. During the transaction any new file data is - ** appended to the archive file, but the central directory is accumulated - ** in main-memory until the transaction is committed. */ - if( pTab->pWriteFd==0 ){ - pTab->pWriteFd = fopen(pTab->zFile, "ab+"); - if( pTab->pWriteFd==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "zipfile: failed to open file %s for writing", pTab->zFile - ); - return SQLITE_ERROR; + i64 iDelete = sqlite3_value_int64(apVal[0]); + ZipfileEntry *p; + for(p=pTab->pFirstEntry; p; p=p->pNext){ + if( p->iRowid==iDelete ){ + p->bDeleted = 1; + break; + } } - fseek(pTab->pWriteFd, 0, SEEK_END); - pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); - rc = zipfileLoadDirectory(pTab); - if( rc!=SQLITE_OK ) return rc; + if( nVal==1 ) return SQLITE_OK; } zPath = (const char*)sqlite3_value_text(apVal[2]); nPath = strlen(zPath); rc = zipfileGetMode(pTab, apVal[3], &mode); @@ -1197,11 +1233,45 @@ pTab->szCurrent = 0; pTab->szOrig = 0; } static int zipfileBegin(sqlite3_vtab *pVtab){ - return SQLITE_OK; + ZipfileTab *pTab = (ZipfileTab*)pVtab; + int rc = SQLITE_OK; + + assert( pTab->pWriteFd==0 ); + + /* This table is only writable if a default archive path was specified + ** as part of the CREATE VIRTUAL TABLE statement. */ + if( pTab->zFile==0 ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "zipfile: writing requires a default archive" + ); + return SQLITE_ERROR; + } + + /* Open a write fd on the file. Also load the entire central directory + ** structure into memory. During the transaction any new file data is + ** appended to the archive file, but the central directory is accumulated + ** in main-memory until the transaction is committed. */ + pTab->pWriteFd = fopen(pTab->zFile, "ab+"); + if( pTab->pWriteFd==0 ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "zipfile: failed to open file %s for writing", pTab->zFile + ); + rc = SQLITE_ERROR; + }else{ + fseek(pTab->pWriteFd, 0, SEEK_END); + pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); + rc = zipfileLoadDirectory(pTab); + } + + if( rc!=SQLITE_OK ){ + zipfileCleanupTransaction(pTab); + } + + return rc; } static int zipfileCommit(sqlite3_vtab *pVtab){ ZipfileTab *pTab = (ZipfileTab*)pVtab; int rc = SQLITE_OK; @@ -1209,12 +1279,13 @@ i64 iOffset = pTab->szCurrent; ZipfileEntry *p; ZipfileEOCD eocd; int nEntry = 0; - /* Write out all entries */ + /* Write out all undeleted entries */ for(p=pTab->pFirstEntry; rc==SQLITE_OK && p; p=p->pNext){ + if( p->bDeleted ) continue; rc = zipfileAppendData(pTab, p->aCdsEntry, p->nCdsEntry); nEntry++; } /* Write out the EOCD record */ Index: test/zipfile.test ================================================================== --- test/zipfile.test +++ test/zipfile.test @@ -27,17 +27,19 @@ 3 sz {} 0 {} 0 4 data {} 0 {} 0 5 method {} 0 {} 0 } -do_execsql_test 1.1 { +do_execsql_test 1.1.1 { INSERT INTO zz VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0); +} +do_execsql_test 1.1.2 { INSERT INTO zz VALUES('g.txt', '-rw-r--r--', 1000000002, 5, '12345', 0); } do_execsql_test 1.2 { - SELECT name, mtime, data FROM zipfile('test.zip'); + SELECT name, mtime, data FROM zipfile('test.zip') } { f.txt 1000000000 abcde g.txt 1000000002 12345 } @@ -53,8 +55,23 @@ } { f.txt 1000000000 abcde 0 g.txt 1000000002 12345 0 h.txt 1000000004 aaaaaaaaaabbbbbbbbbb 8 } + +do_execsql_test 1.5.1 { + BEGIN; + INSERT INTO zz VALUES('i.txt', '-rw-r--r--', 1000000006, 5, 'zxcvb', 0); + SELECT name FROM zz; + COMMIT; +} {f.txt g.txt h.txt i.txt} +do_execsql_test 1.5.2 { + SELECT name FROM zz; +} {f.txt g.txt h.txt i.txt} + +do_execsql_test 1.6.0 { + DELETE FROM zz WHERE name='g.txt'; + SELECT name FROM zz; +} {f.txt h.txt i.txt} finish_test