/ Check-in [b0b7d036]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem in the zipfile module causing it to generate incorrect checksums. Remove the ability to insert compressed data into a zip archive.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256:b0b7d0363acf38c2178e2d3041d8ce2a0de061a51caa64670dbf539ee6d4356b
User & Date: dan 2018-01-15 19:00:35
Context
2018-01-16
02:38
Disable the ".archive" command tests in shell8.test if the CLI is compiled without ZLIB support. check-in: ce8bfe6c user: drh tags: trunk
2018-01-15
21:59
Merge the enhancements associated with the first 3.22.0 beta. check-in: c9d2ec51 user: drh tags: apple-osx
19:00
Fix a problem in the zipfile module causing it to generate incorrect checksums. Remove the ability to insert compressed data into a zip archive. check-in: b0b7d036 user: dan tags: trunk
15:49
Fix a zipfile problem with extracting zero length files compressed using deflate. check-in: cf640872 user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/misc/zipfile.c.

801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
...
822
823
824
825
826
827
828

829

830
831
832

833
834
835
836
837
838
839
...
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
...
904
905
906
907
908
909
910

911
912
913
914
915
916
917
918
...
982
983
984
985
986
987
988



989
990
991
992
993

994
995
996
997
998
999
1000
....
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
....
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
....
1290
1291
1292
1293
1294
1295
1296

1297
1298
1299
1300
1301
1302
1303
....
1308
1309
1310
1311
1312
1313
1314
1315
1316

1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343

1344
1345
1346
1347

1348
1349
1350

1351
1352
1353
1354
1355
1356
1357
....
1366
1367
1368
1369
1370
1371
1372

1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
....
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
static int zipfileColumn(
  sqlite3_vtab_cursor *cur,   /* The cursor */
  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
  int i                       /* Which column to return */
){
  ZipfileCsr *pCsr = (ZipfileCsr*)cur;
  int rc = SQLITE_OK;
  if( i>=3 && sqlite3_vtab_nochange(ctx) ){
    return SQLITE_OK;
  }
  switch( i ){
    case 0:   /* name */
      sqlite3_result_text(ctx, pCsr->cds.zFile, -1, SQLITE_TRANSIENT);
      break;
    case 1:   /* mode */
      /* TODO: Whether or not the following is correct surely depends on
      ** the platform on which the archive was created.  */
................................................................................
        sqlite3_result_int64(ctx, pCsr->mTime);
      }else{
        sqlite3_result_int64(ctx, zipfileMtime(pCsr));
      }
      break;
    }
    case 3: { /* sz */

      sqlite3_result_int64(ctx, pCsr->cds.szUncompressed);

      break;
    }
    case 4:   /* rawdata */

    case 5: { /* data */
      if( i==4 || pCsr->cds.iCompression==0 || pCsr->cds.iCompression==8 ){
        int sz = pCsr->cds.szCompressed;
        int szFinal = pCsr->cds.szUncompressed;
        if( szFinal>0 ){
          u8 *aBuf = sqlite3_malloc(sz);
          if( aBuf==0 ){
................................................................................
      sqlite3_result_int(ctx, pCsr->cds.iCompression);
      break;
    case 7:   /* z */
      sqlite3_result_int64(ctx, pCsr->iId);
      break;
  }

  return SQLITE_OK;
}

/*
** Return the rowid for the current row.
*/
static int zipfileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
  assert( 0 );
................................................................................
  int nRead;                      /* Bytes to read from file */
  i64 iOff;                       /* Offset to read from */
  int rc;

  fseek(pFile, 0, SEEK_END);
  szFile = (i64)ftell(pFile);
  if( szFile==0 ){

    return SQLITE_EMPTY;
  }
  nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE));
  iOff = szFile - nRead;

  rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg);
  if( rc==SQLITE_OK ){
    int i;
................................................................................
    pCsr->pFile = fopen(zFile, "rb");
    if( pCsr->pFile==0 ){
      zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile);
      rc = SQLITE_ERROR;
    }else{
      rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd);
      if( rc==SQLITE_OK ){



        pCsr->iNextOff = pCsr->eocd.iOffset;
        rc = zipfileNext(cur);
      }else if( rc==SQLITE_EMPTY ){
        rc = SQLITE_OK;
        pCsr->bEof = 1;

      }
    }
  }else{
    ZipfileEntry e;
    memset(&e, 0, sizeof(e));
    e.pNext = pTab->pFirstEntry;
    pCsr->pCurrent = &e;
................................................................................
}

static int zipfileLoadDirectory(ZipfileTab *pTab){
  ZipfileEOCD eocd;
  int rc;

  rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd);
  if( rc==SQLITE_OK ){
    int i;
    int iOff = 0;
    u8 *aBuf = sqlite3_malloc(eocd.nSize);
    if( aBuf==0 ){
      rc = SQLITE_NOMEM;
    }else{
      rc = zipfileReadData(
................................................................................
        zipfileAddEntry(pTab, 0, pNew);
      }

      iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
    }

    sqlite3_free(aBuf);
  }else if( rc==SQLITE_EMPTY ){
    rc = SQLITE_OK;
  }

  return rc;
}

static ZipfileEntry *zipfileNewEntry(
  ZipfileCDS *pCds,               /* Values for fixed size part of CDS */
................................................................................
  int nData = 0;                  /* Size of pData buffer in bytes */
  int iMethod = 0;                /* Compression method for new entry */
  u8 *pFree = 0;                  /* Free this */
  char *zFree = 0;                /* Also free this */
  ZipfileCDS cds;                 /* New Central Directory Structure entry */
  ZipfileEntry *pOld = 0;
  int bIsDir = 0;


  assert( pTab->zFile );
  assert( pTab->pWriteFd );

  if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
    const char *zDelete = (const char*)sqlite3_value_text(apVal[0]);
    int nDelete = (int)strlen(zDelete);
................................................................................
        break;
      }
      assert( pOld->pNext );
    }
    if( nVal==1 ) return SQLITE_OK;
  }

  if( sqlite3_value_nochange(apVal[5]) && sqlite3_value_nochange(apVal[6])
   && sqlite3_value_nochange(apVal[7]) && sqlite3_value_nochange(apVal[8])

  ){
    /* Reuse the data from the existing entry. */
    FILE *pFile = pTab->pWriteFd;
    zipfileReadCDS(pOld->aCdsEntry, &cds);

    bIsDir = ((cds.iExternalAttr>>16) & S_IFDIR) ? 1 : 0;
    sz = cds.szUncompressed;
    iMethod = cds.iCompression;
    if( sz>0 ){
      char **pzErr = &pTab->base.zErrMsg;
      ZipfileLFH lfh;
      rc = zipfileReadLFH(pFile, cds.iOffset, pTab->aBuffer, &lfh, pzErr);
      if( rc==SQLITE_OK ){
        nData = lfh.szCompressed;
        pData = pFree = sqlite3_malloc(nData);
        if( pFree==NULL ){
          rc = SQLITE_NOMEM;
        }else{
          i64 iRead = cds.iOffset + ZIPFILE_LFH_FIXED_SZ + lfh.nFile+lfh.nExtra;
          rc = zipfileReadData(pFile, pFree, nData, iRead, pzErr);
        }
      }
    }
  }else{
    int mNull;
    mNull = (sqlite3_value_type(apVal[5])==SQLITE_NULL ? 0x0 : 0x8)  /* sz */
      + (sqlite3_value_type(apVal[6])==SQLITE_NULL ? 0x0 : 0x4)  /* rawdata */

      + (sqlite3_value_type(apVal[7])==SQLITE_NULL ? 0x0 : 0x2)  /* data */
      + (sqlite3_value_type(apVal[8])==SQLITE_NULL ? 0x0 : 0x1); /* method */
    if( mNull==0x00 ){     
      /* All four are NULL - this must be a directory */

      bIsDir = 1;
    }
    else if( mNull==0x2 || mNull==0x3 ){

      /* Value specified for "data", and possibly "method". This must be
      ** a regular file or a symlink. */
      const u8 *aIn = sqlite3_value_blob(apVal[7]);
      int nIn = sqlite3_value_bytes(apVal[7]);
      int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL;

      iMethod = sqlite3_value_int(apVal[8]);
................................................................................
        if( rc==SQLITE_OK ){
          if( iMethod || nCmp<nIn ){
            iMethod = 8;
            pData = pFree;
            nData = nCmp;
          }
        }

      }
    }
    else if( mNull==0x0D ){
      /* Values specified for "sz", "rawdata" and "method". In other words,
      ** pre-compressed data is being inserted.  */
      pData = sqlite3_value_blob(apVal[6]);
      nData = sqlite3_value_bytes(apVal[6]);
      sz = sqlite3_value_int(apVal[5]);
      iMethod = sqlite3_value_int(apVal[8]);
      if( iMethod<0 || iMethod>65535 ){
        pTab->base.zErrMsg = sqlite3_mprintf(
            "zipfile: invalid compression method: %d", iMethod
            );
        rc = SQLITE_ERROR;
      }
    }
    else{
      rc = SQLITE_CONSTRAINT;
    }
  }

  if( rc==SQLITE_OK ){
    rc = zipfileGetMode(pTab, apVal[3], 
        (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)), &mode
    );
    if( rc==SQLITE_OK && (bIsDir == ((mode & S_IFDIR)==0)) ){
................................................................................
    /* Create the new CDS record. */
    memset(&cds, 0, sizeof(cds));
    cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY;
    cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED;
    cds.flags = ZIPFILE_NEWENTRY_FLAGS;
    cds.iCompression = (u16)iMethod;
    zipfileMtimeToDos(&cds, (u32)mTime);
    cds.crc32 = crc32(0, pData, nData);
    cds.szCompressed = nData;
    cds.szUncompressed = (u32)sz;
    cds.iExternalAttr = (mode<<16);
    cds.iOffset = (u32)pTab->szCurrent;
    pNew = zipfileNewEntry(&cds, zPath, nPath, (u32)mTime);
    if( pNew==0 ){
      rc = SQLITE_NOMEM;







<
<
<







 







>
|
>



>







 







|







 







>
|







 







>
>
>
|
|
<
<
<
>







 







|







 







<
<







 







>







 







|
|
>

<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|
<
<
<
|
|
<
<
<
<
<
>
|
<
<
<
>

<
<
>







 







>


<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







|







801
802
803
804
805
806
807



808
809
810
811
812
813
814
...
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
...
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
...
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
...
983
984
985
986
987
988
989
990
991
992
993
994



995
996
997
998
999
1000
1001
1002
....
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
....
1110
1111
1112
1113
1114
1115
1116


1117
1118
1119
1120
1121
1122
1123
....
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
....
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319















1320



1321
1322





1323
1324



1325
1326


1327
1328
1329
1330
1331
1332
1333
1334
....
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352

















1353
1354
1355
1356
1357
1358
1359
....
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
static int zipfileColumn(
  sqlite3_vtab_cursor *cur,   /* The cursor */
  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
  int i                       /* Which column to return */
){
  ZipfileCsr *pCsr = (ZipfileCsr*)cur;
  int rc = SQLITE_OK;



  switch( i ){
    case 0:   /* name */
      sqlite3_result_text(ctx, pCsr->cds.zFile, -1, SQLITE_TRANSIENT);
      break;
    case 1:   /* mode */
      /* TODO: Whether or not the following is correct surely depends on
      ** the platform on which the archive was created.  */
................................................................................
        sqlite3_result_int64(ctx, pCsr->mTime);
      }else{
        sqlite3_result_int64(ctx, zipfileMtime(pCsr));
      }
      break;
    }
    case 3: { /* sz */
      if( sqlite3_vtab_nochange(ctx)==0 ){
        sqlite3_result_int64(ctx, pCsr->cds.szUncompressed);
      }
      break;
    }
    case 4:   /* rawdata */
      if( sqlite3_vtab_nochange(ctx) ) break;
    case 5: { /* data */
      if( i==4 || pCsr->cds.iCompression==0 || pCsr->cds.iCompression==8 ){
        int sz = pCsr->cds.szCompressed;
        int szFinal = pCsr->cds.szUncompressed;
        if( szFinal>0 ){
          u8 *aBuf = sqlite3_malloc(sz);
          if( aBuf==0 ){
................................................................................
      sqlite3_result_int(ctx, pCsr->cds.iCompression);
      break;
    case 7:   /* z */
      sqlite3_result_int64(ctx, pCsr->iId);
      break;
  }

  return rc;
}

/*
** Return the rowid for the current row.
*/
static int zipfileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
  assert( 0 );
................................................................................
  int nRead;                      /* Bytes to read from file */
  i64 iOff;                       /* Offset to read from */
  int rc;

  fseek(pFile, 0, SEEK_END);
  szFile = (i64)ftell(pFile);
  if( szFile==0 ){
    memset(pEOCD, 0, sizeof(ZipfileEOCD));
    return SQLITE_OK;
  }
  nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE));
  iOff = szFile - nRead;

  rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg);
  if( rc==SQLITE_OK ){
    int i;
................................................................................
    pCsr->pFile = fopen(zFile, "rb");
    if( pCsr->pFile==0 ){
      zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile);
      rc = SQLITE_ERROR;
    }else{
      rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd);
      if( rc==SQLITE_OK ){
        if( pCsr->eocd.nEntry==0 ){
          pCsr->bEof = 1;
        }else{
          pCsr->iNextOff = pCsr->eocd.iOffset;
          rc = zipfileNext(cur);



        }
      }
    }
  }else{
    ZipfileEntry e;
    memset(&e, 0, sizeof(e));
    e.pNext = pTab->pFirstEntry;
    pCsr->pCurrent = &e;
................................................................................
}

static int zipfileLoadDirectory(ZipfileTab *pTab){
  ZipfileEOCD eocd;
  int rc;

  rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd);
  if( rc==SQLITE_OK && eocd.nEntry>0 ){
    int i;
    int iOff = 0;
    u8 *aBuf = sqlite3_malloc(eocd.nSize);
    if( aBuf==0 ){
      rc = SQLITE_NOMEM;
    }else{
      rc = zipfileReadData(
................................................................................
        zipfileAddEntry(pTab, 0, pNew);
      }

      iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
    }

    sqlite3_free(aBuf);


  }

  return rc;
}

static ZipfileEntry *zipfileNewEntry(
  ZipfileCDS *pCds,               /* Values for fixed size part of CDS */
................................................................................
  int nData = 0;                  /* Size of pData buffer in bytes */
  int iMethod = 0;                /* Compression method for new entry */
  u8 *pFree = 0;                  /* Free this */
  char *zFree = 0;                /* Also free this */
  ZipfileCDS cds;                 /* New Central Directory Structure entry */
  ZipfileEntry *pOld = 0;
  int bIsDir = 0;
  u32 iCrc32 = 0;

  assert( pTab->zFile );
  assert( pTab->pWriteFd );

  if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
    const char *zDelete = (const char*)sqlite3_value_text(apVal[0]);
    int nDelete = (int)strlen(zDelete);
................................................................................
        break;
      }
      assert( pOld->pNext );
    }
    if( nVal==1 ) return SQLITE_OK;
  }

  /* Check that "sz" and "rawdata" are both NULL: */
  if( sqlite3_value_type(apVal[5])!=SQLITE_NULL
   || sqlite3_value_type(apVal[6])!=SQLITE_NULL
  ){















    rc = SQLITE_CONSTRAINT;



  }






  if( rc==SQLITE_OK ){
    if( sqlite3_value_type(apVal[7])==SQLITE_NULL ){



      /* data=NULL. A directory */
      bIsDir = 1;


    }else{
      /* Value specified for "data", and possibly "method". This must be
      ** a regular file or a symlink. */
      const u8 *aIn = sqlite3_value_blob(apVal[7]);
      int nIn = sqlite3_value_bytes(apVal[7]);
      int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL;

      iMethod = sqlite3_value_int(apVal[8]);
................................................................................
        if( rc==SQLITE_OK ){
          if( iMethod || nCmp<nIn ){
            iMethod = 8;
            pData = pFree;
            nData = nCmp;
          }
        }
        iCrc32 = crc32(0, aIn, nIn);
      }
    }

















  }

  if( rc==SQLITE_OK ){
    rc = zipfileGetMode(pTab, apVal[3], 
        (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)), &mode
    );
    if( rc==SQLITE_OK && (bIsDir == ((mode & S_IFDIR)==0)) ){
................................................................................
    /* Create the new CDS record. */
    memset(&cds, 0, sizeof(cds));
    cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY;
    cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED;
    cds.flags = ZIPFILE_NEWENTRY_FLAGS;
    cds.iCompression = (u16)iMethod;
    zipfileMtimeToDos(&cds, (u32)mTime);
    cds.crc32 = iCrc32;
    cds.szCompressed = nData;
    cds.szUncompressed = (u32)sz;
    cds.iExternalAttr = (mode<<16);
    cds.iOffset = (u32)pTab->szCurrent;
    pNew = zipfileNewEntry(&cds, zPath, nPath, (u32)mTime);
    if( pNew==0 ){
      rc = SQLITE_NOMEM;

Changes to test/zipfile.test.

32
33
34
35
36
37
38
39
40
41





42




43
44
45
46
47
48
49
50
51
52
  2 mtime {} 0 {} 0 
  3 sz {} 0 {} 0 
  4 rawdata {} 0 {} 0
  5 data {} 0 {} 0
  6 method {} 0 {} 0
}

do_execsql_test 1.1.1 {
  INSERT INTO zz(name, mode, mtime, sz, rawdata, method) 
  VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0);





}




do_execsql_test 1.1.2 {
  INSERT INTO zz(name, mtime, sz, rawdata, method) 
  VALUES('g.txt', 1000000002, 5, '12345', 0);
}

do_execsql_test 1.2 {
  SELECT name, mtime, data FROM zipfile('test.zip')
} {
  f.txt 1000000000 abcde 
  g.txt 1000000002 12345







|


>
>
>
>
>
|
>
>
>
>

|
|







32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
  2 mtime {} 0 {} 0 
  3 sz {} 0 {} 0 
  4 rawdata {} 0 {} 0
  5 data {} 0 {} 0
  6 method {} 0 {} 0
}

do_catchsql_test 1.1.0.1 {
  INSERT INTO zz(name, mode, mtime, sz, rawdata, method) 
  VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0);
} {1 {constraint failed}}
do_catchsql_test 1.1.0.1 {
  INSERT INTO zz(name, mtime, sz, rawdata, method) 
  VALUES('g.txt', 1000000002, 5, '12345', 0);
} {1 {constraint failed}}

do_execsql_test 1.1.1 {
  INSERT INTO zz(name, mode, mtime, data, method) 
  VALUES('f.txt', '-rw-r--r--', 1000000000, 'abcde', 0);
}
do_execsql_test 1.1.2 {
  INSERT INTO zz(name, mode, mtime, data, method) 
  VALUES('g.txt', NULL, 1000000002, '12345', 0);
}

do_execsql_test 1.2 {
  SELECT name, mtime, data FROM zipfile('test.zip')
} {
  f.txt 1000000000 abcde 
  g.txt 1000000002 12345