/ Check-in [7e7e472f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Update the zipfile module so that it matches the documentation.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256:7e7e472fa91a2bad2e521d4d67f176c8eb9edc1a07b283e425ea0fa2b6abba1f
User & Date: dan 2018-01-08 19:59:59
Context
2018-01-09
00:26
Fix harmless compiler warnings. check-in: 1adf4e60 user: mistachkin tags: trunk
2018-01-08
19:59
Update the zipfile module so that it matches the documentation. check-in: 7e7e472f user: dan tags: trunk
17:34
Fix problems in the sqlite3expert.c code revealed by -fsanitize. check-in: 7a93dd78 user: dan tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/misc/zipfile.c.

    38     38   typedef sqlite3_int64 i64;
    39     39   typedef unsigned char u8;
    40     40   typedef unsigned short u16;
    41     41   typedef unsigned long u32;
    42     42   #define MIN(a,b) ((a)<(b) ? (a) : (b))
    43     43   #endif
    44     44   
    45         -#define ZIPFILE_SCHEMA "CREATE TABLE y("                           \
    46         -  "name,      /* Name of file in zip archive */"                   \
    47         -  "mode,      /* POSIX mode for file */"                           \
    48         -  "mtime,     /* Last modification time in seconds since epoch */" \
    49         -  "sz,        /* Size of object */"                                \
    50         -  "data,      /* Data stored in zip file (possibly compressed) */" \
    51         -  "method,    /* Compression method (integer) */"                  \
    52         -  "f HIDDEN   /* Name of zip file */"                              \
           45  +#define ZIPFILE_SCHEMA "CREATE TABLE y("                              \
           46  +  "name,      /* 0: Name of file in zip archive */"                   \
           47  +  "mode,      /* 1: POSIX mode for file */"                           \
           48  +  "mtime,     /* 2: Last modification time in seconds since epoch */" \
           49  +  "sz,        /* 3: Size of object */"                                \
           50  +  "rawdata,   /* 4: Raw data */"                                      \
           51  +  "data,      /* 5: Uncompressed data */"                             \
           52  +  "method,    /* 6: Compression method (integer) */"                  \
           53  +  "file HIDDEN   /* Name of zip file */"                              \
    53     54   ");"
    54     55   
    55         -#define ZIPFILE_F_COLUMN_IDX 6    /* Index of column "f" in the above */
           56  +#define ZIPFILE_F_COLUMN_IDX 7    /* Index of column "f" in the above */
    56     57   #define ZIPFILE_BUFFER_SIZE (64*1024)
    57     58   
    58     59   
    59     60   /*
    60     61   ** Magic numbers used to read and write zip files.
    61     62   **
    62     63   ** ZIPFILE_NEWENTRY_MADEBY:
................................................................................
   666    667       (res.tm_hour << 11));
   667    668   
   668    669     pCds->mDate = (u16)(
   669    670       (res.tm_mday-1) +
   670    671       ((res.tm_mon+1) << 5) +
   671    672       ((res.tm_year-80) << 9));
   672    673   }
          674  +
          675  +static void zipfileInflate(
          676  +  sqlite3_context *pCtx,          /* Store error here, if any */
          677  +  const u8 *aIn,                  /* Compressed data */
          678  +  int nIn,                        /* Size of buffer aIn[] in bytes */
          679  +  int nOut                        /* Expected output size */
          680  +){
          681  +  u8 *aRes = sqlite3_malloc(nOut);
          682  +  if( aRes==0 ){
          683  +    sqlite3_result_error_nomem(pCtx);
          684  +  }else{
          685  +    int err;
          686  +    z_stream str;
          687  +    memset(&str, 0, sizeof(str));
          688  +
          689  +    str.next_in = (Byte*)aIn;
          690  +    str.avail_in = nIn;
          691  +    str.next_out = (Byte*)aRes;
          692  +    str.avail_out = nOut;
          693  +
          694  +    err = inflateInit2(&str, -15);
          695  +    if( err!=Z_OK ){
          696  +      zipfileCtxErrorMsg(pCtx, "inflateInit2() failed (%d)", err);
          697  +    }else{
          698  +      err = inflate(&str, Z_NO_FLUSH);
          699  +      if( err!=Z_STREAM_END ){
          700  +        zipfileCtxErrorMsg(pCtx, "inflate() failed (%d)", err);
          701  +      }else{
          702  +        sqlite3_result_blob(pCtx, aRes, nOut, SQLITE_TRANSIENT);
          703  +      }
          704  +    }
          705  +    sqlite3_free(aRes);
          706  +    inflateEnd(&str);
          707  +  }
          708  +}
          709  +
          710  +static int zipfileDeflate(
          711  +  ZipfileTab *pTab,               /* Set error message here */
          712  +  const u8 *aIn, int nIn,         /* Input */
          713  +  u8 **ppOut, int *pnOut          /* Output */
          714  +){
          715  +  int nAlloc = (int)compressBound(nIn);
          716  +  u8 *aOut;
          717  +  int rc;
          718  +
          719  +  aOut = (u8*)sqlite3_malloc(nAlloc);
          720  +  if( aOut==0 ){
          721  +    rc = SQLITE_NOMEM;
          722  +  }else{
          723  +    int res;
          724  +    z_stream str;
          725  +    memset(&str, 0, sizeof(str));
          726  +    str.next_in = (z_const Bytef*)aIn;
          727  +    str.avail_in = nIn;
          728  +    str.next_out = aOut;
          729  +    str.avail_out = nAlloc;
          730  +
          731  +    deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
          732  +    res = deflate(&str, Z_FINISH);
          733  +
          734  +    if( res==Z_STREAM_END ){
          735  +      *ppOut = aOut;
          736  +      *pnOut = (int)str.total_out;
          737  +    }else{
          738  +      sqlite3_free(aOut);
          739  +      pTab->base.zErrMsg = sqlite3_mprintf("zipfile: deflate() error");
          740  +      rc = SQLITE_ERROR;
          741  +    }
          742  +    deflateEnd(&str);
          743  +  }
          744  +
          745  +  return rc;
          746  +}
          747  +
   673    748   
   674    749   /*
   675    750   ** Return values of columns for the row at which the series_cursor
   676    751   ** is currently pointing.
   677    752   */
   678    753   static int zipfileColumn(
   679    754     sqlite3_vtab_cursor *cur,   /* The cursor */
................................................................................
   699    774         }
   700    775         break;
   701    776       }
   702    777       case 3: { /* sz */
   703    778         sqlite3_result_int64(ctx, pCsr->cds.szUncompressed);
   704    779         break;
   705    780       }
   706         -    case 4: { /* data */
   707         -      int sz = pCsr->cds.szCompressed;
   708         -      if( sz>0 ){
   709         -        u8 *aBuf = sqlite3_malloc(sz);
   710         -        if( aBuf==0 ){
   711         -          rc = SQLITE_NOMEM;
   712         -        }else{
   713         -          FILE *pFile = zipfileGetFd(pCsr);
   714         -          rc = zipfileReadData(pFile, aBuf, sz, pCsr->iDataOff,
   715         -              &pCsr->base.pVtab->zErrMsg
   716         -          );
   717         -        }
   718         -        if( rc==SQLITE_OK ){
   719         -          sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT);
   720         -          sqlite3_free(aBuf);
          781  +    case 4:   /* rawdata */
          782  +    case 5: { /* data */
          783  +      if( i==4 || pCsr->cds.iCompression==0 || pCsr->cds.iCompression==8 ){
          784  +        int sz = pCsr->cds.szCompressed;
          785  +        if( sz>0 ){
          786  +          u8 *aBuf = sqlite3_malloc(sz);
          787  +          if( aBuf==0 ){
          788  +            rc = SQLITE_NOMEM;
          789  +          }else{
          790  +            FILE *pFile = zipfileGetFd(pCsr);
          791  +            rc = zipfileReadData(pFile, aBuf, sz, pCsr->iDataOff,
          792  +                &pCsr->base.pVtab->zErrMsg
          793  +            );
          794  +          }
          795  +          if( rc==SQLITE_OK ){
          796  +            if( i==5 && pCsr->cds.iCompression ){
          797  +              zipfileInflate(ctx, aBuf, sz, pCsr->cds.szUncompressed);
          798  +            }else{
          799  +              sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT);
          800  +            }
          801  +            sqlite3_free(aBuf);
          802  +          }
   721    803           }
   722    804         }
   723    805         break;
   724    806       }
   725         -    case 5:   /* method */
          807  +    case 6:   /* method */
   726    808         sqlite3_result_int(ctx, pCsr->cds.iCompression);
   727    809         break;
   728    810     }
   729    811   
   730    812     return SQLITE_OK;
   731    813   }
   732    814   
................................................................................
  1069   1151   
  1070   1152     return rc;
  1071   1153   }
  1072   1154   
  1073   1155   static int zipfileGetMode(ZipfileTab *pTab, sqlite3_value *pVal, int *pMode){
  1074   1156     const char *z = (const char*)sqlite3_value_text(pVal);
  1075   1157     int mode = 0;
  1076         -  if( z==0 || (z[0]>=0 && z[0]<=9) ){
         1158  +  if( z==0 ){
         1159  +    mode = 33188;                 /* -rw-r--r-- */
         1160  +  }else if( z[0]>=0 && z[0]<=9 ){
  1077   1161       mode = sqlite3_value_int(pVal);
  1078   1162     }else{
  1079   1163       const char zTemplate[11] = "-rwxrwxrwx";
  1080   1164       int i;
  1081   1165       if( strlen(z)!=10 ) goto parse_error;
  1082   1166       switch( z[0] ){
  1083   1167         case '-': mode |= S_IFREG; break;
................................................................................
  1114   1198     ZipfileEntry *pNew = 0;         /* New in-memory CDS entry */
  1115   1199   
  1116   1200     int mode;                       /* Mode for new entry */
  1117   1201     i64 mTime;                      /* Modification time for new entry */
  1118   1202     i64 sz;                         /* Uncompressed size */
  1119   1203     const char *zPath;              /* Path for new entry */
  1120   1204     int nPath;                      /* strlen(zPath) */
  1121         -  const u8 *pData;                /* Pointer to buffer containing content */
  1122         -  int nData;                      /* Size of pData buffer in bytes */
         1205  +  const u8 *pData = 0;            /* Pointer to buffer containing content */
         1206  +  int nData = 0;                  /* Size of pData buffer in bytes */
  1123   1207     int iMethod = 0;                /* Compression method for new entry */
  1124   1208     u8 *pFree = 0;                  /* Free this */
  1125   1209     ZipfileCDS cds;                 /* New Central Directory Structure entry */
  1126   1210   
  1127   1211     assert( pTab->zFile );
  1128   1212     assert( pTab->pWriteFd );
  1129   1213   
................................................................................
  1139   1223       if( nVal==1 ) return SQLITE_OK;
  1140   1224     }
  1141   1225   
  1142   1226     zPath = (const char*)sqlite3_value_text(apVal[2]);
  1143   1227     nPath = (int)strlen(zPath);
  1144   1228     rc = zipfileGetMode(pTab, apVal[3], &mode);
  1145   1229     if( rc!=SQLITE_OK ) return rc;
  1146         -  mTime = sqlite3_value_int64(apVal[4]);
  1147         -  sz = sqlite3_value_int(apVal[5]);
  1148         -  pData = sqlite3_value_blob(apVal[6]);
  1149         -  nData = sqlite3_value_bytes(apVal[6]);
  1150         -
  1151         -  /* If a NULL value is inserted into the 'method' column, do automatic
  1152         -  ** compression. */
  1153         -  if( nData>0 && sqlite3_value_type(apVal[7])==SQLITE_NULL ){
  1154         -    pFree = (u8*)sqlite3_malloc(nData);
  1155         -    if( pFree==0 ){
  1156         -      rc = SQLITE_NOMEM;
  1157         -    }else{
  1158         -      int res;
  1159         -      z_stream str;
  1160         -      memset(&str, 0, sizeof(str));
  1161         -      str.next_in = (z_const Bytef*)pData;
  1162         -      str.avail_in = nData;
  1163         -      str.next_out = pFree;
  1164         -      str.avail_out = nData;
  1165         -      deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
  1166         -      res = deflate(&str, Z_FINISH);
  1167         -      if( res==Z_STREAM_END ){
  1168         -        pData = pFree;
  1169         -        nData = str.total_out;
  1170         -        iMethod = 8;
  1171         -      }else if( res!=Z_OK ){
  1172         -        pTab->base.zErrMsg = sqlite3_mprintf("zipfile: deflate() error");
  1173         -        rc = SQLITE_ERROR;
         1230  +  if( sqlite3_value_type(apVal[4])==SQLITE_NULL ){
         1231  +    mTime = (sqlite3_int64)time(0);
         1232  +  }else{
         1233  +    mTime = sqlite3_value_int64(apVal[4]);
         1234  +  }
         1235  +
         1236  +  if( sqlite3_value_type(apVal[5])==SQLITE_NULL    /* sz */
         1237  +   && sqlite3_value_type(apVal[6])==SQLITE_NULL    /* rawdata */
         1238  +   && sqlite3_value_type(apVal[7])!=SQLITE_NULL    /* data */
         1239  +  ){
         1240  +    const u8 *aIn = sqlite3_value_blob(apVal[7]);
         1241  +    int nIn = sqlite3_value_bytes(apVal[7]);
         1242  +    int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL;
         1243  +    
         1244  +    iMethod = sqlite3_value_int(apVal[8]);
         1245  +    sz = nIn;
         1246  +    if( iMethod!=0 && iMethod!=8 ){
         1247  +      rc = SQLITE_CONSTRAINT;
         1248  +    }else if( bAuto || iMethod ){
         1249  +      rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nData);
         1250  +      if( rc==SQLITE_OK ){
         1251  +        if( iMethod || nData<nIn ){
         1252  +          iMethod = 8;
         1253  +          pData = pFree;
         1254  +        }else{
         1255  +          pData = aIn;
         1256  +          nData = nIn;
         1257  +        }
  1174   1258         }
  1175         -      deflateEnd(&str);
  1176   1259       }
  1177         -  }else{
  1178         -    iMethod = sqlite3_value_int(apVal[7]);
         1260  +  }else 
         1261  +  if( sqlite3_value_type(apVal[5])!=SQLITE_NULL    /* sz */
         1262  +   && sqlite3_value_type(apVal[6])!=SQLITE_NULL    /* rawdata */
         1263  +   && sqlite3_value_type(apVal[7])==SQLITE_NULL    /* data */
         1264  +   && sqlite3_value_type(apVal[8])!=SQLITE_NULL    /* method */
         1265  +  ){
         1266  +    pData = sqlite3_value_blob(apVal[6]);
         1267  +    nData = sqlite3_value_bytes(apVal[6]);
         1268  +    sz = sqlite3_value_int(apVal[5]);
         1269  +    iMethod = sqlite3_value_int(apVal[8]);
  1179   1270       if( iMethod<0 || iMethod>65535 ){
  1180   1271         pTab->base.zErrMsg = sqlite3_mprintf(
  1181   1272             "zipfile: invalid compression method: %d", iMethod
  1182   1273         );
  1183   1274         rc = SQLITE_ERROR;
  1184   1275       }
         1276  +  }else{
         1277  +    rc = SQLITE_CONSTRAINT;
  1185   1278     }
  1186   1279   
  1187   1280     if( rc==SQLITE_OK ){
  1188   1281       /* Create the new CDS record. */
  1189   1282       memset(&cds, 0, sizeof(cds));
  1190   1283       cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY;
  1191   1284       cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED;
................................................................................
  1348   1441     int rc = sqlite3_create_module(db, "zipfile"  , &zipfileModule, 0);
  1349   1442     return rc;
  1350   1443   }
  1351   1444   #else         /* SQLITE_OMIT_VIRTUALTABLE */
  1352   1445   # define zipfileRegister(x) SQLITE_OK
  1353   1446   #endif
  1354   1447   
  1355         -/*
  1356         -** zipfile_uncompress(DATA, SZ, METHOD)
  1357         -*/
  1358         -static void zipfileUncompressFunc(
  1359         -  sqlite3_context *context,
  1360         -  int argc,
  1361         -  sqlite3_value **argv
  1362         -){
  1363         -  int iMethod;
  1364         -
  1365         -  iMethod = sqlite3_value_int(argv[2]);
  1366         -  if( iMethod==0 ){
  1367         -    sqlite3_result_value(context, argv[0]);
  1368         -  }else if( iMethod==8 ){
  1369         -    Byte *res;
  1370         -    int sz = sqlite3_value_int(argv[1]);
  1371         -    z_stream str;
  1372         -    memset(&str, 0, sizeof(str));
  1373         -    str.next_in = (Byte*)sqlite3_value_blob(argv[0]);
  1374         -    str.avail_in = sqlite3_value_bytes(argv[0]);
  1375         -    res = str.next_out = (Byte*)sqlite3_malloc(sz);
  1376         -    if( res==0 ){
  1377         -      sqlite3_result_error_nomem(context);
  1378         -    }else{
  1379         -      int err;
  1380         -      str.avail_out = sz;
  1381         -
  1382         -      err = inflateInit2(&str, -15);
  1383         -      if( err!=Z_OK ){
  1384         -        zipfileCtxErrorMsg(context, "inflateInit2() failed (%d)", err);
  1385         -      }else{
  1386         -        err = inflate(&str, Z_NO_FLUSH);
  1387         -        if( err!=Z_STREAM_END ){
  1388         -          zipfileCtxErrorMsg(context, "inflate() failed (%d)", err);
  1389         -        }else{
  1390         -          sqlite3_result_blob(context, res, sz, SQLITE_TRANSIENT);
  1391         -        }
  1392         -      }
  1393         -      sqlite3_free(res);
  1394         -      inflateEnd(&str);
  1395         -    }
  1396         -  }else{
  1397         -    zipfileCtxErrorMsg(context, "unrecognized compression method: %d", iMethod);
  1398         -  }
  1399         -}
  1400         -
  1401   1448   #ifdef _WIN32
  1402   1449   __declspec(dllexport)
  1403   1450   #endif
  1404   1451   int sqlite3_zipfile_init(
  1405   1452     sqlite3 *db, 
  1406   1453     char **pzErrMsg, 
  1407   1454     const sqlite3_api_routines *pApi
  1408   1455   ){
  1409         -  int rc = SQLITE_OK;
  1410   1456     SQLITE_EXTENSION_INIT2(pApi);
  1411   1457     (void)pzErrMsg;  /* Unused parameter */
  1412         -  rc = sqlite3_create_function(db, "zipfile_uncompress", 3,
  1413         -      SQLITE_UTF8, 0, zipfileUncompressFunc, 0, 0
  1414         -  );
  1415         -  if( rc!=SQLITE_OK ) return rc;
  1416   1458     return zipfileRegister(db);
  1417   1459   }
  1418   1460   

Changes to src/shell.c.in.

  4843   4843       "SELECT "
  4844   4844       "  :1 || name, "
  4845   4845       "  writefile(?1 || name, %s, mode, mtime) "
  4846   4846       "FROM %s WHERE (%s) AND (data IS NULL OR ?2 = 0)";
  4847   4847   
  4848   4848     const char *azExtraArg[] = { 
  4849   4849       "sqlar_uncompress(data, sz)",
  4850         -    "zipfile_uncompress(data, sz, method)"
         4850  +    "data"
  4851   4851     };
  4852   4852     const char *azSource[] = {
  4853   4853       "sqlar", "zipfile(?3)"
  4854   4854     };
  4855         -
  4856         -
  4857   4855   
  4858   4856     sqlite3_stmt *pSql = 0;
  4859   4857     int rc = SQLITE_OK;
  4860   4858     char *zDir = 0;
  4861   4859     char *zWhere = 0;
  4862   4860     int i;
  4863   4861   

Changes to test/zipfile.test.

    24     24     CREATE VIRTUAL TABLE temp.zz USING zipfile('test.zip');
    25     25     PRAGMA table_info(zz);
    26     26   } {
    27     27     0 name {} 0 {} 0 
    28     28     1 mode {} 0 {} 0 
    29     29     2 mtime {} 0 {} 0 
    30     30     3 sz {} 0 {} 0 
    31         -  4 data {} 0 {} 0
    32         -  5 method {} 0 {} 0
           31  +  4 rawdata {} 0 {} 0
           32  +  5 data {} 0 {} 0
           33  +  6 method {} 0 {} 0
    33     34   }
    34     35   
    35     36   do_execsql_test 1.1.1 {
    36         -  INSERT INTO zz VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0);
           37  +  INSERT INTO zz(name, mode, mtime, sz, rawdata, method) 
           38  +  VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0);
    37     39   }
    38     40   do_execsql_test 1.1.2 {
    39         -  INSERT INTO zz VALUES('g.txt', '-rw-r--r--', 1000000002, 5, '12345', 0);
           41  +  INSERT INTO zz(name, mtime, sz, rawdata, method) 
           42  +  VALUES('g.txt', 1000000002, 5, '12345', 0);
    40     43   }
    41     44   
    42     45   do_execsql_test 1.2 {
    43     46     SELECT name, mtime, data FROM zipfile('test.zip')
    44     47   } {
    45     48     f.txt 1000000000 abcde 
    46     49     g.txt 1000000002 12345
    47     50   }
    48     51   
    49     52   do_execsql_test 1.3 {
    50         -  INSERT INTO zz VALUES('h.txt', 
    51         -    '-rw-r--r--', 1000000004, 20, 'aaaaaaaaaabbbbbbbbbb', NULL
           53  +  INSERT INTO zz(name, mode, mtime, data) VALUES('h.txt', 
           54  +    '-rw-r--r--', 1000000004, 'aaaaaaaaaabbbbbbbbbb'
    52     55     );
    53     56   }
    54     57   
    55     58   do_execsql_test 1.4 {
    56         -  SELECT name, mtime, zipfile_uncompress(data, sz, method), method
    57         -  FROM zipfile('test.zip');
           59  +  SELECT name, mtime, data, method FROM zipfile('test.zip');
    58     60   } {
    59     61     f.txt 1000000000 abcde 0
    60     62     g.txt 1000000002 12345 0
    61     63     h.txt 1000000004 aaaaaaaaaabbbbbbbbbb 8
    62     64   }
    63     65   
    64     66   do_execsql_test 1.5.1 {
    65     67     BEGIN;
    66         -    INSERT INTO zz VALUES('i.txt', '-rw-r--r--', 1000000006, 5, 'zxcvb', 0);
           68  +    INSERT INTO zz(name, mode, mtime, data, method)
           69  +    VALUES('i.txt', '-rw-r--r--', 1000000006, 'zxcvb', 0);
    67     70       SELECT name FROM zz;
    68     71     COMMIT;
    69     72   } {f.txt g.txt h.txt i.txt}
    70     73   do_execsql_test 1.5.2 {
    71     74     SELECT name FROM zz;
    72     75   } {f.txt g.txt h.txt i.txt}
    73     76