SQLite Archiver
Check-in [d5ea353587]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the -n option to disable compression.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:d5ea3535873bd0a36af3f39766d7e9084b57790f
User & Date: drh 2014-04-07 01:26:52
Context
2014-04-07
13:49
Fix typos in the documentation. check-in: d6387bdf60 user: drh tags: trunk
01:26
Add the -n option to disable compression. check-in: d5ea353587 user: drh tags: trunk
2014-03-14
01:01
Improvements to README.md check-in: a0cf8ef8b7 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to README.md.

39
40
41
42
43
44
45





46
47
48
49
50
51
52
..
54
55
56
57
58
59
60


Without a FILES argument, all files are extracted.

All commands can be supplemented with -v for verbose output. For example:

        sar -v ARCHIVE FILES..
        sar -lv ARCHIVE
        sar -xv ARCHIVE





    
## Storage

The database schema looks like this:

        CREATE TABLE sar(
          name TEXT PRIMARY KEY,  -- name of the file
................................................................................
          mtime INT,              -- last modification time
          sz INT,                 -- original file size
          data BLOB               -- compressed content
        );
        
Both directories and empty files have sar.sz==0.  Directories can be
distinguished from empty file because directories have sar.data IS NULL.









>
>
>
>
>







 







>
>
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
..
59
60
61
62
63
64
65
66
67
Without a FILES argument, all files are extracted.

All commands can be supplemented with -v for verbose output. For example:

        sar -v ARCHIVE FILES..
        sar -lv ARCHIVE
        sar -xv ARCHIVE

File are normally compressed using zlib prior to being stored as BLOBs in
the database.  However, if the file is incompressible or if the -n option
is used on the command-line, then the file is stored in the database exactly
as it appears on disk, without compression.
    
## Storage

The database schema looks like this:

        CREATE TABLE sar(
          name TEXT PRIMARY KEY,  -- name of the file
................................................................................
          mtime INT,              -- last modification time
          sz INT,                 -- original file size
          data BLOB               -- compressed content
        );
        
Both directories and empty files have sar.sz==0.  Directories can be
distinguished from empty file because directories have sar.data IS NULL.
The sar.data file is compressed if length(sar.blob)<sar.sz and is stored
as plaintext if length(sar.blob)==sar.sz.

Changes to sar.c.

25
26
27
28
29
30
31

32
33
34
35
36
37
38
...
119
120
121
122
123
124
125
126

127
128
129

130
131





132
133
134
135
136
137
138
...
143
144
145
146
147
148
149




150
151
152
153
154

155
156
157
158





159
160
161
162
163
164
165
...
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197

198
199
200
201
202
203
204
205
206
207
208







209
210
211
212
213
214
215
216
217
218
219

220
221
222
223
224
225
226
227
228
229
230
...
236
237
238
239
240
241
242
243

244
245
246
247
248
249
250
...
261
262
263
264
265
266
267
268
269
270
271

272
273



274
275
276
277
278
279
280
...
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
...
336
337
338
339
340
341
342

343
344
345
346
347
348
349
350
351
352

353
354
355
356
357
358
359
...
423
424
425
426
427
428
429
430
431
432
433
434
435
/*
** Show a help message and quit.
*/
static void showHelp(const char *argv0){
  fprintf(stderr, "Usage: %s [options] archive [files...]\n", argv0);
  fprintf(stderr, "Options:\n"
                  "   -l      List files in archive\n"

                  "   -x      Extract files from archive\n"
                  "   -v      Verbose output\n"
  );
  exit(1);
}

/*
................................................................................
    errorMsg("Error: %s\nwhile preparing: %s\n",
             sqlite3_errmsg(db), zSql);
  }
}

/*
** Read a file from disk into memory obtained from sqlite3_malloc().
** Compress the file as it is read in.

**
** Return the original size and the compressed size of the file in
** *pSizeOrig and *pSizeCompr, respectively.

*/
static char *read_file(const char *zFilename, int *pSizeOrig, int *pSizeCompr){





  FILE *in;
  char *zIn;
  long int nIn;
  char *zCompr;
  unsigned long int nCompr;
  int rc;

................................................................................
  rewind(in);
  zIn = sqlite3_malloc( nIn+1 );
  if( zIn==0 ) errorMsg("cannot malloc for %d bytes\n", nIn+1);
  if( nIn>0 && fread(zIn, nIn, 1, in)!=1 ){
    errorMsg("unable to read %d bytes of file %s\n", nIn, zFilename);
  }
  fclose(in);




  nCompr = 13 + nIn + (nIn+999)/1000;
  zCompr = sqlite3_malloc( nCompr+1 );
  if( zCompr==0 ) errorMsg("cannot malloc for %d bytes\n", nCompr+1);
  rc = compress(zCompr, &nCompr, zIn, nIn);
  if( rc!=Z_OK ) errorMsg("Cannot compress %s\n", zFilename);

  sqlite3_free(zIn);
  *pSizeOrig = nIn;
  *pSizeCompr = (int)nCompr;
  return zCompr;





}

/*
** Make sure the parent directory for zName exists.  Create it if it does
** not exist.
*/
static void make_parent_directory(const char *zName){
................................................................................

/*
** Write a file or a directory.
**
** Create any missing directories leading up to the given file or directory.
** Also set the access mode and the modification time.
**
** The content is compressed and needs to be decompressed before writing.
*/
static void write_file(
  const char *zFilename,
  int iMode,
  sqlite3_int64 mtime,
  int sz,
  const char *pCompr,
  int nCompr

){
  char *pOut;
  unsigned long int nOut;
  int rc;
  FILE *out;
  make_parent_directory(zFilename);
  if( pCompr==0 ){
    rc = mkdir(zFilename, iMode);
    if( rc ) errorMsg("cannot make directory: %s\n", zFilename);
    return;
  }







  pOut = sqlite3_malloc( sz+1 );
  if( pOut==0 ) errorMsg("cannot allocate %d bytes\n", sz+1);
  nOut = sz;
  rc = uncompress(pOut, &nOut, pCompr, nCompr);
  if( rc!=Z_OK ) errorMsg("uncompress failed for %s\n", zFilename);
  out = fopen(zFilename, "wb");
  if( out==0 ) errorMsg("cannot open for writing: %s\n", zFilename);
  if( nOut>0 && fwrite(pOut, nOut, 1, out)!=1 ){
    errorMsg("failed to write: %s\n", zFilename);
  }
  sqlite3_free(pOut);

  fclose(out);
  rc = chmod(zFilename, iMode&0777);
  if( rc ) errorMsg("cannot change mode to %03o: %s\n", iMode, zFilename);
  
}

/*
** Error out if there are any issues with the given filename
*/
static void check_filename(const char *z){
  if( strncmp(z, "../", 3)==0 || sqlite3_strglob("*/../*", z)==0 ){
................................................................................
}

/*
** Add a file to the database.
*/
static void add_file(
  const char *zFilename,     /* Name of file to add */
  int verboseFlag            /* If true, show each file added */

){
  int rc;
  struct stat x;
  char *zContent;
  int szOrig;
  int szCompr;
  const char *zName;
................................................................................
  }
  zName = zFilename;
  while( zName[0]=='/' ) zName++;
  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
  sqlite3_bind_int(pStmt, 2, x.st_mode);
  sqlite3_bind_int64(pStmt, 3, x.st_mtime);
  if( S_ISREG(x.st_mode) ){
    char *zContent = read_file(zFilename, &szOrig, &szCompr);
    sqlite3_bind_int(pStmt, 4, szOrig);
    sqlite3_bind_blob(pStmt, 5, zContent, szCompr, sqlite3_free);
    if( verboseFlag ){

      int pct = szOrig ? (100*(sqlite3_int64)szCompr)/szOrig : 0;
      printf("  added: %s (deflate %d%%)\n", zFilename, 100-pct);



    } 
  }else{
    sqlite3_bind_int(pStmt, 4, 0);
    sqlite3_bind_null(pStmt, 5);
    if( verboseFlag ) printf("  added: %s\n", zFilename);
  }
  rc = sqlite3_step(pStmt);
................................................................................
    d = opendir(zFilename);
    if( d ){
      while( (pEntry = readdir(d))!=0 ){
        if( strcmp(pEntry->d_name,".")==0 || strcmp(pEntry->d_name,"..")==0 ){
          continue;
        }
        char *zSubpath = sqlite3_mprintf("%s/%s", zFilename, pEntry->d_name);
        add_file(zSubpath, verboseFlag);
        sqlite3_free(zSubpath);
      }
    }
  }
}

/*
................................................................................
int main(int argc, char **argv){
  const char *zArchive = 0;
  char **azFiles = 0;
  int nFiles = 0;
  int listFlag = 0;
  int extractFlag = 0;
  int verboseFlag = 0;

  int i, j;

  if( sqlite3_strglob("*/unsar", argv[0])==0 ){
    extractFlag = 1;
  }
  for(i=1; i<argc; i++){
    if( argv[i][0]=='-' ){
      for(j=1; argv[i][j]; j++){
        switch( argv[i][j] ){
          case 'l':   listFlag = 1;    break;

          case 'v':   verboseFlag = 1; break;
          case 'x':   extractFlag = 1; break;
          case '-':   break;
          default:    showHelp(argv[0]);
        }
      }
    }else if( zArchive==0 ){
................................................................................
                 sqlite3_column_bytes(pStmt,4));
    }
    db_close(1);
  }else{
    if( azFiles==0 ) showHelp(argv[0]);
    db_open(zArchive, 1);
    for(i=0; i<nFiles; i++){
      add_file(azFiles[i], verboseFlag);
    }
    db_close(1);
  }
  return 0;
}







>







 







|
>


|
>

|
>
>
>
>
>







 







>
>
>
>





>
|
|
|
|
>
>
>
>
>







 







|
|
|
|
|
|
|
|
|
>











>
>
>
>
>
>
>
|
|
|
|
|
<
<
|
|
|
|
>



<







 







|
>







 







|



>
|
|
>
>
>







 







|







 







>










>







 







|





25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
...
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
...
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
...
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239


240
241
242
243
244
245
246
247

248
249
250
251
252
253
254
...
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
...
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
...
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
...
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
...
454
455
456
457
458
459
460
461
462
463
464
465
466
/*
** Show a help message and quit.
*/
static void showHelp(const char *argv0){
  fprintf(stderr, "Usage: %s [options] archive [files...]\n", argv0);
  fprintf(stderr, "Options:\n"
                  "   -l      List files in archive\n"
                  "   -n      Do not compress files\n"
                  "   -x      Extract files from archive\n"
                  "   -v      Verbose output\n"
  );
  exit(1);
}

/*
................................................................................
    errorMsg("Error: %s\nwhile preparing: %s\n",
             sqlite3_errmsg(db), zSql);
  }
}

/*
** Read a file from disk into memory obtained from sqlite3_malloc().
** Compress the file as it is read in if doing so reduces the file
** size and if the noCompress flag is false.
**
** Return the original size and the compressed size of the file in
** *pSizeOrig and *pSizeCompr, respectively.  If these two values are
** equal, that means the file was not compressed.
*/
static char *read_file(
  const char *zFilename,    /* Name of file to read */
  int *pSizeOrig,           /* Write original file size here */
  int *pSizeCompr,          /* Write compressed file size here */
  int noCompress            /* Do not compress if true */
){
  FILE *in;
  char *zIn;
  long int nIn;
  char *zCompr;
  unsigned long int nCompr;
  int rc;

................................................................................
  rewind(in);
  zIn = sqlite3_malloc( nIn+1 );
  if( zIn==0 ) errorMsg("cannot malloc for %d bytes\n", nIn+1);
  if( nIn>0 && fread(zIn, nIn, 1, in)!=1 ){
    errorMsg("unable to read %d bytes of file %s\n", nIn, zFilename);
  }
  fclose(in);
  if( noCompress ){
    *pSizeOrig = *pSizeCompr = nIn;
    return zIn;
  }
  nCompr = 13 + nIn + (nIn+999)/1000;
  zCompr = sqlite3_malloc( nCompr+1 );
  if( zCompr==0 ) errorMsg("cannot malloc for %d bytes\n", nCompr+1);
  rc = compress(zCompr, &nCompr, zIn, nIn);
  if( rc!=Z_OK ) errorMsg("Cannot compress %s\n", zFilename);
  if( nIn>nCompr ){
    sqlite3_free(zIn);
    *pSizeOrig = nIn;
    *pSizeCompr = (int)nCompr;
    return zCompr;
  }else{
    sqlite3_free(zCompr);
    *pSizeOrig = *pSizeCompr = nIn;
    return zIn;
  }
}

/*
** Make sure the parent directory for zName exists.  Create it if it does
** not exist.
*/
static void make_parent_directory(const char *zName){
................................................................................

/*
** Write a file or a directory.
**
** Create any missing directories leading up to the given file or directory.
** Also set the access mode and the modification time.
**
** If sz>nCompr that means that the content is compressed and needs to be
** decompressed before writing.
*/
static void write_file(
  const char *zFilename,   /* Store content in this file */
  int iMode,               /* The unix-style access mode */
  sqlite3_int64 mtime,     /* Modification time */
  int sz,                  /* Size of file as stored on disk */
  const char *pCompr,      /* Content (usually compressed) */
  int nCompr               /* Size of content (prior to decompression) */
){
  char *pOut;
  unsigned long int nOut;
  int rc;
  FILE *out;
  make_parent_directory(zFilename);
  if( pCompr==0 ){
    rc = mkdir(zFilename, iMode);
    if( rc ) errorMsg("cannot make directory: %s\n", zFilename);
    return;
  }
  out = fopen(zFilename, "wb");
  if( out==0 ) errorMsg("cannot open for writing: %s\n", zFilename);
  if( sz==nCompr ){
    if( sz>0 && fwrite(pCompr, sz, 1, out)!=1 ){
      errorMsg("failed to write: %s\n", zFilename);
    }
  }else{
    pOut = sqlite3_malloc( sz+1 );
    if( pOut==0 ) errorMsg("cannot allocate %d bytes\n", sz+1);
    nOut = sz;
    rc = uncompress(pOut, &nOut, pCompr, nCompr);
    if( rc!=Z_OK ) errorMsg("uncompress failed for %s\n", zFilename);


    if( nOut>0 && fwrite(pOut, nOut, 1, out)!=1 ){
      errorMsg("failed to write: %s\n", zFilename);
    }
    sqlite3_free(pOut);
  }
  fclose(out);
  rc = chmod(zFilename, iMode&0777);
  if( rc ) errorMsg("cannot change mode to %03o: %s\n", iMode, zFilename);

}

/*
** Error out if there are any issues with the given filename
*/
static void check_filename(const char *z){
  if( strncmp(z, "../", 3)==0 || sqlite3_strglob("*/../*", z)==0 ){
................................................................................
}

/*
** Add a file to the database.
*/
static void add_file(
  const char *zFilename,     /* Name of file to add */
  int verboseFlag,           /* If true, show each file added */
  int noCompress             /* If true, always omit compression */
){
  int rc;
  struct stat x;
  char *zContent;
  int szOrig;
  int szCompr;
  const char *zName;
................................................................................
  }
  zName = zFilename;
  while( zName[0]=='/' ) zName++;
  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
  sqlite3_bind_int(pStmt, 2, x.st_mode);
  sqlite3_bind_int64(pStmt, 3, x.st_mtime);
  if( S_ISREG(x.st_mode) ){
    char *zContent = read_file(zFilename, &szOrig, &szCompr, noCompress);
    sqlite3_bind_int(pStmt, 4, szOrig);
    sqlite3_bind_blob(pStmt, 5, zContent, szCompr, sqlite3_free);
    if( verboseFlag ){
      if( szCompr<szOrig ){
        int pct = szOrig ? (100*(sqlite3_int64)szCompr)/szOrig : 0;
        printf("  added: %s (deflate %d%%)\n", zFilename, 100-pct);
      }else{
        printf("  added: %s\n", zFilename);
      }
    } 
  }else{
    sqlite3_bind_int(pStmt, 4, 0);
    sqlite3_bind_null(pStmt, 5);
    if( verboseFlag ) printf("  added: %s\n", zFilename);
  }
  rc = sqlite3_step(pStmt);
................................................................................
    d = opendir(zFilename);
    if( d ){
      while( (pEntry = readdir(d))!=0 ){
        if( strcmp(pEntry->d_name,".")==0 || strcmp(pEntry->d_name,"..")==0 ){
          continue;
        }
        char *zSubpath = sqlite3_mprintf("%s/%s", zFilename, pEntry->d_name);
        add_file(zSubpath, verboseFlag, noCompress);
        sqlite3_free(zSubpath);
      }
    }
  }
}

/*
................................................................................
int main(int argc, char **argv){
  const char *zArchive = 0;
  char **azFiles = 0;
  int nFiles = 0;
  int listFlag = 0;
  int extractFlag = 0;
  int verboseFlag = 0;
  int noCompress = 0;
  int i, j;

  if( sqlite3_strglob("*/unsar", argv[0])==0 ){
    extractFlag = 1;
  }
  for(i=1; i<argc; i++){
    if( argv[i][0]=='-' ){
      for(j=1; argv[i][j]; j++){
        switch( argv[i][j] ){
          case 'l':   listFlag = 1;    break;
          case 'n':   noCompress = 1;  break;
          case 'v':   verboseFlag = 1; break;
          case 'x':   extractFlag = 1; break;
          case '-':   break;
          default:    showHelp(argv[0]);
        }
      }
    }else if( zArchive==0 ){
................................................................................
                 sqlite3_column_bytes(pStmt,4));
    }
    db_close(1);
  }else{
    if( azFiles==0 ) showHelp(argv[0]);
    db_open(zArchive, 1);
    for(i=0; i<nFiles; i++){
      add_file(azFiles[i], verboseFlag, noCompress);
    }
    db_close(1);
  }
  return 0;
}