/ Check-in [7652b3c2]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Do not use the compress() and uncompress() functions in ext/misc/compress.c - they are not quite compatible with the spec. Instead use new functions in ext/misc/sqlar.c.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | sqlar-shell-support
Files: files | file ages | folders
SHA3-256: 7652b3c2374084047b6c1da3e525e0cac34fe220597f81e793bc4fd9f33358da
User & Date: dan 2017-12-16 19:11:26
Context
2017-12-23
18:34
Merge enhancements from trunk. check-in: 150f07fe user: drh tags: sqlar-shell-support
2017-12-16
19:11
Do not use the compress() and uncompress() functions in ext/misc/compress.c - they are not quite compatible with the spec. Instead use new functions in ext/misc/sqlar.c. check-in: 7652b3c2 user: dan tags: sqlar-shell-support
2017-12-14
19:15
Have the writefile() function optionally set the modification-time of the files it writes or creates. And many small fixes to the new code on this branch. check-in: 7b51269c user: dan tags: sqlar-shell-support
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Added ext/misc/sqlar.c.



































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
/*
** 2017-12-17
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
** Utility functions sqlar_compress() and sqlar_uncompress(). Useful
** for working with sqlar archives and used by the shell tool's built-in
** sqlar support.
*/
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#include <zlib.h>

/*
** Implementation of the "sqlar_compress(X)" SQL function.
**
** If the type of X is SQLITE_BLOB, and compressing that blob using
** zlib utility function compress() yields a smaller blob, return the
** compressed blob. Otherwise, return a copy of X.
*/
static void sqlarCompressFunc(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  assert( argc==1 );
  if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
    const Bytef *pData = sqlite3_value_blob(argv[0]);
    uLong nData = sqlite3_value_bytes(argv[0]);
    uLongf nOut = compressBound(nData);
    Bytef *pOut;

    pOut = (Bytef*)sqlite3_malloc(nOut);
    if( pOut==0 ){
      sqlite3_result_error_nomem(context);
      return;
    }else{
      if( Z_OK!=compress(pOut, &nOut, pData, nData) ){
        sqlite3_result_error(context, "error in compress()", -1);
      }else if( nOut<nData ){
        sqlite3_result_blob(context, pOut, nOut, SQLITE_TRANSIENT);
      }else{
        sqlite3_result_value(context, argv[0]);
      }
      sqlite3_free(pOut);
    }
  }else{
    sqlite3_result_value(context, argv[0]);
  }
}

/*
** Implementation of the "sqlar_uncompress(X,SZ)" SQL function
**
** Parameter SZ is interpreted as an integer. If it is less than or
** equal to zero, then this function returns a copy of X. Or, if
** SZ is equal to the size of X when interpreted as a blob, also
** return a copy of X. Otherwise, decompress blob X using zlib
** utility function uncompress() and return the results (another
** blob).
*/
static void sqlarUncompressFunc(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  uLong nData;
  uLongf sz;

  assert( argc==2 );
  sz = sqlite3_value_int(argv[1]);

  if( sz<=0 || sz==(nData = sqlite3_value_bytes(argv[0])) ){
    sqlite3_result_value(context, argv[0]);
  }else{
    const Bytef *pData= sqlite3_value_blob(argv[0]);
    Bytef *pOut = sqlite3_malloc(sz);
    if( Z_OK!=uncompress(pOut, &sz, pData, nData) ){
      sqlite3_result_error(context, "error in uncompress()", -1);
    }else{
      sqlite3_result_blob(context, pOut, sz, SQLITE_TRANSIENT);
    }
    sqlite3_free(pOut);
  }
}


#ifdef _WIN32
__declspec(dllexport)
#endif
int sqlite3_sqlar_init(
  sqlite3 *db, 
  char **pzErrMsg, 
  const sqlite3_api_routines *pApi
){
  int rc = SQLITE_OK;
  SQLITE_EXTENSION_INIT2(pApi);
  (void)pzErrMsg;  /* Unused parameter */
  rc = sqlite3_create_function(db, "sqlar_compress", 1, SQLITE_UTF8, 0,
                               sqlarCompressFunc, 0, 0);
  if( rc==SQLITE_OK ){
    rc = sqlite3_create_function(db, "sqlar_uncompress", 2, SQLITE_UTF8, 0,
                                 sqlarUncompressFunc, 0, 0);
  }
  return rc;
}

Changes to main.mk.

686
687
688
689
690
691
692
693

694
695
696
697
698
699
700
	./mkkeywordhash >keywordhash.h

# Source files that go into making shell.c
SHELL_SRC = \
	$(TOP)/src/shell.c.in \
	$(TOP)/ext/misc/shathree.c \
	$(TOP)/ext/misc/fileio.c \
	$(TOP)/ext/misc/completion.c


shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
	tclsh $(TOP)/tool/mkshellc.tcl >shell.c



# Rules to build the extension objects.







|
>







686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
	./mkkeywordhash >keywordhash.h

# Source files that go into making shell.c
SHELL_SRC = \
	$(TOP)/src/shell.c.in \
	$(TOP)/ext/misc/shathree.c \
	$(TOP)/ext/misc/fileio.c \
	$(TOP)/ext/misc/completion.c \
	$(TOP)/ext/misc/sqlar.c

shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
	tclsh $(TOP)/tool/mkshellc.tcl >shell.c



# Rules to build the extension objects.

Changes to src/shell.c.in.

793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
....
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
....
4478
4479
4480
4481
4482
4483
4484

4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
....
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
....
4607
4608
4609
4610
4611
4612
4613


4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
....
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
#define SQLITE_EXTENSION_INIT1
#define SQLITE_EXTENSION_INIT2(X) (void)(X)

INCLUDE ../ext/misc/shathree.c
INCLUDE ../ext/misc/fileio.c
INCLUDE ../ext/misc/completion.c
#ifdef SQLITE_HAVE_ZLIB
INCLUDE ../ext/misc/compress.c
#endif

#if defined(SQLITE_ENABLE_SESSION)
/*
** State information for a single open session
*/
typedef struct OpenSession OpenSession;
................................................................................
#ifndef SQLITE_OMIT_LOAD_EXTENSION
    sqlite3_enable_load_extension(p->db, 1);
#endif
    sqlite3_fileio_init(p->db, 0, 0);
    sqlite3_shathree_init(p->db, 0, 0);
    sqlite3_completion_init(p->db, 0, 0);
#ifdef SQLITE_HAVE_ZLIB
    sqlite3_compress_init(p->db, 0, 0);
#endif
    sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0,
                            shellAddSchemaName, 0, 0);
  }
}

#if HAVE_READLINE || HAVE_EDITLINE
................................................................................


/*
** Implementation of .ar "eXtract" command. 
*/
static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
  const char *zSql1 = 

    "SELECT :1 || name, writefile(:1 || name, "
    "CASE WHEN (data AND sz>=0 AND sz!=length(data)) THEN "
    "    uncompress(data) "
    "ELSE"
    "    data "
    "END, "
    "mode, mtime) FROM sqlar WHERE (%s) AND (data IS NULL OR :2 = 0)";

  struct timespec times[2];
  sqlite3_stmt *pSql = 0;
  int rc = SQLITE_OK;
  char *zDir = 0;
  char *zWhere = 0;
  int i;
................................................................................
      "name TEXT PRIMARY KEY,  -- name of the file\n"
      "mode INT,               -- access permissions\n"
      "mtime INT,              -- last modification time\n"
      "sz INT,                 -- original file size\n"
      "data BLOB               -- compressed content\n"
  ")";
  const char *zDrop = "DROP TABLE IF EXISTS sqlar";
  const char *zInsert = "REPLACE INTO sqlar VALUES(?, ?, ?, ?, ?)";

  sqlite3_stmt *pStmt = 0;        /* Directory traverser */
  sqlite3_stmt *pInsert = 0;      /* Compilation of zInsert */
  int i;                          /* For iterating through azFile[] */
  int rc;                         /* Return code */

  Bytef *aCompress = 0;           /* Compression buffer */
  int nCompress = 0;              /* Size of compression buffer */

  rc = sqlite3_exec(db, "SAVEPOINT ar;", 0, 0, 0);
  if( rc!=SQLITE_OK ) return rc;

  if( bUpdate==0 ){
    rc = sqlite3_exec(db, zDrop, 0, 0, 0);
    if( rc!=SQLITE_OK ) return rc;
  }
................................................................................
      sqlite3_bind_text(pInsert, 1, zName, -1, SQLITE_STATIC);
      sqlite3_bind_int(pInsert, 2, mode);
      sqlite3_bind_int64(pInsert, 3, (sqlite3_int64)mtime);

      if( S_ISDIR(mode) ){
        sz = 0;
        sqlite3_bind_null(pInsert, 5);


      }else if( S_ISLNK(mode) ){
        sz = -1;
        sqlite3_bind_value(pInsert, 5, sqlite3_column_value(pStmt, 3));
      }else{
        uLongf nReq;              /* Required size of compression buffer */
        const Bytef *pData = (const Bytef*)sqlite3_column_blob(pStmt, 3);
        sz = sqlite3_column_bytes(pStmt, 3);
        nReq = compressBound(sz);
        if( aCompress==0 || nReq>nCompress ){
          Bytef *aNew = sqlite3_realloc(aCompress, nReq);
          if( aNew==0 ){
            rc = SQLITE_NOMEM;
          }else{
            aCompress = aNew;
            nCompress = nReq;
          }
        }

        if( Z_OK!=compress(aCompress, &nReq, pData, sz) ){
          rc = SQLITE_ERROR;
        }
        if( nReq<sz ){
          sqlite3_bind_blob(pInsert, 5, aCompress, nReq, SQLITE_STATIC);
        }else{
          sqlite3_bind_blob(pInsert, 5, pData, sz, SQLITE_STATIC);
        }
      }

      if( rc==SQLITE_OK ){
        sqlite3_bind_int(pInsert, 4, sz);
        sqlite3_step(pInsert);
        rc = sqlite3_reset(pInsert);
      }
    }
    shellReset(&rc, pStmt);
  }

  if( rc!=SQLITE_OK ){
    sqlite3_exec(db, "ROLLBACK TO ar; RELEASE ar;", 0, 0, 0);
  }else{
    rc = sqlite3_exec(db, "RELEASE ar;", 0, 0, 0);
  }
  shellFinalize(&rc, pStmt);
  shellFinalize(&rc, pInsert);
  sqlite3_free(aCompress);
  return rc;
}

/*
** Implementation of .ar "Create" command. 
**
** Create the "sqlar" table in the database if it does not already exist.
................................................................................
        raw_printf(stderr, "cannot open file: %s (%s)\n", 
            cmd.zFile, sqlite3_errmsg(db)
        );
        sqlite3_close(db);
        return rc;
      }
      sqlite3_fileio_init(db, 0, 0);
      sqlite3_compress_init(db, 0, 0);
    }else{
      db = pState->db;
    }

    switch( cmd.eCmd ){
      case AR_CMD_CREATE:
        rc = arCreateCommand(pState, db, &cmd);







|







 







|







 







>
|
<
|
<
<
<
|







 







|






<
<
<







 







>
>
|
|
<
|
<
<
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<



<
|
|
|
<











<







 







|







793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
....
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
....
4478
4479
4480
4481
4482
4483
4484
4485
4486

4487



4488
4489
4490
4491
4492
4493
4494
4495
....
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575



4576
4577
4578
4579
4580
4581
4582
....
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611

4612


4613


















4614
4615
4616

4617
4618
4619

4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630

4631
4632
4633
4634
4635
4636
4637
....
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
#define SQLITE_EXTENSION_INIT1
#define SQLITE_EXTENSION_INIT2(X) (void)(X)

INCLUDE ../ext/misc/shathree.c
INCLUDE ../ext/misc/fileio.c
INCLUDE ../ext/misc/completion.c
#ifdef SQLITE_HAVE_ZLIB
INCLUDE ../ext/misc/sqlar.c
#endif

#if defined(SQLITE_ENABLE_SESSION)
/*
** State information for a single open session
*/
typedef struct OpenSession OpenSession;
................................................................................
#ifndef SQLITE_OMIT_LOAD_EXTENSION
    sqlite3_enable_load_extension(p->db, 1);
#endif
    sqlite3_fileio_init(p->db, 0, 0);
    sqlite3_shathree_init(p->db, 0, 0);
    sqlite3_completion_init(p->db, 0, 0);
#ifdef SQLITE_HAVE_ZLIB
    sqlite3_sqlar_init(p->db, 0, 0);
#endif
    sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0,
                            shellAddSchemaName, 0, 0);
  }
}

#if HAVE_READLINE || HAVE_EDITLINE
................................................................................


/*
** Implementation of .ar "eXtract" command. 
*/
static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
  const char *zSql1 = 
    "SELECT "
    "  :1 || name, "

    "  writefile(:1 || name, sqlar_uncompress(data, sz), mode, mtime) "



    "FROM sqlar WHERE (%s) AND (data IS NULL OR :2 = 0)";

  struct timespec times[2];
  sqlite3_stmt *pSql = 0;
  int rc = SQLITE_OK;
  char *zDir = 0;
  char *zWhere = 0;
  int i;
................................................................................
      "name TEXT PRIMARY KEY,  -- name of the file\n"
      "mode INT,               -- access permissions\n"
      "mtime INT,              -- last modification time\n"
      "sz INT,                 -- original file size\n"
      "data BLOB               -- compressed content\n"
  ")";
  const char *zDrop = "DROP TABLE IF EXISTS sqlar";
  const char *zInsert = "REPLACE INTO sqlar VALUES(?,?,?,?,sqlar_compress(?))";

  sqlite3_stmt *pStmt = 0;        /* Directory traverser */
  sqlite3_stmt *pInsert = 0;      /* Compilation of zInsert */
  int i;                          /* For iterating through azFile[] */
  int rc;                         /* Return code */




  rc = sqlite3_exec(db, "SAVEPOINT ar;", 0, 0, 0);
  if( rc!=SQLITE_OK ) return rc;

  if( bUpdate==0 ){
    rc = sqlite3_exec(db, zDrop, 0, 0, 0);
    if( rc!=SQLITE_OK ) return rc;
  }
................................................................................
      sqlite3_bind_text(pInsert, 1, zName, -1, SQLITE_STATIC);
      sqlite3_bind_int(pInsert, 2, mode);
      sqlite3_bind_int64(pInsert, 3, (sqlite3_int64)mtime);

      if( S_ISDIR(mode) ){
        sz = 0;
        sqlite3_bind_null(pInsert, 5);
      }else{
        sqlite3_bind_value(pInsert, 5, sqlite3_column_value(pStmt, 3));
        if( S_ISLNK(mode) ){
          sz = -1;

        }else{


          sz = sqlite3_column_bytes(pStmt, 3);


















        }
      }


      sqlite3_bind_int(pInsert, 4, sz);
      sqlite3_step(pInsert);
      rc = sqlite3_reset(pInsert);

    }
    shellReset(&rc, pStmt);
  }

  if( rc!=SQLITE_OK ){
    sqlite3_exec(db, "ROLLBACK TO ar; RELEASE ar;", 0, 0, 0);
  }else{
    rc = sqlite3_exec(db, "RELEASE ar;", 0, 0, 0);
  }
  shellFinalize(&rc, pStmt);
  shellFinalize(&rc, pInsert);

  return rc;
}

/*
** Implementation of .ar "Create" command. 
**
** Create the "sqlar" table in the database if it does not already exist.
................................................................................
        raw_printf(stderr, "cannot open file: %s (%s)\n", 
            cmd.zFile, sqlite3_errmsg(db)
        );
        sqlite3_close(db);
        return rc;
      }
      sqlite3_fileio_init(db, 0, 0);
      sqlite3_sqlar_init(db, 0, 0);
    }else{
      db = pState->db;
    }

    switch( cmd.eCmd ){
      case AR_CMD_CREATE:
        rc = arCreateCommand(pState, db, &cmd);