/ Check-in [5e0129ee]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Enable sqlite3_deserialize() in the CLI. The --deserialize option associated with opening a new database cause the database file to be read into memory and accessed using the sqlite3_deserialize() API. This simplifies running tests on a database without risk of modifying the file on disk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 5e0129ee9afa7c2d707f8ac9e29ef3583c49bb1d0965085c067d58f828ac8cdf
User & Date: drh 2018-10-30 13:19:49
Context
2018-10-30
14:36
In the CLI, when opening a file that begins with the normal SQLite prefix, open the file as a normal database even if it has a ZIP for AppendVFS record at the end. check-in: 7989bbda user: drh tags: trunk
13:19
Enable sqlite3_deserialize() in the CLI. The --deserialize option associated with opening a new database cause the database file to be read into memory and accessed using the sqlite3_deserialize() API. This simplifies running tests on a database without risk of modifying the file on disk. check-in: 5e0129ee user: drh tags: trunk
2018-10-29
21:01
Fix a potential assertion fault that can occur while trying to DROP a table from a corrupted database file. check-in: 147a9429 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   592    592   SHELL_OPT += -DSQLITE_ENABLE_RTREE
   593    593   SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
   594    594   SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
   595    595   SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
   596    596   SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
   597    597   SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
   598    598   SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
          599  +SHELL_OPT += -DSQLITE_ENABLE_DESERIALIZE
   599    600   SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
   600    601   FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
   601    602   FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
   602    603   FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
   603    604   FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
   604    605   FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
   605    606   DBFUZZ_OPT = 

Changes to Makefile.msc.

  1629   1629   # Additional compiler options for the shell.  These are only effective
  1630   1630   # when the shell is not being dynamically linked.
  1631   1631   #
  1632   1632   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
  1633   1633   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
  1634   1634   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
  1635   1635   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
         1636  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
  1636   1637   !ENDIF
  1637   1638   
  1638   1639   # <<mark>>
  1639   1640   # Extra compiler options for various test tools.
  1640   1641   #
  1641   1642   MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5
  1642   1643   FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1

Changes to main.mk.

   519    519   SHELL_OPT += -DSQLITE_ENABLE_RTREE
   520    520   SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
   521    521   SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
   522    522   SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
   523    523   SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
   524    524   SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
   525    525   SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
          526  +SHELL_OPT += -DSQLITE_ENABLE_DESERIALIZE
   526    527   SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
   527    528   FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
   528    529   FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
   529    530   FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
   530    531   FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
   531    532   DBFUZZ_OPT =
   532    533   KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ

Changes to src/shell.c.in.

  1055   1055   #define AUTOEQP_off      0           /* Automatic EXPLAIN QUERY PLAN is off */
  1056   1056   #define AUTOEQP_on       1           /* Automatic EQP is on */
  1057   1057   #define AUTOEQP_trigger  2           /* On and also show plans for triggers */
  1058   1058   #define AUTOEQP_full     3           /* Show full EXPLAIN */
  1059   1059   
  1060   1060   /* Allowed values for ShellState.openMode
  1061   1061   */
  1062         -#define SHELL_OPEN_UNSPEC     0      /* No open-mode specified */
  1063         -#define SHELL_OPEN_NORMAL     1      /* Normal database file */
  1064         -#define SHELL_OPEN_APPENDVFS  2      /* Use appendvfs */
  1065         -#define SHELL_OPEN_ZIPFILE    3      /* Use the zipfile virtual table */
  1066         -#define SHELL_OPEN_READONLY   4      /* Open a normal database read-only */
         1062  +#define SHELL_OPEN_UNSPEC      0      /* No open-mode specified */
         1063  +#define SHELL_OPEN_NORMAL      1      /* Normal database file */
         1064  +#define SHELL_OPEN_APPENDVFS   2      /* Use appendvfs */
         1065  +#define SHELL_OPEN_ZIPFILE     3      /* Use the zipfile virtual table */
         1066  +#define SHELL_OPEN_READONLY    4      /* Open a normal database read-only */
         1067  +#define SHELL_OPEN_DESERIALIZE 5      /* Open using sqlite3_deserialize() */
  1067   1068   
  1068   1069   /*
  1069   1070   ** These are the allowed shellFlgs values
  1070   1071   */
  1071   1072   #define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
  1072   1073   #define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
  1073   1074   #define SHFLG_Backslash      0x00000004 /* The --backslash option is used */
................................................................................
  3428   3429     ".once (-e|-x|FILE)       Output for the next SQL command only to FILE",
  3429   3430     "     If FILE begins with '|' then open as a pipe",
  3430   3431     "     Other options:",
  3431   3432     "       -e    Invoke system text editor",
  3432   3433     "       -x    Open in a spreadsheet",
  3433   3434     ".open ?OPTIONS? ?FILE?   Close existing database and reopen FILE",
  3434   3435     "     Options:",
  3435         -  "        --append      Use appendvfs to append database to the end of FILE",
  3436         -  "        --new         Initialize FILE to an empty database",
  3437         -  "        --readonly    Open FILE readonly",
  3438         -  "        --zip         FILE is a ZIP archive",
         3436  +  "        --append        Use appendvfs to append database to the end of FILE",
         3437  +  "        --deserialize   Load into memory useing sqlite3_deserialize()",
         3438  +  "        --new           Initialize FILE to an empty database",
         3439  +  "        --readonly      Open FILE readonly",
         3440  +  "        --zip           FILE is a ZIP archive",
  3439   3441     ".output ?FILE?           Send output to FILE or stdout if FILE is omitted",
  3440   3442     "     If FILE begins with '|' then open it as a pipe.",
  3441   3443     ".print STRING...         Print literal STRING",
  3442   3444     ".prompt MAIN CONTINUE    Replace the standard prompts",
  3443   3445     ".quit                    Exit this program",
  3444   3446     ".read FILE               Read input from FILE",
  3445   3447     ".restore ?DB? FILE       Restore content of DB (default \"main\") from FILE",
................................................................................
  3719   3721         }
  3720   3722       }
  3721   3723       switch( p->openMode ){
  3722   3724         case SHELL_OPEN_APPENDVFS: {
  3723   3725           sqlite3_open_v2(p->zDbFilename, &p->db, 
  3724   3726              SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, "apndvfs");
  3725   3727           break;
         3728  +      }
         3729  +      case SHELL_OPEN_DESERIALIZE: {
         3730  +        sqlite3_open(0, &p->db);
         3731  +        break;
  3726   3732         }
  3727   3733         case SHELL_OPEN_ZIPFILE: {
  3728   3734           sqlite3_open(":memory:", &p->db);
  3729   3735           break;
  3730   3736         }
  3731   3737         case SHELL_OPEN_READONLY: {
  3732   3738           sqlite3_open_v2(p->zDbFilename, &p->db, SQLITE_OPEN_READONLY, 0);
................................................................................
  3768   3774                               editFunc, 0, 0);
  3769   3775   #endif
  3770   3776       if( p->openMode==SHELL_OPEN_ZIPFILE ){
  3771   3777         char *zSql = sqlite3_mprintf(
  3772   3778            "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename);
  3773   3779         sqlite3_exec(p->db, zSql, 0, 0, 0);
  3774   3780         sqlite3_free(zSql);
         3781  +    }else if( p->openMode==SHELL_OPEN_DESERIALIZE ){
         3782  +      int nData = 0;
         3783  +      unsigned char *aData = (unsigned char*)readFile(p->zDbFilename, &nData);
         3784  +      int rc = sqlite3_deserialize(p->db, "main", aData, nData, nData,
         3785  +                   SQLITE_DESERIALIZE_RESIZEABLE |
         3786  +                   SQLITE_DESERIALIZE_FREEONCLOSE);
         3787  +      if( rc ){
         3788  +        utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc);
         3789  +      }
  3775   3790       }
  3776   3791     }
  3777   3792   }
  3778   3793   
  3779   3794   /*
  3780   3795   ** Attempt to close the databaes connection.  Report errors.
  3781   3796   */
................................................................................
  6649   6664         }else if( optionMatch(z, "zip") ){
  6650   6665           p->openMode = SHELL_OPEN_ZIPFILE;
  6651   6666   #endif
  6652   6667         }else if( optionMatch(z, "append") ){
  6653   6668           p->openMode = SHELL_OPEN_APPENDVFS;
  6654   6669         }else if( optionMatch(z, "readonly") ){
  6655   6670           p->openMode = SHELL_OPEN_READONLY;
         6671  +      }else if( optionMatch(z, "deserialize") ){
         6672  +        p->openMode = SHELL_OPEN_DESERIALIZE;
  6656   6673         }else if( z[0]=='-' ){
  6657   6674           utf8_printf(stderr, "unknown option: %s\n", z);
  6658   6675           rc = 1;
  6659   6676           goto meta_command_exit;
  6660   6677         }
  6661   6678       }
  6662   6679       /* If a filename is specified, try to open it first */
................................................................................
  8617   8634         zVfs = cmdline_option_value(argc, argv, ++i);
  8618   8635   #ifdef SQLITE_HAVE_ZLIB
  8619   8636       }else if( strcmp(z,"-zip")==0 ){
  8620   8637         data.openMode = SHELL_OPEN_ZIPFILE;
  8621   8638   #endif
  8622   8639       }else if( strcmp(z,"-append")==0 ){
  8623   8640         data.openMode = SHELL_OPEN_APPENDVFS;
         8641  +    }else if( strcmp(z,"-deserialize")==0 ){
         8642  +      data.openMode = SHELL_OPEN_DESERIALIZE;
  8624   8643       }else if( strcmp(z,"-readonly")==0 ){
  8625   8644         data.openMode = SHELL_OPEN_READONLY;
  8626   8645   #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)
  8627   8646       }else if( strncmp(z, "-A",2)==0 ){
  8628   8647         /* All remaining command-line arguments are passed to the ".archive"
  8629   8648         ** command, so ignore them */
  8630   8649         break;
................................................................................
  8712   8731         memcpy(data.colSeparator,",",2);
  8713   8732   #ifdef SQLITE_HAVE_ZLIB
  8714   8733       }else if( strcmp(z,"-zip")==0 ){
  8715   8734         data.openMode = SHELL_OPEN_ZIPFILE;
  8716   8735   #endif
  8717   8736       }else if( strcmp(z,"-append")==0 ){
  8718   8737         data.openMode = SHELL_OPEN_APPENDVFS;
         8738  +    }else if( strcmp(z,"-deserialize")==0 ){
         8739  +      data.openMode = SHELL_OPEN_DESERIALIZE;
  8719   8740       }else if( strcmp(z,"-readonly")==0 ){
  8720   8741         data.openMode = SHELL_OPEN_READONLY;
  8721   8742       }else if( strcmp(z,"-ascii")==0 ){
  8722   8743         data.mode = MODE_Ascii;
  8723   8744         sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,
  8724   8745                          SEP_Unit);
  8725   8746         sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,