/ Check-in [5e0129ee]
Login
Home Timeline Branches Tags Tickets Wiki More...
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Enable sqlite3_deserialize() in the CLI. The --deserialize option associated with opening a new database cause the database file to be read into memory and accessed using the sqlite3_deserialize() API. This simplifies running tests on a database without risk of modifying the file on disk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 5e0129ee9afa7c2d707f8ac9e29ef3583c49bb1d0965085c067d58f828ac8cdf
User & Date: drh 2018-10-30 13:19:49
Context
2018-10-30
14:36
In the CLI, when opening a file that begins with the normal SQLite prefix, open the file as a normal database even if it has a ZIP for AppendVFS record at the end. check-in: 7989bbda user: drh tags: trunk
13:19
Enable sqlite3_deserialize() in the CLI. The --deserialize option associated with opening a new database cause the database file to be read into memory and accessed using the sqlite3_deserialize() API. This simplifies running tests on a database without risk of modifying the file on disk. check-in: 5e0129ee user: drh tags: trunk
2018-10-29
21:01
Fix a potential assertion fault that can occur while trying to DROP a table from a corrupted database file. check-in: 147a9429 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to Makefile.in. 

592
593
594
595
596
597
598

599
600
601
602
603
604
605

SHELL_OPT += -DSQLITE_ENABLE_RTREE
SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC

SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
DBFUZZ_OPT = 








>








592
593
594
595
596
597
598
599
600
601
602
603
604
605
606

SHELL_OPT += -DSQLITE_ENABLE_RTREE
SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
SHELL_OPT += -DSQLITE_ENABLE_DESERIALIZE
SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
DBFUZZ_OPT =

Changes to Makefile.msc. 

1629
1630
1631
1632
1633
1634
1635

1636
1637
1638
1639
1640
1641
1642

# Additional compiler options for the shell.  These are only effective
# when the shell is not being dynamically linked.
#
!IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1

!ENDIF

# <<mark>>
# Extra compiler options for various test tools.
#
MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5
FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1








>








1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643

# Additional compiler options for the shell.  These are only effective
# when the shell is not being dynamically linked.
#
!IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
!ENDIF

# <<mark>>
# Extra compiler options for various test tools.
#
MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5
FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1

Changes to main.mk. 

519
520
521
522
523
524
525

526
527
528
529
530
531
532

SHELL_OPT += -DSQLITE_ENABLE_RTREE
SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC

SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
DBFUZZ_OPT =
KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ








>








519
520
521
522
523
524
525
526
527
528
529
530
531
532
533

SHELL_OPT += -DSQLITE_ENABLE_RTREE
SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
SHELL_OPT += -DSQLITE_ENABLE_DESERIALIZE
SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
DBFUZZ_OPT =
KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ

Changes to src/shell.c.in. 

1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066

1067
1068
1069
1070
1071
1072
1073
....
3428
3429
3430
3431
3432
3433
3434
3435

3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
....
3719
3720
3721
3722
3723
3724
3725




3726
3727
3728
3729
3730
3731
3732
....
3768
3769
3770
3771
3772
3773
3774









3775
3776
3777
3778
3779
3780
3781
....
6649
6650
6651
6652
6653
6654
6655


6656
6657
6658
6659
6660
6661
6662
....
8617
8618
8619
8620
8621
8622
8623


8624
8625
8626
8627
8628
8629
8630
....
8712
8713
8714
8715
8716
8717
8718


8719
8720
8721
8722
8723
8724
8725

#define AUTOEQP_off      0           /* Automatic EXPLAIN QUERY PLAN is off */
#define AUTOEQP_on       1           /* Automatic EQP is on */
#define AUTOEQP_trigger  2           /* On and also show plans for triggers */
#define AUTOEQP_full     3           /* Show full EXPLAIN */

/* Allowed values for ShellState.openMode
*/
#define SHELL_OPEN_UNSPEC     0      /* No open-mode specified */
#define SHELL_OPEN_NORMAL     1      /* Normal database file */
#define SHELL_OPEN_APPENDVFS  2      /* Use appendvfs */
#define SHELL_OPEN_ZIPFILE    3      /* Use the zipfile virtual table */
#define SHELL_OPEN_READONLY   4      /* Open a normal database read-only */


/*
** These are the allowed shellFlgs values
*/
#define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
#define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
#define SHFLG_Backslash      0x00000004 /* The --backslash option is used */
................................................................................
  ".once (-e|-x|FILE)       Output for the next SQL command only to FILE",
  "     If FILE begins with '|' then open as a pipe",
  "     Other options:",
  "       -e    Invoke system text editor",
  "       -x    Open in a spreadsheet",
  ".open ?OPTIONS? ?FILE?   Close existing database and reopen FILE",
  "     Options:",
  "        --append      Use appendvfs to append database to the end of FILE",

  "        --new         Initialize FILE to an empty database",
  "        --readonly    Open FILE readonly",
  "        --zip         FILE is a ZIP archive",
  ".output ?FILE?           Send output to FILE or stdout if FILE is omitted",
  "     If FILE begins with '|' then open it as a pipe.",
  ".print STRING...         Print literal STRING",
  ".prompt MAIN CONTINUE    Replace the standard prompts",
  ".quit                    Exit this program",
  ".read FILE               Read input from FILE",
  ".restore ?DB? FILE       Restore content of DB (default \"main\") from FILE",
................................................................................
      }
    }
    switch( p->openMode ){
      case SHELL_OPEN_APPENDVFS: {
        sqlite3_open_v2(p->zDbFilename, &p->db, 
           SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, "apndvfs");
        break;




      }
      case SHELL_OPEN_ZIPFILE: {
        sqlite3_open(":memory:", &p->db);
        break;
      }
      case SHELL_OPEN_READONLY: {
        sqlite3_open_v2(p->zDbFilename, &p->db, SQLITE_OPEN_READONLY, 0);
................................................................................
                            editFunc, 0, 0);
#endif
    if( p->openMode==SHELL_OPEN_ZIPFILE ){
      char *zSql = sqlite3_mprintf(
         "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename);
      sqlite3_exec(p->db, zSql, 0, 0, 0);
      sqlite3_free(zSql);









    }
  }
}

/*
** Attempt to close the databaes connection.  Report errors.
*/
................................................................................
      }else if( optionMatch(z, "zip") ){
        p->openMode = SHELL_OPEN_ZIPFILE;
#endif
      }else if( optionMatch(z, "append") ){
        p->openMode = SHELL_OPEN_APPENDVFS;
      }else if( optionMatch(z, "readonly") ){
        p->openMode = SHELL_OPEN_READONLY;


      }else if( z[0]=='-' ){
        utf8_printf(stderr, "unknown option: %s\n", z);
        rc = 1;
        goto meta_command_exit;
      }
    }
    /* If a filename is specified, try to open it first */
................................................................................
      zVfs = cmdline_option_value(argc, argv, ++i);
#ifdef SQLITE_HAVE_ZLIB
    }else if( strcmp(z,"-zip")==0 ){
      data.openMode = SHELL_OPEN_ZIPFILE;
#endif
    }else if( strcmp(z,"-append")==0 ){
      data.openMode = SHELL_OPEN_APPENDVFS;


    }else if( strcmp(z,"-readonly")==0 ){
      data.openMode = SHELL_OPEN_READONLY;
#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)
    }else if( strncmp(z, "-A",2)==0 ){
      /* All remaining command-line arguments are passed to the ".archive"
      ** command, so ignore them */
      break;
................................................................................
      memcpy(data.colSeparator,",",2);
#ifdef SQLITE_HAVE_ZLIB
    }else if( strcmp(z,"-zip")==0 ){
      data.openMode = SHELL_OPEN_ZIPFILE;
#endif
    }else if( strcmp(z,"-append")==0 ){
      data.openMode = SHELL_OPEN_APPENDVFS;


    }else if( strcmp(z,"-readonly")==0 ){
      data.openMode = SHELL_OPEN_READONLY;
    }else if( strcmp(z,"-ascii")==0 ){
      data.mode = MODE_Ascii;
      sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,
                       SEP_Unit);
      sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,








|
|
|
|
|
>







 







|
>
|
|
|







 







>
>
>
>







 







>
>
>
>
>
>
>
>
>







 







>
>







 







>
>







 







>
>








1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
....
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
....
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
....
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
....
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
6678
6679
....
8634
8635
8636
8637
8638
8639
8640
8641
8642
8643
8644
8645
8646
8647
8648
8649
....
8731
8732
8733
8734
8735
8736
8737
8738
8739
8740
8741
8742
8743
8744
8745
8746

#define AUTOEQP_off      0           /* Automatic EXPLAIN QUERY PLAN is off */
#define AUTOEQP_on       1           /* Automatic EQP is on */
#define AUTOEQP_trigger  2           /* On and also show plans for triggers */
#define AUTOEQP_full     3           /* Show full EXPLAIN */

/* Allowed values for ShellState.openMode
*/
#define SHELL_OPEN_UNSPEC      0      /* No open-mode specified */
#define SHELL_OPEN_NORMAL      1      /* Normal database file */
#define SHELL_OPEN_APPENDVFS   2      /* Use appendvfs */
#define SHELL_OPEN_ZIPFILE     3      /* Use the zipfile virtual table */
#define SHELL_OPEN_READONLY    4      /* Open a normal database read-only */
#define SHELL_OPEN_DESERIALIZE 5      /* Open using sqlite3_deserialize() */

/*
** These are the allowed shellFlgs values
*/
#define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
#define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
#define SHFLG_Backslash      0x00000004 /* The --backslash option is used */
................................................................................
  ".once (-e|-x|FILE)       Output for the next SQL command only to FILE",
  "     If FILE begins with '|' then open as a pipe",
  "     Other options:",
  "       -e    Invoke system text editor",
  "       -x    Open in a spreadsheet",
  ".open ?OPTIONS? ?FILE?   Close existing database and reopen FILE",
  "     Options:",
  "        --append        Use appendvfs to append database to the end of FILE",
  "        --deserialize   Load into memory useing sqlite3_deserialize()",
  "        --new           Initialize FILE to an empty database",
  "        --readonly      Open FILE readonly",
  "        --zip           FILE is a ZIP archive",
  ".output ?FILE?           Send output to FILE or stdout if FILE is omitted",
  "     If FILE begins with '|' then open it as a pipe.",
  ".print STRING...         Print literal STRING",
  ".prompt MAIN CONTINUE    Replace the standard prompts",
  ".quit                    Exit this program",
  ".read FILE               Read input from FILE",
  ".restore ?DB? FILE       Restore content of DB (default \"main\") from FILE",
................................................................................
      }
    }
    switch( p->openMode ){
      case SHELL_OPEN_APPENDVFS: {
        sqlite3_open_v2(p->zDbFilename, &p->db, 
           SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, "apndvfs");
        break;
      }
      case SHELL_OPEN_DESERIALIZE: {
        sqlite3_open(0, &p->db);
        break;
      }
      case SHELL_OPEN_ZIPFILE: {
        sqlite3_open(":memory:", &p->db);
        break;
      }
      case SHELL_OPEN_READONLY: {
        sqlite3_open_v2(p->zDbFilename, &p->db, SQLITE_OPEN_READONLY, 0);
................................................................................
                            editFunc, 0, 0);
#endif
    if( p->openMode==SHELL_OPEN_ZIPFILE ){
      char *zSql = sqlite3_mprintf(
         "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename);
      sqlite3_exec(p->db, zSql, 0, 0, 0);
      sqlite3_free(zSql);
    }else if( p->openMode==SHELL_OPEN_DESERIALIZE ){
      int nData = 0;
      unsigned char *aData = (unsigned char*)readFile(p->zDbFilename, &nData);
      int rc = sqlite3_deserialize(p->db, "main", aData, nData, nData,
                   SQLITE_DESERIALIZE_RESIZEABLE |
                   SQLITE_DESERIALIZE_FREEONCLOSE);
      if( rc ){
        utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc);
      }
    }
  }
}

/*
** Attempt to close the databaes connection.  Report errors.
*/
................................................................................
      }else if( optionMatch(z, "zip") ){
        p->openMode = SHELL_OPEN_ZIPFILE;
#endif
      }else if( optionMatch(z, "append") ){
        p->openMode = SHELL_OPEN_APPENDVFS;
      }else if( optionMatch(z, "readonly") ){
        p->openMode = SHELL_OPEN_READONLY;
      }else if( optionMatch(z, "deserialize") ){
        p->openMode = SHELL_OPEN_DESERIALIZE;
      }else if( z[0]=='-' ){
        utf8_printf(stderr, "unknown option: %s\n", z);
        rc = 1;
        goto meta_command_exit;
      }
    }
    /* If a filename is specified, try to open it first */
................................................................................
      zVfs = cmdline_option_value(argc, argv, ++i);
#ifdef SQLITE_HAVE_ZLIB
    }else if( strcmp(z,"-zip")==0 ){
      data.openMode = SHELL_OPEN_ZIPFILE;
#endif
    }else if( strcmp(z,"-append")==0 ){
      data.openMode = SHELL_OPEN_APPENDVFS;
    }else if( strcmp(z,"-deserialize")==0 ){
      data.openMode = SHELL_OPEN_DESERIALIZE;
    }else if( strcmp(z,"-readonly")==0 ){
      data.openMode = SHELL_OPEN_READONLY;
#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)
    }else if( strncmp(z, "-A",2)==0 ){
      /* All remaining command-line arguments are passed to the ".archive"
      ** command, so ignore them */
      break;
................................................................................
      memcpy(data.colSeparator,",",2);
#ifdef SQLITE_HAVE_ZLIB
    }else if( strcmp(z,"-zip")==0 ){
      data.openMode = SHELL_OPEN_ZIPFILE;
#endif
    }else if( strcmp(z,"-append")==0 ){
      data.openMode = SHELL_OPEN_APPENDVFS;
    }else if( strcmp(z,"-deserialize")==0 ){
      data.openMode = SHELL_OPEN_DESERIALIZE;
    }else if( strcmp(z,"-readonly")==0 ){
      data.openMode = SHELL_OPEN_READONLY;
    }else if( strcmp(z,"-ascii")==0 ){
      data.mode = MODE_Ascii;
      sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,
                       SEP_Unit);
      sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,

This page was generated in about 0.016s by Fossil 2.10 [9d9ef82234] 2019-10-04 21:41:13