/ Check-in [b386fce9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge the dbtotxt enhancement from trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | dbfuzz2-cases
Files: files | file ages | folders
SHA3-256: b386fce9a23e628dce7362dcca2904b8d0af6da58a6fe6eb7f12f058a8363e49
User & Date: drh 2018-12-13 15:52:31
Context
2018-12-13
21:11
Add extra tests for database corruption inside the defragmentPage() routine, as dbfuzz2 has found ways for corruption to leak into that point. Add test cases in fuzzdata7.db. check-in: 997b6511 user: drh tags: trunk
15:52
Merge the dbtotxt enhancement from trunk. Closed-Leaf check-in: b386fce9 user: drh tags: dbfuzz2-cases
15:06
Add the "dbtotxt" utility program and the ability to read "dbtotxt" output as a deserialized input database in the CLI, using the --hexdb option to the ".open" command. check-in: e3bf1d3e user: drh tags: trunk
03:36
New database corruption test cases discovered by dbfuzz2. The new cases have been added to test/fuzzdata7.db, but have not yet all been fixed, so tests will not currently pass. check-in: b4210d32 user: drh tags: dbfuzz2-cases
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   665    665   	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
   666    666   
   667    667   ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
   668    668   	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
   669    669                $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
   670    670   
   671    671   sessionfuzz$(TEXE):	$(TOP)/test/sessionfuzz.c sqlite3.c sqlite3.h
   672         -	$(CC) $(CFLAGS) -I. -o $@ $(TOP)/test/sessionfuzz.c $(TLIBS)
          672  +	$(LTLINK) -o $@ $(TOP)/test/sessionfuzz.c $(TLIBS)
   673    673   
   674    674   dbfuzz$(TEXE):	$(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h
   675    675   	$(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS)
   676    676   
   677    677   DBFUZZ2_OPTS = \
   678    678     -DSQLITE_THREADSAFE=0 \
   679    679     -DSQLITE_OMIT_LOAD_EXTENSION \
................................................................................
   681    681     -DSQLITE_DEBUG \
   682    682     -DSQLITE_ENABLE_DBSTAT_VTAB \
   683    683     -DSQLITE_ENABLE_RTREE \
   684    684     -DSQLITE_ENABLE_FTS4 \
   685    685     -DSQLITE_EANBLE_FTS5
   686    686   
   687    687   dbfuzz2:	$(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h
   688         -	clang-6.0 -I. -g -O0 -fsanitize=fuzzer,undefined,address -o dbfuzz2 \
          688  +	clang-6.0 $(OPT_FEATURE_FLAGS) $(OPTS) -I. -g -O0 \
          689  +		-fsanitize=fuzzer,undefined,address -o dbfuzz2 \
   689    690   		$(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c
   690    691   	mkdir -p dbfuzz2-dir
   691    692   	cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir
   692    693   
   693    694   mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
   694    695   	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
   695    696   		$(TLIBS) -rpath "$(libdir)"
................................................................................
  1286   1287   
  1287   1288   sqlite3_checker$(TEXE):	sqlite3_checker.c
  1288   1289   	$(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS)
  1289   1290   
  1290   1291   dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
  1291   1292   	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
  1292   1293              $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
         1294  +
         1295  +dbtotxt$(TEXE): $(TOP)/tool/dbtotxt.c
         1296  +	$(LTLINK)-o $@ $(TOP)/tool/dbtotxt.c
  1293   1297   
  1294   1298   showdb$(TEXE):	$(TOP)/tool/showdb.c sqlite3.lo
  1295   1299   	$(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS)
  1296   1300   
  1297   1301   showstat4$(TEXE):	$(TOP)/tool/showstat4.c sqlite3.lo
  1298   1302   	$(LTLINK) -o $@ $(TOP)/tool/showstat4.c sqlite3.lo $(TLIBS)
  1299   1303   

Changes to Makefile.msc.

  2421   2421   
  2422   2422   testloadext.lo:	$(TOP)\src\test_loadext.c $(SQLITE3H)
  2423   2423   	$(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c
  2424   2424   
  2425   2425   testloadext.dll:	testloadext.lo
  2426   2426   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo
  2427   2427   
         2428  +dbtotxt.exe:	$(TOP)\tool\dbtotxt.c
         2429  +	$(LTLINK) $(NO_WARN)	$(TOP)\tool\dbtotxt.c /link $(LDFLAGS) $(LTLINKOPTS)
         2430  +
  2428   2431   showdb.exe:	$(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H)
  2429   2432   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2430   2433   		$(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2431   2434   
  2432   2435   showstat4.exe:	$(TOP)\tool\showstat4.c $(SQLITE3C) $(SQLITE3H)
  2433   2436   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2434   2437   		$(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

Changes to main.mk.

   972    972   TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO)
   973    973   $(TEST_EXTENSION): $(TOP)/src/test_loadext.c
   974    974   	$(MKSHLIB) $(TOP)/src/test_loadext.c -o $(TEST_EXTENSION)
   975    975   
   976    976   extensiontest: testfixture$(EXE) $(TEST_EXTENSION)
   977    977   	./testfixture$(EXE) $(TOP)/test/loadext.test
   978    978   
          979  +dbtotxt$(EXE):	$(TOP)/tool/dbtotxt.c
          980  +	$(TCC) -o dbtotxt$(EXE) $(TOP)/tool/dbtotxt.c
          981  +
   979    982   showdb$(EXE):	$(TOP)/tool/showdb.c sqlite3.o
   980    983   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showdb$(EXE) \
   981    984   		$(TOP)/tool/showdb.c sqlite3.o $(THREADLIB)
   982    985   
   983    986   showstat4$(EXE):	$(TOP)/tool/showstat4.c sqlite3.o
   984    987   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showstat4$(EXE) \
   985    988   		$(TOP)/tool/showstat4.c sqlite3.o $(THREADLIB)

Changes to src/shell.c.in.

  1062   1062   */
  1063   1063   #define SHELL_OPEN_UNSPEC      0      /* No open-mode specified */
  1064   1064   #define SHELL_OPEN_NORMAL      1      /* Normal database file */
  1065   1065   #define SHELL_OPEN_APPENDVFS   2      /* Use appendvfs */
  1066   1066   #define SHELL_OPEN_ZIPFILE     3      /* Use the zipfile virtual table */
  1067   1067   #define SHELL_OPEN_READONLY    4      /* Open a normal database read-only */
  1068   1068   #define SHELL_OPEN_DESERIALIZE 5      /* Open using sqlite3_deserialize() */
         1069  +#define SHELL_OPEN_HEXDB       6      /* Use "dbtotxt" output as data source */
  1069   1070   
  1070   1071   /* Allowed values for ShellState.eTraceType
  1071   1072   */
  1072   1073   #define SHELL_TRACE_PLAIN      0      /* Show input SQL text */
  1073   1074   #define SHELL_TRACE_EXPANDED   1      /* Show expanded SQL text */
  1074   1075   #define SHELL_TRACE_NORMALIZED 2      /* Show normalized SQL text */
  1075   1076   
................................................................................
  3440   3441     "       -e    Invoke system text editor",
  3441   3442     "       -x    Open in a spreadsheet",
  3442   3443     ".open ?OPTIONS? ?FILE?   Close existing database and reopen FILE",
  3443   3444     "     Options:",
  3444   3445     "        --append        Use appendvfs to append database to the end of FILE",
  3445   3446   #ifdef SQLITE_ENABLE_DESERIALIZE
  3446   3447     "        --deserialize   Load into memory useing sqlite3_deserialize()",
         3448  +  "        --hexdb         Load the output of \"dbtotxt\" as an in-memory database",
  3447   3449   #endif
  3448   3450     "        --new           Initialize FILE to an empty database",
  3449   3451     "        --readonly      Open FILE readonly",
  3450   3452     "        --zip           FILE is a ZIP archive",
  3451   3453     ".output ?FILE?           Send output to FILE or stdout if FILE is omitted",
  3452   3454     "     If FILE begins with '|' then open it as a pipe.",
  3453   3455     ".print STRING...         Print literal STRING",
................................................................................
  3719   3721         rc = SHELL_OPEN_ZIPFILE;
  3720   3722       }
  3721   3723     }
  3722   3724     fclose(f);
  3723   3725     return rc;  
  3724   3726   }
  3725   3727   
         3728  +#ifdef SQLITE_ENABLE_DESERIALIZE
         3729  +/*
         3730  +** Reconstruct an in-memory database using the output from the "dbtotxt"
         3731  +** program.  Read content from the file in p->zDbFilename.  If p->zDbFilename
         3732  +** is 0, then read from standard input.
         3733  +*/
         3734  +static unsigned char *readHexDb(ShellState *p, int *pnData){
         3735  +  unsigned char *a = 0;
         3736  +  int nLine = 1;
         3737  +  int n = 0;
         3738  +  int pgsz = 0;
         3739  +  int iOffset = 0;
         3740  +  int j, k;
         3741  +  int rc;
         3742  +  FILE *in;
         3743  +  unsigned char x[16];
         3744  +  char zLine[100];
         3745  +  if( p->zDbFilename ){
         3746  +    in = fopen(p->zDbFilename, "r");
         3747  +    if( in==0 ){
         3748  +      utf8_printf(stderr, "cannot open \"%s\" for reading\n", p->zDbFilename);
         3749  +      return 0;
         3750  +    }
         3751  +  }else{
         3752  +    in = stdin;
         3753  +  }
         3754  +  *pnData = 0;
         3755  +  if( fgets(zLine, sizeof(zLine), in)==0 ) goto readHexDb_error;
         3756  +  rc = sscanf(zLine, "| size %d pagesize %d", &n, &pgsz);
         3757  +  if( rc!=2 ) goto readHexDb_error;
         3758  +  if( n<=0 ) goto readHexDb_error;
         3759  +  a = sqlite3_malloc( n );
         3760  +  if( a==0 ){
         3761  +    utf8_printf(stderr, "Out of memory!\n");
         3762  +    goto readHexDb_error;
         3763  +  }
         3764  +  memset(a, 0, n);
         3765  +  if( pgsz<512 || pgsz>65536 || (pgsz & (pgsz-1))!=0 ){
         3766  +    utf8_printf(stderr, "invalid pagesize\n");
         3767  +    goto readHexDb_error;
         3768  +  }
         3769  +  for(nLine=2; fgets(zLine, sizeof(zLine), in)!=0; nLine++){
         3770  +    rc = sscanf(zLine, "| page %d offset %d", &j, &k);
         3771  +    if( rc==2 ){
         3772  +      iOffset = k;
         3773  +      continue;
         3774  +    }
         3775  +    if( strncmp(zLine, "| end ", 6)==0 ){
         3776  +      break;
         3777  +    }
         3778  +    rc = sscanf(zLine,"| %d: %hhx %hhx %hhx %hhx %hhx %hhx %hhx %hhx"
         3779  +                      "  %hhx %hhx %hhx %hhx %hhx %hhx %hhx %hhx",
         3780  +                &j, &x[0], &x[1], &x[2], &x[3], &x[4], &x[5], &x[6], &x[7],
         3781  +                &x[8], &x[9], &x[10], &x[11], &x[12], &x[13], &x[14], &x[15]);
         3782  +    if( rc==17 ){
         3783  +      k = iOffset+j;
         3784  +      if( k+16>n ){
         3785  +        utf8_printf(stderr, "continue exceeds file size\n");
         3786  +        goto readHexDb_error;
         3787  +      }
         3788  +      memcpy(a+k, x, 16);
         3789  +    }
         3790  +  }
         3791  +  *pnData = n;
         3792  +  if( in!=stdin ) fclose(in);
         3793  +  return a;
         3794  +
         3795  +readHexDb_error:
         3796  +  if( in!=stdin ){
         3797  +    fclose(in);
         3798  +  }else{
         3799  +    while( fgets(zLine, sizeof(zLine), in)!=0 ){
         3800  +      if(strncmp(zLine, "| end ", 6)==0 ) break;
         3801  +    }
         3802  +  }
         3803  +  sqlite3_free(a);
         3804  +  utf8_printf(stderr,"Error on line %d of --hexdb input\n", nLine);
         3805  +  return 0;
         3806  +}
         3807  +#endif /* SQLITE_ENABLE_DESERIALIZE */
         3808  +
  3726   3809   /* Flags for open_db().
  3727   3810   **
  3728   3811   ** The default behavior of open_db() is to exit(1) if the database fails to
  3729   3812   ** open.  The OPEN_DB_KEEPALIVE flag changes that so that it prints an error
  3730   3813   ** but still returns without calling exit.
  3731   3814   **
  3732   3815   ** The OPEN_DB_ZIPFILE flag causes open_db() to prefer to open files as a
................................................................................
  3752   3835       }
  3753   3836       switch( p->openMode ){
  3754   3837         case SHELL_OPEN_APPENDVFS: {
  3755   3838           sqlite3_open_v2(p->zDbFilename, &p->db, 
  3756   3839              SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, "apndvfs");
  3757   3840           break;
  3758   3841         }
         3842  +      case SHELL_OPEN_HEXDB:
  3759   3843         case SHELL_OPEN_DESERIALIZE: {
  3760   3844           sqlite3_open(0, &p->db);
  3761   3845           break;
  3762   3846         }
  3763   3847         case SHELL_OPEN_ZIPFILE: {
  3764   3848           sqlite3_open(":memory:", &p->db);
  3765   3849           break;
................................................................................
  3806   3890       if( p->openMode==SHELL_OPEN_ZIPFILE ){
  3807   3891         char *zSql = sqlite3_mprintf(
  3808   3892            "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename);
  3809   3893         sqlite3_exec(p->db, zSql, 0, 0, 0);
  3810   3894         sqlite3_free(zSql);
  3811   3895       }
  3812   3896   #ifdef SQLITE_ENABLE_DESERIALIZE
  3813         -    else if( p->openMode==SHELL_OPEN_DESERIALIZE ){
         3897  +    else
         3898  +    if( p->openMode==SHELL_OPEN_DESERIALIZE || p->openMode==SHELL_OPEN_HEXDB ){
  3814   3899         int nData = 0;
  3815         -      unsigned char *aData = (unsigned char*)readFile(p->zDbFilename, &nData);
         3900  +      unsigned char *aData;
         3901  +      if( p->openMode==SHELL_OPEN_DESERIALIZE ){
         3902  +        aData = (unsigned char*)readFile(p->zDbFilename, &nData);
         3903  +      }else{
         3904  +        aData = readHexDb(p, &nData);
         3905  +        if( aData==0 ){
         3906  +          utf8_printf(stderr, "Error in hexdb input\n");
         3907  +          return;
         3908  +        }
         3909  +      }
  3816   3910         int rc = sqlite3_deserialize(p->db, "main", aData, nData, nData,
  3817   3911                      SQLITE_DESERIALIZE_RESIZEABLE |
  3818   3912                      SQLITE_DESERIALIZE_FREEONCLOSE);
  3819   3913         if( rc ){
  3820   3914           utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc);
  3821   3915         }
  3822   3916       }
................................................................................
  6745   6839         }else if( optionMatch(z, "append") ){
  6746   6840           p->openMode = SHELL_OPEN_APPENDVFS;
  6747   6841         }else if( optionMatch(z, "readonly") ){
  6748   6842           p->openMode = SHELL_OPEN_READONLY;
  6749   6843   #ifdef SQLITE_ENABLE_DESERIALIZE
  6750   6844         }else if( optionMatch(z, "deserialize") ){
  6751   6845           p->openMode = SHELL_OPEN_DESERIALIZE;
  6752         -#endif
         6846  +      }else if( optionMatch(z, "hexdb") ){
         6847  +        p->openMode = SHELL_OPEN_HEXDB;
         6848  +#endif /* SQLITE_ENABLE_DESERIALIZE */
  6753   6849         }else if( z[0]=='-' ){
  6754   6850           utf8_printf(stderr, "unknown option: %s\n", z);
  6755   6851           rc = 1;
  6756   6852           goto meta_command_exit;
  6757   6853         }
  6758   6854       }
  6759   6855       /* If a filename is specified, try to open it first */
  6760   6856       zNewFilename = nArg>iName ? sqlite3_mprintf("%s", azArg[iName]) : 0;
  6761         -    if( zNewFilename ){
         6857  +    if( zNewFilename || p->openMode==SHELL_OPEN_HEXDB ){
  6762   6858         if( newFlag ) shellDeleteFile(zNewFilename);
  6763   6859         p->zDbFilename = zNewFilename;
  6764   6860         open_db(p, OPEN_DB_KEEPALIVE);
  6765   6861         if( p->db==0 ){
  6766   6862           utf8_printf(stderr, "Error: cannot open '%s'\n", zNewFilename);
  6767   6863           sqlite3_free(zNewFilename);
  6768   6864         }else{

Added tool/dbtotxt.c.

            1  +/*
            2  +** Copyright 2008 D. Richard Hipp and Hipp, Wyrick & Company, Inc.
            3  +** All Rights Reserved
            4  +**
            5  +******************************************************************************
            6  +**
            7  +** This file implements a stand-alone utility program that converts
            8  +** a binary file (usually an SQLite database) into a text format that
            9  +** is compact and friendly to human-readers.
           10  +**
           11  +** Usage:
           12  +**
           13  +**         dbtotxt [--pagesize N] FILENAME
           14  +**
           15  +** The translation of the database appears on standard output.  If the
           16  +** --pagesize command-line option is omitted, then the page size is taken
           17  +** from the database header.
           18  +**
           19  +** Compactness is achieved by suppressing lines of all zero bytes.  This
           20  +** works well at compressing test databases that are mostly empty.  But
           21  +** the output will probably be lengthy for a real database containing lots
           22  +** of real content.  For maximum compactness, it is suggested that test
           23  +** databases be constructed with "zeroblob()" rather than "randomblob()"
           24  +** used for filler content and with "PRAGMA secure_delete=ON" selected to
           25  +** zero-out deleted content.
           26  +*/
           27  +#include <stdio.h>
           28  +#include <string.h>
           29  +#include <stdlib.h>
           30  + 
           31  +/* Return true if the line is all zeros */
           32  +static int allZero(unsigned char *aLine){
           33  +  int i;
           34  +  for(i=0; i<16 && aLine[i]==0; i++){}
           35  +  return i==16;
           36  +}
           37  +
           38  +int main(int argc, char **argv){
           39  +  int pgsz = 0;               /* page size */
           40  +  long szFile;                /* Size of the input file in bytes */
           41  +  FILE *in;                   /* Input file */
           42  +  int i, j;                   /* Loop counters */
           43  +  int nErr = 0;               /* Number of errors */
           44  +  const char *zInputFile = 0; /* Name of the input file */
           45  +  const char *zBaseName = 0;  /* Base name of the file */
           46  +  int lastPage = 0;           /* Last page number shown */
           47  +  int iPage;                  /* Current page number */
           48  +  unsigned char aLine[16];    /* A single line of the file */
           49  +  unsigned char aHdr[100];    /* File header */
           50  +  for(i=1; i<argc; i++){
           51  +    if( argv[i][0]=='-' ){
           52  +      const char *z = argv[i];
           53  +      z++;
           54  +      if( z[0]=='-' ) z++;
           55  +      if( strcmp(z,"pagesize")==0 ){
           56  +        i++;
           57  +        pgsz = atoi(argv[i]);
           58  +        if( pgsz<512 || pgsz>65536 || (pgsz&(pgsz-1))!=0 ){
           59  +          fprintf(stderr, "Page size must be a power of two between"
           60  +                          " 512 and 65536.\n");
           61  +          nErr++;
           62  +        }
           63  +        continue;
           64  +      }
           65  +      fprintf(stderr, "Unknown option: %s\n", argv[i]);
           66  +      nErr++;
           67  +    }else if( zInputFile ){
           68  +      fprintf(stderr, "Already using a different input file: [%s]\n", argv[i]);
           69  +      nErr++;
           70  +    }else{
           71  +      zInputFile = argv[i];
           72  +    }
           73  +  }
           74  +  if( zInputFile==0 ){
           75  +    fprintf(stderr, "No input file specified.\n");
           76  +    nErr++;
           77  +  }
           78  +  if( nErr ){
           79  +    fprintf(stderr, "Usage: %s [--pagesize N] FILENAME\n", argv[0]);
           80  +    exit(1);
           81  +  }
           82  +  in = fopen(zInputFile, "rb");
           83  +  if( in==0 ){
           84  +    fprintf(stderr, "Cannot open input file [%s]\n", zInputFile);
           85  +    exit(1);
           86  +  }
           87  +  fseek(in, 0, SEEK_END);
           88  +  szFile = ftell(in);
           89  +  rewind(in);
           90  +  if( szFile<512 ){
           91  +    fprintf(stderr, "File too short. Minimum size is 512 bytes.\n");
           92  +    exit(1);
           93  +  }
           94  +  if( fread(aHdr, 100, 1, in)!=1 ){
           95  +    fprintf(stderr, "Cannot read file header\n");
           96  +    exit(1);
           97  +  }
           98  +  rewind(in);
           99  +  if( pgsz==0 ){
          100  +    pgsz = (aHdr[16]<<8) | aHdr[17];
          101  +    if( pgsz==1 ) pgsz = 65536;
          102  +    if( pgsz<512 || (pgsz&(pgsz-1))!=0 ){
          103  +      fprintf(stderr, "Invalid page size in header: %d\n", pgsz);
          104  +      exit(1);
          105  +    }
          106  +  }
          107  +  zBaseName = zInputFile;
          108  +  for(i=0; zInputFile[i]; i++){
          109  +    if( zInputFile[i]=='/' && zInputFile[i+1]!=0 ) zBaseName = zInputFile+1;
          110  +  }
          111  +  printf("| size %d pagesize %d filename %s\n",(int)szFile,pgsz,zBaseName);
          112  +  for(i=0; i<szFile; i+=16){
          113  +    int got = (int)fread(aLine, 1, 16, in);
          114  +    if( got!=16 ){
          115  +      static int once = 1;
          116  +      if( once ){
          117  +        fprintf(stderr, "Could not read input file starting at byte %d\n",
          118  +                         i+got);
          119  +      }
          120  +      memset(aLine+got, 0, 16-got);
          121  +    }
          122  +    if( allZero(aLine) ) continue;
          123  +    iPage = i/pgsz + 1;
          124  +    if( lastPage!=iPage ){
          125  +      printf("| page %d offset %d\n", iPage, (iPage-1)*pgsz);
          126  +      lastPage = iPage;
          127  +    }
          128  +    printf("|  %5d:", i-(iPage-1)*pgsz);
          129  +    for(j=0; j<16; j++) printf(" %02x", aLine[j]);
          130  +    printf("   ");
          131  +    for(j=0; j<16; j++){
          132  +      char c = aLine[j];
          133  +      fputc(c>=0x20 && c<=0x7e ? c : '.', stdout);
          134  +    }
          135  +    fputc('\n', stdout);
          136  +  }
          137  +  fclose(in);
          138  +  printf("| end %s\n", zBaseName);
          139  +  return 0;
          140  +}

Added tool/dbtotxt.md.

            1  +<h1 align="center">The dbtotxt Tool</h1>
            2  +
            3  +The dbtotxt utility program reads an SQLite database file and writes its
            4  +raw binary content to screen as a hex dump for testing and debugging
            5  +purposes.
            6  +
            7  +The hex-dump output is formatted in such a way as to be easily readable
            8  +both by humans and by software.  The dbtotxt utility has long been a part
            9  +of the TH3 test suite.  The output of dbtotxt can be embedded in TH3 test
           10  +scripts and used to generate very specific database files, perhaps with
           11  +deliberately introduced corruption.  The cov1/corrupt*.test modules in
           12  +TH3 make extensive use of dbtotxt.
           13  +
           14  +More recently (2018-12-13) the dbtotxt utility has been added to the SQLite 
           15  +core and the command-line shell (CLI) has been augmented to be able to read 
           16  +dbtotxt output.  The CLI dot-command is:
           17  +
           18  +>     .open --hexdb  ?OPTIONAL-FILENAME?
           19  +
           20  +If the OPTIONAL-FILENAME is included, then content is read from that file.
           21  +If OPTIONAL-FILENAME is omitted, then the text is taken from the input stream,
           22  +terminated by the "| end" line of the dbtotxt text.  This allows small test
           23  +databases to be embedded directly in scripts.  Consider this example:
           24  +
           25  +>
           26  +    .open --hexdb
           27  +    | size 8192 pagesize 4096 filename x9.db
           28  +    | page 1 offset 0
           29  +    |      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
           30  +    |     16: 10 00 01 01 00 40 20 20 00 00 00 04 00 00 00 02   .....@  ........
           31  +    |     32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04   ................
           32  +    |     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
           33  +    |     80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04   ................
           34  +    |     96: 00 2e 30 38 0d 00 00 00 01 0f c0 00 0f c0 00 00   ..08............
           35  +    |   4032: 3e 01 06 17 11 11 01 69 74 61 62 6c 65 74 31 74   >......itablet1t
           36  +    |   4048: 31 02 43 52 45 41 54 45 20 54 41 42 4c 45 20 74   1.CREATE TABLE t
           37  +    |   4064: 31 28 78 2c 79 20 44 45 46 41 55 4c 54 20 78 27   1(x,y DEFAULT x'
           38  +    |   4080: 66 66 27 2c 7a 20 44 45 46 41 55 4c 54 20 30 29   ff',z DEFAULT 0)
           39  +    | page 2 offset 4096
           40  +    |      0: 0d 08 14 00 04 00 10 00 0e 05 0a 0f 04 15 00 10   ................
           41  +    |     16: 88 02 03 05 90 04 0e 08 00 00 00 00 00 00 00 00   ................
           42  +    |   1040: 00 00 00 00 ff 87 7c 02 05 8f 78 0e 08 00 00 00   ......|...x.....
           43  +    |   2064: 00 00 00 ff 0c 0a 01 fb 00 00 00 00 00 00 00 00   ................
           44  +    |   2560: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 83   ................
           45  +    |   2576: 78 01 05 87 70 0e 08 00 00 00 00 00 00 00 00 00   x...p...........
           46  +    |   3072: 00 00 00 00 00 00 00 00 00 ff 00 00 01 fb 00 00   ................
           47  +    |   3584: 00 00 00 00 00 83 78 00 05 87 70 0e 08 00 00 00   ......x...p.....
           48  +    |   4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff   ................
           49  +    | end x9.db
           50  +    SELECT rowid FROM t1;
           51  +    PRAGMA integrity_check;
           52  +
           53  +You can run this script to see that the database file is correctly decoded 
           54  +and loaded.  Furthermore, you can make subtle corruptions to the input
           55  +database simply by editing the hexadecimal description, then rerun the
           56  +script to verify that SQLite correctly handles the corruption.