/ Check-in [4a5f6f1f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:The "make fuzztest" target now uses fuzzcheck instead of fuzzershell.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | test-using-fuzzcheck
Files: files | file ages | folders
SHA1: 4a5f6f1f0128657fd8d4d99d0682edd5bac2a19e
User & Date: drh 2015-05-26 18:15:08
Context
2015-05-26
18:58
Fix a one-byte buffer overread that may follow a syntax error while preparing an SQL statement. check-in: 07500393 user: dan tags: test-using-fuzzcheck
18:58
Fix fuzzcheck so that it responds correctly to the TEST_FAILURE environment variable. check-in: 76770c9e user: drh tags: test-using-fuzzcheck
18:15
The "make fuzztest" target now uses fuzzcheck instead of fuzzershell. check-in: 4a5f6f1f user: drh tags: test-using-fuzzcheck
17:57
Enhance fuzzcheck with the ability to store descriptions in each source database and to run multiple source databases in a single invocation. check-in: 193364c8 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   518    518   #
   519    519   TESTPROGS = \
   520    520     testfixture$(TEXE) \
   521    521     sqlite3$(TEXE) \
   522    522     sqlite3_analyzer$(TEXE) \
   523    523     sqldiff$(TEXE)
   524    524   
          525  +# Databases containing fuzzer test cases
          526  +#
          527  +FUZZDATA = \
          528  +  $(TOP)/test/fuzzdata1.db \
          529  +  $(TOP)/test/fuzzdata2.db \
          530  +  $(TOP)/test/fuzzdata3.db
   525    531   
   526    532   # This is the default Makefile target.  The objects listed here
   527    533   # are what get build when you type just "make" with no arguments.
   528    534   #
   529    535   all:	sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la)
   530    536   
   531    537   Makefile: $(TOP)/Makefile.in
................................................................................
   971    977   		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)
   972    978   
   973    979   # A very detailed test running most or all test cases
   974    980   fulltest:	$(TESTPROGS) fuzztest
   975    981   	./testfixture$(TEXE) $(TOP)/test/all.test
   976    982   
   977    983   # Really really long testing
   978         -soaktest:	$(TESTPROGS) fuzzoomtest
          984  +soaktest:	$(TESTPROGS)
   979    985   	./testfixture$(TEXE) $(TOP)/test/all.test -soak=1
   980    986   
   981    987   # Do extra testing but not everything.
   982    988   fulltestonly:	$(TESTPROGS)
   983    989   	./testfixture$(TEXE) $(TOP)/test/full.test
   984    990   
   985    991   # Fuzz testing
   986         -fuzztest:	fuzzershell$(TEXE)
   987         -	./fuzzershell$(TEXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt
   988         -
   989         -fuzzoomtest:	fuzzershell$(TEXE)
   990         -	./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt --oom
          992  +fuzztest:	fuzzcheck$(TEXE)
          993  +	./fuzzcheck$(TEXE) $(FUZZDATA)
   991    994   
   992    995   # This is the common case.  Run many tests but not those that take
   993    996   # a really long time.
   994    997   #
   995    998   test:	$(TESTPROGS) fuzztest
   996    999   	./testfixture$(TEXE) $(TOP)/test/veryquick.test
   997   1000   
   998   1001   # Run a test using valgrind.  This can take a really long time
   999   1002   # because valgrind is so much slower than a native machine.
  1000   1003   #
  1001         -valgrindtest:	$(TESTPROGS) fuzzershell$(TEXE)
  1002         -	valgrind -v ./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt
         1004  +valgrindtest:	$(TESTPROGS) fuzzcheck$(TEXE)
         1005  +	valgrind -v ./fuzzcheck$(TEXE) $(FUZZDATA)
  1003   1006   	OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind
  1004   1007   
  1005   1008   # A very fast test that checks basic sanity.  The name comes from
  1006   1009   # the 60s-era electronics testing:  "Turn it on and see if smoke
  1007   1010   # comes out."
  1008   1011   #
  1009         -smoketest:	$(TESTPROGS) fuzzershell$(TEXE)
         1012  +smoketest:	$(TESTPROGS) fuzzcheck$(TEXE)
  1010   1013   	./testfixture$(TEXE) $(TOP)/test/main.test
  1011   1014   
  1012   1015   sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
  1013   1016   	echo "#define TCLSH 2" > $@
  1014   1017   	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
  1015   1018   	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
  1016   1019   	echo "static const char *tclsh_main_loop(void){" >> $@

Changes to Makefile.msc.

  1186   1186   #
  1187   1187   TESTPROGS = \
  1188   1188     testfixture.exe \
  1189   1189     sqlite3.exe \
  1190   1190     sqlite3_analyzer.exe \
  1191   1191     sqldiff.exe
  1192   1192   
         1193  +# Databases containing fuzzer test cases
         1194  +#
         1195  +FUZZDATA = \
         1196  +  $(TOP)\test\fuzzdata1.db \
         1197  +  $(TOP)\test\fuzzdata2.db \
         1198  +  $(TOP)\test\fuzzdata3.db
         1199  +
  1193   1200   
  1194   1201   # This is the default Makefile target.  The objects listed here
  1195   1202   # are what get build when you type just "make" with no arguments.
  1196   1203   #
  1197   1204   all:	dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib
  1198   1205   
  1199   1206   libsqlite3.lib:	$(LIBOBJ)
................................................................................
  1656   1663   
  1657   1664   extensiontest: testfixture.exe testloadext.dll
  1658   1665   	.\testfixture.exe $(TOP)\test\loadext.test
  1659   1666   
  1660   1667   fulltest:	$(TESTPROGS) fuzztest
  1661   1668   	.\testfixture.exe $(TOP)\test\all.test
  1662   1669   
  1663         -soaktest:	$(TESTPROGS) fuzzoomtest
         1670  +soaktest:	$(TESTPROGS)
  1664   1671   	.\testfixture.exe $(TOP)\test\all.test -soak=1
  1665   1672   
  1666   1673   fulltestonly:	$(TESTPROGS) fuzztest
  1667   1674   	.\testfixture.exe $(TOP)\test\full.test
  1668   1675   
  1669   1676   queryplantest:	testfixture.exe sqlite3.exe
  1670   1677   	.\testfixture.exe $(TOP)\test\permutations.test queryplanner
  1671   1678   
  1672         -fuzztest:	fuzzershell.exe
  1673         -	.\fuzzershell.exe $(TOP)\test\fuzzdata1.txt $(TOP)\test\fuzzdata2.txt
  1674         -
  1675         -fuzzoomtest:	fuzzershell.exe
  1676         -	.\fuzzershell.exe -f $(TOP)\test\fuzzdata1.txt --oom
         1679  +fuzztest:	fuzzcheck.exe
         1680  +	.\fuzzcheck.exe $(FUZZDATA)
  1677   1681   
  1678   1682   test:	$(TESTPROGS) fuzztest
  1679   1683   	.\testfixture.exe $(TOP)\test\veryquick.test
  1680   1684   
  1681         -smoketest:	$(TESTPROGS) fuzzershell.exe
         1685  +smoketest:	$(TESTPROGS)
  1682   1686   	.\testfixture.exe $(TOP)\test\main.test
  1683   1687   
  1684   1688   sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
  1685   1689   	echo #define TCLSH 2 > $@
  1686   1690   	echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@
  1687   1691   	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@
  1688   1692   	echo static const char *tclsh_main_loop(void){ >> $@

Changes to main.mk.

   398    398   # executables needed for testing
   399    399   #
   400    400   TESTPROGS = \
   401    401     testfixture$(EXE) \
   402    402     sqlite3$(EXE) \
   403    403     sqlite3_analyzer$(EXE) \
   404    404     sqldiff$(EXE)
          405  +
          406  +# Databases containing fuzzer test cases
          407  +#
          408  +FUZZDATA = \
          409  +  $(TOP)/test/fuzzdata1.db \
          410  +  $(TOP)/test/fuzzdata2.db \
          411  +  $(TOP)/test/fuzzdata3.db
   405    412   
   406    413   # This is the default Makefile target.  The objects listed here
   407    414   # are what get build when you type just "make" with no arguments.
   408    415   #
   409    416   all:	sqlite3.h libsqlite3.a sqlite3$(EXE)
   410    417   
   411    418   libsqlite3.a:	$(LIBOBJ)
................................................................................
   653    660   	-DSQLITE_ENABLE_FTS3=1                                               \
   654    661   		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c       \
   655    662   		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)
   656    663   
   657    664   fulltest:	$(TESTPROGS) fuzztest
   658    665   	./testfixture$(EXE) $(TOP)/test/all.test
   659    666   
   660         -soaktest:	$(TESTPROGS) fuzzoomtest
          667  +soaktest:	$(TESTPROGS)
   661    668   	./testfixture$(EXE) $(TOP)/test/all.test -soak=1
   662    669   
   663    670   fulltestonly:	$(TESTPROGS) fuzztest
   664    671   	./testfixture$(EXE) $(TOP)/test/full.test
   665    672   
   666    673   queryplantest:	testfixture$(EXE) sqlite3$(EXE)
   667    674   	./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner
   668    675   
   669         -fuzztest:	fuzzershell$(EXE)
   670         -	./fuzzershell$(EXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt
   671         -
   672         -fuzzoomtest:	fuzzershell$(EXE)
   673         -	./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt --oom
          676  +fuzztest:	fuzzcheck$(EXE) $(FUZZDATA)
          677  +	./fuzzcheck$(EXE) $(FUZZDATA)
   674    678   
   675    679   test:	$(TESTPROGS) fuzztest
   676    680   	./testfixture$(EXE) $(TOP)/test/veryquick.test
   677    681   
   678    682   # Run a test using valgrind.  This can take a really long time
   679    683   # because valgrind is so much slower than a native machine.
   680    684   #
   681         -valgrindtest:	$(TESTPROGS) fuzzershell$(EXE)
   682         -	valgrind -v ./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt
          685  +valgrindtest:	$(TESTPROGS) fuzzcheck$(EXE) $(FUZZDATA)
          686  +	valgrind -v ./fuzzcheck$(EXE) $(FUZZDATA)
   683    687   	OMIT_MISUSE=1 valgrind -v ./testfixture$(EXE) $(TOP)/test/permutations.test valgrind
   684    688   
   685    689   # A very fast test that checks basic sanity.  The name comes from
   686    690   # the 60s-era electronics testing:  "Turn it on and see if smoke
   687    691   # comes out."
   688    692   #
   689         -smoketest:	$(TESTPROGS) fuzzershell$(EXE)
          693  +smoketest:	$(TESTPROGS) fuzzcheck$(EXE)
   690    694   	./testfixture$(EXE) $(TOP)/test/main.test
   691    695   
   692    696   # The next two rules are used to support the "threadtest" target. Building
   693    697   # threadtest runs a few thread-safety tests that are implemented in C. This
   694    698   # target is invoked by the releasetest.tcl script.
   695    699   # 
   696    700   THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \

Added test/fuzzdata1.db.

cannot compute difference between binary files

Deleted test/fuzzdata1.txt.

cannot compute difference between binary files

Added test/fuzzdata2.db.

cannot compute difference between binary files

Deleted test/fuzzdata2.txt.

cannot compute difference between binary files

Added test/fuzzdata3.db.

cannot compute difference between binary files

Deleted test/mkfuzzdata1.tcl.

     1         -#!/usr/bin/tclsh
     2         -#
     3         -# Run this script in order to rebuild the fuzzdata1.txt file containing
     4         -# fuzzer data for the fuzzershell utility that is create by afl-fuzz.
     5         -#
     6         -# This script gathers all of the test cases identified by afl-fuzz and
     7         -# runs afl-cmin and afl-tmin over them all to try to generate a mimimum
     8         -# set of tests that cover all observed behavior.
     9         -# 
    10         -# Options:
    11         -#
    12         -#    --afl-bin DIR1             DIR1 contains the AFL binaries
    13         -#    --fuzzershell PATH         Full pathname of instrumented fuzzershell
    14         -#    --afl-data DIR3            DIR3 is the "-o" directory from afl-fuzz
    15         -#    -o FILE                    Write results into FILE
    16         -#
    17         -set AFLBIN {}
    18         -set FUZZERSHELL {}
    19         -set AFLDATA {}
    20         -set OUTFILE {}
    21         -
    22         -proc usage {} {
    23         -  puts stderr "Usage: $::argv0 --afl-bin DIR --fuzzershell PATH\
    24         -                  --afl-data DIR -o FILE"
    25         -  exit 1
    26         -}
    27         -proc cmdlineerr {msg} {
    28         -  puts stderr $msg
    29         -  usage
    30         -}
    31         -
    32         -for {set i 0} {$i<[llength $argv]} {incr i} {
    33         -  set x [lindex $argv $i]
    34         -  if {[string index $x 0]!="-"} {cmdlineerr "illegal argument: $x"}
    35         -  set x [string trimleft $x -]
    36         -  incr i
    37         -  if {$i>=[llength $argv]} {cmdlineerr "no argument on --$x"}
    38         -  set a [lindex $argv $i]
    39         -  switch -- $x {
    40         -     afl-bin {set AFLBIN $a}
    41         -     afl-data {set AFLDATA $a}
    42         -     fuzzershell {set FUZZERSHELL $a}
    43         -     o {set OUTFILE $a}
    44         -     default {cmdlineerr "unknown option: --$x"}
    45         -  }
    46         -}
    47         -proc checkarg {varname option} {
    48         -  set val [set ::$varname]
    49         -  if {$val==""} {cmdlineerr "required option missing: --$option"}
    50         -}
    51         -checkarg AFLBIN afl-bin
    52         -checkarg AFLDATA afl-data
    53         -checkarg FUZZERSHELL fuzzershell
    54         -checkarg OUTFILE o
    55         -proc checkexec {x} {
    56         -  if {![file exec $x]} {cmdlineerr "cannot find $x"}
    57         -}
    58         -checkexec $AFLBIN/afl-cmin
    59         -checkexec $AFLBIN/afl-tmin
    60         -checkexec $FUZZERSHELL
    61         -proc checkdir {x} {
    62         -  if {![file isdir $x]} {cmdlineerr "no such directory: $x"}
    63         -}
    64         -checkdir $AFLDATA/queue
    65         -
    66         -proc progress {msg} {
    67         -  puts "******** $msg"
    68         -  flush stdout
    69         -}
    70         -progress "mkdir tmp1 tmp2"
    71         -file mkdir tmp1 tmp2
    72         -progress "copying test cases from $AFLDATA into tmp1..."
    73         -set n 0
    74         -foreach file [glob -nocomplain $AFLDATA/queue/id:*] {
    75         -  incr n
    76         -  file copy $file tmp1/$n
    77         -}
    78         -foreach file [glob -nocomplain $AFLDATA/crash*/id:*] {
    79         -  incr n
    80         -  file copy $file tmp1/$n
    81         -}
    82         -progress "total $n files copied."
    83         -progress "running: $AFLBIN/afl-cmin -i tmp1 -o tmp2 $FUZZERSHELL"
    84         -exec $AFLBIN/afl-cmin -i tmp1 -o tmp2 $FUZZERSHELL >&@ stdout
    85         -progress "afl-cmin complete."
    86         -#
    87         -# Experiments show that running afl-tmin is too slow for this application.
    88         -# And it doesn't really make the test cases that much smaller.  So let's
    89         -# just skip it.
    90         -#
    91         -# foreach file [glob tmp2/*] {
    92         -#   progress "$AFLBIN/afl-tmin -i $file -o tmp3/[file tail $file] $FUZZERSHELL"
    93         -#   exec $AFLBIN/afl-tmin -i $file -o tmp3/[file tail $file] \
    94         -#       $FUZZERSHELL >&@ stdout
    95         -# }
    96         -progress "generating final output into $OUTFILE"
    97         -set out [open $OUTFILE wb]
    98         -puts $out "# Test data for use with fuzzershell.  Automatically
    99         -# generated using $argv0.  This file contains binary data
   100         -#"
   101         -set n 0
   102         -foreach file [glob tmp2/*] {
   103         -  incr n
   104         -  puts -nonewline $out "/****<$n>****/"
   105         -  set in [open $file rb]
   106         -  puts -nonewline $out [read $in]
   107         -  close $in
   108         -}
   109         -close $out
   110         -progress "done.  $n test cases written to $OUTFILE"
   111         -progress "clean-up..."
   112         -file delete -force tmp1
   113         -progress "culled test cases left in the tmp2 directory"